Make driver find all prelude files. Add build rule for //examples:sieve. (#3895)

The driver now looks for all files under core/prelude/ and considers
them all to be part of the prelude. The driver also now only processes
the prelude in `--phase=check` and later, when it would actually be
imported.

With that done, add a simple `carbon_binary` build rule and use it to
build the example in `//examples`. This should cause the example to be
built as part of our continuous integration.

---------

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

bazel/carbon_rules/BUILD

@@ -0,0 +1,3 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

bazel/carbon_rules/defs.bzl

@@ -0,0 +1,47 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+"""Provides rules for building Carbon files using the toolchain."""
+
+load("@bazel_skylib//rules:run_binary.bzl", "run_binary")
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_import")
+
+def carbon_binary(name, srcs):
+    """Compiles a Carbon binary.
+
+    Args:
+      name: The name of the build target.
+      srcs: List of Carbon source files to compile.
+    """
+    for src in srcs:
+        # Build each source file. For now, we pass all sources to each compile
+        # because we don't have visibility into dependencies and have no way to
+        # specify multiple output files. Object code for each input is written
+        # into the output file in turn, so the final carbon source file
+        # specified ends up determining the contents of the object file.
+        #
+        # TODO: This is a hack; replace with something better once the toolchain
+        # supports doing so.
+        out = src + ".o"
+        srcs_reordered = [s for s in srcs if s != src] + [src]
+        run_binary(
+            name = src + ".compile",
+            tool = "//toolchain/driver:carbon",
+            args = (["compile"] +
+                    ["$(location %s)" % s for s in srcs_reordered] +
+                    ["--output=$(location %s)" % out]),
+            srcs = srcs,
+            outs = [out],
+        )
+    cc_import(
+        name = "%s.objs" % name,
+        objects = [src + ".compile" for src in srcs],
+    )
+
+    # For now, we assume that the prelude doesn't produce any necessary object
+    # code, and don't include the .o files for //core/prelude... in the final
+    # linked binary.
+    #
+    # TODO: This will need to be revisited eventually.
+    cc_binary(name = name, deps = ["%s.objs" % name])

core/BUILD

@@ -2,8 +2,9 @@
 # Exceptions. See /LICENSE for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+# The prelude, and all of its dependencies.
 filegroup(
-    name = "core",
-    data = glob(["**/*.carbon"]),
+    name = "prelude",
+    data = ["prelude.carbon"] + glob(["prelude/**/*.carbon"]),
     visibility = ["//visibility:public"],
 )

core/prelude.carbon

@@ -6,6 +6,9 @@
 
 package Core library "prelude" api;
 
+import library "prelude/operators";
+import library "prelude/types";
+
 // TODO: Uncomment once name deduplication works.
 // TODO: These are here for name lookup. Add a way to export import and move them out.
 // fn Int32() -> type = "int.make_type_32";

core/prelude/types.carbon

@@ -0,0 +1,12 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+package Core library "prelude/types" api;
+
+// TODO: Add a mechanism to re-export the names declared here.
+
+// TODO: Start importing `prelude/types/i32` here once we stop eagerly importing
+// all `impl`s from all imported files. Currently, this introduces too much
+// noise in the toolchain tests.
+// import library "prelude/types/i32";

core/prelude/types/i32.carbon

@@ -12,7 +12,7 @@ import library "prelude/operators/bitwise";
 import library "prelude/operators/comparison";
 
 impl i32 as Add {
-  fn Op[self: Self](other: Self) -> Self = "int.add";
+  fn Op[self: Self](other: Self) -> Self = "int.sadd";
 }
 impl i32 as AddAssign {
   fn Op[addr self: Self*](other: Self) {
@@ -63,7 +63,7 @@ impl i32 as BitXorAssign {
 }
 
 impl i32 as Div {
-  fn Op[self: Self](other: Self) -> Self = "int.div";
+  fn Op[self: Self](other: Self) -> Self = "int.sdiv";
 }
 impl i32 as DivAssign {
   fn Op[addr self: Self*](other: Self) {
@@ -88,7 +88,7 @@ impl i32 as LeftShiftAssign {
 }
 
 impl i32 as Mod {
-  fn Op[self: Self](other: Self) -> Self = "int.mod";
+  fn Op[self: Self](other: Self) -> Self = "int.smod";
 }
 impl i32 as ModAssign {
   fn Op[addr self: Self*](other: Self) {
@@ -98,7 +98,7 @@ impl i32 as ModAssign {
 }
 
 impl i32 as Mul {
-  fn Op[self: Self](other: Self) -> Self = "int.mul";
+  fn Op[self: Self](other: Self) -> Self = "int.smul";
 }
 impl i32 as MulAssign {
   fn Op[addr self: Self*](other: Self) {
@@ -108,7 +108,7 @@ impl i32 as MulAssign {
 }
 
 impl i32 as Negate {
-  fn Op[self: Self]() -> Self = "int.negate";
+  fn Op[self: Self]() -> Self = "int.snegate";
 }
 
 impl i32 as Ordered {
@@ -130,7 +130,7 @@ impl i32 as RightShiftAssign {
 }
 
 impl i32 as Sub {
-  fn Op[self: Self](other: Self) -> Self = "int.sub";
+  fn Op[self: Self](other: Self) -> Self = "int.ssub";
 }
 impl i32 as SubAssign {
   fn Op[addr self: Self*](other: Self) {

examples/BUILD

@@ -0,0 +1,10 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+load("//bazel/carbon_rules:defs.bzl", "carbon_binary")
+
+carbon_binary(
+    name = "sieve",
+    srcs = ["sieve.carbon"],
+)

examples/sieve.carbon

@@ -8,11 +8,12 @@ import Core library "prelude/operators/arithmetic";
 import Core library "prelude/operators/comparison";
 
 // TODO: Copied from core/prelude/types/i32.carbon.
-// Remove the following, once we do cross-file impl lookup.
-// import Core library "prelude/types/i32";
+// Because we don't deduplicate interfaces, the implementations in that file are
+// treated as implementing a different interface from the one we import above.
+// Remove the following, once we deduplicate interfaces.
 
 impl i32 as Core.Add {
-  fn Op[self: Self](other: Self) -> Self = "int.add";
+  fn Op[self: Self](other: Self) -> Self = "int.sadd";
 }
 impl i32 as Core.AddAssign {
   fn Op[addr self: Self*](other: Self) {
@@ -26,7 +27,7 @@ impl i32 as Core.Inc {
 }
 
 impl i32 as Core.Div {
-  fn Op[self: Self](other: Self) -> Self = "int.div";
+  fn Op[self: Self](other: Self) -> Self = "int.sdiv";
 }
 impl i32 as Core.DivAssign {
   fn Op[addr self: Self*](other: Self) {
@@ -40,7 +41,7 @@ impl i32 as Core.Eq {
 }
 
 impl i32 as Core.Mod {
-  fn Op[self: Self](other: Self) -> Self = "int.mod";
+  fn Op[self: Self](other: Self) -> Self = "int.smod";
 }
 impl i32 as Core.ModAssign {
   fn Op[addr self: Self*](other: Self) {
@@ -49,7 +50,7 @@ impl i32 as Core.ModAssign {
 }
 
 impl i32 as Core.Mul {
-  fn Op[self: Self](other: Self) -> Self = "int.mul";
+  fn Op[self: Self](other: Self) -> Self = "int.smul";
 }
 impl i32 as Core.MulAssign {
   fn Op[addr self: Self*](other: Self) {
@@ -58,7 +59,7 @@ impl i32 as Core.MulAssign {
 }
 
 impl i32 as Core.Negate {
-  fn Op[self: Self]() -> Self = "int.negate";
+  fn Op[self: Self]() -> Self = "int.snegate";
 }
 
 impl i32 as Core.Ordered {
@@ -70,7 +71,7 @@ impl i32 as Core.Ordered {
 }
 
 impl i32 as Core.Sub {
-  fn Op[self: Self](other: Self) -> Self = "int.sub";
+  fn Op[self: Self](other: Self) -> Self = "int.ssub";
 }
 impl i32 as Core.SubAssign {
   fn Op[addr self: Self*](other: Self) {

toolchain/driver/BUILD

@@ -17,7 +17,7 @@ cc_library(
     name = "driver",
     srcs = ["driver.cpp"],
     hdrs = ["driver.h"],
-    data = ["//core"],
+    data = ["//core:prelude"],
     textual_hdrs = ["flags.def"],
     deps = [
         "//common:command_line",

toolchain/driver/driver.cpp

@@ -31,6 +31,44 @@
 
 namespace Carbon {
 
+auto Driver::FindPreludeFiles(llvm::StringRef data_dir,
+                              llvm::raw_ostream& error_stream)
+    -> llvm::SmallVector<std::string> {
+  llvm::SmallVector<std::string> result;
+
+  // Include <data>/core/prelude.carbon, which is the entry point into the
+  // prelude.
+  {
+    llvm::SmallString<256> prelude_file(data_dir);
+    llvm::sys::path::append(prelude_file, llvm::sys::path::Style::posix,
+                            "core/prelude.carbon");
+    result.push_back(prelude_file.str().str());
+  }
+
+  // Glob for <data>/core/prelude/**/*.carbon and add all the files we find.
+  llvm::SmallString<256> prelude_dir(data_dir);
+  llvm::sys::path::append(prelude_dir, llvm::sys::path::Style::posix,
+                          "core/prelude");
+  std::error_code ec;
+  for (llvm::sys::fs::recursive_directory_iterator prelude_files_it(
+           prelude_dir, ec, /*follow_symlinks=*/false);
+       prelude_files_it != llvm::sys::fs::recursive_directory_iterator();
+       prelude_files_it.increment(ec)) {
+    if (ec) {
+      error_stream << "ERROR: Could not find prelude: " << ec.message() << "\n";
+      result.clear();
+      return result;
+    }
+
+    auto prelude_file = prelude_files_it->path();
+    if (llvm::sys::path::extension(prelude_file) == ".carbon") {
+      result.push_back(prelude_file);
+    }
+  }
+
+  return result;
+}
+
 struct Driver::CompileOptions {
   static constexpr CommandLine::CommandInfo Info = {
       .name = "compile",
@@ -548,12 +586,17 @@ class Driver::CompilationUnit {
     LogCall("CodeGen", [&] { success_ = RunCodeGenHelper(); });
   }
 
-  // Flushes output.
-  auto Flush() -> void { consumer_->Flush(); }
+  // Runs post-compile logic. This is always called, and called after all other
+  // actions on the CompilationUnit.
+  auto PostCompile() const -> void {
+    if (options_.dump_shared_values && IncludeInDumps()) {
+      Yaml::Print(driver_->output_stream_,
+                  value_stores_.OutputYaml(input_filename_));
+    }
 
-  auto PrintSharedValues() const -> void {
-    Yaml::Print(driver_->output_stream_,
-                value_stores_.OutputYaml(input_filename_));
+    // The diagnostics consumer must be flushed before compilation artifacts are
+    // destructed, because diagnostics can refer to their state.
+    consumer_->Flush();
   }
 
   auto input_filename() -> llvm::StringRef { return input_filename_; }
@@ -672,20 +715,26 @@ auto Driver::Compile(const CompileOptions& options) -> RunResult {
     return {.success = false};
   }
 
+  // Find the files comprising the prelude if we are importing it.
+  // TODO: Replace this with a search for library api files in a
+  // package-specific search path based on the library name.
+  bool want_prelude =
+      options.prelude_import && options.phase >= CompileOptions::Phase::Check;
+  auto prelude = want_prelude ? FindPreludeFiles(data_dir_, error_stream_)
+                              : llvm::SmallVector<std::string>{};
+  if (want_prelude && prelude.empty()) {
+    return {.success = false};
+  }
+
   // Prepare CompilationUnits before building scope exit handlers.
   StreamDiagnosticConsumer stream_consumer(error_stream_);
   llvm::SmallVector<std::unique_ptr<CompilationUnit>> units;
-  units.reserve(options.prelude_import + options.input_filenames.size());
+  units.reserve(prelude.size() + options.input_filenames.size());
 
-  // Directly insert the core package into the compilation units.
-  // TODO: Should expand this into a more rich system to search for the core
-  // package source code.
-  if (options.prelude_import) {
-    llvm::SmallString<256> prelude_file(data_dir_);
-    llvm::sys::path::append(prelude_file, llvm::sys::path::Style::posix,
-                            "core/prelude.carbon");
+  // Add the prelude files.
+  for (const auto& input_filename : prelude) {
     units.push_back(std::make_unique<CompilationUnit>(
-        this, options, &stream_consumer, prelude_file));
+        this, options, &stream_consumer, input_filename));
   }
 
   // Add the input source files.
@@ -695,19 +744,12 @@ auto Driver::Compile(const CompileOptions& options) -> RunResult {
   }
 
   auto on_exit = llvm::make_scope_exit([&]() {
-    // Shared values will always be printed after per-file printing.
-    if (options.dump_shared_values) {
-      for (const auto& unit : units) {
-        unit->PrintSharedValues();
-      }
-    }
-
-    // The diagnostics consumer must be flushed before compilation artifacts are
-    // destructed, because diagnostics can refer to their state. This ensures
-    // they're flushed in order of arguments, rather than order of destruction.
+    // Finish compilation units. This flushes their diagnostics in the order in
+    // which they were specified on the command line.
     for (auto& unit : units) {
-      unit->Flush();
+      unit->PostCompile();
     }
+
     stream_consumer.Flush();
   });
 

toolchain/driver/driver.h

@@ -48,6 +48,13 @@ class Driver {
   // error stream (stderr by default).
   auto RunCommand(llvm::ArrayRef<llvm::StringRef> args) -> RunResult;
 
+  // Finds the source files that define the prelude and returns a list of their
+  // filenames. On error, writes a message to `error_stream` and returns an
+  // empty list.
+  static auto FindPreludeFiles(llvm::StringRef data_dir,
+                               llvm::raw_ostream& error_stream)
+      -> llvm::SmallVector<std::string>;
+
  private:
   struct Options;
   struct CompileOptions;

toolchain/driver/testdata/dump_shared_values.carbon

@@ -15,15 +15,6 @@ var real3: f64 = 0.8e9;
 var str1: String = "abc";
 var str2: String = "ab'\"c";
 
-// CHECK:STDOUT: ---
-// CHECK:STDOUT: filename:        'core/prelude.carbon'
-// CHECK:STDOUT: shared_values:
-// CHECK:STDOUT:   ints:            {}
-// CHECK:STDOUT:   reals:           {}
-// CHECK:STDOUT:   strings:
-// CHECK:STDOUT:     str0:            Core
-// CHECK:STDOUT:     str1:            prelude
-// CHECK:STDOUT: ...
 // CHECK:STDOUT: ---
 // CHECK:STDOUT: filename:        dump_shared_values.carbon
 // CHECK:STDOUT: shared_values:

toolchain/testing/file_test.cpp

@@ -29,21 +29,29 @@ class ToolchainFileTest : public FileTestBase {
   auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
            llvm::vfs::InMemoryFileSystem& fs, llvm::raw_pwrite_stream& stdout,
            llvm::raw_pwrite_stream& stderr) -> ErrorOr<RunResult> override {
-    CARBON_RETURN_IF_ERROR(AddFile(fs, "core/prelude.carbon"));
+    const llvm::StringRef data_dir = "";
 
-    Driver driver(fs, "", stdout, stderr);
+    auto prelude = Driver::FindPreludeFiles(data_dir, stderr);
+    if (prelude.empty()) {
+      return Error("Could not find prelude");
+    }
+    for (const auto& file : prelude) {
+      CARBON_RETURN_IF_ERROR(AddFile(fs, file));
+    }
+
+    Driver driver(fs, data_dir, stdout, stderr);
     auto driver_result = driver.RunCommand(test_args);
 
     RunResult result{
         .success = driver_result.success,
         .per_file_success = std::move(driver_result.per_file_success)};
-    // Drop entries that don't look like a file. Note this can empty out the
-    // list.
+    // Drop entries that don't look like a file, and entries corresponding to
+    // the prelude. Note this can empty out the list.
     llvm::erase_if(result.per_file_success,
-                   [](std::pair<llvm::StringRef, bool> entry) {
+                   [&](std::pair<llvm::StringRef, bool> entry) {
                      return entry.first == "." || entry.first == "-" ||
                             entry.first.starts_with("not_file") ||
-                            entry.first.starts_with("core/");
+                            llvm::is_contained(prelude, entry.first);
                    });
     return result;
   }
@@ -52,18 +60,11 @@ class ToolchainFileTest : public FileTestBase {
     llvm::SmallVector<std::string> args = {"compile",
                                            "--phase=" + component_.str()};
 
-    // For `lex` and `parse`, we don't need to import the prelude. Suppress it
-    // to improve the dump output for the tests.
     if (component_ == "lex") {
-      args.insert(args.end(), {"--no-prelude-import", "--dump-tokens", "%s"});
-      return args;
+      args.push_back("--dump-tokens");
     } else if (component_ == "parse") {
-      args.insert(args.end(),
-                  {"--no-prelude-import", "--dump-parse-tree", "%s"});
-      return args;
-    }
-
-    if (component_ == "check") {
+      args.push_back("--dump-parse-tree");
+    } else if (component_ == "check") {
       args.push_back("--dump-sem-ir");
     } else if (component_ == "lower") {
       args.push_back("--dump-llvm-ir");