Switch SourceBuffer to diagnostics. (#3197)

This updates SourceBuffer to diagnostics. Some additional edits to
diagnostics were necessary due to issues moving arguments around, which
seems to stem from a compile error with clang 14 (fixed in later
versions).

---------

Co-authored-by: Richard Smith <richard@metafoo.co.uk>

language_server/language_server.cpp

@@ -94,7 +94,7 @@ void LanguageServer::OnDocumentSymbol(
   vfs.addFile(file, /*mtime=*/0,
               llvm::MemoryBuffer::getMemBufferCopy(files_.at(file)));
 
-  auto buf = SourceBuffer::CreateFromFile(vfs, llvm::nulls(), file);
+  auto buf = SourceBuffer::CreateFromFile(vfs, file, NullDiagnosticConsumer());
   auto lexed = Lex::TokenizedBuffer::Lex(*buf, NullDiagnosticConsumer());
   auto parsed = Parse::Tree::Parse(lexed, NullDiagnosticConsumer(), nullptr);
   std::vector<clang::clangd::DocumentSymbol> result;

toolchain/diagnostics/diagnostic_emitter.h

@@ -54,9 +54,9 @@ struct DiagnosticLocation {
   // A reference to the line of the error.
   llvm::StringRef line;
   // 1-based line number.
-  int32_t line_number;
+  int32_t line_number = -1;
   // 1-based column number.
-  int32_t column_number;
+  int32_t column_number = -1;
 };
 
 // A message composing a diagnostic. This may be the main message, but can also
@@ -215,8 +215,8 @@ class DiagnosticEmitter {
               Internal::NoTypeDeduction<Args>... args) -> DiagnosticBuilder& {
       CARBON_CHECK(diagnostic_base.Level == DiagnosticLevel::Note)
           << static_cast<int>(diagnostic_base.Level);
-      diagnostic_.notes.push_back(
-          MakeMessage(location, diagnostic_base, std::move(args)...));
+      diagnostic_.notes.push_back(MakeMessage(
+          emitter_, location, diagnostic_base, {llvm::Any(args)...}));
       return *this;
     }
 
@@ -234,22 +234,23 @@ class DiagnosticEmitter {
     explicit DiagnosticBuilder(
         DiagnosticEmitter<LocationT>* emitter, LocationT location,
         const Internal::DiagnosticBase<Args...>& diagnostic_base,
-        Internal::NoTypeDeduction<Args>... args)
+        llvm::SmallVector<llvm::Any> args)
         : emitter_(emitter),
-          diagnostic_({.level = diagnostic_base.Level,
-                       .message = MakeMessage(location, diagnostic_base,
-                                              std::move(args)...)}) {
+          diagnostic_(
+              {.level = diagnostic_base.Level,
+               .message = MakeMessage(emitter, location, diagnostic_base,
+                                      std::move(args))}) {
       CARBON_CHECK(diagnostic_base.Level != DiagnosticLevel::Note);
     }
 
     template <typename... Args>
-    auto MakeMessage(LocationT location,
-                     const Internal::DiagnosticBase<Args...>& diagnostic_base,
-                     Internal::NoTypeDeduction<Args>... args)
-        -> DiagnosticMessage {
+    static auto MakeMessage(
+        DiagnosticEmitter<LocationT>* emitter, LocationT location,
+        const Internal::DiagnosticBase<Args...>& diagnostic_base,
+        llvm::SmallVector<llvm::Any> args) -> DiagnosticMessage {
       return DiagnosticMessage(
-          diagnostic_base.Kind, emitter_->translator_->GetLocation(location),
-          diagnostic_base.Format, {std::move(args)...},
+          diagnostic_base.Kind, emitter->translator_->GetLocation(location),
+          diagnostic_base.Format, std::move(args),
           [&diagnostic_base](const DiagnosticMessage& message) -> std::string {
             return diagnostic_base.FormatFn(message);
           });
@@ -275,7 +276,7 @@ class DiagnosticEmitter {
   auto Emit(LocationT location,
             const Internal::DiagnosticBase<Args...>& diagnostic_base,
             Internal::NoTypeDeduction<Args>... args) -> void {
-    DiagnosticBuilder(this, location, diagnostic_base, std::move(args)...)
+    DiagnosticBuilder(this, location, diagnostic_base, {llvm::Any(args)...})
         .Emit();
   }
 
@@ -290,7 +291,7 @@ class DiagnosticEmitter {
              const Internal::DiagnosticBase<Args...>& diagnostic_base,
              Internal::NoTypeDeduction<Args>... args) -> DiagnosticBuilder {
     return DiagnosticBuilder(this, location, diagnostic_base,
-                             std::move(args)...);
+                             {llvm::Any(args)...});
   }
 
  private:
@@ -310,13 +311,19 @@ class StreamDiagnosticConsumer : public DiagnosticConsumer {
     }
   }
   auto Print(const DiagnosticMessage& message) -> void {
-    *stream_ << message.location.file_name << ":"
-             << message.location.line_number << ":"
-             << message.location.column_number << ": "
-             << message.format_fn(message) << "\n"
-             << message.location.line << "\n";
-    stream_->indent(message.location.column_number - 1);
-    *stream_ << "^\n";
+    *stream_ << message.location.file_name;
+    if (message.location.line_number > 0) {
+      *stream_ << ":" << message.location.line_number;
+      if (message.location.column_number > 0) {
+        *stream_ << ":" << message.location.column_number;
+      }
+    }
+    *stream_ << ": " << message.format_fn(message) << "\n";
+    if (message.location.column_number > 0) {
+      *stream_ << message.location.line << "\n";
+      stream_->indent(message.location.column_number - 1);
+      *stream_ << "^\n";
+    }
   }
 
  private:

toolchain/diagnostics/diagnostic_kind.def

@@ -16,6 +16,15 @@
 #error "Must define the x-macro to use this file."
 #endif
 
+// ============================================================================
+// SourceBuffer diagnostics
+// ============================================================================
+
+CARBON_DIAGNOSTIC_KIND(ErrorOpeningFile)
+CARBON_DIAGNOSTIC_KIND(ErrorStattingFile)
+CARBON_DIAGNOSTIC_KIND(FileTooLarge)
+CARBON_DIAGNOSTIC_KIND(ErrorReadingFile)
+
 // ============================================================================
 // Lexer diagnostics
 // ============================================================================

toolchain/driver/driver.cpp

@@ -401,8 +401,8 @@ class Driver::CompilationUnit {
   // Loads source and lexes it. Returns true on success.
   auto RunLex() -> bool {
     LogCall("SourceBuffer::CreateFromFile", [&] {
-      source_ = SourceBuffer::CreateFromFile(
-          driver_->fs_, driver_->error_stream_, input_file_name_);
+      source_ = SourceBuffer::CreateFromFile(driver_->fs_, input_file_name_,
+                                             *consumer_);
     });
     if (!source_) {
       return false;

toolchain/driver/testdata/fail_missing_file.carbon

@@ -0,0 +1,9 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// ARGS: compile --phase=lex nonexistent.carbon
+//
+// AUTOUPDATE
+
+// CHECK:STDERR: nonexistent.carbon: Error opening file for read: No such file or directory

toolchain/lex/BUILD

@@ -241,6 +241,7 @@ cc_binary(
         ":token_kind",
         ":tokenized_buffer",
         "//common:check",
+        "//toolchain/diagnostics:diagnostic_emitter",
         "//toolchain/diagnostics:null_diagnostics",
         "@com_github_google_benchmark//:benchmark_main",
         "@com_google_absl//absl/random",

toolchain/lex/tokenized_buffer_benchmark.cpp

@@ -10,6 +10,7 @@
 #include "common/check.h"
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/StringExtras.h"
+#include "toolchain/diagnostics/diagnostic_emitter.h"
 #include "toolchain/diagnostics/null_diagnostics.h"
 #include "toolchain/lex/token_kind.h"
 #include "toolchain/lex/tokenized_buffer.h"
@@ -297,8 +298,8 @@ class LexerBenchHelper {
   auto MakeSourceBuffer(llvm::StringRef text) -> SourceBuffer {
     CARBON_CHECK(fs_.addFile(filename_, /*ModificationTime=*/0,
                              llvm::MemoryBuffer::getMemBuffer(text)));
-    return std::move(
-        *SourceBuffer::CreateFromFile(fs_, llvm::errs(), filename_));
+    return std::move(*SourceBuffer::CreateFromFile(
+        fs_, filename_, ConsoleDiagnosticConsumer()));
   }
 
   llvm::vfs::InMemoryFileSystem fs_;

toolchain/lex/tokenized_buffer_fuzzer.cpp

@@ -30,7 +30,8 @@ extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data,
       TestFileName, /*ModificationTime=*/0,
       llvm::MemoryBuffer::getMemBuffer(data_ref, /*BufferName=*/TestFileName,
                                        /*RequiresNullTerminator=*/false)));
-  auto source = SourceBuffer::CreateFromFile(fs, llvm::nulls(), TestFileName);
+  auto source =
+      SourceBuffer::CreateFromFile(fs, TestFileName, NullDiagnosticConsumer());
 
   auto buffer = Lex::TokenizedBuffer::Lex(*source, NullDiagnosticConsumer());
   if (buffer.has_errors()) {

toolchain/lex/tokenized_buffer_test.cpp

@@ -34,8 +34,8 @@ class LexerTest : public ::testing::Test {
     std::string filename = llvm::formatv("test{0}.carbon", ++file_index_);
     CARBON_CHECK(fs_.addFile(filename, /*ModificationTime=*/0,
                              llvm::MemoryBuffer::getMemBuffer(text)));
-    source_storage_.push_front(
-        std::move(*SourceBuffer::CreateFromFile(fs_, llvm::errs(), filename)));
+    source_storage_.push_front(std::move(*SourceBuffer::CreateFromFile(
+        fs_, filename, ConsoleDiagnosticConsumer())));
     return source_storage_.front();
   }
 

toolchain/parse/parse_fuzzer.cpp

@@ -27,7 +27,8 @@ extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data,
       TestFileName, /*ModificationTime=*/0,
       llvm::MemoryBuffer::getMemBuffer(data_ref, /*BufferName=*/TestFileName,
                                        /*RequiresNullTerminator=*/false)));
-  auto source = SourceBuffer::CreateFromFile(fs, llvm::nulls(), TestFileName);
+  auto source =
+      SourceBuffer::CreateFromFile(fs, TestFileName, NullDiagnosticConsumer());
 
   // Lex the input.
   auto tokens = Lex::TokenizedBuffer::Lex(*source, NullDiagnosticConsumer());

toolchain/parse/tree_test.cpp

@@ -26,8 +26,8 @@ class TreeTest : public ::testing::Test {
   auto GetSourceBuffer(llvm::StringRef t) -> SourceBuffer& {
     CARBON_CHECK(fs.addFile("test.carbon", /*ModificationTime=*/0,
                             llvm::MemoryBuffer::getMemBuffer(t)));
-    source_storage.push_front(std::move(
-        *SourceBuffer::CreateFromFile(fs, llvm::errs(), "test.carbon")));
+    source_storage.push_front(
+        std::move(*SourceBuffer::CreateFromFile(fs, "test.carbon", consumer)));
     return source_storage.front();
   }
 

toolchain/source/BUILD

@@ -12,6 +12,7 @@ cc_library(
     hdrs = ["source_buffer.h"],
     deps = [
         "//common:error",
+        "//toolchain/diagnostics:diagnostic_emitter",
         "@llvm-project//llvm:Support",
     ],
 )
@@ -24,6 +25,7 @@ cc_test(
         ":source_buffer",
         "//common:check",
         "//testing/base:gtest_main",
+        "//toolchain/diagnostics:diagnostic_emitter",
         "@com_google_googletest//:gtest",
         "@llvm-project//llvm:Support",
     ],

toolchain/source/source_buffer.cpp

@@ -10,36 +10,52 @@
 
 namespace Carbon {
 
+namespace {
+struct FilenameTranslator : DiagnosticLocationTranslator<llvm::StringRef> {
+  auto GetLocation(llvm::StringRef filename) -> DiagnosticLocation override {
+    return {.file_name = filename};
+  }
+};
+}  // namespace
+
 auto SourceBuffer::CreateFromFile(llvm::vfs::FileSystem& fs,
-                                  llvm::raw_ostream& error_stream,
-                                  llvm::StringRef filename)
+                                  llvm::StringRef filename,
+                                  DiagnosticConsumer& consumer)
     -> std::optional<SourceBuffer> {
+  FilenameTranslator translator;
+  DiagnosticEmitter<llvm::StringRef> emitter(translator, consumer);
+
   llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> file =
       fs.openFileForRead(filename);
   if (file.getError()) {
-    error_stream << "Error opening `" << filename
-                 << "`: " << file.getError().message();
+    CARBON_DIAGNOSTIC(ErrorOpeningFile, Error,
+                      "Error opening file for read: {0}", std::string);
+    emitter.Emit(filename, ErrorOpeningFile, file.getError().message());
     return std::nullopt;
   }
 
   llvm::ErrorOr<llvm::vfs::Status> status = (*file)->status();
   if (status.getError()) {
-    error_stream << "Error getting status for `" << filename
-                 << "`: " << file.getError().message();
+    CARBON_DIAGNOSTIC(ErrorStattingFile, Error, "Error statting file: {0}",
+                      std::string);
+    emitter.Emit(filename, ErrorStattingFile, file.getError().message());
     return std::nullopt;
   }
-  auto size = status->getSize();
+  int64_t size = status->getSize();
   if (size >= std::numeric_limits<int32_t>::max()) {
-    error_stream << "Cannot load `" << filename
-                 << "`: file is over the 2GiB input limit.";
+    CARBON_DIAGNOSTIC(FileTooLarge, Error,
+                      "File is over the 2GiB input limit; size is {0} bytes.",
+                      int64_t);
+    emitter.Emit(filename, FileTooLarge, size);
     return std::nullopt;
   }
 
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> buffer =
       (*file)->getBuffer(filename, size, /*RequiresNullTerminator=*/false);
   if (buffer.getError()) {
-    error_stream << "Error reading `" << filename
-                 << "`: " << file.getError().message();
+    CARBON_DIAGNOSTIC(ErrorReadingFile, Error, "Error reading file: {0}",
+                      std::string);
+    emitter.Emit(filename, ErrorReadingFile, file.getError().message());
     return std::nullopt;
   }
 

toolchain/source/source_buffer.h

@@ -11,6 +11,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/VirtualFileSystem.h"
+#include "toolchain/diagnostics/diagnostic_emitter.h"
 
 namespace Carbon {
 
@@ -35,10 +36,9 @@ class SourceBuffer {
  public:
   // Opens the requested file. Returns a SourceBuffer on success. Prints an
   // error and returns nullopt on failure.
-  // TODO: Switch to using diagnostics.
   static auto CreateFromFile(llvm::vfs::FileSystem& fs,
-                             llvm::raw_ostream& error_stream,
-                             llvm::StringRef filename)
+                             llvm::StringRef filename,
+                             DiagnosticConsumer& consumer)
       -> std::optional<SourceBuffer>;
 
   // Use one of the factory functions above to create a source buffer.

toolchain/source/source_buffer_test.cpp

@@ -8,6 +8,7 @@
 
 #include "common/check.h"
 #include "llvm/Support/VirtualFileSystem.h"
+#include "toolchain/diagnostics/diagnostic_emitter.h"
 
 namespace Carbon::Testing {
 namespace {
@@ -16,7 +17,8 @@ static constexpr llvm::StringLiteral TestFileName = "test.carbon";
 
 TEST(SourceBufferTest, MissingFile) {
   llvm::vfs::InMemoryFileSystem fs;
-  auto buffer = SourceBuffer::CreateFromFile(fs, llvm::errs(), TestFileName);
+  auto buffer = SourceBuffer::CreateFromFile(fs, TestFileName,
+                                             ConsoleDiagnosticConsumer());
   EXPECT_FALSE(buffer);
 }
 
@@ -25,7 +27,8 @@ TEST(SourceBufferTest, SimpleFile) {
   CARBON_CHECK(fs.addFile(TestFileName, /*ModificationTime=*/0,
                           llvm::MemoryBuffer::getMemBuffer("Hello World")));
 
-  auto buffer = SourceBuffer::CreateFromFile(fs, llvm::errs(), TestFileName);
+  auto buffer = SourceBuffer::CreateFromFile(fs, TestFileName,
+                                             ConsoleDiagnosticConsumer());
   ASSERT_TRUE(buffer);
 
   EXPECT_EQ(TestFileName, buffer->filename());
@@ -41,7 +44,8 @@ TEST(SourceBufferTest, NoNull) {
                                        /*BufferName=*/"",
                                        /*RequiresNullTerminator=*/false)));
 
-  auto buffer = SourceBuffer::CreateFromFile(fs, llvm::errs(), TestFileName);
+  auto buffer = SourceBuffer::CreateFromFile(fs, TestFileName,
+                                             ConsoleDiagnosticConsumer());
   ASSERT_TRUE(buffer);
 
   EXPECT_EQ(TestFileName, buffer->filename());
@@ -53,7 +57,8 @@ TEST(SourceBufferTest, EmptyFile) {
   CARBON_CHECK(fs.addFile(TestFileName, /*ModificationTime=*/0,
                           llvm::MemoryBuffer::getMemBuffer("")));
 
-  auto buffer = SourceBuffer::CreateFromFile(fs, llvm::errs(), TestFileName);
+  auto buffer = SourceBuffer::CreateFromFile(fs, TestFileName,
+                                             ConsoleDiagnosticConsumer());
   ASSERT_TRUE(buffer);
 
   EXPECT_EQ(TestFileName, buffer->filename());