Initial Reverse Interop implementation (#6901)

Add a clang::ExternalASTSource to begin exposing Carbon entities to
Clang - initially only a single `Carbon` top level namespace.

Subsequent work will add Carbon entities to this namespace.

Likely this CarbonExternalASTSource will be refactored into another
file, tie into/reference SemIR::File and CppFile, etc eventually - but
that'll wait for future patches.

If there's mechanical problems with the current implementation - how I'm
creating the new NamespaceDecl, etc - I'm all ears. It's very much in
the "it seems to work" state, not much more than that.

This does break Clang Modules (header modules, C++20 modules,
precompiled headers, etc) since they're implemented as an
ExternalASTSource as well, and Clang's ASTContext only supports one
ExternalASTSource at a time. To fix that regression we'll need to
implement some kind of ExternalASTSource multiplexing support - either
in Clang or Carbon (unclear which).

This regression of modules support can be observed by the following:
`A.h`
```
inline void f1() { }
```
`module.modulemap`
```
module A {
  header "A.h"
  export *
}
```
`test.carbon`
```
import Cpp inline '''
// Hardcode the pragma to ensure this isn't silently falling back to
// textual inclusion.
void f2() {
  f1();
}
''';
```
```
carbon compile test.carbon -- -I . -fmodules -fimplicit-modules -fmodules-cache-path=module_cache
```

I wrote a `file_test` test for this, but it doesn't /quite/ work because
`file_test` provides an in-memory filesystem for tests to make them more
hermetic, but Clang's Filesystem abstrtaction is for reading only - so
the module that's written out successfully can't be found when it needs
to be read back in - so the test doesn't pass as a baseline. Clang does
have support for `llvm::vfs::OutputBackend` which allows virtualizing
output - which I guess we could tie together with the InMemoryFilesystem
we use for input to make such a test work. But I guess that's not worth
the effort here?

---------

Co-authored-by: Richard Smith <richard@metafoo.co.uk>
Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

toolchain/check/cpp/generate_ast.cpp

@@ -17,6 +17,7 @@
 #include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Parse/Parser.h"
 #include "clang/Sema/ExternalSemaSource.h"
+#include "clang/Sema/MultiplexExternalSemaSource.h"
 #include "clang/Sema/Sema.h"
 #include "common/check.h"
 #include "common/raw_string_ostream.h"
@@ -331,6 +332,56 @@ class ShallowCopyCompilerInvocation : public clang::CompilerInvocation {
   }
 };
 
+class CarbonExternalASTSource : public clang::ExternalASTSource {
+ public:
+  explicit CarbonExternalASTSource(clang::ASTContext* ast_context)
+      : ast_context_(*ast_context) {}
+
+  auto FindExternalVisibleDeclsByName(
+      const clang::DeclContext* decl_context, clang::DeclarationName decl_name,
+      const clang::DeclContext* original_decl_context) -> bool override;
+
+  auto StartTranslationUnit(clang::ASTConsumer* consumer) -> void override;
+
+ private:
+  clang::ASTContext& ast_context_;
+};
+
+void CarbonExternalASTSource::StartTranslationUnit(
+    clang::ASTConsumer* /*Consumer*/) {
+  auto& translation_unit = *ast_context_.getTranslationUnitDecl();
+  // Mark the translation unit as having external storage so we get a query for
+  // the `Carbon` namespace in the top level/translation unit scope.
+  translation_unit.setHasExternalVisibleStorage();
+}
+
+auto CarbonExternalASTSource::FindExternalVisibleDeclsByName(
+    const clang::DeclContext* decl_context, clang::DeclarationName decl_name,
+    const clang::DeclContext* /*OriginalDC*/) -> bool {
+  if (decl_context->getDeclKind() != clang::Decl::Kind::TranslationUnit) {
+    return false;
+  }
+
+  static const llvm::StringLiteral carbon_namespace_name = "Carbon";
+  if (auto* identifier = decl_name.getAsIdentifierInfo();
+      !identifier || !identifier->isStr(carbon_namespace_name)) {
+    return false;
+  }
+
+  auto& ast_context = decl_context->getParentASTContext();
+  auto& mutable_tu_decl_context = *ast_context.getTranslationUnitDecl();
+  SetExternalVisibleDeclsForName(
+      decl_context, decl_name,
+      {
+          clang::NamespaceDecl::Create(
+              ast_context, &mutable_tu_decl_context, false,
+              clang::SourceLocation(), clang::SourceLocation(),
+              &ast_context.Idents.get(carbon_namespace_name), nullptr, false),
+      });
+
+  return true;
+}
+
 // An action and a set of registered Clang callbacks used to generate an AST
 // from a set of Cpp imports.
 class GenerateASTAction : public clang::ASTFrontendAction {
@@ -358,8 +409,6 @@ class GenerateASTAction : public clang::ASTFrontendAction {
 
   auto BeginSourceFileAction(clang::CompilerInstance& /*clang_instance*/)
       -> bool override {
-    // TODO: Consider creating an `ExternalSemaSource` here and attaching it to
-    // the compilation.
     // TODO: `clang.getPreprocessor().enableIncrementalProcessing();` to avoid
     // the TU scope getting torn down before we're done parsing macros.
     return true;
@@ -470,6 +519,15 @@ auto GenerateAst(Context& context,
     return false;
   }
 
+  auto& ast = clang_instance.getASTContext();
+  // TODO: Clang's modules support is implemented as an ExternalASTSource
+  // (ASTReader) and there's no multiplexing support for ExternalASTSources at
+  // the moment - so registering CarbonExternalASTSource breaks Clang modules
+  // support. Implement multiplexing support (possibly in Clang) to restore
+  // modules functionality.
+  ast.setExternalSource(
+      llvm::makeIntrusiveRefCnt<CarbonExternalASTSource>(&ast));
+
   if (llvm::Error error = action.Execute()) {
     // `Execute` currently never fails, but its contract allows it to.
     context.TODO(SemIR::LocId::None, "failed to execute clang action: " +

toolchain/check/testdata/interop/cpp/reverse/simple.carbon

@@ -0,0 +1,25 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// INCLUDE-FILE: toolchain/testing/testdata/min_prelude/none.carbon
+//
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/check/testdata/interop/cpp/reverse/simple.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/check/testdata/interop/cpp/reverse/simple.carbon
+
+// --- fail_simple.carbon
+
+library "[[@TEST_NAME]]";
+
+// Demonstrate that the 'Carbon' namespace has been injected into the C++ compilation.
+
+import Cpp inline '''
+// CHECK:STDERR: fail_simple.carbon:[[@LINE+4]]:9: error: no type named 'NonExistent' in namespace 'Carbon' [CppInteropParseError]
+// CHECK:STDERR:    11 | Carbon::NonExistent v;
+// CHECK:STDERR:       | ~~~~~~~~^
+// CHECK:STDERR:
+Carbon::NonExistent v;
+''';