Add a flag to dump the C++ AST (#5918)

Use it to dump the AST that includes a generated C++ thunk.
Based on #5917.

Also added printing of the full actual text when check fails to make
debugging easier.
Changed line replacement to allow removing complete lines.

Part of #5514.

testing/file_test/autoupdate.cpp

@@ -201,6 +201,9 @@ auto FileTestAutoupdater::BuildCheckLines(llvm::StringRef output,
     }
 
     do_extra_check_replacements_(check_line);
+    if (check_line.empty()) {
+      continue;
+    }
 
     if (default_file_re_) {
       absl::string_view filename;

testing/file_test/file_test_base.cpp

@@ -246,15 +246,23 @@ auto FileTestCase::TestBody() -> void {
   }
   if (test_file.check_subset) {
     EXPECT_THAT(SplitOutput(test_file.actual_stdout),
-                IsSupersetOf(test_file.expected_stdout));
+                IsSupersetOf(test_file.expected_stdout))
+        << "Actual text:\n"
+        << test_file.actual_stdout;
     EXPECT_THAT(SplitOutput(test_file.actual_stderr),
-                IsSupersetOf(test_file.expected_stderr));
+                IsSupersetOf(test_file.expected_stderr))
+        << "Actual text:\n"
+        << test_file.actual_stderr;
 
   } else {
     EXPECT_THAT(SplitOutput(test_file.actual_stdout),
-                ElementsAreArray(test_file.expected_stdout));
+                ElementsAreArray(test_file.expected_stdout))
+        << "Actual text:\n"
+        << test_file.actual_stdout;
     EXPECT_THAT(SplitOutput(test_file.actual_stderr),
-                ElementsAreArray(test_file.expected_stderr));
+                ElementsAreArray(test_file.expected_stderr))
+        << "Actual text:\n"
+        << test_file.actual_stderr;
   }
 
   if (HasFailure()) {

toolchain/check/check.cpp

@@ -392,6 +392,22 @@ static auto MaybeDumpSemIR(
   }
 }
 
+// Handles options for dumping C++ AST.
+static auto MaybeDumpCppAST(llvm::ArrayRef<Unit> units,
+                            const CheckParseTreesOptions& options) -> void {
+  if (!options.dump_cpp_ast_stream) {
+    return;
+  }
+
+  for (const Unit& unit : units) {
+    if (!unit.cpp_ast || !*unit.cpp_ast) {
+      continue;
+    }
+    clang::ASTContext& ast_context = (*unit.cpp_ast)->getASTContext();
+    ast_context.getTranslationUnitDecl()->dump(*options.dump_cpp_ast_stream);
+  }
+}
+
 auto CheckParseTrees(
     llvm::MutableArrayRef<Unit> units,
     const Parse::GetTreeAndSubtreesStore& tree_and_subtrees_getters,
@@ -509,6 +525,7 @@ auto CheckParseTrees(
   }
 
   MaybeDumpSemIR(units, tree_and_subtrees_getters, options);
+  MaybeDumpCppAST(units, options);
 }
 
 }  // namespace Carbon::Check

toolchain/check/check.h

@@ -60,6 +60,9 @@ struct CheckParseTreesOptions {
   // If set, SemIR will be dumped to this.
   llvm::raw_ostream* dump_stream = nullptr;
 
+  // If set, C++ AST will be dumped to this.
+  llvm::raw_ostream* dump_cpp_ast_stream = nullptr;
+
   // When dumping textual SemIR (or printing it to for verbose output), whether
   // to use ranges.
   enum class DumpSemIRRanges : int8_t {

toolchain/check/cpp_thunk.cpp

@@ -195,11 +195,13 @@ static auto CreateThunkFunctionDecl(
       callee_function_decl.getReturnType(), thunk_param_types,
       callee_function_type->getExtProtoInfo());
 
+  clang::DeclContext* decl_context = ast_context.getTranslationUnitDecl();
   // TODO: Thunks should not have external linkage, consider using `SC_Static`.
   clang::FunctionDecl* thunk_function_decl = clang::FunctionDecl::Create(
-      ast_context, ast_context.getTranslationUnitDecl(), clang_loc, clang_loc,
+      ast_context, decl_context, clang_loc, clang_loc,
       clang::DeclarationName(&identifier_info), thunk_function_type,
       /*TInfo=*/nullptr, clang::SC_Extern);
+  decl_context->addDecl(thunk_function_decl);
 
   thunk_function_decl->setParams(BuildThunkParameters(
       ast_context, callee_function_decl, thunk_function_decl));

toolchain/check/testdata/interop/cpp/function/full_semir.carbon

@@ -26,7 +26,6 @@ library "[[@TEST_NAME]]";
 import Cpp library "short_param.h";
 
 fn F() {
-  // TODO: Find a way to test the full C++ thunk AST.
   Cpp.foo(1 as i16);
 }
 
@@ -168,21 +167,21 @@ fn F() {
 // CHECK:STDOUT:   %int_16: Core.IntLiteral = int_value 16 [concrete = constants.%int_16]
 // CHECK:STDOUT:   %i16: type = class_type @Int, @Int(constants.%int_16) [concrete = constants.%i16]
 // CHECK:STDOUT:   %impl.elem0: %.91d = impl_witness_access constants.%As.impl_witness.0ef, element0 [concrete = constants.%Core.IntLiteral.as.As.impl.Convert.489]
-// CHECK:STDOUT:   %bound_method.loc8_13.1: <bound method> = bound_method %int_1, %impl.elem0 [concrete = constants.%Core.IntLiteral.as.As.impl.Convert.bound]
+// CHECK:STDOUT:   %bound_method.loc7_13.1: <bound method> = bound_method %int_1, %impl.elem0 [concrete = constants.%Core.IntLiteral.as.As.impl.Convert.bound]
 // CHECK:STDOUT:   %specific_fn: <specific function> = specific_function %impl.elem0, @Core.IntLiteral.as.As.impl.Convert(constants.%int_16) [concrete = constants.%Core.IntLiteral.as.As.impl.Convert.specific_fn]
-// CHECK:STDOUT:   %bound_method.loc8_13.2: <bound method> = bound_method %int_1, %specific_fn [concrete = constants.%bound_method]
-// CHECK:STDOUT:   %Core.IntLiteral.as.As.impl.Convert.call: init %i16 = call %bound_method.loc8_13.2(%int_1) [concrete = constants.%int_1.f90]
-// CHECK:STDOUT:   %.loc8_13.1: %i16 = value_of_initializer %Core.IntLiteral.as.As.impl.Convert.call [concrete = constants.%int_1.f90]
-// CHECK:STDOUT:   %.loc8_13.2: %i16 = converted %int_1, %.loc8_13.1 [concrete = constants.%int_1.f90]
-// CHECK:STDOUT:   %.loc8_19.1: ref %i16 = temporary_storage
-// CHECK:STDOUT:   %.loc8_19.2: init %i16 = initialize_from %.loc8_13.2 to %.loc8_19.1 [concrete = constants.%int_1.f90]
-// CHECK:STDOUT:   %addr.loc8_19.1: %ptr.251 = addr_of %.loc8_19.1
-// CHECK:STDOUT:   %foo__carbon_thunk.call: init %empty_tuple.type = call imports.%foo__carbon_thunk.decl(%addr.loc8_19.1)
-// CHECK:STDOUT:   %T.as.Destroy.impl.Op.bound: <bound method> = bound_method %.loc8_19.1, constants.%T.as.Destroy.impl.Op.507
+// CHECK:STDOUT:   %bound_method.loc7_13.2: <bound method> = bound_method %int_1, %specific_fn [concrete = constants.%bound_method]
+// CHECK:STDOUT:   %Core.IntLiteral.as.As.impl.Convert.call: init %i16 = call %bound_method.loc7_13.2(%int_1) [concrete = constants.%int_1.f90]
+// CHECK:STDOUT:   %.loc7_13.1: %i16 = value_of_initializer %Core.IntLiteral.as.As.impl.Convert.call [concrete = constants.%int_1.f90]
+// CHECK:STDOUT:   %.loc7_13.2: %i16 = converted %int_1, %.loc7_13.1 [concrete = constants.%int_1.f90]
+// CHECK:STDOUT:   %.loc7_19.1: ref %i16 = temporary_storage
+// CHECK:STDOUT:   %.loc7_19.2: init %i16 = initialize_from %.loc7_13.2 to %.loc7_19.1 [concrete = constants.%int_1.f90]
+// CHECK:STDOUT:   %addr.loc7_19.1: %ptr.251 = addr_of %.loc7_19.1
+// CHECK:STDOUT:   %foo__carbon_thunk.call: init %empty_tuple.type = call imports.%foo__carbon_thunk.decl(%addr.loc7_19.1)
+// CHECK:STDOUT:   %T.as.Destroy.impl.Op.bound: <bound method> = bound_method %.loc7_19.1, constants.%T.as.Destroy.impl.Op.507
 // CHECK:STDOUT:   %T.as.Destroy.impl.Op.specific_fn: <specific function> = specific_function constants.%T.as.Destroy.impl.Op.507, @T.as.Destroy.impl.Op(constants.%i16) [concrete = constants.%T.as.Destroy.impl.Op.specific_fn]
-// CHECK:STDOUT:   %bound_method.loc8_19: <bound method> = bound_method %.loc8_19.1, %T.as.Destroy.impl.Op.specific_fn
-// CHECK:STDOUT:   %addr.loc8_19.2: %ptr.251 = addr_of %.loc8_19.1
-// CHECK:STDOUT:   %T.as.Destroy.impl.Op.call: init %empty_tuple.type = call %bound_method.loc8_19(%addr.loc8_19.2)
+// CHECK:STDOUT:   %bound_method.loc7_19: <bound method> = bound_method %.loc7_19.1, %T.as.Destroy.impl.Op.specific_fn
+// CHECK:STDOUT:   %addr.loc7_19.2: %ptr.251 = addr_of %.loc7_19.1
+// CHECK:STDOUT:   %T.as.Destroy.impl.Op.call: init %empty_tuple.type = call %bound_method.loc7_19(%addr.loc7_19.2)
 // CHECK:STDOUT:   return
 // CHECK:STDOUT: }
 // CHECK:STDOUT:

toolchain/check/testdata/interop/cpp/function/thunk_ast.carbon

@@ -0,0 +1,42 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// INCLUDE-FILE: toolchain/testing/testdata/min_prelude/int.carbon
+// EXTRA-ARGS: --dump-cpp-ast
+// SET-CHECK-SUBSET
+//
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/check/testdata/interop/cpp/function/thunk_ast.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/check/testdata/interop/cpp/function/thunk_ast.carbon
+// CHECK:STDOUT: TranslationUnitDecl {{0x[a-f0-9]+}} <<invalid sloc>> <invalid sloc>
+
+// --- thunk_required.h
+
+auto foo(short a) -> void;
+// CHECK:STDOUT: |-FunctionDecl {{0x[a-f0-9]+}} <./thunk_required.h:[[@LINE-1]]:1, col:22> col:6 used foo 'auto (short) -> void'
+// CHECK:STDOUT: | `-ParmVarDecl {{0x[a-f0-9]+}} <col:10, col:16> col:16 a 'short'
+// CHECK:STDOUT: `-FunctionDecl {{0x[a-f0-9]+}} <col:6> col:6 foo__carbon_thunk 'auto (short * _Nonnull) -> void' extern
+// CHECK:STDOUT:   |-ParmVarDecl {{0x[a-f0-9]+}} <col:6> col:6 used a 'short * _Nonnull':'short *'
+// CHECK:STDOUT:   |-ReturnStmt {{0x[a-f0-9]+}} <col:6>
+// CHECK:STDOUT:   | `-CallExpr {{0x[a-f0-9]+}} <col:6> 'void'
+// CHECK:STDOUT:   |   |-ImplicitCastExpr {{0x[a-f0-9]+}} <col:6> 'auto (*)(short) -> void' <FunctionToPointerDecay>
+// CHECK:STDOUT:   |   | `-DeclRefExpr {{0x[a-f0-9]+}} <col:6> 'auto (short) -> void' Function {{0x[a-f0-9]+}} 'foo' 'auto (short) -> void'
+// CHECK:STDOUT:   |   `-ImplicitCastExpr {{0x[a-f0-9]+}} <col:6> 'short' <LValueToRValue>
+// CHECK:STDOUT:   |     `-UnaryOperator {{0x[a-f0-9]+}} <col:6> 'short' lvalue prefix '*' cannot overflow
+// CHECK:STDOUT:   |       `-ImplicitCastExpr {{0x[a-f0-9]+}} <col:6> 'short * _Nonnull':'short *' <LValueToRValue>
+// CHECK:STDOUT:   |         `-DeclRefExpr {{0x[a-f0-9]+}} <col:6> 'short * _Nonnull':'short *' lvalue ParmVar {{0x[a-f0-9]+}} 'a' 'short * _Nonnull':'short *'
+// CHECK:STDOUT:   |-AlwaysInlineAttr {{0x[a-f0-9]+}} <<invalid sloc>> Implicit always_inline
+// CHECK:STDOUT:   `-AsmLabelAttr {{0x[a-f0-9]+}} <col:6> Implicit "_Z3foos.carbon_thunk"
+
+// --- import_thunk_required.carbon
+
+library "[[@TEST_NAME]]";
+
+import Cpp library "thunk_required.h";
+
+fn F() {
+  Cpp.foo(1 as i16);
+}

toolchain/driver/compile_subcommand.cpp

@@ -208,6 +208,14 @@ Dump the full SemIR to stdout when built.
 )""",
       },
       [&](auto& arg_b) { arg_b.Set(&dump_sem_ir); });
+  b.AddFlag(
+      {
+          .name = "dump-cpp-ast",
+          .help = R"""(
+Dump the full C++ AST to stdout when built.
+)""",
+      },
+      [&](auto& arg_b) { arg_b.Set(&dump_cpp_ast); });
 
   b.AddOneOfOption(
       {
@@ -399,6 +407,11 @@ auto CompileSubcommand::ValidateOptions(
                      PhaseToString(options_.phase));
         return false;
       }
+      if (options_.dump_cpp_ast) {
+        emitter.Emit(CompilePhaseFlagConflict, "C++ AST",
+                     PhaseToString(options_.phase));
+        return false;
+      }
       [[fallthrough]];
     case Phase::Check:
       if (options_.dump_llvm_ir) {
@@ -1015,11 +1028,15 @@ auto CompileSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
   options.gen_implicit_type_impls = options_.gen_implicit_type_impls;
   options.vlog_stream = driver_env.vlog_stream;
   options.fuzzing = driver_env.fuzzing;
-  if (options.vlog_stream || options_.dump_sem_ir || options_.dump_raw_sem_ir) {
+  if (options.vlog_stream || options_.dump_sem_ir || options_.dump_cpp_ast ||
+      options_.dump_raw_sem_ir) {
     options.include_in_dumps = &cache.include_in_dumps();
     if (options_.dump_sem_ir) {
       options.dump_stream = driver_env.output_stream;
     }
+    if (options_.dump_cpp_ast) {
+      options.dump_cpp_ast_stream = driver_env.output_stream;
+    }
     if (options.vlog_stream || options_.dump_sem_ir) {
       options.dump_sem_ir_ranges = options_.dump_sem_ir_ranges;
     }

toolchain/driver/compile_subcommand.h

@@ -52,6 +52,7 @@ struct CompileOptions {
   bool dump_parse_tree = false;
   bool dump_raw_sem_ir = false;
   bool dump_sem_ir = false;
+  bool dump_cpp_ast = false;
   bool dump_llvm_ir = false;
   bool dump_asm = false;
   bool dump_mem_usage = false;

toolchain/testing/file_test.cpp

@@ -263,6 +263,32 @@ auto ToolchainFileTest::GetLineNumberReplacements(
   return replacements;
 }
 
+// For Clang AST dump lines, we remove references to builtins because they're
+// inconsistent between systems and replace the ids since they're inconsistent
+// between runs.
+static auto DoClangASTCheckReplacements(std::string& check_line) -> void {
+  static constexpr llvm::StringRef ClangDeclIdRegex = "0x[a-f0-9]+";
+  static const RE2 is_clang_ast_line_re(
+      R"(^// CHECK:STDOUT: (TranslationUnitDecl|[ |]*`?\-))");
+  if (!RE2::PartialMatch(check_line, is_clang_ast_line_re)) {
+    return;
+  }
+
+  // Filter out references to builtins.
+  static const RE2 is_builtin_referring_re(
+      R"(`-BuiltinType |[ ']__[a-zA-Z]|\| `\-PointerType 0x[a-f0-9]+ 'char \*'$)");
+  if (RE2::PartialMatch(check_line, is_builtin_referring_re)) {
+    check_line.clear();
+    return;
+  }
+
+  // Replace the ids.
+  static const RE2 clang_decl_id_re(llvm::formatv(" {0} ", ClangDeclIdRegex));
+  static const std::string& clang_decl_id_replacement =
+      *new std::string(llvm::formatv(" {{{{{0}}} ", ClangDeclIdRegex));
+  RE2::GlobalReplace(&check_line, clang_decl_id_re, clang_decl_id_replacement);
+}
+
 auto ToolchainFileTest::DoExtraCheckReplacements(std::string& check_line) const
     -> void {
   if (component_ == "driver") {
@@ -286,6 +312,9 @@ auto ToolchainFileTest::DoExtraCheckReplacements(std::string& check_line) const
     // package to the VFS with a fixed name.
     absl::StrReplaceAll({{data_->installation.core_package(), "{{.*}}"}},
                         &check_line);
+    if (component_ == "check") {
+      DoClangASTCheckReplacements(check_line);
+    }
   } else {
     FileTestBase::DoExtraCheckReplacements(check_line);
   }