// Part of the Carbon Language project, under the Apache License v2.0 with LLVM // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "testing/file_test/file_test_base.h" #include #include #include "common/check.h" #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/Twine.h" #include "llvm/Support/InitLLVM.h" namespace Carbon::Testing { // The length of the base directory. static int base_dir_len = 0; // The name of the `.subset` target. static std::string* subset_target = nullptr; // The original working directory for restoration after each test. static std::filesystem::path* orig_working_dir = nullptr; using ::testing::Eq; FileTestBase::FileTestBase(const std::filesystem::path& path) : path_(&path) {} FileTestBase::~FileTestBase() { // Restore the original working directory. std::error_code ec; std::filesystem::current_path(*orig_working_dir, ec); CARBON_CHECK(!ec) << ec.message(); } void FileTestBase::RegisterTests( const char* fixture_label, const llvm::SmallVector& paths, std::function factory) { // Use RegisterTest instead of INSTANTIATE_TEST_CASE_P because of ordering // issues between container initialization and test instantiation by // InitGoogleTest. for (const auto& path : paths) { std::string test_name = path.string().substr(base_dir_len); testing::RegisterTest(fixture_label, test_name.c_str(), nullptr, test_name.c_str(), __FILE__, __LINE__, [=]() { return factory(path); }); } } // Splits outputs to string_view because gtest handles string_view by default. static auto SplitOutput(llvm::StringRef output) -> llvm::SmallVector { if (output.empty()) { return {}; } llvm::SmallVector lines; llvm::StringRef(output).split(lines, "\n"); return llvm::SmallVector(lines.begin(), lines.end()); } // Runs a test and compares output. This keeps output split by line so that // issues are a little easier to identify by the different line. auto FileTestBase::TestBody() -> void { const char* src_dir = getenv("TEST_SRCDIR"); CARBON_CHECK(src_dir); std::string test_file = path().lexically_relative( std::filesystem::path(src_dir).append("carbon")); llvm::errs() << "\nTo test this file alone, run:\n bazel test " << *subset_target << " --test_arg=" << test_file << "\n\n"; SetTmpAsWorkingDir(); // Load expected output. llvm::SmallVector test_files; auto cleanup_test_files = llvm::make_scope_exit([&]() { // Remove all the created files. Even if no files are split, there will be a // symlink to remove. for (const auto& file : test_files) { std::filesystem::remove(file); } }); llvm::SmallVector> expected_stdout; llvm::SmallVector> expected_stderr; ProcessTestFile(test_files, expected_stdout, expected_stderr); if (HasFailure()) { return; } // Capture trace streaming, but only when in debug mode. std::string stdout; std::string stderr; llvm::raw_string_ostream stdout_ostream(stdout); llvm::raw_string_ostream stderr_ostream(stderr); bool run_succeeded = RunWithFiles(test_files, stdout_ostream, stderr_ostream); if (HasFailure()) { return; } EXPECT_THAT(!llvm::StringRef(path().filename()).starts_with("fail_"), Eq(run_succeeded)) << "Tests should be prefixed with `fail_` if and only if running them " "is expected to fail."; // Check results. EXPECT_THAT(SplitOutput(stdout), ElementsAreArray(expected_stdout)); EXPECT_THAT(SplitOutput(stderr), ElementsAreArray(expected_stderr)); } auto FileTestBase::SetTmpAsWorkingDir() -> void { char* tmpdir = getenv("TEST_TMPDIR"); CARBON_CHECK(tmpdir); // Run from the tmpdir. std::error_code ec; std::filesystem::current_path(tmpdir, ec); CARBON_CHECK(!ec) << ec.message(); } auto FileTestBase::ProcessTestFile( llvm::SmallVector& test_files, llvm::SmallVector>& expected_stdout, llvm::SmallVector>& expected_stderr) -> void { std::ifstream file_content(path()); std::ofstream split_file; bool found_content_pre_split = false; int line_index = 0; std::string line_str; while (std::getline(file_content, line_str)) { llvm::StringRef line = line_str; if (line.consume_front("// ---")) { ASSERT_FALSE(found_content_pre_split) << "When using split files, there must be no content before the " "first split file."; test_files.push_back(line.trim().str()); const auto& test_file = test_files.back(); ASSERT_FALSE(std::filesystem::path(test_file).has_parent_path()) << "Only filenames are supported, not subdirectories: " << test_file; if (split_file.is_open()) { split_file.close(); } split_file.open(test_file); ASSERT_TRUE(split_file.is_open()) << "Failed to open " << test_file; line_index = 0; continue; } else if (split_file.is_open()) { split_file << line_str << "\n"; } else if (!line.starts_with("//") && !line.trim().empty()) { found_content_pre_split = true; } ++line_index; line = line.ltrim(); if (!line.consume_front("// CHECK")) { continue; } if (line.consume_front(":STDOUT:")) { expected_stdout.push_back(TransformExpectation(line_index, line)); } else if (line.consume_front(":STDERR:")) { expected_stderr.push_back(TransformExpectation(line_index, line)); } else { FAIL() << "Unexpected CHECK in input: " << line_str; } } if (test_files.empty()) { // Symlink the main test file and provide it in test_files. test_files.push_back(path().filename()); std::error_code ec; std::filesystem::create_symlink(path(), path().filename(), ec); CARBON_CHECK(!ec) << ec.message(); } else { split_file.close(); } // Assume there is always a suffix `\n` in output. if (!expected_stdout.empty()) { expected_stdout.push_back(testing::StrEq("")); } if (!expected_stderr.empty()) { expected_stderr.push_back(testing::StrEq("")); } } auto FileTestBase::TransformExpectation(int line_index, llvm::StringRef in) -> testing::Matcher { if (in.empty()) { return testing::StrEq(""); } CARBON_CHECK(in[0] == ' ') << "Malformated input: " << in; std::string str = in.substr(1).str(); for (int pos = 0; pos < static_cast(str.size());) { switch (str[pos]) { case '(': case ')': case ']': case '}': case '.': case '^': case '$': case '*': case '+': case '?': case '|': case '\\': { // Escape regex characters. str.insert(pos, "\\"); pos += 2; break; } case '[': { llvm::StringRef line_keyword_cursor = llvm::StringRef(str).substr(pos); if (line_keyword_cursor.consume_front("[[")) { static constexpr llvm::StringLiteral LineKeyword = "@LINE"; if (line_keyword_cursor.consume_front(LineKeyword)) { // Allow + or - here; consumeInteger handles -. line_keyword_cursor.consume_front("+"); int offset; // consumeInteger returns true for errors, not false. CARBON_CHECK(!line_keyword_cursor.consumeInteger(10, offset) && line_keyword_cursor.consume_front("]]")) << "Unexpected @LINE offset at `" << line_keyword_cursor.substr(0, 5) << "` in: " << in; std::string int_str = llvm::Twine(line_index + offset).str(); int remove_len = (line_keyword_cursor.data() - str.data()) - pos; str.replace(pos, remove_len, int_str); pos += int_str.size(); } else { CARBON_FATAL() << "Unexpected [[, should be {{\\[\\[}} at `" << line_keyword_cursor.substr(0, 5) << "` in: " << in; } } else { // Escape the `[`. str.insert(pos, "\\"); pos += 2; } break; } case '{': { if (pos + 1 == static_cast(str.size()) || str[pos + 1] != '{') { // Single `{`, escape it. str.insert(pos, "\\"); pos += 2; } else { // Replace the `{{...}}` regex syntax with standard `(...)` syntax. str.replace(pos, 2, "("); for (++pos; pos < static_cast(str.size() - 1); ++pos) { if (str[pos] == '}' && str[pos + 1] == '}') { str.replace(pos, 2, ")"); ++pos; break; } } } break; } default: { ++pos; } } } return testing::MatchesRegex(str); } } // namespace Carbon::Testing auto main(int argc, char** argv) -> int { testing::InitGoogleTest(&argc, argv); llvm::setBugReportMsg( "Please report issues to " "https://github.com/carbon-language/carbon-lang/issues and include the " "crash backtrace.\n"); llvm::InitLLVM init_llvm(argc, argv); if (argc < 2) { llvm::errs() << "At least one test file must be provided.\n"; return EXIT_FAILURE; } const char* target = getenv("TEST_TARGET"); CARBON_CHECK(target != nullptr); // Configure the name of the subset target. std::string subset_target_storage = target; static constexpr char SubsetSuffix[] = ".subset"; if (!llvm::StringRef(subset_target_storage).ends_with(SubsetSuffix)) { subset_target_storage += SubsetSuffix; } Carbon::Testing::subset_target = &subset_target_storage; // Save the working directory for later restoration. std::error_code ec; std::filesystem::path orig_working_dir_storage = std::filesystem::current_path(ec); CARBON_CHECK(!ec) << ec.message(); Carbon::Testing::orig_working_dir = &orig_working_dir_storage; // Configure the base directory for test names. llvm::StringRef target_dir = target; // Leaves one slash. CARBON_CHECK(target_dir.consume_front("/")); target_dir = target_dir.substr(0, target_dir.rfind(":")); std::string base_dir = orig_working_dir_storage.string() + target_dir.str() + "/"; Carbon::Testing::base_dir_len = base_dir.size(); // Register tests based on their absolute path. llvm::SmallVector paths; for (int i = 1; i < argc; ++i) { auto path = std::filesystem::absolute(argv[i], ec); CARBON_CHECK(!ec) << argv[i] << ": " << ec.message(); CARBON_CHECK(llvm::StringRef(path.string()).starts_with(base_dir)) << "\n " << path << "\n should start with\n " << base_dir; paths.push_back(path); } Carbon::Testing::RegisterFileTests(paths); return RUN_ALL_TESTS(); }