Fix the installable archive build. (#4064)

There was a file that got into the testing filegroups but not the actual
built tarball. The result was that the installation failed to locate
itself correctly.

We also didn't have any testing to catch this. I'd like to add a bit
more end-to-end testing long-term, but for now just add a Python test
that validates the same set of files are in both.

Sadly, building and testing the compressed release is really slow and
not likely worthwhile outside of the actual release, but it would be
good to catch this stuff earlier. I tried switching from `bz2` to `gz`,
which made very little file size difference (so we should do it
anyways), and it is still too slow to do routinely. Instead, I've added
a Starlark macro that builds both the `pkg_tar` and the `py_test` to
validate it for both uncompressed `tar` and compressed `tar.gz`. This
lets us continuously test the `tar` version, and just test the `tar.gz`
in the nightly release run.

Updates the nightly release workflow to both test, run this specific
test, and use `gz`.

Last but not least, removes the `pkg_zip` as I don't think we have a use
case for this yet and some TODOs that should have been removed when the
version got added.

.github/workflows/nightly-release.yaml

@@ -65,10 +65,11 @@ jobs:
         run: |
           ./scripts/run_bazel.py \
             --attempts=5 --jobs-on-last-attempt=4 \
-            build -c opt --remote_download_toplevel \
+            test -c opt --remote_download_toplevel \
             --pre_release=nightly --nightly_date=${{ env.nightly_date }} \
             //toolchain/install:prefix_root/bin/carbon \
-            //toolchain/install:carbon_toolchain_tar_rule
+            //toolchain/install:carbon_toolchain_tar_gz_rule \
+            //toolchain/install:carbon_toolchain_tar_gz_test
 
       - name: Extract the release version
         run: |
@@ -90,4 +91,4 @@ jobs:
             --notes 'A nightly development build of Carbon.' \
             --prerelease \
             v${{ env.release_version }} \
-            "bazel-bin/toolchain/install/carbon_toolchain-${{ env.release_version }}.tar.bz2"
+            "bazel-bin/toolchain/install/carbon_toolchain-${{ env.release_version }}.tar.gz"

toolchain/install/BUILD

@@ -6,9 +6,7 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file")
 load("@llvm-project//llvm:binary_alias.bzl", "binary_alias")
 load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test")
 load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files", "pkg_mklink", "strip_prefix")
-load("@rules_pkg//pkg:tar.bzl", "pkg_tar")
-load("@rules_pkg//pkg:zip.bzl", "pkg_zip")
-load("pkg_naming.bzl", "pkg_naming_variables")
+load("pkg_helpers.bzl", "pkg_naming_variables", "pkg_tar_and_test")
 load("symlink_filegroup.bzl", "symlink_filegroup")
 
 package(default_visibility = ["//visibility:public"])
@@ -32,10 +30,22 @@ write_file(
     ],
 )
 
+# Create symlinks for the core library. Note that this should *not* be depended
+# on directly, use `:core_data` instead.
 symlink_filegroup(
-    name = "core_data",
+    name = "symlink_core",
     srcs = ["//core:prelude"],
     out_prefix = "prefix_root/lib/carbon/",
+    visibility = ["//visibility:private"],
+)
+
+# The filegroup to get the core library.
+filegroup(
+    name = "core_data",
+    srcs = [
+        ":install_marker",
+        ":symlink_core",
+    ],
 )
 
 # Copy Clang and LLVM toolchain files into a synthetic LLVM installation under
@@ -170,15 +180,17 @@ pkg_files(
 ]
 
 pkg_files(
-    name = "packaging_core_files",
-    srcs = [":core_data"],
+    name = "packaging_data_files",
+    srcs = [
+        ":core_data",
+    ],
     strip_prefix = strip_prefix.from_pkg("prefix_root"),
 )
 
 pkg_filegroup(
     name = "packaging_files",
     srcs = [
-        ":packaging_core_files",
+        ":packaging_data_files",
         ":packaging_exe_files",
     ] + [
         ":packaging_link_lld_alias_" + bin_name
@@ -190,28 +202,18 @@ pkg_naming_variables(
     name = "packaging_variables",
 )
 
-# TODO: We should add support for injecting a version string into both the
-# output filename and the package directory name.
-pkg_tar(
-    name = "carbon_toolchain_tar_rule",
+# We build both a compressed and uncompressed tar file with the same code here.
+# This lets us use the tar file in testing as it is fast to create, but ship the
+# compressed version as a release.
+pkg_tar_and_test(
     srcs = [":packaging_files"],
-    extension = "tar.bz2",
+    name_base = "carbon_toolchain",
     package_dir = "carbon_toolchain-$(version)",
-    package_file_name = "carbon_toolchain-$(version).tar.bz2",
+    package_file_name_base = "carbon_toolchain-$(version)",
     package_variables = ":packaging_variables",
     stamp = -1,  # Allow `--stamp` builds to produce file timestamps.
-    tags = ["manual"],  # Slow, exclude from wildcard builds.
-)
-
-# TODO: We should add support for injecting a version string into both the
-# output filename and the package directory name.
-pkg_zip(
-    name = "carbon_toolchain_zip_rule",
-    srcs = [":packaging_files"],
-    out = "carbon_toolchain.zip",
-    package_dir = "carbon_toolchain-$(version)",
-    package_file_name = "carbon_toolchain-$(version).zip",
-    package_variables = ":packaging_variables",
-    stamp = -1,  # Allow `--stamp` builds to produce file timestamps.
-    tags = ["manual"],  # Slow, exclude from wildcard builds.
+    test_data = [
+        ":install_data",
+    ],
+    test_install_marker = ":install_marker",
 )

toolchain/install/pkg_helpers.bzl

@@ -0,0 +1,69 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+"""Rule to create variables for package naming."""
+
+load("@rules_pkg//pkg:providers.bzl", "PackageVariablesInfo")
+load("@rules_pkg//pkg:tar.bzl", "pkg_tar")
+load("@rules_python//python:defs.bzl", "py_test")
+load("//bazel/version:compute_version.bzl", "VERSION_ATTRS", "compute_version")
+
+def _pkg_naming_variables_impl(ctx):
+    # TODO: Add support for digging the target CPU out of the toolchain here,
+    # remapping it to a more canonical name, and add that to the variables. The
+    # Bazel target CPU is already directly available, but it isn't likely
+    # canonical.
+    # TODO: Include the target OS as well as the target CPU. This likely needs
+    # similar re-mapping as the CPU does.
+    return PackageVariablesInfo(values = {
+        "version": compute_version(ctx),
+    })
+
+pkg_naming_variables = rule(
+    implementation = _pkg_naming_variables_impl,
+    attrs = VERSION_ATTRS,
+)
+
+def pkg_tar_and_test(name_base, package_file_name_base, test_data, test_install_marker, **kwargs):
+    """Create a `pkg_tar` and a test for both `.tar` and `.tar.gz` extensions.
+
+    Args:
+        name_base:
+            The base name of the rules and tests. Will have `tar` or `tar_gz` added
+            and then `_rule` for the `pkg_tar` and `_test` for the test.
+        package_file_name_base:
+            The base of the `package_file_name` attribute to `pkg_tar`. The file
+            extensions will be appended after a `.`.
+        test_data:
+            The test data to verify the tar file against.
+        test_install_marker:
+            The install marker within the test data to locate the installation.
+        **kwargs:
+            Passed to `pkg_tar` for all the rest of its attributes.
+    """
+    for file_ext in ["tar", "tar.gz"]:
+        target_ext = file_ext.replace(".", "_")
+        tar_target = name_base + "_" + target_ext + "_rule"
+        pkg_tar(
+            name = tar_target,
+            extension = file_ext,
+            package_file_name = package_file_name_base + "." + file_ext,
+            # The compressed tar is slow, exclude building and testing that.
+            tags = ["manual"] if file_ext == "tar.gz" else [],
+            **kwargs
+        )
+
+        py_test(
+            name = name_base + "_" + target_ext + "_test",
+            size = "small",
+            srcs = ["toolchain_tar_test.py"],
+            data = [":" + tar_target, test_install_marker] + test_data,
+            args = [
+                "$(location :{})".format(tar_target),
+                "$(location {})".format(test_install_marker),
+            ],
+            main = "toolchain_tar_test.py",
+            # The compressed tar is slow, exclude building and testing that.
+            tags = ["manual"] if file_ext == "tar.gz" else [],
+        )

toolchain/install/pkg_naming.bzl

@@ -1,24 +0,0 @@
-# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-# Exceptions. See /LICENSE for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-"""Rule to create variables for package naming."""
-
-load("@rules_pkg//pkg:providers.bzl", "PackageVariablesInfo")
-load("//bazel/version:compute_version.bzl", "VERSION_ATTRS", "compute_version")
-
-def _pkg_naming_variables_impl(ctx):
-    # TODO: Add support for digging the target CPU out of the toolchain here,
-    # remapping it to a more canonical name, and add that to the variables. The
-    # Bazel target CPU is already directly available, but it isn't likely
-    # canonical.
-    # TODO: Include the target OS as well as the target CPU. This likely needs
-    # similar re-mapping as the CPU does.
-    return PackageVariablesInfo(values = {
-        "version": compute_version(ctx),
-    })
-
-pkg_naming_variables = rule(
-    implementation = _pkg_naming_variables_impl,
-    attrs = VERSION_ATTRS,
-)

toolchain/install/toolchain_tar_test.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+"""Check that a release tar contains the same files as a prefix root."""
+
+__copyright__ = """
+Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+Exceptions. See /LICENSE for license information.
+SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""
+
+import argparse
+import sys
+from pathlib import Path
+import tarfile
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "tar_file",
+        type=Path,
+        help="The tar file to test.",
+    )
+    parser.add_argument(
+        "install_marker",
+        type=Path,
+        help="The path of the install marker in a prefix root to test against.",
+    )
+    args = parser.parse_args()
+
+    # Locate the prefix root from the install marker.
+    if not args.install_marker.exists():
+        sys.exit("ERROR: No install marker: " + args.install_marker)
+    prefix_root_path = args.install_marker.parent.parent.parent
+
+    # First check that every file and directory in the tar file exists in our
+    # prefix root, and build a set of those paths.
+    installed_paths = set()
+    with tarfile.open(args.tar_file) as tar:
+        for tarinfo in tar:
+            relative_path = Path(*Path(tarinfo.name).parts[1:])
+            installed_paths.add(relative_path)
+            if not prefix_root_path.joinpath(relative_path).exists():
+                sys.exit(
+                    "ERROR: File `{0}` is not in prefix root: `{1}`".format(
+                        tarinfo.name, prefix_root_path
+                    )
+                )
+
+    # If we found an empty tar file, it's always an error.
+    if len(installed_paths) == 0:
+        sys.exit("ERROR: Tar file `{0}` was empty.".format(args.tar_file))
+
+    # Now check that every file and directory in the prefix root is in that set.
+    for prefix_path in prefix_root_path.glob("**/*"):
+        relative_path = prefix_path.relative_to(prefix_root_path)
+        if relative_path not in installed_paths:
+            sys.exit(
+                "ERROR: File `{0}` is not in tar file.".format(relative_path)
+            )
+
+
+if __name__ == "__main__":
+    main()