Move the jekyll sidebar to a python script for consistent generation. (#205)

One implication here is that both proposals/README.md and the website should use basically the same format for their proposal lists. Another subtle change is that the website sidebar was previously sorted by *filename*, and is now sorted by *title*, which seems better because it's something that readers can see.

Enumerating why files change:

- gen_sidebar.py is the centerpiece, with a couple helpful functions for re-use elsewhere
- Delete the old sidebar html includes
- Modify the Makefile to handle gen_sidebar.py reasonably well (let's be honest... I'm not great at Makefiles)
- Move proposal listing out to its own file (proposals.py) for re-use
- Add a few PYTHONPATH things + `__init__.py` files to get modules importing correctly (may be a better way at this, I've hit a wall though).

.github/workflows/publish-docs.yaml

@@ -36,4 +36,4 @@ jobs:
         run: make build
         working-directory: website/jekyll
       - name: Publish to www.carbon-lang.dev
-        run: gsutil cp -R website/jekyll/.gen-site/* gs://www.carbon-lang.dev/
+        run: gsutil cp -R website/jekyll/build/site/* gs://www.carbon-lang.dev/

.pre-commit-config.yaml

@@ -17,6 +17,7 @@ repos:
       - id: check-executables-have-shebangs
       - id: check-merge-conflict
       - id: check-symlinks
+        exclude: '^website/jekyll/site/_includes$'
       - id: check-yaml
       - id: detect-private-key
       - id: end-of-file-fixer

docs/design/README.md

@@ -1,4 +1,4 @@
-# Language design overview
+# Language design
 
 <!--
 Part of the Carbon Language project, under the Apache License v2.0 with LLVM

proposals/README.md

@@ -21,36 +21,36 @@ request:
 ## Proposal list
 
 <!-- proposals -->
-<!-- This list is updated by src/scripts/update_proposal_list.py. -->
+<!-- Generated by ./scripts/update_proposal_list.py -->
 
 -   [0029 - Linear, rebase, and pull-request GitHub workflow](p0029.md)
-    -   [Decision](p0029_decision.md)
+    -   [0029 - Decision](p0029_decision.md)
 -   [0042 - Create code review guidelines](p0042.md)
-    -   [Decision](p0042_decision.md)
+    -   [0042 - Decision](p0042_decision.md)
 -   [0044 - Proposal tracking](p0044.md)
-    -   [Decision](p0044_decision.md)
+    -   [0044 - Decision](p0044_decision.md)
 -   [0051 - Goals](p0051.md)
-    -   [Decision](p0051_decision.md)
+    -   [0051 - Decision](p0051_decision.md)
 -   [0063 - Criteria for Carbon to go public](p0063.md)
-    -   [Decision](p0063_decision.md)
+    -   [0063 - Decision](p0063_decision.md)
 -   [0074 - Change comment/decision timelines in proposal process](p0074.md)
-    -   [Decision](p0074_decision.md)
+    -   [0074 - Decision](p0074_decision.md)
 -   [0083 - In-progress design overview](p0083.md)
-    -   [Decision](p0083_decision.md)
+    -   [0083 - Decision](p0083_decision.md)
 -   [0107 - Code and name organization](p0107.md)
-    -   [Decision](p0107_decision.md)
+    -   [0107 - Decision](p0107_decision.md)
 -   [0113 - Add a C++ style guide](p0113.md)
-    -   [Decision](p0113_decision.md)
+    -   [0113 - Decision](p0113_decision.md)
 -   [0120 - Add idiomatic code performance and developer-facing docs to goals](p0120.md)
-    -   [Decision](p0120_decision.md)
+    -   [0120 - Decision](p0120_decision.md)
 -   [0142 - Unicode source files](p0142.md)
-    -   [Decision](p0142_decision.md)
+    -   [0142 - Decision](p0142_decision.md)
 -   [0143 - Numeric literals](p0143.md)
-    -   [Decision](p0143_decision.md)
+    -   [0143 - Decision](p0143_decision.md)
 -   [0149 - Change documentation style guide](p0149.md)
-    -   [Decision](p0149_decision.md)
+    -   [0149 - Decision](p0149_decision.md)
 -   [0162 - Basic Syntax](p0162.md)
-    -   [Decision](p0162_decision.md)
+    -   [0162 - Decision](p0162_decision.md)
 -   [0175 - C++ interoperability goals](p0175.md)
 
 <!-- endproposals -->

proposals/scripts/new_proposal_test.py

@@ -12,7 +12,7 @@ import os
 import unittest
 from unittest import mock
 
-import new_proposal
+from proposals.scripts import new_proposal
 
 
 class FakeExitError(Exception):
@@ -60,7 +60,9 @@ class TestNewProposal(unittest.TestCase):
         new_proposal._run(["true"])
 
     def test_run_failure(self):
-        with mock.patch("new_proposal._exit", side_effect=_fake_exit):
+        with mock.patch(
+            "proposals.scripts.new_proposal._exit", side_effect=_fake_exit
+        ):
             self.assertRaises(FakeExitError, new_proposal._run, ["false"])
 
 

proposals/scripts/proposals.py

@@ -0,0 +1,42 @@
+"""Provides a list of proposal files."""
+
+__copyright__ = """
+Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+Exceptions. See /LICENSE for license information.
+SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""
+
+import os
+import re
+import sys
+
+
+def get_title(parent_dir, entry):
+    """Returns the title from the requested file."""
+    path = os.path.join(parent_dir, entry)
+    with open(path) as md:
+        titles = [t for t in md.readlines() if t.startswith("# ")]
+        if not titles:
+            sys.exit("%r is missing a title." % path)
+        return titles[0][2:-1]
+
+
+def get_path():
+    return os.path.realpath(os.path.join(os.path.dirname(__file__), ".."))
+
+
+def get_list(proposals_path):
+    proposals = []
+    proposals_list = os.listdir(proposals_path)
+    proposals_list.sort()
+    for f in proposals_list:
+        match = re.match(r"^p([0-9]{4})\.md$", f)
+        if not match:
+            continue
+        number = match[1]
+        title = get_title(proposals_path, f)
+        proposals.append(("%s - %s" % (number, title), f))
+        decision_file = "p%s_decision.md" % number
+        if decision_file in proposals_list:
+            proposals.append(("%s - Decision" % number, decision_file))
+    return proposals

proposals/scripts/proposals_test.py

@@ -0,0 +1,25 @@
+"""Tests for proposals.py."""
+
+__copyright__ = """
+Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+Exceptions. See /LICENSE for license information.
+SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""
+
+import unittest
+
+from proposals.scripts import proposals
+
+
+class TestProposal(unittest.TestCase):
+    def test_get_path(self):
+        proposals_path = proposals.get_path()
+        p = proposals.get_list(proposals_path)
+        self.assertEqual(
+            p[0],
+            (
+                "0029 - Linear, rebase, and pull-request GitHub workflow",
+                "p0029.md",
+            ),
+        )
+        self.assertEqual(p[1], ("0029 - Decision", "p0029_decision.md"))

proposals/scripts/update_proposal_list.py

@@ -8,48 +8,33 @@ Exceptions. See /LICENSE for license information.
 SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 """
 
+import io
 import os
 import re
 import sys
 
+# To support direct runs, ensure the pythonpath has the repo root.
+_PYTHONPATH = os.path.realpath(os.path.join(os.path.dirname(__file__), "../.."))
+if _PYTHONPATH not in sys.path:
+    sys.path.insert(0, _PYTHONPATH)
+
+from proposals.scripts import proposals
+
 if __name__ == "__main__":
-    # Read the proposal dir relative to this script.
-    proposal_dir = os.path.realpath(
-        os.path.join(os.path.dirname(sys.argv[0]), "../../proposals/")
-    )
-    print(proposal_dir)
+    proposals_path = proposals.get_path()
 
-    # Identify proposal titles in the file list.
-    proposals = [
-        "<!-- This list is updated by src/scripts/update_proposal_list.py. "
-        "-->",
-        "",
-    ]
-    error = False
-    for file in sorted(os.listdir(proposal_dir)):
-        file_match = re.match(r"^p([0-9]{4})\.md$", file)
-        if not file_match:
-            continue
-        with open(os.path.join(proposal_dir, file)) as f:
-            content = f.read()
-        title = content.split("\n")[0]
-        title_match = re.match(r"^# (.*)$", title)
-        if not title_match:
-            print("ERROR: %s doesn't have a title on the first line." % file)
-            error = True
-        proposals.append(
-            "-   [%s - %s](%s)" % (file_match[1], title_match[1], file)
-        )
-        decision_file = "p%s_decision.md" % file_match[1]
-        if os.path.exists(os.path.join(proposal_dir, decision_file)):
-            proposals.append("    -   [Decision](%s)" % decision_file)
-    # We print batched errors for usability, but still need to exit with
-    # failure.
-    if error:
-        sys.exit(1)
+    with io.StringIO() as out:
+        out.write("<!-- Generated by ./scripts/update_proposal_list.py -->\n\n")
+        results = out.getvalue()
+        for title, filename in proposals.get_list(proposals_path):
+            indent = ""
+            if filename.endswith("decision.md"):
+                indent = "    "
+            out.write("%s-   [%s](%s)\n" % (indent, title, filename))
+        toc = out.getvalue()
 
     # Replace the README content if needed.
-    readme_path = os.path.join(proposal_dir, "README.md")
+    readme_path = os.path.join(proposals_path, "README.md")
     with open(readme_path) as f:
         old_content = f.read()
     proposals_re = re.compile(
@@ -62,9 +47,7 @@ if __name__ == "__main__":
             "<!-- endproposals --> marker."
         )
         sys.exit(1)
-    new_content = proposals_re.sub(
-        r"\1\n%s\n\n\2" % "\n".join(proposals), old_content
-    )
+    new_content = proposals_re.sub(r"\1\n%s\n\2" % toc, old_content)
     if old_content != new_content:
         print("Updating proposals/README.md")
         with open(readme_path, "w") as f:

setup.cfg

@@ -4,3 +4,7 @@
 
 [flake8]
 max-line-length = 80
+exclude = website/jekyll/build
+# E402: Allow the pythonpath modifications before repo-local imports.
+# W503: flake8 v3.8.4 is inconsistent with black v20.8b1 (pre-commit run -a).
+ignore = E402,W503