Add GHC 9.8 to update-ghc workflow

.github/update-ghc.py

@@ -78,44 +78,23 @@ def main():
 
         print("found update:", latest_release, file=sys.stderr)
 
-        replace = re.compile(
-            r'^(?P<indent>\s+)\{\s*"version"\s*:\s*"'
-            + re.escape(GHC_MAJOR_MINOR + "."),
-            re.MULTILINE,
-        )
-
-        print(" 1. modify haskell/gen_ghc_bindist.py", file=sys.stderr)
-
-        gen_script_path = SCRIPT_PATH.parent.parent.joinpath(
-            "haskell/gen_ghc_bindist.py"
-        )
-        with open(gen_script_path, "r+") as gen:
-            gen_script = gen.read()
-
-            added_version = replace.sub(
-                rf"""\g<indent>{{ "version": { repr(latest_release) },
-\g<indent>  "ignore_suffixes": [".bz2", ".lz", ".zip"] }},
-\g<0>""",
-                gen_script,
-                count=1,
-            )
-
-            if added_version is gen_script:
-                sys.exit(
-                    f"could not add new version {latest_release} using regex {replace}"
-                )
-
-            gen.truncate(0)
-            gen.seek(0)
-            gen.write(added_version)
-
-        print(" 2. call haskell/gen_ghc_bindist.py", file=sys.stderr)
-
-        check_call([sys.executable, "haskell/gen_ghc_bindist.py"])
-
-        if "GITHUB_OUTPUT" in os.environ:
-            with open(os.environ["GITHUB_OUTPUT"], "a") as output:
-                print(f"latest={ latest_release }", file=output)
+        print(" 1. modify haskell/private/ghc_bindist_generated.json", file=sys.stderr)
+
+        bindists[latest_release] = {}
+
+        bindist_json.truncate(0)
+        bindist_json.seek(0)
+        json.dump(bindists, bindist_json)
+
+    gen_script_path = SCRIPT_PATH.parent.parent.joinpath("haskell/gen_ghc_bindist.py")
+
+    print(" 2. call haskell/gen_ghc_bindist.py", file=sys.stderr)
+
+    check_call([sys.executable, "haskell/gen_ghc_bindist.py"])
+
+    if "GITHUB_OUTPUT" in os.environ:
+        with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+            print(f"latest={ latest_release }", file=output)
 
 
 if __name__ == "__main__":

.github/workflows/update-ghc.yaml

@@ -16,8 +16,11 @@ jobs:
           - '9.2'
           - '9.4'
           - '9.6'
+          - '9.8'
     steps:
       - uses: actions/checkout@v4
+        with:
+          ref: master
       - uses: cachix/install-nix-action@v23
         with:
           nix_path: nixpkgs=nixpkgs/default.nix

haskell/gen_ghc_bindist.py

@@ -10,180 +10,158 @@
 from urllib.request import urlopen
 from distutils.version import StrictVersion
 
-# All GHC versions we generate.
-# `version` is the version number
-# `distribution_version` is a corrected name
-# (sometimes bindists have errors and are updated by new bindists)
-# `ignore_prefixes` is the prefix of files to ignore
-# `ignore_suffixes` is the suffix of files to ignore
-VERSIONS = [
-    { "version": '9.6.3',
-      "ignore_suffixes": [".bz2", ".lz", ".zip"] },
-    { "version": "9.6.2",
-      "ignore_suffixes": [".bz2", ".lz", ".zip"] },
-    { "version": "9.6.1",
-      "ignore_suffixes": [".bz2", ".lz", ".zip"] },
-    { "version": '9.4.6',
-      "ignore_suffixes": [".bz2", ".lz", ".zip"] },
-    { "version": '9.4.7',
-      "ignore_suffixes": [".bz2", ".lz", ".zip"] },
-    { "version": "9.4.5",
-      "ignore_suffixes": [".bz2", ".lz", ".zip"] },
-    { "version": '9.2.8',
-      "ignore_suffixes": [".bz2", ".lz", ".zip"] },
-    { "version": "9.2.5",
-      "ignore_suffixes": [".bz2", ".lz", ".zip"] },
-    { "version": "9.2.4",
-      "ignore_suffixes": [".bz2", ".lz", ".zip"] },
-    { "version": "9.2.3",
-      "ignore_suffixes": [".bz2", ".lz", ".zip"] },
-    { "version": "9.2.1",
-      "ignore_suffixes": [".bz2", ".lz", ".zip"] },
-    { "version": "9.0.2",
-      "ignore_prefixes": ["ghc-9.0.2a"],
-      "ignore_suffixes": [".bz2", ".lz", ".zip"] },
-    { "version": "9.0.1",
-      "ignore_suffixes": [".bz2", ".lz", ".zip"] },
-    { "version": "8.10.7",
-      "ignore_suffixes": [".bz2", ".lz", ".zip"] },
-    { "version": "8.10.4" },
-    { "version": "8.10.3" },
-    { "version": "8.8.4" },
-    { "version": "8.6.5" },
-    { "version": "8.4.4" },
-    { "version": "8.4.3" },
-    { "version": "8.4.2" },
-    { "version": "8.4.1" },
-    { "version": "8.2.2" },
-]
+# Sometimes bindists have errors and are updated by new bindists.
+# This dict is used to keep a version -> corrected_version mapping.
+VERSIONS_CORRECTED = {}
 
 # All architectures we generate.
 # bazel: bazel name
 # upstream: list of download.haskell.org name
 ARCHES = [
-    { "bazel": "linux_amd64",
-      "upstream": ["x86_64-deb8-linux", "x86_64-deb9-linux", "x86_64-deb10-linux"], },
-    { "bazel": "linux_arm64",
-      "upstream": ["aarch64-deb10-linux"], },
-    { "bazel": "darwin_amd64",
-      "upstream": ["x86_64-apple-darwin"] },
-    { "bazel": "darwin_arm64",
-      "upstream": ["aarch64-apple-darwin"] },
-    { "bazel": "windows_amd64",
-      "upstream": ["x86_64-unknown-mingw32"] },
+    {
+        "bazel": "linux_amd64",
+        "upstream": ["x86_64-deb8-linux", "x86_64-deb9-linux", "x86_64-deb10-linux"],
+    },
+    {
+        "bazel": "linux_arm64",
+        "upstream": ["aarch64-deb10-linux"],
+    },
+    {"bazel": "darwin_amd64", "upstream": ["x86_64-apple-darwin"]},
+    {"bazel": "darwin_arm64", "upstream": ["aarch64-apple-darwin"]},
+    {"bazel": "windows_amd64", "upstream": ["x86_64-unknown-mingw32"]},
 ]
 
 
 # An url to a bindist tarball.
 def link_for_tarball(arch, version):
     return "https://downloads.haskell.org/~ghc/{ver}/ghc-{ver}-{arch}.tar.xz".format(
-        ver = version,
-        arch = arch,
+        ver=version,
+        arch=arch,
     )
 
+
 # An url to a version's tarball hashsum file.
 # The files contain the hashsums for all arches.
 def link_for_sha256_file(version):
-    return "https://downloads.haskell.org/~ghc/{ver}/SHA256SUMS".format(
-        ver = version
-    )
+    return "https://downloads.haskell.org/~ghc/{ver}/SHA256SUMS".format(ver=version)
+
 
 # Parses the tarball hashsum file for a distribution version.
 def parse_sha256_file(content, version, url):
     res = {}
-    errs = []
+
+    prefix = "ghc-{ver}-".format(ver=VERSIONS_CORRECTED.get(version, version))
+    suffix = ".tar.xz"
+
     for line in content:
         # f5763983a26dedd88b65a0b17267359a3981b83a642569b26334423f684f8b8c  ./ghc-8.4.3-i386-deb8-linux.tar.xz
         (hash, file_) = line.decode().strip().split("  ./")
-        prefix = "ghc-{ver}-".format(ver = version.get("distribution_version", version['version']))
-        suffix = ".tar.xz"
-
-        # filter ignored files
-        if   any([file_.startswith(p) for p in version.get("ignore_prefixes", [])]) \
-          or any([file_.endswith(s)   for s in version.get("ignore_suffixes", [])]):
-            continue
 
         if file_.startswith(prefix) and file_.endswith(suffix):
             # i386-deb8-linux
-            name = file_[len(prefix):-len(suffix)]
+            name = file_[len(prefix) : -len(suffix)]
             res[name] = hash
-        else:
-            errs.append("Can't parse the sha256 field for {ver}: {entry}".format(
-                ver = version['version'], entry = line.strip()))
 
-    if errs:
-        eprint("Errors parsing file at " + url + ". Either fix or ignore the lines (ignore_suffixes/ignore_prefixes).")
-        for e in errs:
-            eprint(e)
+    if not res:
+        eprint(
+            f"Errors parsing file at {url}. Could not find entries for {prefix}…{suffix}"
+        )
         exit(1)
 
     return res
 
+
 # Print to stderr.
 def eprint(mes):
-    print(mes, file = sys.stderr)
+    print(mes, file=sys.stderr)
+
 
 def select_one(xs, ys):
     """Select a single item from xs, prefer the first item also in ys."""
     items = [x for x in xs if x in ys]
     return items[0] if items else xs[0]
 
-# Main.
-if __name__ == "__main__":
+
+def fetch_hashsums(versions):
     # Fetch all hashsum files
     # grab : { version: { arch: sha256 } }
     grab = {}
-    for ver in VERSIONS:
-        eprint("fetching " + ver['version'])
-        url = link_for_sha256_file(ver['version'])
+    for ver in versions:
+        eprint("fetching " + ver)
+        url = link_for_sha256_file(ver)
         res = urlopen(url)
         if res.getcode() != 200:
             eprint("download of {} failed with status {}".format(url, res.getcode()))
             sys.exit(1)
         else:
-            grab[ver['version']] = parse_sha256_file(res, ver, url)
+            grab[ver] = parse_sha256_file(res, ver, url)
 
     # check whether any version is missing arches we need
     # errs : { version: set(missing_arches) }
     errs = {}
     for ver, hashes in grab.items():
-      real_arches = frozenset(hashes.keys())
-      upstreams = [select_one(a['upstream'], real_arches) for a in ARCHES]
-      needed_arches = frozenset(upstreams)
-      missing_arches = needed_arches.difference(real_arches)
-      if missing_arches:
-          errs[ver] = missing_arches
+        real_arches = frozenset(hashes.keys())
+        upstreams = [select_one(a["upstream"], real_arches) for a in ARCHES]
+        needed_arches = frozenset(upstreams)
+        missing_arches = needed_arches.difference(real_arches)
+        if missing_arches:
+            errs[ver] = missing_arches
     if errs:
         for ver, missing in errs.items():
             print(
                 "WARN: version {ver} is missing hashes for architectures {arches}".format(
-                    ver = ver,
-                    arches = ','.join(missing)),
-                file=sys.stderr
+                    ver=ver, arches=",".join(missing)
+                ),
+                file=sys.stderr,
             )
 
+    return grab
+
+
+def fetch_bindists(grab):
     # fetch the arches we need and create the GHC_BINDISTS dict
     # ghc_bindists : { version: { bazel_arch: (tarball_url, sha256_hash) } }
     ghc_bindists = {}
     for ver, hashes in grab.items():
         # { bazel_arch: (tarball_url, sha256_hash) }
         arch_dists = {}
         for arch in ARCHES:
-            upstream = select_one(arch['upstream'], hashes)
+            upstream = select_one(arch["upstream"], hashes)
 
             if upstream in hashes:
-                arch_dists[arch['bazel']] = (
+                arch_dists[arch["bazel"]] = (
                     link_for_tarball(upstream, ver),
-                    hashes[upstream]
+                    hashes[upstream],
                 )
         ghc_bindists[ver] = arch_dists
 
-    ghc_versions = { version: ghc_bindists[version] for version in sorted(ghc_bindists.keys(), key=StrictVersion) }
+    return ghc_bindists
+
 
+# Main.
+if __name__ == "__main__":
     working_directory = os.environ.get("BUILD_WORKING_DIRECTORY", ".")
 
-    with open(os.path.join(working_directory, "haskell/private/ghc_bindist_generated.json"), "w", encoding="utf-8") as json_file:
-        json.dump(ghc_versions, json_file, indent=4)
-        json_file.write('\n')
+    with open(
+        os.path.join(working_directory, "haskell/private/ghc_bindist_generated.json"),
+        "r+",
+        encoding="utf-8",
+    ) as json_file:
+        ghc_versions = json.load(json_file)
+
+        # All GHC versions we generate.
+        versions = ghc_versions.keys()
 
+        grab = fetch_hashsums(versions)
 
+        ghc_bindists = fetch_bindists(grab)
+
+        ghc_versions = {
+            version: ghc_bindists[version]
+            for version in sorted(ghc_bindists.keys(), key=StrictVersion)
+        }
+
+        json_file.truncate(0)
+        json_file.seek(0)
+        json.dump(ghc_versions, json_file, indent=4)
+        json_file.write("\n")