From 4fb3d3ec0e0ddfe1f3e1d9b5e9f6d1446882f832 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Wed, 18 Oct 2023 13:01:49 +0200 Subject: [PATCH 1/7] CI: Always run update-ghc workflow on current master --- .github/workflows/update-ghc.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/update-ghc.yaml b/.github/workflows/update-ghc.yaml index 17d6b15c9..9c70f69ea 100644 --- a/.github/workflows/update-ghc.yaml +++ b/.github/workflows/update-ghc.yaml @@ -18,6 +18,8 @@ jobs: - '9.6' steps: - uses: actions/checkout@v4 + with: + ref: master - uses: cachix/install-nix-action@v23 with: nix_path: nixpkgs=nixpkgs/default.nix From a5b9d2e67f5cc02b045e788aaf29dab9f28c4e76 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Wed, 18 Oct 2023 15:27:37 +0200 Subject: [PATCH 2/7] Simplify gen_ghc_bindist.py script Remove the `ignore_prefixes` and `ignore_suffixes` handling. We only process .tar.xz files from the `SHA256SUMS` file expecting a specific prefix. --- .github/update-ghc.py | 3 +- haskell/gen_ghc_bindist.py | 66 ++++++++++++-------------------------- 2 files changed, 21 insertions(+), 48 deletions(-) diff --git a/.github/update-ghc.py b/.github/update-ghc.py index dc646d075..5585aa360 100644 --- a/.github/update-ghc.py +++ b/.github/update-ghc.py @@ -93,8 +93,7 @@ def main(): gen_script = gen.read() added_version = replace.sub( - rf"""\g{{ "version": { repr(latest_release) }, -\g "ignore_suffixes": [".bz2", ".lz", ".zip"] }}, + rf"""\g{{ "version": "{ latest_release }" }}, \g<0>""", gen_script, count=1, diff --git a/haskell/gen_ghc_bindist.py b/haskell/gen_ghc_bindist.py index 5fe5810ae..39a640e4a 100755 --- a/haskell/gen_ghc_bindist.py +++ b/haskell/gen_ghc_bindist.py @@ -14,38 +14,21 @@ # `version` is the version number # `distribution_version` is a corrected name # (sometimes bindists have errors and are updated by new bindists) -# `ignore_prefixes` is the prefix of files to ignore -# `ignore_suffixes` is the suffix of files to ignore VERSIONS = [ - { "version": '9.6.3', - "ignore_suffixes": [".bz2", ".lz", ".zip"] }, - { "version": "9.6.2", - "ignore_suffixes": [".bz2", ".lz", ".zip"] }, - { "version": "9.6.1", - "ignore_suffixes": [".bz2", ".lz", ".zip"] }, - { "version": '9.4.6', - "ignore_suffixes": [".bz2", ".lz", ".zip"] }, - { "version": '9.4.7', - "ignore_suffixes": [".bz2", ".lz", ".zip"] }, - { "version": "9.4.5", - "ignore_suffixes": [".bz2", ".lz", ".zip"] }, - { "version": '9.2.8', - "ignore_suffixes": [".bz2", ".lz", ".zip"] }, - { "version": "9.2.5", - "ignore_suffixes": [".bz2", ".lz", ".zip"] }, - { "version": "9.2.4", - "ignore_suffixes": [".bz2", ".lz", ".zip"] }, - { "version": "9.2.3", - "ignore_suffixes": [".bz2", ".lz", ".zip"] }, - { "version": "9.2.1", - "ignore_suffixes": [".bz2", ".lz", ".zip"] }, - { "version": "9.0.2", - "ignore_prefixes": ["ghc-9.0.2a"], - "ignore_suffixes": [".bz2", ".lz", ".zip"] }, - { "version": "9.0.1", - "ignore_suffixes": [".bz2", ".lz", ".zip"] }, - { "version": "8.10.7", - "ignore_suffixes": [".bz2", ".lz", ".zip"] }, + { "version": "9.6.3" }, + { "version": "9.6.2" }, + { "version": "9.6.1" }, + { "version": "9.4.7" }, + { "version": "9.4.6" }, + { "version": "9.4.5" }, + { "version": "9.2.8" }, + { "version": "9.2.5" }, + { "version": "9.2.4" }, + { "version": "9.2.3" }, + { "version": "9.2.1" }, + { "version": "9.0.2" }, + { "version": "9.0.1" }, + { "version": "8.10.7" }, { "version": "8.10.4" }, { "version": "8.10.3" }, { "version": "8.8.4" }, @@ -91,30 +74,21 @@ def link_for_sha256_file(version): # Parses the tarball hashsum file for a distribution version. def parse_sha256_file(content, version, url): res = {} - errs = [] + + prefix = "ghc-{ver}-".format(ver = version.get("distribution_version", version['version'])) + suffix = ".tar.xz" + for line in content: # f5763983a26dedd88b65a0b17267359a3981b83a642569b26334423f684f8b8c ./ghc-8.4.3-i386-deb8-linux.tar.xz (hash, file_) = line.decode().strip().split(" ./") - prefix = "ghc-{ver}-".format(ver = version.get("distribution_version", version['version'])) - suffix = ".tar.xz" - - # filter ignored files - if any([file_.startswith(p) for p in version.get("ignore_prefixes", [])]) \ - or any([file_.endswith(s) for s in version.get("ignore_suffixes", [])]): - continue if file_.startswith(prefix) and file_.endswith(suffix): # i386-deb8-linux name = file_[len(prefix):-len(suffix)] res[name] = hash - else: - errs.append("Can't parse the sha256 field for {ver}: {entry}".format( - ver = version['version'], entry = line.strip())) - if errs: - eprint("Errors parsing file at " + url + ". Either fix or ignore the lines (ignore_suffixes/ignore_prefixes).") - for e in errs: - eprint(e) + if not res: + eprint(f"Errors parsing file at {url}. Could not find entries for {prefix}…{suffix}") exit(1) return res From 95c047a37772e849b3f7c0f8567de87958207f28 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Wed, 18 Oct 2023 16:34:40 +0200 Subject: [PATCH 3/7] Keep a simple list of GHC versions in the gen_ghc_bindist.py script --- haskell/gen_ghc_bindist.py | 61 +++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/haskell/gen_ghc_bindist.py b/haskell/gen_ghc_bindist.py index 39a640e4a..e7a7f51a5 100755 --- a/haskell/gen_ghc_bindist.py +++ b/haskell/gen_ghc_bindist.py @@ -11,35 +11,36 @@ from distutils.version import StrictVersion # All GHC versions we generate. -# `version` is the version number -# `distribution_version` is a corrected name -# (sometimes bindists have errors and are updated by new bindists) VERSIONS = [ - { "version": "9.6.3" }, - { "version": "9.6.2" }, - { "version": "9.6.1" }, - { "version": "9.4.7" }, - { "version": "9.4.6" }, - { "version": "9.4.5" }, - { "version": "9.2.8" }, - { "version": "9.2.5" }, - { "version": "9.2.4" }, - { "version": "9.2.3" }, - { "version": "9.2.1" }, - { "version": "9.0.2" }, - { "version": "9.0.1" }, - { "version": "8.10.7" }, - { "version": "8.10.4" }, - { "version": "8.10.3" }, - { "version": "8.8.4" }, - { "version": "8.6.5" }, - { "version": "8.4.4" }, - { "version": "8.4.3" }, - { "version": "8.4.2" }, - { "version": "8.4.1" }, - { "version": "8.2.2" }, + "9.6.3", + "9.6.2", + "9.6.1", + "9.4.7", + "9.4.6", + "9.4.5", + "9.2.8", + "9.2.5", + "9.2.4", + "9.2.3", + "9.2.1", + "9.0.2", + "9.0.1", + "8.10.7", + "8.10.4", + "8.10.3", + "8.8.4", + "8.6.5", + "8.4.4", + "8.4.3", + "8.4.2", + "8.4.1", + "8.2.2", ] +# Sometimes bindists have errors and are updated by new bindists. +# This dict is used to keep a version -> corrected_version mapping. +VERSIONS_CORRECTED = {} + # All architectures we generate. # bazel: bazel name # upstream: list of download.haskell.org name @@ -75,7 +76,7 @@ def link_for_sha256_file(version): def parse_sha256_file(content, version, url): res = {} - prefix = "ghc-{ver}-".format(ver = version.get("distribution_version", version['version'])) + prefix = "ghc-{ver}-".format(ver=VERSIONS_CORRECTED.get(version, version)) suffix = ".tar.xz" for line in content: @@ -108,14 +109,14 @@ def select_one(xs, ys): # grab : { version: { arch: sha256 } } grab = {} for ver in VERSIONS: - eprint("fetching " + ver['version']) - url = link_for_sha256_file(ver['version']) + eprint("fetching " + ver) + url = link_for_sha256_file(ver) res = urlopen(url) if res.getcode() != 200: eprint("download of {} failed with status {}".format(url, res.getcode())) sys.exit(1) else: - grab[ver['version']] = parse_sha256_file(res, ver, url) + grab[ver] = parse_sha256_file(res, ver, url) # check whether any version is missing arches we need # errs : { version: set(missing_arches) } From c59e88d7366dcdc37625b9e23117bdca9a6d84b2 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Wed, 18 Oct 2023 16:38:31 +0200 Subject: [PATCH 4/7] Format with black --- haskell/gen_ghc_bindist.py | 81 ++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 35 deletions(-) diff --git a/haskell/gen_ghc_bindist.py b/haskell/gen_ghc_bindist.py index e7a7f51a5..7077f94cc 100755 --- a/haskell/gen_ghc_bindist.py +++ b/haskell/gen_ghc_bindist.py @@ -45,32 +45,33 @@ # bazel: bazel name # upstream: list of download.haskell.org name ARCHES = [ - { "bazel": "linux_amd64", - "upstream": ["x86_64-deb8-linux", "x86_64-deb9-linux", "x86_64-deb10-linux"], }, - { "bazel": "linux_arm64", - "upstream": ["aarch64-deb10-linux"], }, - { "bazel": "darwin_amd64", - "upstream": ["x86_64-apple-darwin"] }, - { "bazel": "darwin_arm64", - "upstream": ["aarch64-apple-darwin"] }, - { "bazel": "windows_amd64", - "upstream": ["x86_64-unknown-mingw32"] }, + { + "bazel": "linux_amd64", + "upstream": ["x86_64-deb8-linux", "x86_64-deb9-linux", "x86_64-deb10-linux"], + }, + { + "bazel": "linux_arm64", + "upstream": ["aarch64-deb10-linux"], + }, + {"bazel": "darwin_amd64", "upstream": ["x86_64-apple-darwin"]}, + {"bazel": "darwin_arm64", "upstream": ["aarch64-apple-darwin"]}, + {"bazel": "windows_amd64", "upstream": ["x86_64-unknown-mingw32"]}, ] # An url to a bindist tarball. def link_for_tarball(arch, version): return "https://downloads.haskell.org/~ghc/{ver}/ghc-{ver}-{arch}.tar.xz".format( - ver = version, - arch = arch, + ver=version, + arch=arch, ) + # An url to a version's tarball hashsum file. # The files contain the hashsums for all arches. def link_for_sha256_file(version): - return "https://downloads.haskell.org/~ghc/{ver}/SHA256SUMS".format( - ver = version - ) + return "https://downloads.haskell.org/~ghc/{ver}/SHA256SUMS".format(ver=version) + # Parses the tarball hashsum file for a distribution version. def parse_sha256_file(content, version, url): @@ -85,24 +86,29 @@ def parse_sha256_file(content, version, url): if file_.startswith(prefix) and file_.endswith(suffix): # i386-deb8-linux - name = file_[len(prefix):-len(suffix)] + name = file_[len(prefix) : -len(suffix)] res[name] = hash if not res: - eprint(f"Errors parsing file at {url}. Could not find entries for {prefix}…{suffix}") + eprint( + f"Errors parsing file at {url}. Could not find entries for {prefix}…{suffix}" + ) exit(1) return res + # Print to stderr. def eprint(mes): - print(mes, file = sys.stderr) + print(mes, file=sys.stderr) + def select_one(xs, ys): """Select a single item from xs, prefer the first item also in ys.""" items = [x for x in xs if x in ys] return items[0] if items else xs[0] + # Main. if __name__ == "__main__": # Fetch all hashsum files @@ -122,19 +128,19 @@ def select_one(xs, ys): # errs : { version: set(missing_arches) } errs = {} for ver, hashes in grab.items(): - real_arches = frozenset(hashes.keys()) - upstreams = [select_one(a['upstream'], real_arches) for a in ARCHES] - needed_arches = frozenset(upstreams) - missing_arches = needed_arches.difference(real_arches) - if missing_arches: - errs[ver] = missing_arches + real_arches = frozenset(hashes.keys()) + upstreams = [select_one(a["upstream"], real_arches) for a in ARCHES] + needed_arches = frozenset(upstreams) + missing_arches = needed_arches.difference(real_arches) + if missing_arches: + errs[ver] = missing_arches if errs: for ver, missing in errs.items(): print( "WARN: version {ver} is missing hashes for architectures {arches}".format( - ver = ver, - arches = ','.join(missing)), - file=sys.stderr + ver=ver, arches=",".join(missing) + ), + file=sys.stderr, ) # fetch the arches we need and create the GHC_BINDISTS dict @@ -144,21 +150,26 @@ def select_one(xs, ys): # { bazel_arch: (tarball_url, sha256_hash) } arch_dists = {} for arch in ARCHES: - upstream = select_one(arch['upstream'], hashes) + upstream = select_one(arch["upstream"], hashes) if upstream in hashes: - arch_dists[arch['bazel']] = ( + arch_dists[arch["bazel"]] = ( link_for_tarball(upstream, ver), - hashes[upstream] + hashes[upstream], ) ghc_bindists[ver] = arch_dists - ghc_versions = { version: ghc_bindists[version] for version in sorted(ghc_bindists.keys(), key=StrictVersion) } + ghc_versions = { + version: ghc_bindists[version] + for version in sorted(ghc_bindists.keys(), key=StrictVersion) + } working_directory = os.environ.get("BUILD_WORKING_DIRECTORY", ".") - with open(os.path.join(working_directory, "haskell/private/ghc_bindist_generated.json"), "w", encoding="utf-8") as json_file: + with open( + os.path.join(working_directory, "haskell/private/ghc_bindist_generated.json"), + "w", + encoding="utf-8", + ) as json_file: json.dump(ghc_versions, json_file, indent=4) - json_file.write('\n') - - + json_file.write("\n") From b53a7c0f19da2694aa6e7df260e91aa6515c83aa Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Wed, 18 Oct 2023 17:32:27 +0200 Subject: [PATCH 5/7] Determine GHC bindist versions from JSON file --- haskell/gen_ghc_bindist.py | 62 +++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/haskell/gen_ghc_bindist.py b/haskell/gen_ghc_bindist.py index 7077f94cc..a9e1d3075 100755 --- a/haskell/gen_ghc_bindist.py +++ b/haskell/gen_ghc_bindist.py @@ -10,33 +10,6 @@ from urllib.request import urlopen from distutils.version import StrictVersion -# All GHC versions we generate. -VERSIONS = [ - "9.6.3", - "9.6.2", - "9.6.1", - "9.4.7", - "9.4.6", - "9.4.5", - "9.2.8", - "9.2.5", - "9.2.4", - "9.2.3", - "9.2.1", - "9.0.2", - "9.0.1", - "8.10.7", - "8.10.4", - "8.10.3", - "8.8.4", - "8.6.5", - "8.4.4", - "8.4.3", - "8.4.2", - "8.4.1", - "8.2.2", -] - # Sometimes bindists have errors and are updated by new bindists. # This dict is used to keep a version -> corrected_version mapping. VERSIONS_CORRECTED = {} @@ -109,12 +82,11 @@ def select_one(xs, ys): return items[0] if items else xs[0] -# Main. -if __name__ == "__main__": +def fetch_hashsums(versions): # Fetch all hashsum files # grab : { version: { arch: sha256 } } grab = {} - for ver in VERSIONS: + for ver in versions: eprint("fetching " + ver) url = link_for_sha256_file(ver) res = urlopen(url) @@ -143,6 +115,10 @@ def select_one(xs, ys): file=sys.stderr, ) + return grab + + +def fetch_bindists(grab): # fetch the arches we need and create the GHC_BINDISTS dict # ghc_bindists : { version: { bazel_arch: (tarball_url, sha256_hash) } } ghc_bindists = {} @@ -159,17 +135,33 @@ def select_one(xs, ys): ) ghc_bindists[ver] = arch_dists - ghc_versions = { - version: ghc_bindists[version] - for version in sorted(ghc_bindists.keys(), key=StrictVersion) - } + return ghc_bindists + +# Main. +if __name__ == "__main__": working_directory = os.environ.get("BUILD_WORKING_DIRECTORY", ".") with open( os.path.join(working_directory, "haskell/private/ghc_bindist_generated.json"), - "w", + "r+", encoding="utf-8", ) as json_file: + ghc_versions = json.load(json_file) + + # All GHC versions we generate. + versions = ghc_versions.keys() + + grab = fetch_hashsums(versions) + + ghc_bindists = fetch_bindists(grab) + + ghc_versions = { + version: ghc_bindists[version] + for version in sorted(ghc_bindists.keys(), key=StrictVersion) + } + + json_file.truncate(0) + json_file.seek(0) json.dump(ghc_versions, json_file, indent=4) json_file.write("\n") From 412576401737e6bbddb4e5a3db81184ae3ffc6b0 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Thu, 19 Oct 2023 07:37:05 +0200 Subject: [PATCH 6/7] Directly modify the JSON file and run gen_ghc_bindist.py When updating a GHC version, just add an empty entry for the version to the `ghc_bindist_generated.json` file and run the gen_ghc_bindist.py to update it. --- .github/update-ghc.py | 54 ++++++++++++++----------------------------- 1 file changed, 17 insertions(+), 37 deletions(-) diff --git a/.github/update-ghc.py b/.github/update-ghc.py index 5585aa360..5026ae056 100644 --- a/.github/update-ghc.py +++ b/.github/update-ghc.py @@ -78,43 +78,23 @@ def main(): print("found update:", latest_release, file=sys.stderr) - replace = re.compile( - r'^(?P\s+)\{\s*"version"\s*:\s*"' - + re.escape(GHC_MAJOR_MINOR + "."), - re.MULTILINE, - ) - - print(" 1. modify haskell/gen_ghc_bindist.py", file=sys.stderr) - - gen_script_path = SCRIPT_PATH.parent.parent.joinpath( - "haskell/gen_ghc_bindist.py" - ) - with open(gen_script_path, "r+") as gen: - gen_script = gen.read() - - added_version = replace.sub( - rf"""\g{{ "version": "{ latest_release }" }}, -\g<0>""", - gen_script, - count=1, - ) - - if added_version is gen_script: - sys.exit( - f"could not add new version {latest_release} using regex {replace}" - ) - - gen.truncate(0) - gen.seek(0) - gen.write(added_version) - - print(" 2. call haskell/gen_ghc_bindist.py", file=sys.stderr) - - check_call([sys.executable, "haskell/gen_ghc_bindist.py"]) - - if "GITHUB_OUTPUT" in os.environ: - with open(os.environ["GITHUB_OUTPUT"], "a") as output: - print(f"latest={ latest_release }", file=output) + print(" 1. modify haskell/private/ghc_bindist_generated.json", file=sys.stderr) + + bindists[latest_release] = {} + + bindist_json.truncate(0) + bindist_json.seek(0) + json.dump(bindists, bindist_json) + + gen_script_path = SCRIPT_PATH.parent.parent.joinpath("haskell/gen_ghc_bindist.py") + + print(" 2. call haskell/gen_ghc_bindist.py", file=sys.stderr) + + check_call([sys.executable, "haskell/gen_ghc_bindist.py"]) + + if "GITHUB_OUTPUT" in os.environ: + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + print(f"latest={ latest_release }", file=output) if __name__ == "__main__": From a4b8b1e35cb376c2c225e11d62f20d72302d365b Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Wed, 18 Oct 2023 13:02:05 +0200 Subject: [PATCH 7/7] Add GHC 9.8 to update-ghc workflow --- .github/workflows/update-ghc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/update-ghc.yaml b/.github/workflows/update-ghc.yaml index 9c70f69ea..a169801f2 100644 --- a/.github/workflows/update-ghc.yaml +++ b/.github/workflows/update-ghc.yaml @@ -16,6 +16,7 @@ jobs: - '9.2' - '9.4' - '9.6' + - '9.8' steps: - uses: actions/checkout@v4 with: