From 86eb77fce68a36c7d8ae149884d1aaa637790ffb Mon Sep 17 00:00:00 2001 From: Adrian Clay Lake Date: Fri, 20 Dec 2024 16:44:32 +0100 Subject: [PATCH] Fix: Block Release of EOL Images (#320) --------- Co-authored-by: clay-lake --- oci/mock-rock/_releases.json | 38 +- oci/mock-rock/image.yaml | 9 + src/image/release.py | 393 ++++++++++++--------- tests/data/mock-rock_circular_release.json | 8 + tests/data/mock-rock_release.json | 99 ++++++ tests/fixtures/sample_data.py | 19 +- tests/unit/test_release.py | 80 +++++ 7 files changed, 483 insertions(+), 163 deletions(-) create mode 100644 tests/data/mock-rock_circular_release.json create mode 100644 tests/data/mock-rock_release.json create mode 100644 tests/unit/test_release.py diff --git a/oci/mock-rock/_releases.json b/oci/mock-rock/_releases.json index a831d07d..a12f7671 100644 --- a/oci/mock-rock/_releases.json +++ b/oci/mock-rock/_releases.json @@ -65,5 +65,41 @@ "target": "1.2-22.04_beta" } }, - "1.0.0-22.04": {} + "1.0.0-22.04": {}, + "eol": { + "end-of-life": "2030-05-01T00:00:00Z", + "beta": { + "target": "1.0-22.04_beta" + }, + "edge": { + "target": "eol_beta" + } + }, + "eol-release": { + "end-of-life": "2000-05-01T00:00:00Z", + "beta": { + "target": "1.1-22.04_beta" + }, + "edge": { + "target": "eol-release_beta" + } + }, + "eol-upload": { + "end-of-life": "2030-05-01T00:00:00Z", + "beta": { + "target": "1.0-22.04_beta" + }, + "edge": { + "target": "eol-upload_beta" + } + }, + "eol-all": { + "end-of-life": "2000-05-01T00:00:00Z", + "beta": { + "target": "1.0-22.04_beta" + }, + "edge": { + "target": "eol-all_beta" + } + } } \ No newline at end of file diff --git a/oci/mock-rock/image.yaml b/oci/mock-rock/image.yaml index 72382b32..23727130 100644 --- a/oci/mock-rock/image.yaml +++ b/oci/mock-rock/image.yaml @@ -7,6 +7,15 @@ release: test: end-of-life: "2030-05-01T00:00:00Z" beta: 1.1-22.04_beta + eol-upload: + end-of-life: "2030-05-01T00:00:00Z" + beta: 1.0-22.04_beta + eol-release: + end-of-life: "2000-05-01T00:00:00Z" + beta: 1.1-22.04_beta + eol-all: + end-of-life: "2000-05-01T00:00:00Z" + beta: 1.0-22.04_beta upload: - source: "canonical/rocks-toolbox" diff --git a/src/image/release.py b/src/image/release.py index 1ba19e0e..ea8807b9 100755 --- a/src/image/release.py +++ b/src/image/release.py @@ -11,11 +11,17 @@ import re import subprocess from collections import defaultdict +from datetime import datetime, timezone + import yaml -from .utils.encoders import DateTimeEncoder -from .utils.schema.triggers import ImageSchema, KNOWN_RISKS_ORDERED + import src.shared.release_info as shared +from .utils.encoders import DateTimeEncoder +from .utils.schema.triggers import KNOWN_RISKS_ORDERED, ImageSchema + +# generate single date for consistent EOL checking +execution_timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") parser = argparse.ArgumentParser() parser.add_argument( "--image-trigger", @@ -44,187 +50,252 @@ required=True, ) -args = parser.parse_args() -img_name = ( - args.image_name - if args.image_name - else os.path.abspath(args.image_trigger).split("/")[-2] -) - -print(f"Preparing to release revision tags for {img_name}") -all_revision_tags = shared.get_all_revision_tags(args.all_revision_tags) -revision_to_track = shared.get_revision_to_track(all_revision_tags) -print( - "Revision (aka 'canonical') tags grouped by revision:\n" - f"{json.dumps(revision_to_track, indent=2)}" -) - -print(f"Reading all previous releases from {args.all_releases}...") - -all_releases = shared.read_json_file(args.all_releases) -tag_mapping_from_all_releases = shared.get_tag_mapping_from_all_releases(all_releases) - -print(f"Parsing image trigger {args.image_trigger}") -with open(args.image_trigger, encoding="UTF-8") as trigger: - image_trigger = yaml.load(trigger, Loader=yaml.BaseLoader) +def remove_eol_tags(tag_to_revision, all_releases): + """Remove all EOL tags from tag to revision mapping.""" + + filtered_tag_to_revision = tag_to_revision.copy() + for base_tag, _ in tag_to_revision.items(): + path = [] # track revisions to prevent inf loop + tag = base_tag # init state + while True: + if tag in path: + raise shared.BadChannel( + f"Circular tracks found in release JSON:\n {all_releases}" + ) + + path.append(tag) + + # if we find a numeric revision, break since we reached the end of the path + if tag.isdigit(): + break + + # we allways expect len == 2 unless we reach the final numeric tag + if not len(split := tag.split("_")) == 2: + raise shared.BadChannel( + f"Malformed tag. Expected format is _. Found tag {repr(tag)}." + ) + + track, risk = split + + # if we do not end on a numeric revision, we have a dangling tag. + if track not in all_releases or risk not in all_releases[track]: + raise shared.BadChannel( + f"Dangling tag found. Tag {repr(tag)} does not point to any revision." + ) + + # if EOL date is specified and expired, pop the tag from the map + if ( + "end-of-life" in all_releases[track] + and (eol_date := all_releases[track]["end-of-life"]) + < execution_timestamp + and base_tag in filtered_tag_to_revision + ): + print(f"Warning: Removing EOL tag {repr(base_tag)}, date: {eol_date}") + filtered_tag_to_revision.pop(base_tag) + + # prep next iteration + tag = all_releases[track][risk]["target"] + + return filtered_tag_to_revision + + +def main(): + args = parser.parse_args() + img_name = ( + args.image_name + if args.image_name + else os.path.abspath(args.image_trigger).split("/")[-2] + ) -_ = ImageSchema(**image_trigger) + print(f"Preparing to release revision tags for {img_name}") + all_revision_tags = shared.get_all_revision_tags(args.all_revision_tags) + revision_to_track = shared.get_revision_to_track(all_revision_tags) -tag_mapping_from_trigger = {} -for track, risks in image_trigger["release"].items(): - if track not in all_releases: - print(f"Track {track} will be created for the 1st time") - all_releases[track] = {} + print( + "Revision (aka 'canonical') tags grouped by revision:\n" + f"{json.dumps(revision_to_track, indent=2)}" + ) - for risk, value in risks.items(): - if value is None: - continue + print(f"Reading all previous releases from {args.all_releases}...") - if risk in ["end-of-life", "end_of_life"]: - all_releases[track]["end-of-life"] = value - continue + all_releases = shared.read_json_file(args.all_releases) + tag_mapping_from_all_releases = shared.get_tag_mapping_from_all_releases( + all_releases + ) - if risk not in KNOWN_RISKS_ORDERED: - print(f"Skipping unknown risk {risk} in track {track}") - continue + print(f"Parsing image trigger {args.image_trigger}") + with open(args.image_trigger, encoding="UTF-8") as trigger: + image_trigger = yaml.load(trigger, Loader=yaml.BaseLoader) - all_releases[track][risk] = {"target": value} - tag = f"{track}_{risk}" - print(f"Channel {tag} points to {value}") - tag_mapping_from_trigger[tag] = value + _ = ImageSchema(**image_trigger) -# update EOL dates from upload dictionary -for upload in image_trigger["upload"] or []: - for track, upload_release_dict in (upload["release"] or {}).items(): + tag_mapping_from_trigger = {} + for track, risks in image_trigger["release"].items(): if track not in all_releases: print(f"Track {track} will be created for the 1st time") all_releases[track] = {} - if isinstance(upload_release_dict, dict) and "end-of-life" in upload_release_dict: - all_releases[track]["end-of-life"] = upload_release_dict["end-of-life"] - -print( - "Going to update channels according to the following:\n" - f"{json.dumps(tag_mapping_from_trigger, indent=2)}" -) + for risk, value in risks.items(): + if value is None: + continue + + if risk in ["end-of-life", "end_of_life"]: + all_releases[track]["end-of-life"] = value + continue + + if risk not in KNOWN_RISKS_ORDERED: + print(f"Skipping unknown risk {risk} in track {track}") + continue + + all_releases[track][risk] = {"target": value} + tag = f"{track}_{risk}" + print(f"Channel {tag} points to {value}") + tag_mapping_from_trigger[tag] = value + + # update EOL dates from upload dictionary + for upload in image_trigger["upload"] or []: + for track, upload_release_dict in (upload["release"] or {}).items(): + if track not in all_releases: + print(f"Track {track} will be created for the 1st time") + all_releases[track] = {} + + if ( + isinstance(upload_release_dict, dict) + and "end-of-life" in upload_release_dict + ): + all_releases[track]["end-of-life"] = upload_release_dict["end-of-life"] + + print( + "Going to update channels according to the following:\n" + f"{json.dumps(tag_mapping_from_trigger, indent=2)}" + ) -# combine all tags -all_tags_mapping = { - **tag_mapping_from_all_releases, - **tag_mapping_from_trigger, -} - -# we need to validate the release request, to make sure that: -# - the target revisions exist -# - the target tags (when following) do not incur in a circular dependency -# - the target tags (when following) exist -tag_to_revision = tag_mapping_from_trigger.copy() -for channel_tag, target in tag_mapping_from_trigger.items(): - # a target cannot follow its own tag - if target == channel_tag: - msg = f"A tag cannot follow itself ({target})" - raise shared.BadChannel(msg) - - # we need to map tags to a revision number, - # even those that point to other tags - follow_tag = target - followed_tags = [] - while not follow_tag.isdigit(): - # does the parent tag exist? - if follow_tag not in all_tags_mapping: - msg = ( - f"The tag {channel_tag} wants to follow channel {follow_tag}," - " which is undefined and doesn't point to a revision" - ) + # combine all tags + all_tags_mapping = { + **tag_mapping_from_all_releases, + **tag_mapping_from_trigger, + } + + # we need to validate the release request, to make sure that: + # - the target revisions exist + # - the target tags (when following) do not incur in a circular dependency + # - the target tags (when following) exist + tag_to_revision = tag_mapping_from_trigger.copy() + for channel_tag, target in tag_mapping_from_trigger.items(): + # a target cannot follow its own tag + if target == channel_tag: + msg = f"A tag cannot follow itself ({target})" raise shared.BadChannel(msg) - if follow_tag in followed_tags: - # then we have a circular dependency, tags are following each - # other but we cannot pinpoint the exact revision - msg = ( - f"The tag {channel_tag} was caught is a circular dependency, " - "following tags that follow themselves. Cannot pin a revision." + # we need to map tags to a revision number, + # even those that point to other tags + follow_tag = target + followed_tags = [] + while not follow_tag.isdigit(): + # does the parent tag exist? + if follow_tag not in all_tags_mapping: + msg = ( + f"The tag {channel_tag} wants to follow channel {follow_tag}," + " which is undefined and doesn't point to a revision" + ) + raise shared.BadChannel(msg) + + if follow_tag in followed_tags: + # then we have a circular dependency, tags are following each + # other but we cannot pinpoint the exact revision + msg = ( + f"The tag {channel_tag} was caught is a circular dependency, " + "following tags that follow themselves. Cannot pin a revision." + ) + raise shared.BadChannel(msg) + followed_tags.append(follow_tag) + + # follow the parent tag until it is a digit (ie. revision number) + parent_tag = all_tags_mapping[follow_tag] + + print(f"Tag {follow_tag} is following tag {parent_tag}.") + follow_tag = parent_tag + + if int(follow_tag) not in revision_to_track: + msg = str( + f"The tag {channel_tag} points to revision {follow_tag}, " + "which doesn't exist!" ) raise shared.BadChannel(msg) - followed_tags.append(follow_tag) - - # follow the parent tag until it is a digit (ie. revision number) - parent_tag = all_tags_mapping[follow_tag] - print(f"Tag {follow_tag} is following tag {parent_tag}.") - follow_tag = parent_tag + tag_to_revision[channel_tag] = int(follow_tag) + + # if we get here, it is a valid (tag, revision) + + # remove all EOL tags to be released + filtered_tag_to_revision = remove_eol_tags(tag_to_revision, all_releases) + + # we now need to add tag aliases + release_tags = filtered_tag_to_revision.copy() + for base_tag, revision in tag_to_revision.items(): + # "latest" is a special tag for OCI + if re.match( + rf"latest_({'|'.join(KNOWN_RISKS_ORDERED)})$", + base_tag, + ): + latest_alias = base_tag.split("_")[-1] + print(f"Exceptionally converting tag {base_tag} to {latest_alias}.") + release_tags[latest_alias] = revision + release_tags.pop(base_tag) + + # stable risks have an alias with any risk string + if base_tag.endswith("_stable"): + stable_alias = "_".join(base_tag.split("_")[:-1]) + print(f"Adding stable tag alias {stable_alias} for {base_tag}") + release_tags[stable_alias] = revision + + # we finally have all the OCI tags to be released, + # and which revisions to release for each tag. Let's release! + group_by_revision = defaultdict(list) + for tag, revision in sorted(release_tags.items()): + group_by_revision[revision].append(tag) + + print( + "Processed tag aliases and ready to release the following revisions:\n" + f"{json.dumps(group_by_revision, indent=2)}" + ) - if int(follow_tag) not in revision_to_track: - msg = str( - f"The tag {channel_tag} points to revision {follow_tag}, " - "which doesn't exist!" + github_tags = [] + for revision, tags in group_by_revision.items(): + revision_track = revision_to_track[revision] + source_img = ( + "docker://ghcr.io/" + f"{args.ghcr_repo}/{img_name}:{revision_track}_{revision}" ) - raise shared.BadChannel(msg) - - tag_to_revision[channel_tag] = int(follow_tag) - -# if we get here, it is a valid (tag, revision) - -# we now need to add tag aliases -release_tags = tag_to_revision.copy() -for base_tag, revision in tag_to_revision.items(): - # "latest" is a special tag for OCI - if re.match( - rf"latest_({'|'.join(KNOWN_RISKS_ORDERED)})$", - base_tag, - ): - latest_alias = base_tag.split("_")[-1] - print(f"Exceptionally converting tag {base_tag} to {latest_alias}.") - release_tags[latest_alias] = revision - release_tags.pop(base_tag) - - # stable risks have an alias with any risk string - if base_tag.endswith("_stable"): - stable_alias = "_".join(base_tag.split("_")[:-1]) - print(f"Adding stable tag alias {stable_alias} for {base_tag}") - release_tags[stable_alias] = revision - -# we finally have all the OCI tags to be released, -# and which revisions to release for each tag. Let's release! -group_by_revision = defaultdict(list) -for tag, revision in sorted(release_tags.items()): - group_by_revision[revision].append(tag) - -print( - "Processed tag aliases and ready to release the following revisions:\n" - f"{json.dumps(group_by_revision, indent=2)}" -) -github_tags = [] -for revision, tags in group_by_revision.items(): - revision_track = revision_to_track[revision] - source_img = ( - "docker://ghcr.io/" f"{args.ghcr_repo}/{img_name}:{revision_track}_{revision}" - ) - this_dir = os.path.dirname(__file__) - print(f"Releasing {source_img} with tags:\n{tags}") - subprocess.check_call( - [f"{this_dir}/tag_and_publish.sh", source_img, img_name] + tags + this_dir = os.path.dirname(__file__) + print(f"Releasing {source_img} with tags:\n{tags}") + subprocess.check_call( + [f"{this_dir}/tag_and_publish.sh", source_img, img_name] + tags + ) + + for tag in tags: + gh_release_info = {} + gh_release_info["canonical-tag"] = f"{img_name}_{revision_track}_{revision}" + gh_release_info["release-name"] = f"{img_name}_{tag}" + gh_release_info["name"] = f"{img_name}" + gh_release_info["revision"] = f"{revision}" + gh_release_info["channel"] = f"{tag}" + github_tags.append(gh_release_info) + + print( + f"Updating {args.all_releases} file with:\n" + f"{json.dumps(all_releases, indent=2, cls=DateTimeEncoder)}" ) - for tag in tags: - gh_release_info = {} - gh_release_info["canonical-tag"] = f"{img_name}_{revision_track}_{revision}" - gh_release_info["release-name"] = f"{img_name}_{tag}" - gh_release_info["name"] = f"{img_name}" - gh_release_info["revision"] = f"{revision}" - gh_release_info["channel"] = f"{tag}" - github_tags.append(gh_release_info) - -print( - f"Updating {args.all_releases} file with:\n" - f"{json.dumps(all_releases, indent=2, cls=DateTimeEncoder)}" -) + with open(args.all_releases, "w", encoding="UTF-8") as fd: + json.dump(all_releases, fd, indent=4, cls=DateTimeEncoder) + + matrix = {"include": github_tags} -with open(args.all_releases, "w", encoding="UTF-8") as fd: - json.dump(all_releases, fd, indent=4, cls=DateTimeEncoder) + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="UTF-8") as gh_out: + print(f"gh-releases-matrix={matrix}", file=gh_out) -matrix = {"include": github_tags} -with open(os.environ["GITHUB_OUTPUT"], "a", encoding="UTF-8") as gh_out: - print(f"gh-releases-matrix={matrix}", file=gh_out) +if __name__ == "__main__": + main() diff --git a/tests/data/mock-rock_circular_release.json b/tests/data/mock-rock_circular_release.json new file mode 100644 index 00000000..780ccdfb --- /dev/null +++ b/tests/data/mock-rock_circular_release.json @@ -0,0 +1,8 @@ +{ + "circular": { + "edge": { + "target": "circular_edge" + }, + "end-of-life": "2030-05-01T00:00:00Z" + } + } \ No newline at end of file diff --git a/tests/data/mock-rock_release.json b/tests/data/mock-rock_release.json new file mode 100644 index 00000000..79697a4b --- /dev/null +++ b/tests/data/mock-rock_release.json @@ -0,0 +1,99 @@ +{ + "latest": { + "candidate": { + "target": "1.2-22.04_beta" + }, + "beta": { + "target": "latest_candidate" + }, + "edge": { + "target": "latest_beta" + }, + "end-of-life": "2030-05-01T00:00:00Z" + }, + "1.0-22.04": { + "candidate": { + "target": "878" + }, + "beta": { + "target": "878" + }, + "edge": { + "target": "878" + }, + "end-of-life": "2024-05-01T00:00:00Z" + }, + "test": { + "beta": { + "target": "1.1-22.04_beta" + }, + "edge": { + "target": "test_beta" + }, + "end-of-life": "2030-05-01T00:00:00Z" + }, + "1.1-22.04": { + "end-of-life": "2030-05-01T00:00:00Z", + "candidate": { + "target": "1032" + }, + "beta": { + "target": "1032" + }, + "edge": { + "target": "1032" + } + }, + "1-22.04": { + "end-of-life": "2030-05-01T00:00:00Z", + "candidate": { + "target": "1032" + }, + "beta": { + "target": "1032" + }, + "edge": { + "target": "1032" + } + }, + "1.2-22.04": { + "end-of-life": "2030-05-01T00:00:00Z", + "beta": { + "target": "1033" + }, + "edge": { + "target": "1.2-22.04_beta" + } + }, + "1.0.0-22.04": {}, + "eol": { + "end-of-life": "2030-05-01T00:00:00Z", + "beta": { + "target": "1.0-22.04_beta" + }, + "edge": { + "target": "eol_beta" + } + }, + "eol-release": { + "end-of-life": "2000-05-01T00:00:00Z", + "beta": { + "target": "1.1-22.04_beta" + }, + "edge": { + "target": "eol-release_beta" + } + }, + "eol-upload": { + "end-of-life": "2030-05-01T00:00:00Z", + "beta": { + "target": "1.0-22.04_beta" + } + }, + "eol-all": { + "end-of-life": "2000-05-01T00:00:00Z", + "beta": { + "target": "1.0-22.04_beta" + } + } + } \ No newline at end of file diff --git a/tests/fixtures/sample_data.py b/tests/fixtures/sample_data.py index c3df9af1..7996797f 100644 --- a/tests/fixtures/sample_data.py +++ b/tests/fixtures/sample_data.py @@ -1,9 +1,26 @@ -import pytest +import json import xml.etree.ElementTree as ET + +import pytest import yaml + from .. import DATA_DIR +@pytest.fixture +def release_json(): + """Load a sample of _release.json from mock-rock""" + release_str = (DATA_DIR / "mock-rock_release.json").read_text() + return json.loads(release_str) + + +@pytest.fixture +def circular_release_json(): + """Load a sample of _release.json from mock-rock""" + release_str = (DATA_DIR / "mock-rock_circular_release.json").read_text() + return json.loads(release_str) + + @pytest.fixture def junit_with_failure(): """Load ET of junit xml report with failure.""" diff --git a/tests/unit/test_release.py b/tests/unit/test_release.py new file mode 100644 index 00000000..90013484 --- /dev/null +++ b/tests/unit/test_release.py @@ -0,0 +1,80 @@ +import pytest + +import src.shared.release_info as shared +from src.image.release import remove_eol_tags + +from ..fixtures.sample_data import circular_release_json, release_json + + +def test_remove_eol_tags_no_change(release_json): + """Ensure format of non-EOL tags are preserved""" + + revision_to_tag = { + "latest_candidate": "1033", + "1.1-22.04_beta": "1032", + } + + result = remove_eol_tags(revision_to_tag, release_json) + + assert revision_to_tag == result, "No change should have occured" + + +def test_remove_eol_tags_malformed_tag(release_json): + """Ensure malformed tag raises BadChannel exception.""" + + revision_to_tag = { + "malformed-tag": "1033", + } + + with pytest.raises(shared.BadChannel): + remove_eol_tags(revision_to_tag, release_json) + + +def test_remove_eol_tags_dangling_tag(release_json): + """Ensure dangling tag raises BadChannel exception.""" + + dangling_track = { + "1.0.0-22.04_beta": "", # the track for this tag does not exist + } + + dangling_risk = { + "1.0-22.04_gamma": "", # the risk for this tag does not exist + } + + with pytest.raises(shared.BadChannel): + remove_eol_tags(dangling_track, release_json) + + with pytest.raises(shared.BadChannel): + remove_eol_tags(dangling_risk, release_json) + + +def test_remove_eol_tags(release_json): + """Ensure EOL tags are removed.""" + + revision_to_tag = { + "latest_candidate": "1033", + "1.1-22.04_beta": "1032", + "eol-release_beta": "1032", + "eol-upload_beta": "878", + "eol-all_beta": "878", + } + + excepted_result = { + "latest_candidate": "1033", + "1.1-22.04_beta": "1032", + } + + result = remove_eol_tags(revision_to_tag, release_json) + + assert excepted_result == result, "All EOL tags should have been removed" + + +def test_remove_eol_tags_circular_release(circular_release_json): + """Ensure circular releases are handled.""" + + revision_to_tag = { + "circular_edge": "1033", + } + + with pytest.raises(shared.BadChannel): + remove_eol_tags(revision_to_tag, circular_release_json)