Skip to content

Commit

Permalink
Retire obsolete Akamai cache purge support
Browse files Browse the repository at this point in the history 
Exodus-gw is now responsible for Akamai CDN cache flush
during publish in all the supported workflows. It makes
the code for CDN cache flush obsolete. Hence, remove the
code pertaining to CDN cache flush.
  • Loading branch information
@rajulkumar
rajulkumar committed Jun 21, 2024
1 parent d939d59 commit ef55f22
Show file tree
Hide file tree
Showing 4 changed files with 9 additions and 563 deletions.
209 changes: 3 additions & 206 deletions tests/test_cdn.py
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,9 @@
from concurrent.futures import as_completed
from datetime import datetime

import pytest
from pubtools.pulplib import Distributor, FakeController, YumRepository

from ubipop._cdn import CdnClient, Publisher

EDGERC_FAKE_CONF = {
"client_secret": "some-secret",
"host": "some-host",
"access_token": "some-access-token",
"client_token": "some-client-token",
}
from ubipop._cdn import Publisher


@pytest.fixture(name="pulp")
Expand All @@ -28,16 +20,9 @@ def create_and_insert_repo(pulp, repos):
return out


def setup_fastpurge_mock(requests_mock):
url = "https://some-host/ccu/v3/delete/url/production"
seconds = 0.1
response = {"some": ["return", "value"], "estimatedSeconds": seconds}
requests_mock.register_uri("POST", url, status_code=201, json=response)


def test_publisher_without_cache_purge(pulp, requests_mock):
def test_publisher(pulp):
"""
Tests a basic scenario of repository publish without cache purge.
Tests a basic scenario of repository publish.
"""
dt = datetime(2019, 9, 12, 0, 0, 0)
repos_to_insert = []
Expand Down Expand Up @@ -68,191 +53,3 @@ def test_publisher_without_cache_purge(pulp, requests_mock):
assert [hist.repository.id for hist in pulp.publish_history] == [
repo.id for repo in repos
]

hist = requests_mock.request_history

assert len(hist) == 0 # no requests expected as CDN cache purge is disabled


def test_publisher_with_cache_purge(pulp, requests_mock):
"""
Tests a scenario of repository publish with cache purge by URL and ARLs generated with
data from CDN service.
"""
dt = datetime(2019, 9, 12, 0, 0, 0)
repos_to_insert = []
repo_id = "repo-1"
distributor = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id=repo_id,
last_publish=dt,
relative_url="content/unit/1/client",
)
repo = YumRepository(
id=repo_id,
eng_product_id=102,
distributors=[distributor],
relative_url="content/unit/1/client",
mutable_urls=["repomd.xml"],
)
repos_to_insert.append(repo)

repos = create_and_insert_repo(pulp, repos_to_insert)
publisher_args = {
"edgerc": EDGERC_FAKE_CONF,
"publish_options": {
"clean": True,
},
"cdn_root": "https://cdn.example.com",
"arl_templates": [
"/arl/1/test/{ttl}/{path}",
"/arl/2/test/{ttl}/{path}",
# extra case with whitespace chars
" /arl/3/test/{ttl}/{path} /extra// \n\n \t ",
],
"max_retry_sleep": 0.001,
}
setup_fastpurge_mock(requests_mock)

url_ttl = ("https://cdn.example.com/content/unit/1/client/repomd.xml", "33s")
# ARLs are generated from the template using the {ttl} placeholder, which is replaced with
# the real TTL value. The real TTL value is extracted from the cache key header of the real
# request for the given path using '/(\d+[smhd])/' regex.
# The /1h/foo in the mocked header here is to test that if the path contains a component
# that also matches the TTL regex ('/1h/'), it will still find the correct value ('/33s/').
headers = {"X-Cache-Key": f"/fake/cache-key/{url_ttl[1]}/something/1h/foo"}
requests_mock.register_uri("HEAD", url_ttl[0], headers=headers)

# enqueue repos to publish and wait for publish and purge to finish
with Publisher(**publisher_args) as publisher:
publisher.enqueue(*repos)
publisher.wait_publish_and_purge_cache()

assert [hist.repository.id for hist in pulp.publish_history] == [
repo.id for repo in repos
]

hist = requests_mock.request_history

assert len(hist) == 2 # 1 request to cdn service for headers and 1 purge request

assert hist[0].url == "https://cdn.example.com/content/unit/1/client/repomd.xml"
assert hist[1].json()["objects"] == [
"https://cdn.example.com/content/unit/1/client/repomd.xml",
"/arl/1/test/33s/content/unit/1/client/repomd.xml",
"/arl/2/test/33s/content/unit/1/client/repomd.xml",
"/arl/3/test/33s/content/unit/1/client/repomd.xml /extra//",
]


def test_cdn_client_retries(pulp, requests_mock):
"""
Tests a scenario when some request to CDN service for TTL fails but
it's retried with success.
"""
dt = datetime(2019, 9, 12, 0, 0, 0)
repos_to_insert = []
repo_id = "repo-1"
distributor = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id=repo_id,
last_publish=dt,
relative_url="content/unit/1/client",
)
repo = YumRepository(
id=repo_id,
eng_product_id=102,
distributors=[distributor],
relative_url="content/unit/1/client",
mutable_urls=["repomd.xml"],
)
repos_to_insert.append(repo)

repos = create_and_insert_repo(pulp, repos_to_insert)

url = "https://cdn.example.com/content/unit/1/client/repomd.xml"
publisher_args = {
"edgerc": EDGERC_FAKE_CONF,
"publish_options": {
"clean": True,
},
"cdn_root": "https://cdn.example.com",
"arl_templates": ["/arl/1/test/{ttl}/{path}", "/arl/2/test/{ttl}/{path}"],
"max_retry_sleep": 0.001,
}
setup_fastpurge_mock(requests_mock)
requests_mock.register_uri(
"HEAD",
url,
[
# Fails on first try
{"status_code": 500},
# Then succeeds
{
"status_code": 200,
"headers": {"X-Cache-Key": "/fake/cache-key/10h/something"},
},
],
)

# enqueue repos for publish and wait to finish
with Publisher(**publisher_args) as publisher:
publisher.enqueue(*repos)
publisher.wait_publish_and_purge_cache()

# all published repos should be recorded in history
assert [hist.repository.id for hist in pulp.publish_history] == [
repo.id for repo in repos
]

hist = requests_mock.request_history

# there should be 2 requests to cdn service (failure and success) for headers and 1 CDN cache purge request
assert len(hist) == 3

for i in range(2):
assert hist[i].url == "https://cdn.example.com/content/unit/1/client/repomd.xml"

assert hist[-1].json()["objects"] == [
"https://cdn.example.com/content/unit/1/client/repomd.xml",
"/arl/1/test/10h/content/unit/1/client/repomd.xml",
"/arl/2/test/10h/content/unit/1/client/repomd.xml",
]


@pytest.mark.parametrize(
"path, expected_ttl",
[
("content/test/repodata/repomd.xml", "4h"),
("content/test/repodata/", "10m"),
("/ostree/repo/refs/heads/test-path/base", "10m"),
("content/test/PULP_MANIFEST", "10m"),
("content/test/", "4h"),
("content/test/some-file", "30d"),
],
)
def test_arl_fallback(requests_mock, path, expected_ttl):
"""
Tests fallback to default TTL values when TTL cannot be
fetched from CDN service.
"""
templates = [
"/fake/template-1/{ttl}/{path}",
]
with CdnClient(
"https://cdn.example.com/", arl_templates=templates, max_retry_sleep=0.001
) as client:
url = "https://cdn.example.com/content/foo/test-path-1/some-file"

requests_mock.register_uri("HEAD", url, status_code=500)

# Request ARLs
arls_ft = client.get_arl_for_path(path)

# It should be successful
arl = [item.result() for item in as_completed(arls_ft)][0]

# It should fallback to default ttl value
assert arl == "/fake/template-1/{ttl}/{path}".format(ttl=expected_ttl, path=path)
78 changes: 2 additions & 76 deletions tests/test_ubipop.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import shutil
import sys
import tempfile
import textwrap
from concurrent.futures import wait
from datetime import datetime
from tempfile import NamedTemporaryFile
Expand Down Expand Up @@ -1050,37 +1049,9 @@ def test_get_current_content(mock_ubipop_runner, pulp, skip_debug_repo):
assert source_rpms[0].name == "test-srpm"


@pytest.fixture(name="cache_purge_env")
def fixture_cache_purge_env(monkeypatch):
with tempfile.NamedTemporaryFile("w") as fp:
monkeypatch.setenv("UBIPOP_EDGERC_CFG", fp.name)
monkeypatch.setenv("UBIPOP_CDN_ROOT", "https://test.root-url.com/")
monkeypatch.setenv(
"UBIPOP_ARL_TEMPLATES",
"/test/template-1/{ttl}/{path},/test/template-2/{ttl}/{path}",
)

fp.write(
textwrap.dedent(
"""
[default]
client_secret = some-secret
host = some-host
access_token = some-access-token
client_token = some-client-token
"""
)
)
fp.flush()

yield


@patch("pubtools.pulplib.YumRepository.get_debug_repository")
@patch("pubtools.pulplib.YumRepository.get_source_repository")
def test_populate_ubi_repos(
get_debug_repository, get_source_repository, cache_purge_env, requests_mock
):
def test_populate_ubi_repos(get_debug_repository, get_source_repository, requests_mock):
# pylint: disable=unused-argument
"""Test run of populate_ubi_repos that checks correct number of repo publication.
It's simplified to contain only actions on RPM packages."""
Expand Down Expand Up @@ -1208,8 +1179,6 @@ def test_populate_ubi_repos(

# mock calls to ubi-manifest service
_create_ubi_manifest_mocks(requests_mock)
_create_fastpurge_mocks(requests_mock)
_create_cdn_mocks(requests_mock)

# let's run actual population
ubi_populate.populate_ubi_repos()
Expand All @@ -1235,24 +1204,6 @@ def test_populate_ubi_repos(

assert repo_ids_published == expected_published_repo_ids

request = requests_mock.request_history[
-1
] # last request should be for purge cache request

expected_req = [
"https://test.root-url.com/content/unit/3/client/repodata/repomd.xml",
"/test/template-1/24h/content/unit/3/client/repodata/repomd.xml",
"/test/template-2/24h/content/unit/3/client/repodata/repomd.xml",
"https://test.root-url.com/content/unit/2/client/repodata/repomd.xml",
"/test/template-1/30m/content/unit/2/client/repodata/repomd.xml",
"/test/template-2/30m/content/unit/2/client/repodata/repomd.xml",
"https://test.root-url.com/content/unit/1/client/repodata/repomd.xml",
"/test/template-1/10s/content/unit/1/client/repodata/repomd.xml",
"/test/template-2/10s/content/unit/1/client/repodata/repomd.xml",
]

assert sorted(request.json()["objects"]) == sorted(expected_req)


@patch("ubipop._cdn.Publisher")
@patch("pubtools.pulplib.YumRepository.get_debug_repository")
Expand Down Expand Up @@ -1531,8 +1482,6 @@ def test_populate_ubi_repos_dry_run(

# mock calls to ubi-manifest service
_create_ubi_manifest_mocks(requests_mock)
_create_fastpurge_mocks(requests_mock)
_create_cdn_mocks(requests_mock)

# let's run actual population
ubi_populate.populate_ubi_repos()
Expand All @@ -1553,9 +1502,7 @@ def test_populate_ubi_repos_dry_run(


@patch("pubtools.pulplib.YumRepository.get_source_repository")
def test_populate_ubi_repos_missing_repos(
get_source_repository, cache_purge_env, requests_mock, caplog
):
def test_populate_ubi_repos_missing_repos(get_source_repository, requests_mock, caplog):
# pylint: disable=unused-argument
"""Test run of populate_ubi_repos with a missing repo."""
dt = datetime(2019, 9, 12, 0, 0, 0)
Expand Down Expand Up @@ -1666,8 +1613,6 @@ def test_populate_ubi_repos_missing_repos(

# mock calls to ubi-manifest service
_create_ubi_manifest_mocks(requests_mock)
_create_fastpurge_mocks(requests_mock)
_create_cdn_mocks(requests_mock)

# let's run actual population
ubi_populate.populate_ubi_repos()
Expand All @@ -1680,13 +1625,6 @@ def test_populate_ubi_repos_missing_repos(
assert "Skipping current content triplet, some repos are missing" in caplog.text


def _create_fastpurge_mocks(requests_mock):
url = "https://some-host/ccu/v3/delete/url/production"
seconds = 0.1
response = {"some": ["return", "value"], "estimatedSeconds": seconds}
requests_mock.register_uri("POST", url, status_code=201, json=response)


def _create_ubi_manifest_mocks(requests_mock):
# mock requesting manifest generation
url = "api/v1/manifest"
Expand Down Expand Up @@ -1740,18 +1678,6 @@ def _create_ubi_manifest_mocks(requests_mock):
requests_mock.register_uri("GET", url, json=response)


def _create_cdn_mocks(requests_mock):
url_ttl = [
("https://test.root-url.com/content/unit/1/client/repodata/repomd.xml", "10s"),
("https://test.root-url.com/content/unit/2/client/repodata/repomd.xml", "30m"),
("https://test.root-url.com/content/unit/3/client/repodata/repomd.xml", "24h"),
]

for url, ttl in url_ttl:
headers = {"X-Cache-Key": f"/fake/cache-key/{ttl}/something"}
requests_mock.register_uri("HEAD", url, headers=headers)


@patch("ubipop.Client")
def test_make_pulp_client_cert(mocked_client):
"""Tests that pulp client can be created with cert/key used for auth"""
Expand Down
9 changes: 0 additions & 9 deletions ubipop/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,19 +124,10 @@ def __init__(
self._content_set_regex = kwargs.get("content_set_regex", None)
self._ubi_manifest_url = kwargs.get("ubi_manifest_url") or None
self._action_batch_size = kwargs.get("action_batch_size", 100)
arl_templates = os.getenv("UBIPOP_ARL_TEMPLATES", "")
self._publisher_args = {
"edgerc": os.getenv("UBIPOP_EDGERC_CFG", "/etc/.edgerc"),
"publish_options": {
"clean": True,
},
"cdn_root": os.getenv("UBIPOP_CDN_ROOT", ""),
"arl_templates": arl_templates.split(",") if arl_templates else [],
"cert": (
os.getenv("UBIPOP_CDN_CERT", ""),
os.getenv("UBIPOP_CDN_KEY", ""),
),
"verify": os.getenv("UBIPOP_CDN_CA_CERT", ""),
}
self._skip_publish = os.getenv("UBIPOP_SKIP_PUBLISH", "false")

Expand Down
Loading

0 comments on commit ef55f22

Please sign in to comment.