Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Retire obsolete Akamai cache purge support #233

Merged
merged 1 commit into from
Jun 25, 2024
Merged

Retire obsolete Akamai cache purge support #233

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
211 changes: 4 additions & 207 deletions tests/test_cdn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,9 @@
from concurrent.futures import as_completed
from datetime import datetime

import pytest
from pubtools.pulplib import Distributor, FakeController, YumRepository

from ubipop._cdn import CdnClient, Publisher

EDGERC_FAKE_CONF = {
"client_secret": "some-secret",
"host": "some-host",
"access_token": "some-access-token",
"client_token": "some-client-token",
}
from ubipop._cdn import Publisher


@pytest.fixture(name="pulp")
Expand All @@ -28,16 +20,9 @@ def create_and_insert_repo(pulp, repos):
return out


def setup_fastpurge_mock(requests_mock):
url = "https://some-host/ccu/v3/delete/url/production"
seconds = 0.1
response = {"some": ["return", "value"], "estimatedSeconds": seconds}
requests_mock.register_uri("POST", url, status_code=201, json=response)


def test_publisher_without_cache_purge(pulp, requests_mock):
def test_publisher(pulp):
"""
Tests a basic scenario of repository publish without cache purge.
Tests a basic scenario of repository publish.
"""
dt = datetime(2019, 9, 12, 0, 0, 0)
repos_to_insert = []
Expand All @@ -62,197 +47,9 @@ def test_publisher_without_cache_purge(pulp, requests_mock):
# enqueue repos for publish and wait for publish to finish
with Publisher(**{}) as publisher:
publisher.enqueue(*repos)
publisher.wait_publish_and_purge_cache()
publisher.wait_publish()

# all repos are properly published
assert [hist.repository.id for hist in pulp.publish_history] == [
repo.id for repo in repos
]

hist = requests_mock.request_history

assert len(hist) == 0 # no requests expected as CDN cache purge is disabled


def test_publisher_with_cache_purge(pulp, requests_mock):
"""
Tests a scenario of repository publish with cache purge by URL and ARLs generated with
data from CDN service.
"""
dt = datetime(2019, 9, 12, 0, 0, 0)
repos_to_insert = []
repo_id = "repo-1"
distributor = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id=repo_id,
last_publish=dt,
relative_url="content/unit/1/client",
)
repo = YumRepository(
id=repo_id,
eng_product_id=102,
distributors=[distributor],
relative_url="content/unit/1/client",
mutable_urls=["repomd.xml"],
)
repos_to_insert.append(repo)

repos = create_and_insert_repo(pulp, repos_to_insert)
publisher_args = {
"edgerc": EDGERC_FAKE_CONF,
"publish_options": {
"clean": True,
},
"cdn_root": "https://cdn.example.com",
"arl_templates": [
"/arl/1/test/{ttl}/{path}",
"/arl/2/test/{ttl}/{path}",
# extra case with whitespace chars
" /arl/3/test/{ttl}/{path} /extra// \n\n \t ",
],
"max_retry_sleep": 0.001,
}
setup_fastpurge_mock(requests_mock)

url_ttl = ("https://cdn.example.com/content/unit/1/client/repomd.xml", "33s")
# ARLs are generated from the template using the {ttl} placeholder, which is replaced with
# the real TTL value. The real TTL value is extracted from the cache key header of the real
# request for the given path using '/(\d+[smhd])/' regex.
# The /1h/foo in the mocked header here is to test that if the path contains a component
# that also matches the TTL regex ('/1h/'), it will still find the correct value ('/33s/').
headers = {"X-Cache-Key": f"/fake/cache-key/{url_ttl[1]}/something/1h/foo"}
requests_mock.register_uri("HEAD", url_ttl[0], headers=headers)

# enqueue repos to publish and wait for publish and purge to finish
with Publisher(**publisher_args) as publisher:
publisher.enqueue(*repos)
publisher.wait_publish_and_purge_cache()

assert [hist.repository.id for hist in pulp.publish_history] == [
repo.id for repo in repos
]

hist = requests_mock.request_history

assert len(hist) == 2 # 1 request to cdn service for headers and 1 purge request

assert hist[0].url == "https://cdn.example.com/content/unit/1/client/repomd.xml"
assert hist[1].json()["objects"] == [
"https://cdn.example.com/content/unit/1/client/repomd.xml",
"/arl/1/test/33s/content/unit/1/client/repomd.xml",
"/arl/2/test/33s/content/unit/1/client/repomd.xml",
"/arl/3/test/33s/content/unit/1/client/repomd.xml /extra//",
]


def test_cdn_client_retries(pulp, requests_mock):
"""
Tests a scenario when some request to CDN service for TTL fails but
it's retried with success.
"""
dt = datetime(2019, 9, 12, 0, 0, 0)
repos_to_insert = []
repo_id = "repo-1"
distributor = Distributor(
id="yum_distributor",
type_id="yum_distributor",
repo_id=repo_id,
last_publish=dt,
relative_url="content/unit/1/client",
)
repo = YumRepository(
id=repo_id,
eng_product_id=102,
distributors=[distributor],
relative_url="content/unit/1/client",
mutable_urls=["repomd.xml"],
)
repos_to_insert.append(repo)

repos = create_and_insert_repo(pulp, repos_to_insert)

url = "https://cdn.example.com/content/unit/1/client/repomd.xml"
publisher_args = {
"edgerc": EDGERC_FAKE_CONF,
"publish_options": {
"clean": True,
},
"cdn_root": "https://cdn.example.com",
"arl_templates": ["/arl/1/test/{ttl}/{path}", "/arl/2/test/{ttl}/{path}"],
"max_retry_sleep": 0.001,
}
setup_fastpurge_mock(requests_mock)
requests_mock.register_uri(
"HEAD",
url,
[
# Fails on first try
{"status_code": 500},
# Then succeeds
{
"status_code": 200,
"headers": {"X-Cache-Key": "/fake/cache-key/10h/something"},
},
],
)

# enqueue repos for publish and wait to finish
with Publisher(**publisher_args) as publisher:
publisher.enqueue(*repos)
publisher.wait_publish_and_purge_cache()

# all published repos should be recorded in history
assert [hist.repository.id for hist in pulp.publish_history] == [
repo.id for repo in repos
]

hist = requests_mock.request_history

# there should be 2 requests to cdn service (failure and success) for headers and 1 CDN cache purge request
assert len(hist) == 3

for i in range(2):
assert hist[i].url == "https://cdn.example.com/content/unit/1/client/repomd.xml"

assert hist[-1].json()["objects"] == [
"https://cdn.example.com/content/unit/1/client/repomd.xml",
"/arl/1/test/10h/content/unit/1/client/repomd.xml",
"/arl/2/test/10h/content/unit/1/client/repomd.xml",
]


@pytest.mark.parametrize(
"path, expected_ttl",
[
("content/test/repodata/repomd.xml", "4h"),
("content/test/repodata/", "10m"),
("/ostree/repo/refs/heads/test-path/base", "10m"),
("content/test/PULP_MANIFEST", "10m"),
("content/test/", "4h"),
("content/test/some-file", "30d"),
],
)
def test_arl_fallback(requests_mock, path, expected_ttl):
"""
Tests fallback to default TTL values when TTL cannot be
fetched from CDN service.
"""
templates = [
"/fake/template-1/{ttl}/{path}",
]
with CdnClient(
"https://cdn.example.com/", arl_templates=templates, max_retry_sleep=0.001
) as client:
url = "https://cdn.example.com/content/foo/test-path-1/some-file"

requests_mock.register_uri("HEAD", url, status_code=500)

# Request ARLs
arls_ft = client.get_arl_for_path(path)

# It should be successful
arl = [item.result() for item in as_completed(arls_ft)][0]

# It should fallback to default ttl value
assert arl == "/fake/template-1/{ttl}/{path}".format(ttl=expected_ttl, path=path)
78 changes: 2 additions & 76 deletions tests/test_ubipop.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import shutil
import sys
import tempfile
import textwrap
from concurrent.futures import wait
from datetime import datetime
from tempfile import NamedTemporaryFile
Expand Down Expand Up @@ -1050,37 +1049,9 @@ def test_get_current_content(mock_ubipop_runner, pulp, skip_debug_repo):
assert source_rpms[0].name == "test-srpm"


@pytest.fixture(name="cache_purge_env")
def fixture_cache_purge_env(monkeypatch):
with tempfile.NamedTemporaryFile("w") as fp:
monkeypatch.setenv("UBIPOP_EDGERC_CFG", fp.name)
monkeypatch.setenv("UBIPOP_CDN_ROOT", "https://test.root-url.com/")
monkeypatch.setenv(
"UBIPOP_ARL_TEMPLATES",
"/test/template-1/{ttl}/{path},/test/template-2/{ttl}/{path}",
)

fp.write(
textwrap.dedent(
"""
[default]
client_secret = some-secret
host = some-host
access_token = some-access-token
client_token = some-client-token
"""
)
)
fp.flush()

yield


@patch("pubtools.pulplib.YumRepository.get_debug_repository")
@patch("pubtools.pulplib.YumRepository.get_source_repository")
def test_populate_ubi_repos(
get_debug_repository, get_source_repository, cache_purge_env, requests_mock
):
def test_populate_ubi_repos(get_debug_repository, get_source_repository, requests_mock):
# pylint: disable=unused-argument
"""Test run of populate_ubi_repos that checks correct number of repo publication.
It's simplified to contain only actions on RPM packages."""
Expand Down Expand Up @@ -1208,8 +1179,6 @@ def test_populate_ubi_repos(

# mock calls to ubi-manifest service
_create_ubi_manifest_mocks(requests_mock)
_create_fastpurge_mocks(requests_mock)
_create_cdn_mocks(requests_mock)

# let's run actual population
ubi_populate.populate_ubi_repos()
Expand All @@ -1235,24 +1204,6 @@ def test_populate_ubi_repos(

assert repo_ids_published == expected_published_repo_ids

request = requests_mock.request_history[
-1
] # last request should be for purge cache request

expected_req = [
"https://test.root-url.com/content/unit/3/client/repodata/repomd.xml",
"/test/template-1/24h/content/unit/3/client/repodata/repomd.xml",
"/test/template-2/24h/content/unit/3/client/repodata/repomd.xml",
"https://test.root-url.com/content/unit/2/client/repodata/repomd.xml",
"/test/template-1/30m/content/unit/2/client/repodata/repomd.xml",
"/test/template-2/30m/content/unit/2/client/repodata/repomd.xml",
"https://test.root-url.com/content/unit/1/client/repodata/repomd.xml",
"/test/template-1/10s/content/unit/1/client/repodata/repomd.xml",
"/test/template-2/10s/content/unit/1/client/repodata/repomd.xml",
]

assert sorted(request.json()["objects"]) == sorted(expected_req)


@patch("ubipop._cdn.Publisher")
@patch("pubtools.pulplib.YumRepository.get_debug_repository")
Expand Down Expand Up @@ -1531,8 +1482,6 @@ def test_populate_ubi_repos_dry_run(

# mock calls to ubi-manifest service
_create_ubi_manifest_mocks(requests_mock)
_create_fastpurge_mocks(requests_mock)
_create_cdn_mocks(requests_mock)

# let's run actual population
ubi_populate.populate_ubi_repos()
Expand All @@ -1553,9 +1502,7 @@ def test_populate_ubi_repos_dry_run(


@patch("pubtools.pulplib.YumRepository.get_source_repository")
def test_populate_ubi_repos_missing_repos(
get_source_repository, cache_purge_env, requests_mock, caplog
):
def test_populate_ubi_repos_missing_repos(get_source_repository, requests_mock, caplog):
# pylint: disable=unused-argument
"""Test run of populate_ubi_repos with a missing repo."""
dt = datetime(2019, 9, 12, 0, 0, 0)
Expand Down Expand Up @@ -1666,8 +1613,6 @@ def test_populate_ubi_repos_missing_repos(

# mock calls to ubi-manifest service
_create_ubi_manifest_mocks(requests_mock)
_create_fastpurge_mocks(requests_mock)
_create_cdn_mocks(requests_mock)

# let's run actual population
ubi_populate.populate_ubi_repos()
Expand All @@ -1680,13 +1625,6 @@ def test_populate_ubi_repos_missing_repos(
assert "Skipping current content triplet, some repos are missing" in caplog.text


def _create_fastpurge_mocks(requests_mock):
url = "https://some-host/ccu/v3/delete/url/production"
seconds = 0.1
response = {"some": ["return", "value"], "estimatedSeconds": seconds}
requests_mock.register_uri("POST", url, status_code=201, json=response)


def _create_ubi_manifest_mocks(requests_mock):
# mock requesting manifest generation
url = "api/v1/manifest"
Expand Down Expand Up @@ -1740,18 +1678,6 @@ def _create_ubi_manifest_mocks(requests_mock):
requests_mock.register_uri("GET", url, json=response)


def _create_cdn_mocks(requests_mock):
url_ttl = [
("https://test.root-url.com/content/unit/1/client/repodata/repomd.xml", "10s"),
("https://test.root-url.com/content/unit/2/client/repodata/repomd.xml", "30m"),
("https://test.root-url.com/content/unit/3/client/repodata/repomd.xml", "24h"),
]

for url, ttl in url_ttl:
headers = {"X-Cache-Key": f"/fake/cache-key/{ttl}/something"}
requests_mock.register_uri("HEAD", url, headers=headers)


@patch("ubipop.Client")
def test_make_pulp_client_cert(mocked_client):
"""Tests that pulp client can be created with cert/key used for auth"""
Expand Down
Loading