From 0df102a416b28147a69dde392df765d5c4e820b3 Mon Sep 17 00:00:00 2001 From: Jordan Cook Date: Sun, 3 Mar 2024 14:12:41 -0600 Subject: [PATCH 1/2] Update dependencies; run pre-commit; use requests-cache 1.1 for python 3.7 support --- README.md | 14 +++-- poetry.lock | 93 ++++++++++++++++++++++++++++++- pyproject.toml | 2 +- test/test_requests_ratelimiter.py | 15 ++++- 4 files changed, 115 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index e4040ab..aa3603a 100644 --- a/README.md +++ b/README.md @@ -253,10 +253,16 @@ class CachedLimiterSession(CacheMixin, LimiterMixin, Session): # Optionally use SQLite as both the bucket backend and the cache backend -session = CachedLimiterSession(per_second=5, - cache_name='cache.db', - bucket_class=SQLiteBucket, - bucket_kwargs={"path": "cache.db", 'isolation_level': "EXCLUSIVE", 'check_same_thread': False}) +session = CachedLimiterSession( + per_second=5, + cache_name='cache.db', + bucket_class=SQLiteBucket, + bucket_kwargs={ + "path": "cache.db", + 'isolation_level': "EXCLUSIVE", + 'check_same_thread': False, + }, +) ``` This example has an extra benefit: cache hits won't count against your rate limit! diff --git a/poetry.lock b/poetry.lock index df50749..3a9f0e2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -11,6 +11,28 @@ files = [ {file = "alabaster-0.7.13.tar.gz", hash = "sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2"}, ] +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.dependencies] +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + [[package]] name = "babel" version = "2.14.0" @@ -49,6 +71,31 @@ charset-normalizer = ["charset-normalizer"] html5lib = ["html5lib"] lxml = ["lxml"] +[[package]] +name = "cattrs" +version = "23.1.2" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cattrs-23.1.2-py3-none-any.whl", hash = "sha256:b2bb14311ac17bed0d58785e5a60f022e5431aca3932e3fc5cc8ed8639de50a4"}, + {file = "cattrs-23.1.2.tar.gz", hash = "sha256:db1c821b8c537382b2c7c66678c3790091ca0275ac486c76f3c8f3920e83c657"}, +] + +[package.dependencies] +attrs = ">=20" +exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} +typing_extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.2.0,<5.0.0)"] +cbor2 = ["cbor2 (>=5.4.6,<6.0.0)"] +msgpack = ["msgpack (>=1.0.2,<2.0.0)"] +orjson = ["orjson (>=3.5.2,<4.0.0)"] +pyyaml = ["PyYAML (>=6.0,<7.0)"] +tomlkit = ["tomlkit (>=0.11.4,<0.12.0)"] +ujson = ["ujson (>=5.4.0,<6.0.0)"] + [[package]] name = "certifi" version = "2024.2.2" @@ -825,6 +872,36 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + [[package]] name = "requests-mock" version = "1.11.0" @@ -1093,6 +1170,20 @@ files = [ {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, ] +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + [[package]] name = "urllib3" version = "2.0.7" @@ -1152,4 +1243,4 @@ docs = ["furo", "myst-parser", "sphinx", "sphinx-autodoc-typehints", "sphinx-cop [metadata] lock-version = "2.0" python-versions = "^3.7" -content-hash = "75e6b4196a7beb4f9379fb89a61c3bb8929c5136260af40dd97111023787c9c6" +content-hash = "2e186267aed610829a83143f8d98c6adbea6c179377109b04472e8d7a0b84f40" diff --git a/pyproject.toml b/pyproject.toml index 81fb976..2233f54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ pytest = "^7.2" pytest-cov = ">=4.0" pytest-xdist = ">=3.1" requests-mock = ">=1.11" -requests_cache = ">=1.2" +requests-cache = ">=1.1" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/test/test_requests_ratelimiter.py b/test/test_requests_ratelimiter.py index c5e7262..93adda9 100644 --- a/test/test_requests_ratelimiter.py +++ b/test/test_requests_ratelimiter.py @@ -2,8 +2,8 @@ General rate-limiting behavior is covered by pyrate-limiter unit tests. These tests should cover additional behavior specific to requests-ratelimiter. """ -import os +import os from test.conftest import ( MOCKED_URL, MOCKED_URL_429, @@ -19,7 +19,8 @@ from pyrate_limiter import Duration, Limiter, RequestRate, SQLiteBucket from requests import Response, Session from requests.adapters import HTTPAdapter -from requests_cache import CacheMixin, SQLiteCache +from requests_cache import CacheMixin + from requests_ratelimiter import LimiterAdapter, LimiterMixin, LimiterSession from requests_ratelimiter.requests_ratelimiter import _convert_rate @@ -160,7 +161,15 @@ def test_convert_rate(limit, interval, expected_limit, expected_interval): @patch_sleep def test_sqlite_backend(mock_sleep): """Check that the SQLite backend works as expected""" - session = get_mock_session(per_second=5, bucket_class=SQLiteBucket, bucket_kwargs={"path": "rate_limit.db", 'isolation_level': "EXCLUSIVE", 'check_same_thread': False}) + session = get_mock_session( + per_second=5, + bucket_class=SQLiteBucket, + bucket_kwargs={ + "path": "rate_limit.db", + 'isolation_level': "EXCLUSIVE", + 'check_same_thread': False, + }, + ) for _ in range(5): session.get(MOCKED_URL) From 6399696c8b297f3383a376ed9517997030d30d1f Mon Sep 17 00:00:00 2001 From: Jordan Cook Date: Sun, 3 Mar 2024 14:12:45 -0600 Subject: [PATCH 2/2] Use temp dir for SQLite files --- test/test_requests_ratelimiter.py | 70 +++++++++++++++++++------------ 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/test/test_requests_ratelimiter.py b/test/test_requests_ratelimiter.py index 93adda9..80966ec 100644 --- a/test/test_requests_ratelimiter.py +++ b/test/test_requests_ratelimiter.py @@ -3,7 +3,6 @@ additional behavior specific to requests-ratelimiter. """ -import os from test.conftest import ( MOCKED_URL, MOCKED_URL_429, @@ -97,6 +96,7 @@ def test_custom_session(mock_sleep): session.get(MOCKED_URL) assert mock_sleep.called is True + @patch_sleep def test_429(mock_sleep): """After receiving a 429 response, the bucket should be filled, allowing no more requests""" @@ -159,14 +159,14 @@ def test_convert_rate(limit, interval, expected_limit, expected_interval): @patch_sleep -def test_sqlite_backend(mock_sleep): +def test_sqlite_backend(mock_sleep, tmp_path): """Check that the SQLite backend works as expected""" session = get_mock_session( per_second=5, bucket_class=SQLiteBucket, bucket_kwargs={ - "path": "rate_limit.db", - 'isolation_level': "EXCLUSIVE", + 'path': tmp_path / 'rate_limit.db', + 'isolation_level': 'EXCLUSIVE', 'check_same_thread': False, }, ) @@ -178,16 +178,34 @@ def test_sqlite_backend(mock_sleep): session.get(MOCKED_URL) assert mock_sleep.called is True - # force the database to close & clean up - del session - os.remove("rate_limit.db") - @patch_sleep -def test_custom_bucket(mock_sleep): - """With custom buckets, each session can be called independently without triggering rate limiting but requires a common backend such as sqlite""" - session_a = get_mock_session(per_second=5, bucket_name="a", bucket_class=SQLiteBucket, bucket_kwargs={"path": "rate_limit.db", 'isolation_level': "EXCLUSIVE", 'check_same_thread': False}) - session_b = get_mock_session(per_second=5, bucket_name="b", bucket_class=SQLiteBucket, bucket_kwargs={"path": "rate_limit.db", 'isolation_level': "EXCLUSIVE", 'check_same_thread': False}) +def test_custom_bucket(mock_sleep, tmp_path): + """With custom buckets, each session can be called independently without triggering rate + limiting but requires a common backend such as sqlite + """ + ratelimit_path = tmp_path / 'rate_limit.db' + + session_a = get_mock_session( + per_second=5, + bucket_name="a", + bucket_class=SQLiteBucket, + bucket_kwargs={ + 'path': ratelimit_path, + 'isolation_level': 'EXCLUSIVE', + 'check_same_thread': False, + }, + ) + session_b = get_mock_session( + per_second=5, + bucket_name='b', + bucket_class=SQLiteBucket, + bucket_kwargs={ + 'path': ratelimit_path, + 'isolation_level': 'EXCLUSIVE', + 'check_same_thread': False, + }, + ) for _ in range(5): session_a.get(MOCKED_URL) @@ -197,14 +215,9 @@ def test_custom_bucket(mock_sleep): session_a.get(MOCKED_URL) assert mock_sleep.called is True - # force the database to close & clean up - del session_a - del session_b - os.remove("rate_limit.db") - @patch_sleep -def test_caching(mock_sleep): +def test_cache_with_limiter(mock_sleep, tmp_path_factory): """Check that caching integration works as expected""" class CachedLimiterSession(CacheMixin, LimiterMixin, Session): @@ -213,16 +226,21 @@ class CachedLimiterSession(CacheMixin, LimiterMixin, Session): LimiterSession and CachedSession. """ - session = CachedLimiterSession(per_second=5, cache_name='cache.db', bucket_class=SQLiteBucket, bucket_kwargs={"path": "cache.db", 'isolation_level': "EXCLUSIVE", 'check_same_thread': False}) + cache_path = tmp_path_factory.mktemp('pytest') / 'cache.db' + ratelimit_path = tmp_path_factory.mktemp('pytest') / 'rate_limit.db' + + session = CachedLimiterSession( + per_second=5, + cache_name=str(cache_path), + bucket_class=SQLiteBucket, + bucket_kwargs={ + 'path': str(ratelimit_path), + 'isolation_level': 'EXCLUSIVE', + 'check_same_thread': False, + }, + ) session = mount_mock_adapter(session) for _ in range(10): session.get(MOCKED_URL) assert mock_sleep.called is False - - - # force the database to close & clean up - del session - os.remove("cache.db") - -