From cef77d2655ec38e2fd56431fb2aefd7add9129bd Mon Sep 17 00:00:00 2001 From: Peng-Yu Chen Date: Fri, 13 Dec 2024 13:22:49 +0100 Subject: [PATCH 1/3] Update warning message on installation of msgpack-python (#172) See also 0a422fec --- scrapinghub/hubstorage/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapinghub/hubstorage/client.py b/scrapinghub/hubstorage/client.py index f9e48b53..afe5ef85 100644 --- a/scrapinghub/hubstorage/client.py +++ b/scrapinghub/hubstorage/client.py @@ -90,7 +90,7 @@ def __init__(self, auth=None, endpoint=None, connection_timeout=None, self.use_msgpack = MSGPACK_AVAILABLE and use_msgpack if use_msgpack != self.use_msgpack: logger.warning('Messagepack is not available, please ensure that ' - 'msgpack-python library is properly installed.') + 'msgpack library is properly installed.') def request(self, is_idempotent=False, **kwargs): """ From 35264ac2b6eb7a25bb2efdec17c1081127771f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 13 Dec 2024 14:53:43 +0100 Subject: [PATCH 2/3] Freeze vcrpy (#178) --- requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-test.txt b/requirements-test.txt index 06032721..6622cf42 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,5 @@ mock -vcrpy +vcrpy<6 # https://github.com/kevin1024/vcrpy/issues/885 pytest pytest-cov responses From 6354a96126522763f198d045343c0a52dca5a8ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 13 Dec 2024 14:54:08 +0100 Subject: [PATCH 3/3] Add .readthedocs.yml (#173) --- .readthedocs.yml | 15 ++++++++++ docs/client/overview.rst | 2 +- docs/conf.py | 50 ++----------------------------- docs/index.rst | 3 -- docs/requirements.txt | 2 ++ scrapinghub/client/__init__.py | 2 +- scrapinghub/client/collections.py | 6 ++-- scrapinghub/client/frontiers.py | 8 ++--- scrapinghub/client/jobs.py | 20 ++++++------- scrapinghub/client/logs.py | 2 +- tox.ini | 7 +++++ 11 files changed, 46 insertions(+), 71 deletions(-) create mode 100644 .readthedocs.yml create mode 100644 docs/requirements.txt diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..f9a5741e --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,15 @@ +version: 2 +formats: all +sphinx: + configuration: docs/conf.py + fail_on_warning: true +build: + os: ubuntu-22.04 + tools: + # For available versions, see: + # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python + python: "3.10" # Keep in sync with .github/workflows/main.yml +python: + install: + - requirements: docs/requirements.txt + - path: . diff --git a/docs/client/overview.rst b/docs/client/overview.rst index 6bdbe01e..1bb7b1f2 100644 --- a/docs/client/overview.rst +++ b/docs/client/overview.rst @@ -145,7 +145,7 @@ For example, to run a new job for a given spider with custom parameters:: Getting job information -^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^ To select a specific job for a project, use ``.jobs.get()``:: diff --git a/docs/conf.py b/docs/conf.py index 7bada0aa..c3b9b3f6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,10 +20,6 @@ import sys from datetime import datetime -from docutils import nodes -from sphinx.util.docfields import TypedField -from sphinx import addnodes - sys.path.insert(0, os.path.abspath('..')) @@ -76,7 +72,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -94,8 +90,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -# -html_theme = 'alabaster' +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -171,44 +166,3 @@ html_theme = 'sphinx_rtd_theme' html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - - -# disable cross-reference for ivar -# patch taken from http://stackoverflow.com/a/41184353/1932023 -def patched_make_field(self, types, domain, items, env=None): - # type: (List, unicode, Tuple) -> nodes.field - def handle_item(fieldarg, content): - par = nodes.paragraph() - par += addnodes.literal_strong('', fieldarg) # Patch: this line added - # par.extend(self.make_xrefs(self.rolename, domain, fieldarg, - # addnodes.literal_strong)) - if fieldarg in types: - par += nodes.Text(' (') - # NOTE: using .pop() here to prevent a single type node to be - # inserted twice into the doctree, which leads to - # inconsistencies later when references are resolved - fieldtype = types.pop(fieldarg) - if len(fieldtype) == 1 and isinstance(fieldtype[0], nodes.Text): - typename = u''.join(n.astext() for n in fieldtype) - par.extend(self.make_xrefs(self.typerolename, domain, typename, - addnodes.literal_emphasis)) - else: - par += fieldtype - par += nodes.Text(')') - par += nodes.Text(' -- ') - par += content - return par - - fieldname = nodes.field_name('', self.label) - if len(items) == 1 and self.can_collapse: - fieldarg, content = items[0] - bodynode = handle_item(fieldarg, content) - else: - bodynode = self.list_type() - for fieldarg, content in items: - bodynode += nodes.list_item('', handle_item(fieldarg, content)) - fieldbody = nodes.field_body('', bodynode) - return nodes.field('', fieldname, fieldbody) - - -TypedField.make_field = patched_make_field diff --git a/docs/index.rst b/docs/index.rst index 043e2a18..6af68db8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,9 +2,6 @@ Client interface for Scrapinghub API ==================================== -.. image:: https://secure.travis-ci.org/scrapinghub/python-scrapinghub.svg?branch=master - :target: https://travis-ci.org/scrapinghub/python-scrapinghub - The ``scrapinghub`` is a Python library for communicating with the `Scrapinghub API`_. .. _Scrapinghub API: https://doc.scrapinghub.com/scrapy-cloud.html#scrapycloud diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..d5476d84 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx==7.2.6 +sphinx-rtd-theme==2.0.0 diff --git a/scrapinghub/client/__init__.py b/scrapinghub/client/__init__.py index 8a386736..80825852 100644 --- a/scrapinghub/client/__init__.py +++ b/scrapinghub/client/__init__.py @@ -38,7 +38,7 @@ class ScrapinghubClient(object): If you need full access to *Scrapy Cloud* features, you'll need to provide a Scrapinghub APIKEY through this argument or deploying ``SH_APIKEY``. :param dash_endpoint: (optional) Scrapinghub Dash panel url. - :param \*\*kwargs: (optional) Additional arguments for + :param kwargs: (optional) Additional arguments for :class:`~scrapinghub.hubstorage.HubstorageClient` constructor. :ivar projects: projects collection, diff --git a/scrapinghub/client/collections.py b/scrapinghub/client/collections.py index b88c04df..9cdc061b 100644 --- a/scrapinghub/client/collections.py +++ b/scrapinghub/client/collections.py @@ -160,7 +160,7 @@ def get(self, key, **params): """Get item from collection by key. :param key: string item key. - :param \*\*params: (optional) additional query params for the request. + :param params: (optional) additional query params for the request. :return: an item dictionary if exists. :rtype: :class:`dict` """ @@ -217,7 +217,7 @@ def iter(self, key=None, prefix=None, prefixcount=None, startts=None, :param startts: UNIX timestamp at which to begin results. :param endts: UNIX timestamp at which to end results. :param requests_params: (optional) a dict with optional requests params. - :param \*\*params: (optional) additional query params for the request. + :param params: (optional) additional query params for the request. :return: an iterator over items list. :rtype: :class:`collections.abc.Iterable[dict]` """ @@ -243,7 +243,7 @@ def list(self, key=None, prefix=None, prefixcount=None, startts=None, :param startts: UNIX timestamp at which to begin results. :param endts: UNIX timestamp at which to end results. :param requests_params: (optional) a dict with optional requests params. - :param \*\*params: (optional) additional query params for the request. + :param params: (optional) additional query params for the request. :return: a list of items where each item is represented with a dict. :rtype: :class:`list[dict]` """ diff --git a/scrapinghub/client/frontiers.py b/scrapinghub/client/frontiers.py index 3fd70c76..38298a0b 100644 --- a/scrapinghub/client/frontiers.py +++ b/scrapinghub/client/frontiers.py @@ -319,7 +319,7 @@ def add(self, fps): def iter(self, **params): """Iterate through fingerprints in the slot. - :param \*\*params: (optional) additional query params for the request. + :param params: (optional) additional query params for the request. :return: an iterator over fingerprints. :rtype: :class:`collections.abc.Iterable[str]` """ @@ -331,7 +331,7 @@ def iter(self, **params): def list(self, **params): """List fingerprints in the slot. - :param \*\*params: (optional) additional query params for the request. + :param params: (optional) additional query params for the request. :return: a list of fingerprints. :rtype: :class:`list[str]` """ @@ -355,7 +355,7 @@ def iter(self, mincount=None, **params): """Iterate through batches in the queue. :param mincount: (optional) limit results with min amount of requests. - :param \*\*params: (optional) additional query params for the request. + :param params: (optional) additional query params for the request. :return: an iterator over request batches in the queue where each batch is represented with a dict with ('id', 'requests') field. :rtype: :class:`collections.abc.Iterable[dict]` @@ -369,7 +369,7 @@ def list(self, mincount=None, **params): """List request batches in the queue. :param mincount: (optional) limit results with min amount of requests. - :param \*\*params: (optional) additional query params for the request. + :param params: (optional) additional query params for the request. :return: a list of request batches in the queue where each batch is represented with a dict with ('id', 'requests') field. :rtype: :class:`list[dict]` diff --git a/scrapinghub/client/jobs.py b/scrapinghub/client/jobs.py index d99ed0d0..75d0acbc 100644 --- a/scrapinghub/client/jobs.py +++ b/scrapinghub/client/jobs.py @@ -57,7 +57,7 @@ def count(self, spider=None, state=None, has_tag=None, lacks_tag=None, in milliseconds. :param endts: (optional) UNIX timestamp at which to end results, in milliseconds. - :param \*\*params: (optional) other filter params. + :param params: (optional) other filter params. :return: jobs count. :rtype: :class:`int` @@ -156,7 +156,7 @@ def iter(self, count=None, start=None, spider=None, state=None, in millisecons. :param meta: (optional) request for additional fields, a single field name or a list of field names to return. - :param \*\*params: (optional) other filter params. + :param params: (optional) other filter params. :return: a generator object over a list of dictionaries of jobs summary for a given filter params. @@ -227,7 +227,7 @@ def list(self, count=None, start=None, spider=None, state=None, in milliseconds. :param meta: (optional) request for additional fields, a single field name or a list of field names to return. - :param \*\*params: (optional) other filter params. + :param params: (optional) other filter params. :return: list of dictionaries of jobs summary for a given filter params. :rtype: :class:`list[dict]` @@ -262,7 +262,7 @@ def run(self, spider=None, units=None, priority=None, meta=None, :param job_settings: (optional) a dictionary with job settings. :param cmd_args: (optional) a string with script command args. :param environment: (option) a dictionary with custom environment - :param \*\*params: (optional) additional keyword args. + :param params: (optional) additional keyword args. :return: a job instance, representing the scheduled job. :rtype: :class:`Job` @@ -334,7 +334,7 @@ def summary(self, state=None, spider=None, **params): :param state: (optional) a string state to filter jobs. :param spider: (optional) a spider name (not needed if instantiated with :class:`~scrapinghub.client.spiders.Spider`). - :param \*\*params: (optional) additional keyword args. + :param params: (optional) additional keyword args. :return: a list of dictionaries of jobs summary for a given filter params grouped by job state. :rtype: :class:`list[dict]` @@ -362,7 +362,7 @@ def iter_last(self, start=None, start_after=None, count=None, :param count: (optional) :param spider: (optional) a spider name (not needed if instantiated with :class:`~scrapinghub.client.spiders.Spider`). - :param \*\*params: (optional) additional keyword args. + :param params: (optional) additional keyword args. :return: a generator object over a list of dictionaries of jobs summary for a given filter params. :rtype: :class:`types.GeneratorType[dict]` @@ -512,7 +512,7 @@ def close_writers(self): def start(self, **params): """Move job to running state. - :param \*\*params: (optional) keyword meta parameters to update. + :param params: (optional) keyword meta parameters to update. :return: a previous string job state. :rtype: :class:`str` @@ -526,7 +526,7 @@ def start(self, **params): def finish(self, **params): """Move running job to finished state. - :param \*\*params: (optional) keyword meta parameters to update. + :param params: (optional) keyword meta parameters to update. :return: a previous string job state. :rtype: :class:`str` @@ -540,7 +540,7 @@ def finish(self, **params): def delete(self, **params): """Mark finished job for deletion. - :param \*\*params: (optional) keyword meta parameters to update. + :param params: (optional) keyword meta parameters to update. :return: a previous string job state. :rtype: :class:`str` @@ -555,7 +555,7 @@ def update(self, state, **params): """Update job state. :param state: a new job state. - :param \*\*params: (optional) keyword meta parameters to update. + :param params: (optional) keyword meta parameters to update. :return: a previous string job state. :rtype: :class:`str` diff --git a/scrapinghub/client/logs.py b/scrapinghub/client/logs.py index d9dd0cae..3ab37e96 100644 --- a/scrapinghub/client/logs.py +++ b/scrapinghub/client/logs.py @@ -55,7 +55,7 @@ def log(self, message, level=logging.INFO, ts=None, **other): :param message: a string message. :param level: (optional) logging level, default to INFO. :param ts: (optional) UNIX timestamp in milliseconds. - :param \*\*other: other optional kwargs. + :param other: other optional kwargs. """ self._origin.log(message, level=level, ts=ts, **other) diff --git a/tox.ini b/tox.ini index f8a512ab..d13e85d9 100644 --- a/tox.ini +++ b/tox.ini @@ -13,3 +13,10 @@ deps = msgpack: -r{toxinidir}/requirements.txt pypy-msgpack: -r{toxinidir}/requirements-pypy.txt commands = py.test --cov=scrapinghub --cov-report=xml {posargs: scrapinghub tests} + +[testenv:docs] +changedir = docs +deps = + -r docs/requirements.txt +commands = + sphinx-build -W -b html . {envtmpdir}/html