From a217dd5a71cb1d777851feeec589832f9044059f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 1 Oct 2024 17:49:40 +0500 Subject: [PATCH] Add job posting navigation classes. --- docs/reference/items.rst | 12 +++-- docs/reference/pages.rst | 12 +++++ tests/test_pages.py | 1 + zyte_common_items/__init__.py | 5 ++ zyte_common_items/items/__init__.py | 1 + .../items/job_posting_navigation.py | 46 ++++++++++++++++ zyte_common_items/pages/__init__.py | 5 ++ .../pages/job_posting_navigation.py | 53 +++++++++++++++++++ 8 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 zyte_common_items/items/job_posting_navigation.py create mode 100644 zyte_common_items/pages/job_posting_navigation.py diff --git a/docs/reference/items.rst b/docs/reference/items.rst index 8cc12381..3a7746d3 100644 --- a/docs/reference/items.rst +++ b/docs/reference/items.rst @@ -52,7 +52,6 @@ Article .. autoclass:: zyte_common_items.ArticleMetadata(**kwargs) :members: dateDownloaded, probability, validationMessages - Article list ============ @@ -87,7 +86,6 @@ Business place .. autoclass:: zyte_common_items.BusinessPlaceMetadata(**kwargs) :members: dateDownloaded, probability, searchText, validationMessages - Real estate =========== @@ -108,6 +106,15 @@ Job posting .. autoclass:: zyte_common_items.JobPostingMetadata(**kwargs) :members: dateDownloaded, probability, validationMessages +Job posting navigation +====================== + +.. autoclass:: zyte_common_items.JobPostingNavigation(**kwargs) + :members: + :inherited-members: + +.. autoclass:: zyte_common_items.JobPostingNavigationMetadata(**kwargs) + :members: dateDownloaded, validationMessages Search engine results ===================== @@ -119,7 +126,6 @@ Search engine results .. autoclass:: zyte_common_items.SerpMetadata(**kwargs) :members: dateDownloaded, displayedQuery, searchedQuery, totalOrganicResults, validationMessages - Social media post ================= diff --git a/docs/reference/pages.rst b/docs/reference/pages.rst index 30090f49..27d0b4e9 100644 --- a/docs/reference/pages.rst +++ b/docs/reference/pages.rst @@ -114,6 +114,18 @@ Job posting .. autoclass:: zyte_common_items.AutoJobPostingPage(**kwargs) :show-inheritance: +Job posting navigation +====================== + +.. autoclass:: zyte_common_items.BaseJobPostingNavigationPage(**kwargs) + :show-inheritance: + +.. autoclass:: zyte_common_items.JobPostingNavigationPage(**kwargs) + :show-inheritance: + +.. autoclass:: zyte_common_items.AutoJobPostingNavigationPage(**kwargs) + :show-inheritance: + Search engine results ===================== diff --git a/tests/test_pages.py b/tests/test_pages.py index d8416bbb..f13caeeb 100644 --- a/tests/test_pages.py +++ b/tests/test_pages.py @@ -220,6 +220,7 @@ def test_matching_items(): "ProductNavigation": {"dateDownloaded", "validationMessages"}, "RealEstate": {"dateDownloaded", "probability", "validationMessages"}, "JobPosting": {"dateDownloaded", "probability", "searchText", "validationMessages"}, + "JobPostingNavigation": {"dateDownloaded", "validationMessages"}, "Serp": { "dateDownloaded", "displayedQuery", diff --git a/zyte_common_items/__init__.py b/zyte_common_items/__init__.py index e81c4a70..545b4354 100644 --- a/zyte_common_items/__init__.py +++ b/zyte_common_items/__init__.py @@ -52,6 +52,8 @@ CustomAttributesValues, JobPosting, JobPostingMetadata, + JobPostingNavigation, + JobPostingNavigationMetadata, Product, ProductFromList, ProductList, @@ -78,6 +80,7 @@ AutoArticleNavigationPage, AutoArticlePage, AutoBusinessPlacePage, + AutoJobPostingNavigationPage, AutoJobPostingPage, AutoProductListPage, AutoProductNavigationPage, @@ -89,6 +92,7 @@ BaseArticleNavigationPage, BaseArticlePage, BaseBusinessPlacePage, + BaseJobPostingNavigationPage, BaseJobPostingPage, BasePage, BaseProductListPage, @@ -99,6 +103,7 @@ BaseSocialMediaPostPage, BusinessPlacePage, HasMetadata, + JobPostingNavigationPage, JobPostingPage, MetadataT, Page, diff --git a/zyte_common_items/items/__init__.py b/zyte_common_items/items/__init__.py index f60cac27..2fac206d 100644 --- a/zyte_common_items/items/__init__.py +++ b/zyte_common_items/items/__init__.py @@ -10,6 +10,7 @@ CustomAttributesValues, ) from .job_posting import JobPosting, JobPostingMetadata +from .job_posting_navigation import JobPostingNavigation, JobPostingNavigationMetadata from .product import Product, ProductMetadata, ProductVariant from .product_list import ProductFromList, ProductList, ProductListMetadata from .product_navigation import ProductNavigation, ProductNavigationMetadata diff --git a/zyte_common_items/items/job_posting_navigation.py b/zyte_common_items/items/job_posting_navigation.py new file mode 100644 index 00000000..e0755b0e --- /dev/null +++ b/zyte_common_items/items/job_posting_navigation.py @@ -0,0 +1,46 @@ +from typing import List, Optional + +import attrs + +from zyte_common_items.base import Item +from zyte_common_items.components import ListMetadata, ProbabilityRequest, Request +from zyte_common_items.converters import ( + to_metadata_optional, + to_probability_request_list_optional, + url_to_str, +) + + +@attrs.define(kw_only=True) +class JobPostingNavigationMetadata(ListMetadata): + """Metadata class for :data:`zyte_common_items.JobPostingNavigation.metadata`.""" + + +@attrs.define(kw_only=True) +class JobPostingNavigation(Item): + """Represents the navigational aspects of a job posting listing page on a + job website""" + + #: Main URL from which the data is extracted. + url: str = attrs.field(converter=url_to_str) + + #: List of job postings available on this page. + items: Optional[List[ProbabilityRequest]] = attrs.field( + default=None, converter=to_probability_request_list_optional, kw_only=True # type: ignore[misc] + ) + + #: A link to the next page, if available. + nextPage: Optional[Request] = None + + #: Number of the current page. + #: + #: It should only be extracted if the webpage shows a page number. + #: + #: It must be 1-based. For example, if the first page of a listing is + #: numbered as 0 on the website, it should be extracted as `1` nonetheless. + pageNumber: Optional[int] = None + + #: Data extraction process metadata. + metadata: Optional[JobPostingNavigationMetadata] = attrs.field( + default=None, converter=to_metadata_optional(JobPostingNavigationMetadata), kw_only=True # type: ignore[misc] + ) diff --git a/zyte_common_items/pages/__init__.py b/zyte_common_items/pages/__init__.py index 6eaeb63f..b5f7729a 100644 --- a/zyte_common_items/pages/__init__.py +++ b/zyte_common_items/pages/__init__.py @@ -13,6 +13,11 @@ BusinessPlacePage, ) from .job_posting import AutoJobPostingPage, BaseJobPostingPage, JobPostingPage +from .job_posting_navigation import ( + AutoJobPostingNavigationPage, + BaseJobPostingNavigationPage, + JobPostingNavigationPage, +) from .mixins import DescriptionMixin, HasMetadata, MetadataT, PriceMixin from .product import AutoProductPage, BaseProductPage, ProductPage from .product_list import AutoProductListPage, BaseProductListPage, ProductListPage diff --git a/zyte_common_items/pages/job_posting_navigation.py b/zyte_common_items/pages/job_posting_navigation.py new file mode 100644 index 00000000..5c5cbfce --- /dev/null +++ b/zyte_common_items/pages/job_posting_navigation.py @@ -0,0 +1,53 @@ +from typing import List, Optional + +import attrs +from web_poet import Returns + +from zyte_common_items.components import ProbabilityRequest, Request +from zyte_common_items.fields import auto_field +from zyte_common_items.items import JobPostingNavigation, JobPostingNavigationMetadata +from zyte_common_items.processors import probability_request_list_processor + +from .base import BasePage, Page +from .mixins import HasMetadata + + +class BaseJobPostingNavigationPage( + BasePage, Returns[JobPostingNavigation], HasMetadata[JobPostingNavigationMetadata] +): + """:class:`BasePage` subclass for :class:`JobPostingNavigation`.""" + + class Processors(BasePage.Processors): + subCategories = [probability_request_list_processor] + items = [probability_request_list_processor] + + +class JobPostingNavigationPage( + Page, Returns[JobPostingNavigation], HasMetadata[JobPostingNavigationMetadata] +): + """:class:`Page` subclass for :class:`JobPostingNavigation`.""" + + +@attrs.define +class AutoJobPostingNavigationPage(BaseJobPostingNavigationPage): + job_posting_navigation: JobPostingNavigation + + @auto_field + def items(self) -> Optional[List[ProbabilityRequest]]: + return self.job_posting_navigation.items + + @auto_field + def metadata(self) -> Optional[JobPostingNavigationMetadata]: + return self.job_posting_navigation.metadata + + @auto_field + def nextPage(self) -> Optional[Request]: + return self.job_posting_navigation.nextPage + + @auto_field + def pageNumber(self) -> Optional[int]: + return self.job_posting_navigation.pageNumber + + @auto_field + def url(self) -> Optional[str]: + return self.job_posting_navigation.url