From 31340ead6d88f4a8aad97be93d89d1cdbe5f9f95 Mon Sep 17 00:00:00 2001 From: Uchechukwu Orji Date: Mon, 3 Jun 2024 16:03:28 +0100 Subject: [PATCH] add comments on field types, use py3.10+ annotations --- .gitignore | 1 + backend/pyproject.toml | 15 ++-- backend/src/backend/db/models.py | 81 ++++++++++--------- backend/src/backend/migrations/script.py.mako | 9 +-- ...7d19a79bcb_add_isp_field_to_tests_model.py | 31 ------- ...=> ef3115ef505c_set_up_database_models.py} | 22 ++--- worker/pyproject.toml | 8 +- 7 files changed, 65 insertions(+), 102 deletions(-) delete mode 100644 backend/src/backend/migrations/versions/307d19a79bcb_add_isp_field_to_tests_model.py rename backend/src/backend/migrations/versions/{d45beab913d1_set_up_database_models.py => ef3115ef505c_set_up_database_models.py} (87%) diff --git a/.gitignore b/.gitignore index d26ba83..c366b7b 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST +.python-version # PyInstaller # Usually these files are written by a python script from a template diff --git a/backend/pyproject.toml b/backend/pyproject.toml index b84c1b1..5aefed1 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "mirrors_qa_backend" -requires-python = ">=3.11,<3.13" +requires-python = ">=3.12,<3.13" description = "mirrors-qa Backend API" readme = "README.md" authors = [ @@ -21,7 +21,7 @@ dependencies = [ license = {text = "GPL-3.0-or-later"} classifiers = [ "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", ] @@ -74,9 +74,6 @@ features = ["dev"] [tool.hatch.envs.test] features = ["scripts", "test"] -[[tool.hatch.envs.test.matrix]] -python = ["3.11", "3.12"] - [tool.hatch.envs.test.scripts] run = "inv test --args '{args}'" run-cov = "inv test-cov --args '{args}'" @@ -106,10 +103,10 @@ all = "inv checkall --args '{args}'" [tool.black] line-length = 88 -target-version = ['py310'] +target-version = ['py312'] [tool.ruff] -target-version = "py311" +target-version = "py312" line-length = 88 src = ["src"] @@ -184,8 +181,6 @@ ignore = [ "S603", # Ignore complexity "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", - # Allow use of typing.Optional, typing.List for type annotations - "UP007", "UP035", "UP006" ] unfixable = [ # Don't touch unused imports @@ -234,6 +229,6 @@ exclude_lines = [ include = ["src", "tests", "tasks.py"] exclude = [".env/**", ".venv/**"] extraPaths = ["src"] -pythonVersion = "3.11" +pythonVersion = "3.12" typeCheckingMode="strict" disableBytesTypePromotions = true diff --git a/backend/src/backend/db/models.py b/backend/src/backend/db/models.py index 9d49117..a5dc792 100644 --- a/backend/src/backend/db/models.py +++ b/backend/src/backend/db/models.py @@ -1,6 +1,7 @@ +from __future__ import annotations + from datetime import datetime from ipaddress import IPv4Address -from typing import List, Optional from uuid import UUID from sqlalchemy import DateTime, Enum, ForeignKey, UniqueConstraint, text @@ -23,10 +24,10 @@ class Base(MappedAsDataclass, DeclarativeBase): # type has to be used. type_annotation_map = { # noqa: RUF012 - str: CITEXT, # transform Python str to PostgreSQL CITEXT - List[str]: ARRAY( + str: CITEXT, # use case-insensitive strings on PostgreSQL backend + list[str]: ARRAY( item_type=CITEXT - ), # transform Python List[str] into PostgreSQL Array of strings + ), # transform Python list[str] into PostgreSQL Array of strings datetime: DateTime( timezone=False ), # transform Python datetime into PostgreSQL Datetime without timezone @@ -50,16 +51,15 @@ class Base(MappedAsDataclass, DeclarativeBase): class Country(Base): __tablename__ = "country" - code: Mapped[str] = mapped_column(primary_key=True) - name: Mapped[str] + code: Mapped[str] = mapped_column( + primary_key=True + ) # two-letter country codes as defined in ISO 3166-1 - worker_id: Mapped[Optional[UUID]] = mapped_column( - ForeignKey("worker.id"), init=False - ) - worker: Mapped[Optional["Worker"]] = relationship( - back_populates="countries", init=False - ) - mirrors: Mapped[List["Mirror"]] = relationship( + name: Mapped[str] # full name of the country (in English) + + worker_id: Mapped[UUID | None] = mapped_column(ForeignKey("worker.id"), init=False) + worker: Mapped[Worker | None] = relationship(back_populates="countries", init=False) + mirrors: Mapped[list[Mirror]] = relationship( back_populates="country", init=False, cascade="all, delete-orphan", @@ -71,25 +71,25 @@ class Country(Base): class Mirror(Base): __tablename__ = "mirror" - base_url: Mapped[str] = mapped_column(primary_key=True) + id: Mapped[str] = mapped_column(primary_key=True) # hostname of a mirror URL + base_url: Mapped[str] enabled: Mapped[bool] # metadata of a mirror from MirroBrain (https://mirrorbrain-docs.readthedocs.io/en/latest/mirrors.html#displaying-details-about-a-mirror) - id: Mapped[Optional[str]] - region: Mapped[Optional[str]] - asn: Mapped[Optional[str]] - score: Mapped[Optional[int]] - latitude: Mapped[Optional[float]] - longitude: Mapped[Optional[float]] - country_only: Mapped[Optional[bool]] - region_only: Mapped[Optional[bool]] - as_only: Mapped[Optional[bool]] - other_countries: Mapped[Optional[List[str]]] + region: Mapped[str | None] + asn: Mapped[str | None] + score: Mapped[int | None] + latitude: Mapped[float | None] + longitude: Mapped[float | None] + country_only: Mapped[bool | None] + region_only: Mapped[bool | None] + as_only: Mapped[bool | None] + other_countries: Mapped[list[str] | None] country_code: Mapped[str] = mapped_column( ForeignKey("country.code"), init=False, ) - country: Mapped["Country"] = relationship(back_populates="mirrors", init=False) + country: Mapped[Country] = relationship(back_populates="mirrors", init=False) class Worker(Base): @@ -97,11 +97,11 @@ class Worker(Base): id: Mapped[UUID] = mapped_column( init=False, primary_key=True, server_default=text("uuid_generate_v4()") ) + # RSA public key for generating access tokens needed to make request to + # the web server auth_info: Mapped[str] - last_seen: Mapped[Optional[datetime]] - countries: Mapped[List["Country"]] = relationship( - back_populates="worker", init=False - ) + last_seen_on: Mapped[datetime | None] + countries: Mapped[list[Country]] = relationship(back_populates="worker", init=False) class Test(Base): @@ -110,8 +110,8 @@ class Test(Base): init=False, primary_key=True, server_default=text("uuid_generate_v4()") ) requested_on: Mapped[datetime] - started_on: Mapped[Optional[datetime]] - status: Mapped[Optional[StatusEnum]] = mapped_column( + started_on: Mapped[datetime | None] + status: Mapped[StatusEnum | None] = mapped_column( Enum( native_enum=False, validate_strings=True, @@ -119,12 +119,13 @@ class Test(Base): name="status", ) ) - error: Mapped[Optional[str]] - ip_address: Mapped[Optional[IPv4Address]] - asn: Mapped[Optional[str]] - isp: Mapped[Optional[str]] - location: Mapped[Optional[str]] - latency: Mapped[Optional[int]] # milliseconds - download_size: Mapped[Optional[int]] # bytes - duration: Mapped[Optional[int]] # seconds - speed: Mapped[Optional[float]] # bytes per second + error: Mapped[str | None] + isp: Mapped[str | None] + ip_address: Mapped[IPv4Address | None] + asn: Mapped[str | None] # autonomous system based on IP + country: Mapped[str | None] # country based on IP + location: Mapped[str | None] # city based on IP + latency: Mapped[int | None] # milliseconds + download_size: Mapped[int | None] # bytes + duration: Mapped[int | None] # seconds + speed: Mapped[float | None] # bytes per second diff --git a/backend/src/backend/migrations/script.py.mako b/backend/src/backend/migrations/script.py.mako index fbc4b07..24db54b 100644 --- a/backend/src/backend/migrations/script.py.mako +++ b/backend/src/backend/migrations/script.py.mako @@ -5,17 +5,16 @@ Revises: ${down_revision | comma,n} Create Date: ${create_date} """ -from typing import Sequence, Union from alembic import op import sqlalchemy as sa ${imports if imports else ""} # revision identifiers, used by Alembic. -revision: str = ${repr(up_revision)} -down_revision: Union[str, None] = ${repr(down_revision)} -branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} -depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} def upgrade() -> None: diff --git a/backend/src/backend/migrations/versions/307d19a79bcb_add_isp_field_to_tests_model.py b/backend/src/backend/migrations/versions/307d19a79bcb_add_isp_field_to_tests_model.py deleted file mode 100644 index 48b8730..0000000 --- a/backend/src/backend/migrations/versions/307d19a79bcb_add_isp_field_to_tests_model.py +++ /dev/null @@ -1,31 +0,0 @@ -"""add isp field to tests model - -Revision ID: 307d19a79bcb -Revises: d45beab913d1 -Create Date: 2024-06-03 12:22:41.041501 - -""" - -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision: str = "307d19a79bcb" -down_revision: Union[str, None] = "d45beab913d1" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.add_column("test", sa.Column("isp", postgresql.CITEXT(), nullable=True)) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_column("test", "isp") - # ### end Alembic commands ### diff --git a/backend/src/backend/migrations/versions/d45beab913d1_set_up_database_models.py b/backend/src/backend/migrations/versions/ef3115ef505c_set_up_database_models.py similarity index 87% rename from backend/src/backend/migrations/versions/d45beab913d1_set_up_database_models.py rename to backend/src/backend/migrations/versions/ef3115ef505c_set_up_database_models.py index 6e745d5..84a60ba 100644 --- a/backend/src/backend/migrations/versions/d45beab913d1_set_up_database_models.py +++ b/backend/src/backend/migrations/versions/ef3115ef505c_set_up_database_models.py @@ -1,22 +1,20 @@ """set up database models -Revision ID: d45beab913d1 +Revision ID: ef3115ef505c Revises: -Create Date: 2024-06-03 07:46:35.661038 +Create Date: 2024-06-03 15:53:20.253276 """ -from typing import Sequence, Union - import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision: str = "d45beab913d1" -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None +revision = "ef3115ef505c" +down_revision = None +branch_labels = None +depends_on = None def upgrade() -> None: @@ -37,8 +35,10 @@ def upgrade() -> None: nullable=True, ), sa.Column("error", postgresql.CITEXT(), nullable=True), + sa.Column("isp", postgresql.CITEXT(), nullable=True), sa.Column("ip_address", postgresql.INET(), nullable=True), sa.Column("asn", postgresql.CITEXT(), nullable=True), + sa.Column("country", postgresql.CITEXT(), nullable=True), sa.Column("location", postgresql.CITEXT(), nullable=True), sa.Column("latency", sa.Integer(), nullable=True), sa.Column("download_size", sa.Integer(), nullable=True), @@ -55,7 +55,7 @@ def upgrade() -> None: nullable=False, ), sa.Column("auth_info", postgresql.CITEXT(), nullable=False), - sa.Column("last_seen", sa.DateTime(), nullable=True), + sa.Column("last_seen_on", sa.DateTime(), nullable=True), sa.PrimaryKeyConstraint("id", name=op.f("pk_worker")), ) op.create_table( @@ -71,9 +71,9 @@ def upgrade() -> None: ) op.create_table( "mirror", + sa.Column("id", postgresql.CITEXT(), nullable=False), sa.Column("base_url", postgresql.CITEXT(), nullable=False), sa.Column("enabled", sa.Boolean(), nullable=False), - sa.Column("id", postgresql.CITEXT(), nullable=True), sa.Column("region", postgresql.CITEXT(), nullable=True), sa.Column("asn", postgresql.CITEXT(), nullable=True), sa.Column("score", sa.Integer(), nullable=True), @@ -91,7 +91,7 @@ def upgrade() -> None: ["country.code"], name=op.f("fk_mirror_country_code_country"), ), - sa.PrimaryKeyConstraint("base_url", name=op.f("pk_mirror")), + sa.PrimaryKeyConstraint("id", name=op.f("pk_mirror")), ) # ### end Alembic commands ### diff --git a/worker/pyproject.toml b/worker/pyproject.toml index d7e6877..478f8bf 100644 --- a/worker/pyproject.toml +++ b/worker/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "mirrors_qa_worker" -requires-python = ">=3.11,<3.13" +requires-python = ">=3.12,<3.13" description = "mirrors-qa Worker" readme = "README.md" authors = [ @@ -17,7 +17,7 @@ dependencies = [ license = {text = "GPL-3.0-or-later"} classifiers = [ "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", ] @@ -70,8 +70,6 @@ features = ["dev"] [tool.hatch.envs.test] features = ["scripts", "test"] -[[tool.hatch.envs.test.matrix]] -python = ["3.10", "3.11"] [tool.hatch.envs.test.scripts] run = "inv test --args '{args}'" @@ -228,6 +226,6 @@ exclude_lines = [ include = ["src", "tests", "tasks.py"] exclude = [".env/**", ".venv/**"] extraPaths = ["src"] -pythonVersion = "3.11" +pythonVersion = "3.12" typeCheckingMode="strict" disableBytesTypePromotions = true