From 7f7a145608c1326be0c1e20c020c5b1c54c5495f Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Thu, 1 Feb 2024 16:39:22 +0100 Subject: [PATCH 01/79] remove dev-dependencies except for pytest for kraken-wrapper because the format/lint dependencies are now Pdm installed by Kraken --- kraken-wrapper/pyproject.toml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/kraken-wrapper/pyproject.toml b/kraken-wrapper/pyproject.toml index 61ec7055..500ee5e0 100644 --- a/kraken-wrapper/pyproject.toml +++ b/kraken-wrapper/pyproject.toml @@ -28,14 +28,7 @@ python = ">=3.10" termcolor = "^1.1.0" [tool.poetry.dev-dependencies] -black = "^23.10.1" -flake8 = "^6.1.0" -isort = "^5.12.0" -mypy = "^1.6.1" -pycln = "^2.1.3" -pylint = "^3.0.2" pytest = ">=6.0.0" -pyupgrade = "^3.15.0" [tool.poetry.scripts] krakenw = "kraken.wrapper.main:main" From cecb34cd9f1f093ff7ce4c99e9b1718349f86948 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 2 Feb 2024 13:35:42 +0100 Subject: [PATCH 02/79] feature: Introduce `kraken.std.python.project.python_project()` function which creates all tasks for a Python project. --- .changelog/_unreleased.toml | 5 + .kraken.py | 1 + examples/pdm-project/.kraken.py | 16 +- kraken-build/src/kraken/common/_fs.py | 6 +- kraken-build/src/kraken/common/_importlib.py | 6 +- .../src/kraken/common/_requirements.py | 6 +- kraken-build/src/kraken/common/_runner.py | 3 +- kraken-build/src/kraken/common/_text.py | 3 +- kraken-build/src/kraken/common/supplier.py | 1 - .../src/kraken/core/system/context.py | 6 +- .../kraken/core/system/executor/__init__.py | 30 +-- .../kraken/core/system/executor/default.py | 6 +- .../src/kraken/core/system/project.py | 2 +- .../src/kraken/core/system/project_test.py | 12 +- kraken-build/src/kraken/core/system/task.py | 14 +- .../kraken/std/cargo/tasks/cargo_deny_task.py | 8 +- .../kraken/std/git/tasks/check_file_task.py | 8 +- .../kraken/std/python/buildsystem/__init__.py | 1 - .../src/kraken/std/python/buildsystem/pdm.py | 10 +- .../kraken/std/python/buildsystem/poetry.py | 22 +- kraken-build/src/kraken/std/python/project.py | 243 ++++++++++++++++++ .../src/kraken/std/python/pyproject.py | 4 +- .../src/kraken/std/python/settings.py | 19 +- .../src/kraken/std/python/tasks/black_task.py | 85 +++++- .../src/kraken/std/python/tasks/build_task.py | 3 +- .../kraken/std/python/tasks/flake8_task.py | 102 +++++++- .../src/kraken/std/python/tasks/info_task.py | 32 ++- .../kraken/std/python/tasks/install_task.py | 3 +- .../src/kraken/std/python/tasks/isort_task.py | 77 ++++-- .../src/kraken/std/python/tasks/mypy_task.py | 116 +++++++-- .../kraken/std/python/tasks/publish_task.py | 5 +- .../src/kraken/std/python/tasks/pycln_task.py | 40 ++- .../kraken/std/python/tasks/pylint_task.py | 5 +- .../kraken/std/python/tasks/pytest_task.py | 42 ++- .../kraken/std/python/tasks/pyupgrade_task.py | 30 ++- .../std/python/tasks/update_lockfile_task.py | 3 +- kraken-build/src/kraken/std/sccache.py | 6 +- kraken-build/tests/resources.py | 1 - kraken-wrapper/pyproject.toml | 40 +-- 39 files changed, 790 insertions(+), 232 deletions(-) create mode 100644 .changelog/_unreleased.toml create mode 100644 kraken-build/src/kraken/std/python/project.py diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml new file mode 100644 index 00000000..884beb4c --- /dev/null +++ b/.changelog/_unreleased.toml @@ -0,0 +1,5 @@ +[[entries]] +id = "fb3fcc4a-a38a-445f-b124-571395b7ac86" +type = "feature" +description = "Introduce `kraken.std.python.project.python_project()` function which creates all tasks for a Python project." +author = "niklas.rosenstein@helsing.ai" diff --git a/.kraken.py b/.kraken.py index 8043de1d..8e172e25 100644 --- a/.kraken.py +++ b/.kraken.py @@ -91,6 +91,7 @@ def configure_project() -> None: from kraken.build import project +from kraken.std.python.project import python_project try: project.subproject("docs") diff --git a/examples/pdm-project/.kraken.py b/examples/pdm-project/.kraken.py index 2eadfdcc..690ed9cb 100644 --- a/examples/pdm-project/.kraken.py +++ b/examples/pdm-project/.kraken.py @@ -1,12 +1,18 @@ import os -from kraken.std import python +from kraken.std.python.project import python_library, python_package_index -python.python_settings(always_use_managed_env=True).add_package_index( +python_package_index( alias="local", index_url=os.environ["LOCAL_PACKAGE_INDEX"], + is_package_source=False, credentials=(os.environ["LOCAL_USER"], os.environ["LOCAL_PASSWORD"]), ) -python.install() -python.mypy() -python.publish(package_index="local", distributions=python.build(as_version="0.1.0").output_files) + +python_library() + +# python.python_settings(always_use_managed_env=True).add_package_index( +# ) +# python.install() +# python.mypy() +# python.publish(package_index="local", distributions=python.build(as_version="0.1.0").output_files) diff --git a/kraken-build/src/kraken/common/_fs.py b/kraken-build/src/kraken/common/_fs.py index 0655b91a..56b9eb80 100644 --- a/kraken-build/src/kraken/common/_fs.py +++ b/kraken-build/src/kraken/common/_fs.py @@ -14,8 +14,7 @@ def atomic_file_swap( mode: Literal["w"], always_revert: bool = ..., create_dirs: bool = ..., -) -> ContextManager[TextIO]: - ... +) -> ContextManager[TextIO]: ... @overload @@ -24,8 +23,7 @@ def atomic_file_swap( mode: Literal["wb"], always_revert: bool = ..., create_dirs: bool = ..., -) -> ContextManager[BinaryIO]: - ... +) -> ContextManager[BinaryIO]: ... @contextlib.contextmanager # type: ignore diff --git a/kraken-build/src/kraken/common/_importlib.py b/kraken-build/src/kraken/common/_importlib.py index 59bf814d..aa8363c6 100644 --- a/kraken-build/src/kraken/common/_importlib.py +++ b/kraken-build/src/kraken/common/_importlib.py @@ -8,13 +8,11 @@ @overload -def import_class(fqn: str) -> type: - ... +def import_class(fqn: str) -> type: ... @overload -def import_class(fqn: str, base_type: type[T]) -> type[T]: - ... +def import_class(fqn: str, base_type: type[T]) -> type[T]: ... def import_class(fqn: str, base_type: "Type[T] | None" = None) -> "Type[T] | type": diff --git a/kraken-build/src/kraken/common/_requirements.py b/kraken-build/src/kraken/common/_requirements.py index f613278a..18c462ab 100644 --- a/kraken-build/src/kraken/common/_requirements.py +++ b/kraken-build/src/kraken/common/_requirements.py @@ -121,9 +121,9 @@ def replace( requirements=self.requirements if requirements is None else tuple(requirements), index_url=self.index_url if index_url is NotSet.Value else index_url, extra_index_urls=self.extra_index_urls if extra_index_urls is None else tuple(extra_index_urls), - interpreter_constraint=self.interpreter_constraint - if interpreter_constraint is NotSet.Value - else interpreter_constraint, + interpreter_constraint=( + self.interpreter_constraint if interpreter_constraint is NotSet.Value else interpreter_constraint + ), pythonpath=self.pythonpath if pythonpath is None else tuple(pythonpath), ) diff --git a/kraken-build/src/kraken/common/_runner.py b/kraken-build/src/kraken/common/_runner.py index 5a2f0681..0076ab13 100644 --- a/kraken-build/src/kraken/common/_runner.py +++ b/kraken-build/src/kraken/common/_runner.py @@ -68,8 +68,7 @@ class ProjectFinder(ABC): """ @abstractmethod - def find_project(self, directory: Path) -> "ProjectInfo | None": - ... + def find_project(self, directory: Path) -> "ProjectInfo | None": ... ## diff --git a/kraken-build/src/kraken/common/_text.py b/kraken-build/src/kraken/common/_text.py index c456ee58..ff47df94 100644 --- a/kraken-build/src/kraken/common/_text.py +++ b/kraken-build/src/kraken/common/_text.py @@ -7,8 +7,7 @@ class SupportsLen(Protocol): - def __len__(self) -> int: - ... + def __len__(self) -> int: ... def pluralize(word: str, count: "int | SupportsLen") -> str: diff --git a/kraken-build/src/kraken/common/supplier.py b/kraken-build/src/kraken/common/supplier.py index 1e745891..7e20334b 100644 --- a/kraken-build/src/kraken/common/supplier.py +++ b/kraken-build/src/kraken/common/supplier.py @@ -1,7 +1,6 @@ """ This module provides provides the :class:`Supplier` interface which is used to represent values that can be calculated lazily and track provenance of such computations. """ - import abc from collections.abc import Callable, Iterable, Mapping, Sequence from typing import Any, Generic, TypeVar diff --git a/kraken-build/src/kraken/core/system/context.py b/kraken-build/src/kraken/core/system/context.py index 3bd99603..3ee33fc4 100644 --- a/kraken-build/src/kraken/core/system/context.py +++ b/kraken-build/src/kraken/core/system/context.py @@ -389,12 +389,10 @@ def execute(self, tasks: list[str | Task] | TaskGraph | None = None) -> None: @overload def listen( self, event_type: str | ContextEvent.Type - ) -> Callable[[ContextEvent.T_Listener], ContextEvent.T_Listener]: - ... + ) -> Callable[[ContextEvent.T_Listener], ContextEvent.T_Listener]: ... @overload - def listen(self, event_type: str | ContextEvent.Type, listener: ContextEvent.Listener) -> None: - ... + def listen(self, event_type: str | ContextEvent.Type, listener: ContextEvent.Listener) -> None: ... def listen(self, event_type: str | ContextEvent.Type, listener: ContextEvent.Listener | None = None) -> Any: """Registers a listener to the context for the given event type.""" diff --git a/kraken-build/src/kraken/core/system/executor/__init__.py b/kraken-build/src/kraken/core/system/executor/__init__.py index aa64b92d..a73f3711 100644 --- a/kraken-build/src/kraken/core/system/executor/__init__.py +++ b/kraken-build/src/kraken/core/system/executor/__init__.py @@ -56,35 +56,25 @@ def tasks( class GraphExecutorObserver(abc.ABC): """Observes events in a Kraken task executor.""" - def before_execute_graph(self, graph: Graph) -> None: - ... + def before_execute_graph(self, graph: Graph) -> None: ... - def before_prepare_task(self, task: Task) -> None: - ... + def before_prepare_task(self, task: Task) -> None: ... - def after_prepare_task(self, task: Task, status: TaskStatus) -> None: - ... + def after_prepare_task(self, task: Task, status: TaskStatus) -> None: ... - def before_execute_task(self, task: Task, status: TaskStatus) -> None: - ... + def before_execute_task(self, task: Task, status: TaskStatus) -> None: ... - def on_task_output(self, task: Task, chunk: bytes) -> None: - ... + def on_task_output(self, task: Task, chunk: bytes) -> None: ... - def after_execute_task(self, task: Task, status: TaskStatus) -> None: - ... + def after_execute_task(self, task: Task, status: TaskStatus) -> None: ... - def before_teardown_task(self, task: Task) -> None: - ... + def before_teardown_task(self, task: Task) -> None: ... - def after_teardown_task(self, task: Task, status: TaskStatus) -> None: - ... + def after_teardown_task(self, task: Task, status: TaskStatus) -> None: ... - def after_execute_graph(self, graph: Graph) -> None: - ... + def after_execute_graph(self, graph: Graph) -> None: ... class GraphExecutor(abc.ABC): @abc.abstractmethod - def execute_graph(self, graph: Graph, observer: GraphExecutorObserver) -> None: - ... + def execute_graph(self, graph: Graph, observer: GraphExecutorObserver) -> None: ... diff --git a/kraken-build/src/kraken/core/system/executor/default.py b/kraken-build/src/kraken/core/system/executor/default.py index 926f38a7..980a3dc7 100644 --- a/kraken-build/src/kraken/core/system/executor/default.py +++ b/kraken-build/src/kraken/core/system/executor/default.py @@ -18,12 +18,10 @@ class TaskExecutor(abc.ABC): @abc.abstractmethod - def execute_task(self, task: Task, done: Callable[[TaskStatus], None]) -> None: - ... + def execute_task(self, task: Task, done: Callable[[TaskStatus], None]) -> None: ... @abc.abstractmethod - def teardown_task(self, task: Task, done: Callable[[TaskStatus], None]) -> None: - ... + def teardown_task(self, task: Task, done: Callable[[TaskStatus], None]) -> None: ... class DefaultTaskExecutor(TaskExecutor): diff --git a/kraken-build/src/kraken/core/system/project.py b/kraken-build/src/kraken/core/system/project.py index 4a71ad76..615efc6b 100644 --- a/kraken-build/src/kraken/core/system/project.py +++ b/kraken-build/src/kraken/core/system/project.py @@ -64,7 +64,7 @@ def __init__(self, name: str, directory: Path, parent: Project | None, context: gen_group = self.group("gen", description="Tasks that perform code generation.", default=True) lint_group = self.group("lint", description="Tasks that perform code linting.", default=True) - lint_group.depends_on(check_group, mode="strict") + lint_group.depends_on(check_group, mode="order-only") lint_group.depends_on(gen_group, mode="strict") build_group = self.group("build", description="Tasks that produce build artefacts.") diff --git a/kraken-build/src/kraken/core/system/project_test.py b/kraken-build/src/kraken/core/system/project_test.py index 7f17727e..2f359365 100644 --- a/kraken-build/src/kraken/core/system/project_test.py +++ b/kraken-build/src/kraken/core/system/project_test.py @@ -21,8 +21,7 @@ def test__Project__resolve_outputs__can_find_dataclass_in_properties(kraken_proj class MyTask(Task): out_prop: Property[MyDescriptor] = Property.output() - def execute(self) -> None: - ... + def execute(self) -> None: ... task = kraken_project.task("carrier", MyTask) task.out_prop = MyDescriptor("foobar") @@ -33,8 +32,7 @@ def test__Project__resolve_outputs__can_not_find_input_property(kraken_project: class MyTask(Task): out_prop: Property[MyDescriptor] - def execute(self) -> None: - ... + def execute(self) -> None: ... task = kraken_project.task("carrier", MyTask) task.out_prop = MyDescriptor("foobar") @@ -45,8 +43,7 @@ def test__Project__resolve_outputs_supplier(kraken_project: Project) -> None: class MyTask(Task): out_prop: Property[MyDescriptor] = Property.output() - def execute(self) -> None: - ... + def execute(self) -> None: ... task = kraken_project.task("carrier", MyTask) task.out_prop = MyDescriptor("foobar") @@ -69,8 +66,7 @@ def test__Project__do__does_not_set_property_on_None_value(kraken_project: Proje class MyTask(Task): in_prop: Property[str] - def execute(self) -> None: - ... + def execute(self) -> None: ... kraken_project.task("carrier", MyTask) assert kraken_project.resolve_tasks(":carrier").select(str).supplier().get() == [] diff --git a/kraken-build/src/kraken/core/system/task.py b/kraken-build/src/kraken/core/system/task.py index 60973c99..56275fac 100644 --- a/kraken-build/src/kraken/core/system/task.py +++ b/kraken-build/src/kraken/core/system/task.py @@ -165,9 +165,11 @@ def warning(message: str | None = None) -> TaskStatus: def from_exit_code(command: list[str] | None, code: int) -> TaskStatus: return TaskStatus( TaskStatusType.SUCCEEDED if code == 0 else TaskStatusType.FAILED, - None - if code == 0 or command is None - else 'command "' + " ".join(map(shlex.quote, command)) + f'" returned exit code {code}', + ( + None + if code == 0 or command is None + else 'command "' + " ".join(map(shlex.quote, command)) + f'" returned exit code {code}' + ), ) @@ -742,12 +744,10 @@ def __iter__(self) -> Iterable[str]: return iter(self._ptt) @overload - def __getitem__(self, partition: str) -> Collection[Task]: - ... + def __getitem__(self, partition: str) -> Collection[Task]: ... @overload - def __getitem__(self, partition: Task) -> Collection[str]: - ... + def __getitem__(self, partition: Task) -> Collection[str]: ... def __getitem__(self, partition: str | Task) -> Collection[str] | Collection[Task]: if isinstance(partition, str): diff --git a/kraken-build/src/kraken/std/cargo/tasks/cargo_deny_task.py b/kraken-build/src/kraken/std/cargo/tasks/cargo_deny_task.py index fb99652a..6dcc856d 100644 --- a/kraken-build/src/kraken/std/cargo/tasks/cargo_deny_task.py +++ b/kraken-build/src/kraken/std/cargo/tasks/cargo_deny_task.py @@ -24,7 +24,9 @@ def execute(self) -> TaskStatus: return TaskStatus.succeeded() return self.error_message.map( - lambda message: TaskStatus.failed(message) - if message is not None - else TaskStatus.from_exit_code(command, result.returncode) + lambda message: ( + TaskStatus.failed(message) + if message is not None + else TaskStatus.from_exit_code(command, result.returncode) + ) ).get() diff --git a/kraken-build/src/kraken/std/git/tasks/check_file_task.py b/kraken-build/src/kraken/std/git/tasks/check_file_task.py index 1b3c3971..ca333c37 100644 --- a/kraken-build/src/kraken/std/git/tasks/check_file_task.py +++ b/kraken-build/src/kraken/std/git/tasks/check_file_task.py @@ -70,9 +70,11 @@ def execute(self) -> TaskStatus: assert False, f"Unknown status: {status}" return TaskStatus( - TaskStatusType.SUCCEEDED - if success - else (TaskStatusType.WARNING if self.warn_only.get() else TaskStatusType.FAILED), + ( + TaskStatusType.SUCCEEDED + if success + else (TaskStatusType.WARNING if self.warn_only.get() else TaskStatusType.FAILED) + ), status.to_description(self.file_to_check.get()), ) diff --git a/kraken-build/src/kraken/std/python/buildsystem/__init__.py b/kraken-build/src/kraken/std/python/buildsystem/__init__.py index 98f762e0..3677937b 100644 --- a/kraken-build/src/kraken/std/python/buildsystem/__init__.py +++ b/kraken-build/src/kraken/std/python/buildsystem/__init__.py @@ -1,6 +1,5 @@ """ Abstraction of Python build systems such as Poetry and Slap. """ - from __future__ import annotations import abc diff --git a/kraken-build/src/kraken/std/python/buildsystem/pdm.py b/kraken-build/src/kraken/std/python/buildsystem/pdm.py index 42f2ed91..8138370d 100644 --- a/kraken-build/src/kraken/std/python/buildsystem/pdm.py +++ b/kraken-build/src/kraken/std/python/buildsystem/pdm.py @@ -92,11 +92,11 @@ def set_package_indexes(self, indexes: Sequence[PackageIndex]) -> None: ) indexes = sorted( indexes, - key=lambda x: 0 - if x.priority == PackageIndex.Priority.default - else 1 - if x.priority == PackageIndex.Priority.primary - else 2, + key=lambda x: ( + 0 + if x.priority == PackageIndex.Priority.default + else 1 if x.priority == PackageIndex.Priority.primary else 2 + ), ) for index in indexes: diff --git a/kraken-build/src/kraken/std/python/buildsystem/poetry.py b/kraken-build/src/kraken/std/python/buildsystem/poetry.py index 39a10886..aaee253d 100644 --- a/kraken-build/src/kraken/std/python/buildsystem/poetry.py +++ b/kraken-build/src/kraken/std/python/buildsystem/poetry.py @@ -81,15 +81,19 @@ def get_package_indexes(self) -> list[PackageIndex]: PackageIndex( alias=source["name"], index_url=source["url"], - priority=PackageIndex.Priority[source["priority"]] - if "priority" in source - else ( - PackageIndex.Priority.default - # Support deprecated source configurations. - if source.get("default") - else PackageIndex.Priority.secondary - if source.get("secondary") - else PackageIndex.Priority.supplemental + priority=( + PackageIndex.Priority[source["priority"]] + if "priority" in source + else ( + PackageIndex.Priority.default + # Support deprecated source configurations. + if source.get("default") + else ( + PackageIndex.Priority.secondary + if source.get("secondary") + else PackageIndex.Priority.supplemental + ) + ) ), verify_ssl=True, ) diff --git a/kraken-build/src/kraken/std/python/project.py b/kraken-build/src/kraken/std/python/project.py new file mode 100644 index 00000000..b378d598 --- /dev/null +++ b/kraken-build/src/kraken/std/python/project.py @@ -0,0 +1,243 @@ +""" New-style API and template for defining the tasks for an entire Python project.""" + +from collections.abc import Sequence +import logging +from pathlib import Path +import re +from kraken.common.supplier import Supplier +from kraken.std.python.buildsystem import detect_build_system +from kraken.std.python.pyproject import PackageIndex +from kraken.std.python.tasks.mypy_task import MypyConfig +from nr.stream import Optional + +from kraken.std.python.tasks.pytest_task import CoverageFormat + +logger = logging.getLogger(__name__) + + +def python_package_index( + *, + alias: str, + index_url: str | None = None, + upload_url: str | None = None, + credentials: tuple[str, str] | None = None, + is_package_source: bool = True, + priority: PackageIndex.Priority | str = PackageIndex.Priority.supplemental, + publish: bool = False, +) -> None: + """ + Add a Python package index that can be used to consume packages and/or to publish Python packages to. + + Args: + alias: An alias for the package index. If no *index_url* is specified, it must be either `pypi` or `testpypi` + to refer to `pypi.org` and `test.pypi.org`, respectively. + index_url: The URL of the package index, incl. the trailing `/simple` portion. + upload_url: For some Python registries, the upload URL differs from the index URL, often without `/simple`. + If it is not specified, the *index_url* must end with `/simple` and the suffix will be removed to + derive the *upload_url*. + credentials: Username/password to read/write to the package index, if needed. + is_package_source: Whether this package index should be considered as a source for packages when resolving + the dependencies of the project. Kraken will inject these settings into the `pyproject.toml` in the section + corresponding to the Python build system being used (e.g. Poetry or PDM) when running `krakenw run apply`. + priority: The package priority. This is inspired by Poetry and may not be fully applicable to other Python + build systems. Kraken does a best-effort to translate the priority for the corresponding build system. + publish: Whether a publish task for the package registry should be created. The publish task will be called + `python.publish.{alias}`. + """ + + # NOTE(@niklas): Currently this function is a simple wrapper, but it may replace the wrapped method eventually. + + from .settings import python_settings + + python_settings().add_package_index( + alias=alias, + index_url=index_url, + upload_url=upload_url, + credentials=credentials, + is_package_source=is_package_source, + priority=priority, + publish=publish, + ) + + +def python_project( + *, + source_directory: str = "src", + tests_directory: str = "tests", + additional_lint_directories: Sequence[str] | None = None, + exclude_lint_directories: Sequence[str] = (), + line_length: int = 120, + pyupgrade_keep_runtime_typing: bool = False, + pycln_remove_all_unused_imports: bool = False, + pytest_ignore_dirs: Sequence[str] = (), + mypy_use_daemon: bool = True, + isort_version_spec: str = ">=5.13.2,<6.0.0", + black_version_spec: str = ">=24.1.1,<25.0.0", + flake8_version_spec: str = ">=7.0.0,<8.0.0", + flake8_additional_requirements: Sequence[str] = (), + flake8_extend_ignore: Sequence[str] = ("W503", "W504", "E203", "E704"), + mypy_version_spec: str = ">=1.8.0,<2.0.0", + pycln_version_spec: str = ">=2.4.0,<3.0.0", + pyupgrade_version_spec: str = ">=3.15.0,<4.0.0", +) -> None: + """ + Use this function in a Python project. + + The Python build system used for the library is automatically detected. Supported build systems are: + + * [Slap](https://github.com/NiklasRosenstein/slap) + * [Poetry](https://python-poetry.org/docs) + * [PDM](https://pdm-project.org/latest/) + + Note: Pytest dependencies + Your project should have the `pytest` dependency in it's development dependencies. Kraken does not currently + automatically inject this dependency into your project. If you would like to utilize parallel test execution, + you should also add `pytest-xdist[psutil]`. Unlike linters and formatters, Pytest needs to be available at + runtime in the Python environment that the tests are being run in, so Kraken cannot install it separately with + Pex. + + Args: + source_directory: The directory that contains all Python source files. + tests_directory: The directory that contains test files that are not in the source directory. + additional_lint_directories: Additional directories in the project that contain Python source code + that should be linted and formatted. If not specified, it will default to `["examples"]` if the + directory exists. + exclude_lint_directories: Directories in the project that contain Python sourec code that should not be + linted and formatted but would otherwise be included via *source_directory*, *tests_directory* + and *additional_directories*. + line_length: The line length to assume for all formatters and linters. + pyupgrade_keep_runtime_typing: Whether to not replace `typing` type hints. This is required for example + for projects using Typer as it does not support all modern type hints at runtime. + pycln_remove_all_unused_imports: Remove all unused imports, including these with side effects. + For reference, see https://hadialqattan.github.io/pycln/#/?id=-a-all-flag. + + flake8_additional_requirements: Additional Python requirements to install alongside Flake8. This should + be used to add Flake8 plugins. + flake8_extend_ignore: Flake8 lints to ignore. The default ignores lints that would otherwise conflict with + the way Black formats code. + """ + + from kraken.build import project + from kraken.common import not_none + from .settings import python_settings + from .tasks.login_task import login as login_task + from .tasks.update_lockfile_task import update_lockfile_task + from .tasks.update_pyproject_task import update_pyproject_task + from .tasks.install_task import install as install_task + from .tasks.info_task import info as info_task + from .tasks.pyupgrade_task import pyupgrade as pyupgrade_task + from .tasks.pycln_task import pycln as pycln_task + from .tasks.isort_task import isort as isort_tasks, IsortConfig + from .tasks.black_task import black as black_tasks, BlackConfig + from .tasks.flake8_task import flake8 as flake8_tasks, Flake8Config + from .tasks.mypy_task import mypy as mypy_task, MypyConfig + from .tasks.pytest_task import pytest as pytest_task + from .pyproject import Pyproject + + if additional_lint_directories is None: + additional_lint_directories = [] + if project.directory.joinpath("examples").is_dir(): + additional_lint_directories.append("examples") + + source_paths = [source_directory, *additional_lint_directories] + if project.directory.joinpath(tests_directory).is_dir(): + source_paths.insert(1, tests_directory) + logger.info("Source paths for Python project %s: %s", project.address, source_paths) + + settings = python_settings() + build_system = not_none(detect_build_system(project.directory)) + pyproject = Pyproject.read(project.directory / "pyproject.toml") + handler = build_system.get_pyproject_reader(pyproject) + project_version = handler.get_version() + + # NOTE(@niklas): This is not entirely correct, but good enough in practice. We assume that for a version range, + # the lowest Python version comes first in the version spec. We also need to support Poetry-style semver + # range selectors here. + if python_version := handler.get_python_version_constraint(): + python_version = Optional(re.search(r"[\d\.]+", python_version)).map(lambda m: m.group(0)).or_else(None) + if not python_version: + logger.warning( + "Unable to determine minimum Python version for project %s, fallback to '3'", + project.directory, + ) + python_version = "3" + + login = login_task() + update_lockfile = update_lockfile_task() + update_pyproject = update_pyproject_task() + install = install_task() + info = info_task(build_system=build_system) + + pyupgrade = pyupgrade_task( + python_version=python_version, + keep_runtime_typing=pyupgrade_keep_runtime_typing, + exclude=[Path(x) for x in exclude_lint_directories], + paths=[Path(x) for x in source_paths], + version_spec=pyupgrade_version_spec, + ) + + pycln = pycln_task( + paths=source_paths, + exclude_directories=exclude_lint_directories, + remove_all_unused_imports=pycln_remove_all_unused_imports, + version_spec=pycln_version_spec, + ) + + black = black_tasks( + paths=source_paths, + config=BlackConfig(line_length=line_length, exclude_directories=exclude_lint_directories), + version_spec=black_version_spec, + ) + + isort = isort_tasks( + paths=source_paths, + config=IsortConfig( + profile="black", + line_length=line_length, + extend_skip=exclude_lint_directories, + ), + version_spec=isort_version_spec, + ) + isort.format.depends_on(black.format) + + flake8 = flake8_tasks( + paths=source_paths, + config=Flake8Config(extend_ignore=flake8_extend_ignore, exclude=exclude_lint_directories), + version_spec=flake8_version_spec, + additional_requirements=flake8_additional_requirements, + ) + flake8.depends_on(black.format, isort.format, mode="order-only") + + mypy = mypy_task( + paths=source_paths, + config=MypyConfig( + exclude_directories=exclude_lint_directories, + global_overrides={}, + module_overrides={}, + ), + version_spec=mypy_version_spec, + python_version=python_version, + use_daemon=mypy_use_daemon, + ) + + # TODO(@niklas): Improve this heuristic to check whether Coverage reporting should be enabled. + if "pytest-cov" in str(dict(pyproject)): + coverage = CoverageFormat.XML + else: + coverage = None + + pytest = pytest_task( + paths=source_paths, + ignore_dirs=pytest_ignore_dirs, + coverage=coverage, + doctest_modules=True, + allow_no_tests=True, + ) + + # TODO(@niklas): Support auto-detecting when Mypy stubtests need to be run or + # accept arguments for stubtests. + + # version: str | None = None, + # version: The current version number of the package. If not specified, it will be automatically derived + # from Git using `git describe --tags`. Kraken will automatically bump the version number on-disk for + # you for relevant operations so you can keep the version number in your `pyproject.toml` at `0.0.0`. diff --git a/kraken-build/src/kraken/std/python/pyproject.py b/kraken-build/src/kraken/std/python/pyproject.py index 382ec683..cb25f8f7 100644 --- a/kraken-build/src/kraken/std/python/pyproject.py +++ b/kraken-build/src/kraken/std/python/pyproject.py @@ -14,7 +14,7 @@ logger = logging.getLogger(__name__) -class _PackageIndexPriority(str, Enum): +class PackageIndexPriority(str, Enum): """ Poetry has a very granular representation of priorities for indices, so we inherit that. The priority should to be interpreted in the spirit of its definition in other tools. @@ -36,7 +36,7 @@ class PackageIndex: conflicts with the field value. """ - Priority: ClassVar[TypeAlias] = _PackageIndexPriority + Priority: ClassVar[TypeAlias] = PackageIndexPriority #: A name for the package index. alias: str diff --git a/kraken-build/src/kraken/std/python/settings.py b/kraken-build/src/kraken/std/python/settings.py index 99e5cb62..978a0c42 100644 --- a/kraken-build/src/kraken/std/python/settings.py +++ b/kraken-build/src/kraken/std/python/settings.py @@ -6,6 +6,7 @@ from kraken.core import Project from kraken.std.python.pyproject import PackageIndex +from nr.stream import Optional from .buildsystem import PythonBuildSystem, detect_build_system @@ -57,7 +58,8 @@ def get_tests_directory(self) -> Path | None: def get_tests_directory_as_args(self) -> list[str]: """Returns a list with a single item that is the test directory, or an empty list. This is convenient - when constructing command-line arguments where you want to pass the test directory if it exists.""" + when constructing command-line arguments where you want to pass the test directory if it exists. + """ test_dir = self.get_tests_directory() return [] if test_dir is None else [str(test_dir)] @@ -133,6 +135,15 @@ def add_package_index( ) return self + def get_source_paths(self) -> list[str]: + return [ + str(self.source_directory), + *Optional(self.get_tests_directory()) + .map(lambda p: [str(p.relative_to(self.project.directory))]) + .or_else([]), + *map(str, self.lint_enforced_directories), + ] + def python_settings( project: Project | None = None, @@ -164,7 +175,11 @@ def python_settings( # Autodetect the environment handler. build_system = detect_build_system(project.directory) if build_system: - logger.info("Detected Python build system %r for %s", type(build_system).__name__, project) + logger.info( + "Detected Python build system %r for %s", + type(build_system).__name__, + project, + ) if build_system is not None: if settings.build_system: diff --git a/kraken-build/src/kraken/std/python/tasks/black_task.py b/kraken-build/src/kraken/std/python/tasks/black_task.py index 6cd3d208..91a1e1d2 100644 --- a/kraken-build/src/kraken/std/python/tasks/black_task.py +++ b/kraken-build/src/kraken/std/python/tasks/black_task.py @@ -1,16 +1,49 @@ from __future__ import annotations -import dataclasses +from dataclasses import dataclass from collections.abc import Sequence from pathlib import Path +import re +from typing import Any + +import tomli_w from kraken.common import Supplier from kraken.core import Project, Property +from kraken.core.system.task import TaskStatus +from kraken.std.python.settings import python_settings from kraken.std.python.tasks.pex_build_task import pex_build from .base_task import EnvironmentAwareDispatchTask +@dataclass +class BlackConfig: + line_length: int + exclude_directories: Sequence[str] = () + + def dump(self) -> dict[str, Any]: + config = {} + + # TODO(@niklas): For some reason Black doesn't recognize this option the way we try to pass it.. It definitely + # worked in kraken-hs, but I can't tell what's differenet (we write in the same format to a `black.cfg` + # file). + config["line_length"] = str(self.line_length) + + # Apply overrides from the project config. + if self.exclude_directories: + exclude_patterns = [] + for dirname in self.exclude_directories or (): + exclude_patterns.append("^/" + re.escape(dirname.strip("/")) + "/.*$") + exclude_regex = "(" + "|".join(exclude_patterns) + ")" + config["exclude"] = exclude_regex + + return config + + def to_file(self, path: Path) -> None: + path.write_text(tomli_w.dumps(self.dump())) + + class BlackTask(EnvironmentAwareDispatchTask): """A task to run the `black` formatter to either check for necessary changes or apply changes.""" @@ -18,21 +51,24 @@ class BlackTask(EnvironmentAwareDispatchTask): black_bin: Property[str] = Property.default("black") check_only: Property[bool] = Property.default(False) - config_file: Property[Path] + config: Property[BlackConfig | None] = Property.default(None) + config_file: Property[Path | None] = Property.default(None) + paths: Property[Sequence[str]] = Property.default_factory(list) additional_args: Property[Sequence[str]] = Property.default_factory(list) - additional_files: Property[Sequence[Path]] = Property.default_factory(list) + + __config_file: Path | None = None # EnvironmentAwareDispatchTask def get_execute_command(self) -> list[str]: - command = [self.black_bin.get(), str(self.settings.source_directory)] - command += self.settings.get_tests_directory_as_args() - command += [str(directory) for directory in self.settings.lint_enforced_directories] - command += [str(p) for p in self.additional_files.get()] + command = [self.black_bin.get(), *self.paths.get()] if self.check_only.get(): command += ["--check", "--diff"] - if self.config_file.is_filled(): - command += ["--config", str(self.config_file.get().absolute())] + config = self.config.get() + if config: + command += ["--line-length", str(config.line_length)] + if self.__config_file: + command += ["--config", str(self.__config_file.absolute())] command += self.additional_args.get() return command @@ -44,8 +80,20 @@ def get_description(self) -> str | None: else: return "Format Python source files with Black." + def prepare(self) -> TaskStatus | None: + config = self.config.get() + config_file = self.config_file.get() + if config is not None and config_file is not None: + raise RuntimeError(f"BlackTask.config and .config_file cannot be mixed") + if config is not None: + config_file = self.project.build_directory / self.name / "black.cfg" + config_file.parent.mkdir(parents=True, exist_ok=True) + config.to_file(config_file) + self.__config_file = config_file + return super().prepare() -@dataclasses.dataclass + +@dataclass class BlackTasks: check: BlackTask format: BlackTask @@ -55,9 +103,10 @@ def black( *, name: str = "python.black", project: Project | None = None, + config: BlackConfig | None = None, config_file: Path | Supplier[Path] | None = None, additional_args: Sequence[str] | Supplier[Sequence[str]] = (), - additional_files: Sequence[Path] | Supplier[Sequence[Path]] = (), + paths: Sequence[str] | Supplier[Sequence[str]] | None = None, version_spec: str | None = None, ) -> BlackTasks: """Creates two black tasks, one to check and another to format. The check task will be grouped under `"lint"` @@ -71,23 +120,31 @@ def black( if version_spec is not None: black_bin = pex_build( - "black", requirements=[f"black{version_spec}"], console_script="black", project=project + "black", + requirements=[f"black{version_spec}"], + console_script="black", + project=project, ).output_file.map(str) else: black_bin = Supplier.of("black") + if paths is None: + paths = python_settings(project).get_source_paths() + check_task = project.task(f"{name}.check", BlackTask, group="lint") check_task.black_bin = black_bin check_task.check_only = True + check_task.config = config check_task.config_file = config_file check_task.additional_args = additional_args - check_task.additional_files = additional_files + check_task.paths = paths format_task = project.task(name, BlackTask, group="fmt") format_task.black_bin = black_bin format_task.check_only = False + format_task.config = config format_task.config_file = config_file format_task.additional_args = additional_args - format_task.additional_files = additional_files + format_task.paths = paths return BlackTasks(check_task, format_task) diff --git a/kraken-build/src/kraken/std/python/tasks/build_task.py b/kraken-build/src/kraken/std/python/tasks/build_task.py index 36a7eda6..59694f2c 100644 --- a/kraken-build/src/kraken/std/python/tasks/build_task.py +++ b/kraken-build/src/kraken/std/python/tasks/build_task.py @@ -46,7 +46,8 @@ def build( ) -> BuildTask: """Creates a build task for the given project. - The build task relies on the build system configured in the Python project settings.""" + The build task relies on the build system configured in the Python project settings. + """ project = project or Project.current() task = project.task(name, BuildTask, group=group) diff --git a/kraken-build/src/kraken/std/python/tasks/flake8_task.py b/kraken-build/src/kraken/std/python/tasks/flake8_task.py index d49973f0..08ae547c 100644 --- a/kraken-build/src/kraken/std/python/tasks/flake8_task.py +++ b/kraken-build/src/kraken/std/python/tasks/flake8_task.py @@ -1,14 +1,59 @@ from __future__ import annotations +from collections.abc import Sequence +from configparser import ConfigParser +from dataclasses import dataclass +import os from pathlib import Path from kraken.common import Supplier from kraken.core import Project, Property +from kraken.core.system.task import TaskStatus +from kraken.std.python.settings import python_settings +from kraken.std.python.tasks.isort_task import IsortConfig from kraken.std.python.tasks.pex_build_task import pex_build from .base_task import EnvironmentAwareDispatchTask +@dataclass +class Flake8Config: + extend_ignore: Sequence[str] + exclude: Sequence[str] + + def dump(self, project_root: Path, config_file_dir: Path) -> ConfigParser: + """Dump to a `ConfigParser` object. + + Args: + project_root: The root of the Python project from which Flake8 will be run. + config_file_dir: The directory that will contain the config file. This needs to be specified + because exclude patterns need to be relative to the configuration file. + """ + + flake8_section = "flake8" + + # Compute the exclude patterns, which need to be relative to the config file. + exclude = [os.path.relpath(project_root / x, config_file_dir.absolute()) for x in self.exclude] + # If we find just a '.' or '*', we need to adjust it due to https://github.com/PyCQA/flake8/issues/298 + exclude = ["./*" if x == "." else "./*" if x == "*" else x for x in exclude] + + config = ConfigParser() + config.add_section(flake8_section) + config.set(flake8_section, "extend-ignore", ",".join(self.extend_ignore)) + config.set(flake8_section, "exclude", ",".join(exclude)) + + # config.add_section(LOCAL_PLUGINS_SECTION) + # for key, value in local_plugins_config.items(): + # config.set(LOCAL_PLUGINS_SECTION, key, value) + + return config + + def to_file(self, project_root: Path, config_file: Path) -> None: + config = self.dump(project_root, config_file.parent) + with config_file.open("w") as fp: + config.write(fp) + + class Flake8Task(EnvironmentAwareDispatchTask): """ Lint Python source files with Flake8. @@ -18,47 +63,80 @@ class Flake8Task(EnvironmentAwareDispatchTask): python_dependencies = ["flake8"] flake8_bin: Property[str] = Property.default("flake8") - config_file: Property[Path] - additional_args: Property[list[str]] = Property.default_factory(list) + config: Property[Flake8Config | None] = Property.default(None) + config_file: Property[Path | None] = Property.default(None) + paths: Property[Sequence[str]] = Property.default_factory(list) + additional_args: Property[Sequence[str]] = Property.default_factory(list) + + __config_file: Path | None = None # EnvironmentAwareDispatchTask def get_execute_command(self) -> list[str]: command = [ self.flake8_bin.get(), - str(self.settings.source_directory), - ] + self.settings.get_tests_directory_as_args() - command += [str(directory) for directory in self.settings.lint_enforced_directories] - if self.config_file.is_filled(): - command += ["--config", str(self.config_file.get().absolute())] + *self.paths.get(), + *self.additional_args.get(), + ] + if self.__config_file: + command += ["--config", str(self.__config_file.absolute())] command += self.additional_args.get() return command + # Task + + def prepare(self) -> TaskStatus | None: + config = self.config.get() + config_file = self.config_file.get() + if config is not None and config_file is not None: + raise RuntimeError("Flake8Task.config and .config_file cannot be mixed") + if config is not None: + config_file = self.project.build_directory / self.name / ".flake8" + config_file.parent.mkdir(parents=True, exist_ok=True) + config.to_file(self.project.directory, config_file) + self.__config_file = config_file + return super().prepare() + def flake8( *, name: str = "python.flake8", project: Project | None = None, + config: Flake8Config | None = None, config_file: Path | Supplier[Path] | None = None, + paths: Sequence[str] | None = None, + additional_args: Sequence[str] = (), version_spec: str | None = None, + additional_requirements: Sequence[str] = (), ) -> Flake8Task: """Creates a task for linting your Python project with Flake8. - :param version_spec: If specified, the Flake8 tool will be installed as a PEX and does not need to be installed - into the Python project's virtual env. + Args: + paths: A list of paths to pass to Flake8. If not specified, defaults to the source and test directories from + the project's `PythonSettings`. + version_spec: If specified, the Flake8 tool will be installed as a PEX and does not need to be installed + into the Python project's virtual env. """ project = project or Project.current() if version_spec is not None: flake8_bin = pex_build( - "flake8", requirements=[f"flake8{version_spec}"], console_script="flake8", project=project + "flake8", + requirements=[f"flake8{version_spec}", *additional_requirements], + console_script="flake8", + project=project, ).output_file.map(str) else: flake8_bin = Supplier.of("flake8") + if paths is None: + paths = python_settings(project).get_source_paths() + task = project.task(name, Flake8Task, group="lint") task.flake8_bin = flake8_bin - if config_file is not None: - task.config_file = config_file + task.config = config + task.config_file = config_file + task.paths = paths + task.additional_args = additional_args return task diff --git a/kraken-build/src/kraken/std/python/tasks/info_task.py b/kraken-build/src/kraken/std/python/tasks/info_task.py index 1da4daed..75f933f7 100644 --- a/kraken-build/src/kraken/std/python/tasks/info_task.py +++ b/kraken-build/src/kraken/std/python/tasks/info_task.py @@ -25,12 +25,29 @@ def execute(self) -> TaskStatus: return TaskStatus.failed(f"Error while getting the version of the current Python interpreter: {error}") print( - colored(f" ---------- {self.build_system.get().name}-managed environment information ----------", "magenta") + colored( + f" ---------- {self.build_system.get().name}-managed environment information ----------", + "magenta", + ) + ) + print( + colored("Python version: ", "cyan"), + colored(f"{version.strip()}", "blue"), + ) + print( + colored("Python path: ", "cyan"), + colored(f"{python_path}", "blue"), + ) + print( + colored("Virtual environment path: ", "cyan"), + colored(f"{virtual_env_path}", "blue"), + ) + print( + colored( + " ------------------------------------------------------------", + "magenta", + ) ) - print(colored("Python version: ", "cyan"), colored(f"{version.strip()}", "blue")) - print(colored("Python path: ", "cyan"), colored(f"{python_path}", "blue")) - print(colored("Virtual environment path: ", "cyan"), colored(f"{virtual_env_path}", "blue")) - print(colored(" ------------------------------------------------------------", "magenta")) return TaskStatus.succeeded() @@ -47,7 +64,10 @@ def get_python_path(self) -> Path: def get_python_version(self) -> str: """Returns the version of the Python interpreter of the Kraken-managed environment.""" return subprocess.run( - [self.get_python_path(), "--version"], stdout=subprocess.PIPE, shell=False, check=True + [self.get_python_path(), "--version"], + stdout=subprocess.PIPE, + shell=False, + check=True, ).stdout.decode("utf-8") diff --git a/kraken-build/src/kraken/std/python/tasks/install_task.py b/kraken-build/src/kraken/std/python/tasks/install_task.py index 9756dccc..cbc8f66e 100644 --- a/kraken-build/src/kraken/std/python/tasks/install_task.py +++ b/kraken-build/src/kraken/std/python/tasks/install_task.py @@ -64,7 +64,8 @@ def execute(self) -> TaskStatus: def install(*, name: str = "python.install", project: Project | None = None) -> InstallTask: """Get or create the `python.install` task for the given project. - The install task relies on the build system configured in the Python project settings.""" + The install task relies on the build system configured in the Python project settings. + """ project = project or Project.current() task = cast(Union[InstallTask, None], project.tasks().get(name)) diff --git a/kraken-build/src/kraken/std/python/tasks/isort_task.py b/kraken-build/src/kraken/std/python/tasks/isort_task.py index 0b95082e..155d6b0d 100644 --- a/kraken-build/src/kraken/std/python/tasks/isort_task.py +++ b/kraken-build/src/kraken/std/python/tasks/isort_task.py @@ -1,37 +1,61 @@ from __future__ import annotations -import dataclasses from collections.abc import Sequence +from configparser import ConfigParser +from dataclasses import dataclass from pathlib import Path from kraken.common import Supplier from kraken.core import Project, Property +from kraken.core.system.task import TaskStatus +from kraken.std.python.settings import python_settings from kraken.std.python.tasks.pex_build_task import pex_build from .base_task import EnvironmentAwareDispatchTask +@dataclass +class IsortConfig: + profile: str + line_length: int = 80 + combine_as_imports: bool = True + extend_skip: Sequence[str] = () + + def to_config(self) -> ConfigParser: + section_name = "isort" + config = ConfigParser() + config.add_section(section_name) + config.set(section_name, "profile", self.profile) + config.set(section_name, "line_length", str(self.line_length)) + config.set(section_name, "combine_as_imports", str(self.combine_as_imports).lower()) + config.set(section_name, "extend_skip", ",".join(self.extend_skip)) + return config + + def to_file(self, path: Path) -> None: + config = self.to_config() + with path.open("w") as fp: + config.write(fp) + + class IsortTask(EnvironmentAwareDispatchTask): python_dependencies = ["isort"] isort_bin: Property[str] = Property.default("isort") check_only: Property[bool] = Property.default(False) - config_file: Property[Path] - additional_files: Property[Sequence[Path]] = Property.default_factory(list) + config: Property[IsortConfig | None] = Property.default(None) + config_file: Property[Path | None] = Property.default(None) + paths: Property[Sequence[str]] = Property.default_factory(list) + + __config_file: Path | None = None # EnvironmentAwareDispatchTask def get_execute_command(self) -> list[str]: - command = [ - self.isort_bin.get(), - str(self.settings.source_directory), - ] + self.settings.get_tests_directory_as_args() - command += [str(directory) for directory in self.settings.lint_enforced_directories] - command += [str(p) for p in self.additional_files.get()] + command = [self.isort_bin.get(), *self.paths.get()] if self.check_only.get(): command += ["--check-only", "--diff"] - if self.config_file.is_filled(): - command += ["--settings-file", str(self.config_file.get().absolute())] + if self.__config_file: + command += ["--settings-file", str(self.__config_file)] return command # Task @@ -42,8 +66,20 @@ def get_description(self) -> str | None: else: return "Format Python source files with isort." + def prepare(self) -> TaskStatus | None: + config = self.config.get() + config_file = self.config_file.get() + if config is not None and config_file is not None: + raise RuntimeError(f"IsortTask.config and .config_file cannot be mixed") + if config is not None: + config_file = self.project.build_directory / self.name / "isort.ini" + config_file.parent.mkdir(parents=True, exist_ok=True) + config.to_file(config_file) + self.__config_file = config_file + return super().prepare() + -@dataclasses.dataclass +@dataclass class IsortTasks: check: IsortTask format: IsortTask @@ -53,8 +89,9 @@ def isort( *, name: str = "python.isort", project: Project | None = None, + config: IsortConfig | None = None, config_file: Path | Supplier[Path] | None = None, - additional_files: Sequence[Path] | Supplier[Sequence[Path]] = (), + paths: Sequence[str] | Supplier[Sequence[str]] | None = None, version_spec: str | None = None, ) -> IsortTasks: """ @@ -68,21 +105,29 @@ def isort( if version_spec is not None: isort_bin = pex_build( - "isort", requirements=[f"isort{version_spec}"], console_script="isort", project=project + "isort", + requirements=[f"isort{version_spec}"], + console_script="isort", + project=project, ).output_file.map(str) else: isort_bin = Supplier.of("isort") + if paths is None: + paths = python_settings(project).get_source_paths() + check_task = project.task(f"{name}.check", IsortTask, group="lint") check_task.isort_bin = isort_bin check_task.check_only = True + check_task.config = config check_task.config_file = config_file - check_task.additional_files = additional_files + check_task.paths = paths format_task = project.task(name, IsortTask, group="fmt") format_task.isort_bin = isort_bin + format_task.config = config format_task.check_only = False format_task.config_file = config_file - format_task.additional_files = additional_files + format_task.paths = paths return IsortTasks(check_task, format_task) diff --git a/kraken-build/src/kraken/std/python/tasks/mypy_task.py b/kraken-build/src/kraken/std/python/tasks/mypy_task.py index 3a6df8c2..ecadd4fc 100644 --- a/kraken-build/src/kraken/std/python/tasks/mypy_task.py +++ b/kraken-build/src/kraken/std/python/tasks/mypy_task.py @@ -1,26 +1,96 @@ from __future__ import annotations from collections.abc import MutableMapping, Sequence +from configparser import ConfigParser +from dataclasses import dataclass, field from pathlib import Path +import re +from typing import Mapping from kraken.common import Supplier from kraken.core import Project, Property +from kraken.core.system.task import TaskStatus +from kraken.std.python.settings import python_settings from kraken.std.python.tasks.pex_build_task import pex_build from .base_task import EnvironmentAwareDispatchTask +MYPY_BASE_CONFIG = """ +[mypy] +explicit_package_bases = true +ignore_missing_imports = true +namespace_packages = true +pretty = true +show_column_numbers = true +show_error_codes = true +show_error_context = true +strict = true +warn_no_return = true +warn_redundant_casts = true +warn_unreachable = true +warn_unused_configs = true +warn_unused_ignores = true +""" + + +@dataclass +class MypyConfig: + exclude_directories: Sequence[str] = () + """ A list of directories to exclude. """ + + exclude_patterns: Sequence[str] = () + """ A list of regular expressions to match on files to exclude them.""" + + global_overrides: Mapping[str, str | Sequence[str]] = field(default_factory=dict) + """ Global overrides to place into the Mypy config under the `[mypy]` section.""" + + module_overrides: Mapping[str, Mapping[str, str | Sequence[str]]] = field(default_factory=dict) + """ Per-module overrides to place into the `[mypy-{key}]` section, where `{key}` is the key of the root mapping.""" + + def dump(self) -> ConfigParser: + config = ConfigParser(inline_comment_prefixes="#;") + config.read(MYPY_BASE_CONFIG) + + if self.exclude_directories or self.exclude_patterns: + exclude_patterns = [] + for dirname in self.exclude_directories or (): + exclude_patterns.append("^" + re.escape(dirname.rstrip("/")) + "/.*$") + exclude_patterns += self.exclude_patterns or () + exclude_regex = "(" + "|".join(exclude_patterns) + ")" + config.set("mypy", "exclude", exclude_regex) + for key, value in self.global_overrides.items(): + config.set("mypy", key, value if isinstance(value, str) else ",".join(value)) + for patterns, options in self.module_overrides.items(): + section = f"mypy-{patterns}" + config.add_section(section) + for key, value in options.items(): + config.set(section, key, value if isinstance(value, str) else ",".join(value)) + + # config.set("mypy", "mypy_path", str(source_directory)) + + return config + + def to_file(self, path: Path) -> None: + config = self.dump() + with path.open("w") as fp: + config.write(fp) + class MypyTask(EnvironmentAwareDispatchTask): description = "Static type checking for Python code using Mypy." python_dependencies = ["mypy"] mypy_pex_bin: Property[Path | None] = Property.default(None) - config_file: Property[Path] + config: Property[MypyConfig | None] = Property.default(None) + config_file: Property[Path | None] = Property.default(None) + paths: Property[Sequence[str]] = Property.default_factory(list) additional_args: Property[Sequence[str]] = Property.default_factory(list) check_tests: Property[bool] = Property.default(True) use_daemon: Property[bool] = Property.default(True) python_version: Property[str] + __config_file: Path | None = None + # EnvironmentAwareDispatchTask def get_execute_command_v2(self, env: MutableMapping[str, str]) -> list[str]: @@ -45,33 +115,41 @@ def get_execute_command_v2(self, env: MutableMapping[str, str]) -> list[str]: # during the execution of this task as this is an "EnvironmentAwareDispatchTask". If we don't supply this # option, MyPy will only know the packages in its PEX. command += ["--python-executable", "python"] - if self.config_file.is_filled(): - command += ["--config-file", str(self.config_file.get().absolute())] + if self.__config_file: + command += ["--config-file", str(self.__config_file.absolute())] else: command += ["--show-error-codes", "--namespace-packages"] # Sane defaults. 🙏 if self.python_version.is_filled(): command += ["--python-version", self.python_version.get()] - source_dir = self.settings.source_directory - command += [str(source_dir)] - if self.check_tests.get(): - # We only want to add the tests directory if it is not already in the source directory. Otherwise - # Mypy will find the test files twice and error. - tests_dir = self.settings.get_tests_directory() - if tests_dir: - try: - tests_dir.relative_to(source_dir) - except ValueError: - command += [str(tests_dir)] + + command += [*self.paths.get()] command += [str(directory) for directory in self.settings.lint_enforced_directories] command += self.additional_args.get() return command + # Task + + def prepare(self) -> TaskStatus | None: + config_file = self.config.get() + config_file = self.config_file.get() + if config_file is not None and config_file is not None: + raise RuntimeError(f"IsortTask.config and .config_file cannot be mixed") + if config_file is None: + config = config_file or MypyConfig() + config_file = self.project.build_directory / self.name / "isort.ini" + config_file.parent.mkdir(parents=True, exist_ok=True) + config.to_file(config_file) + self.__config_file = config_file + return super().prepare() + def mypy( *, name: str = "python.mypy", project: Project | None = None, + config: MypyConfig | None = None, config_file: Path | Supplier[Path] | None = None, + paths: Sequence[str] | None = None, additional_args: Sequence[str] | Supplier[Sequence[str]] = (), check_tests: bool = True, use_daemon: bool = True, @@ -87,14 +165,22 @@ def mypy( if version_spec is not None: mypy_pex_bin = pex_build( - "mypy", requirements=[f"mypy{version_spec}"], console_script="mypy", project=project + "mypy", + requirements=[f"mypy{version_spec}"], + console_script="mypy", + project=project, ).output_file else: mypy_pex_bin = None + if paths is None: + paths = python_settings(project).get_source_paths() + task = project.task(name, MypyTask, group="lint") task.mypy_pex_bin = mypy_pex_bin + task.config = config task.config_file = config_file + task.paths = paths task.additional_args = additional_args task.check_tests = check_tests task.use_daemon = use_daemon diff --git a/kraken-build/src/kraken/std/python/tasks/publish_task.py b/kraken-build/src/kraken/std/python/tasks/publish_task.py index 6b4a576b..dcefa03d 100644 --- a/kraken-build/src/kraken/std/python/tasks/publish_task.py +++ b/kraken-build/src/kraken/std/python/tasks/publish_task.py @@ -81,7 +81,10 @@ def publish( raise ValueError(f"package index {package_index!r} is not defined") twine_bin = pex_build( - "twine", requirements=[f"twine{twine_version}"], console_script="twine", project=project + "twine", + requirements=[f"twine{twine_version}"], + console_script="twine", + project=project, ).output_file index = settings.package_indexes[package_index] diff --git a/kraken-build/src/kraken/std/python/tasks/pycln_task.py b/kraken-build/src/kraken/std/python/tasks/pycln_task.py index d1587e11..4ec364b6 100644 --- a/kraken-build/src/kraken/std/python/tasks/pycln_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pycln_task.py @@ -2,12 +2,14 @@ import dataclasses from pathlib import Path +import re +from typing import Sequence from kraken.common.supplier import Supplier from kraken.core import Project, Property from kraken.std.python.tasks.pex_build_task import pex_build -from .base_task import EnvironmentAwareDispatchTask +from .base_task import EnvironmentAwareDispatchTask, python_settings class PyclnTask(EnvironmentAwareDispatchTask): @@ -48,24 +50,52 @@ class PyclnTasks: format: PyclnTask -def pycln(*, name: str = "python.pycln", project: Project | None = None, version_spec: str | None = None) -> PyclnTasks: +def pycln( + *, + name: str = "python.pycln", + project: Project | None = None, + remove_all_unused_imports: bool = False, + paths: Sequence[str] | None = None, + exclude_directories: Sequence[str] = (), + version_spec: str | None = None, +) -> PyclnTasks: """Creates two pycln tasks, one to check and another to format. The check task will be grouped under `"lint"` whereas the format task will be grouped under `"fmt"`. - :param version_spec: If specified, the pycln tool will be installed as a PEX and does not need to be installed - into the Python project's virtual env. + Args: + paths: A list of paths to pass to Pycln. If not specified, the source and test directories from the project's + `PythonSettings` are used. + version_spec: If specified, the pycln tool will be installed as a PEX and does not need to be installed + into the Python project's virtual env. """ project = project or Project.current() if version_spec is not None: pycln_bin = pex_build( - "pycln", requirements=[f"pycln{version_spec}"], console_script="pycln", project=project + "pycln", + requirements=[f"pycln{version_spec}"], + console_script="pycln", + project=project, ).output_file.map(str) else: pycln_bin = Supplier.of("pycln") + if paths is None: + paths = python_settings(project).get_source_paths() + + additional_args = [*paths] + if remove_all_unused_imports: + additional_args.append("--all") + for path in exclude_directories: + additional_args.extend(["--extend-exclude", re.escape(path.rstrip("/")) + "/.*"]) + check_task = project.task(f"{name}.check", PyclnTask, group="lint") check_task.pycln_bin = pycln_bin check_task.check_only = True + check_task.additional_args = additional_args + format_task = project.task(name, PyclnTask, group="fmt") + format_task.pycln_bin = pycln_bin + format_task.additional_args = additional_args + return PyclnTasks(check_task, format_task) diff --git a/kraken-build/src/kraken/std/python/tasks/pylint_task.py b/kraken-build/src/kraken/std/python/tasks/pylint_task.py index 71e26139..bba3210d 100644 --- a/kraken-build/src/kraken/std/python/tasks/pylint_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pylint_task.py @@ -48,7 +48,10 @@ def pylint( project = project or Project.current() if version_spec is not None: pylint_bin = pex_build( - "pylint", requirements=[f"pylint{version_spec}"], console_script="pylint", project=project + "pylint", + requirements=[f"pylint{version_spec}"], + console_script="pylint", + project=project, ).output_file.map(str) else: pylint_bin = Supplier.of("pylint") diff --git a/kraken-build/src/kraken/std/python/tasks/pytest_task.py b/kraken-build/src/kraken/std/python/tasks/pytest_task.py index 87e2e836..083a6e58 100644 --- a/kraken-build/src/kraken/std/python/tasks/pytest_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pytest_task.py @@ -8,6 +8,7 @@ from kraken.common import flatten from kraken.core import Project, Property, TaskStatus +from kraken.std.python.settings import python_settings from .base_task import EnvironmentAwareDispatchTask @@ -30,8 +31,7 @@ class PytestTask(EnvironmentAwareDispatchTask): description = "Run unit tests using Pytest." python_dependencies = ["pytest"] - tests_dir: Property[Path] - include_dirs: Property[Sequence[Path]] = Property.default(()) + paths: Property[Sequence[str]] ignore_dirs: Property[Sequence[Path]] = Property.default_factory(list) allow_no_tests: Property[bool] = Property.default(False) doctest_modules: Property[bool] = Property.default(True) @@ -40,23 +40,12 @@ class PytestTask(EnvironmentAwareDispatchTask): # EnvironmentAwareDispatchTask - def is_skippable(self) -> bool: - return self.allow_no_tests.get() and self.tests_dir.is_empty() and not self.settings.get_tests_directory() - def get_execute_command(self) -> list[str] | TaskStatus: - tests_dir = self.tests_dir.get_or(None) - tests_dir = tests_dir or self.settings.get_tests_directory() - if not tests_dir: - print("error: no test directory configured and none could be detected") - return TaskStatus.failed("no test directory configured and none could be detected") - command = [ - "pytest", - "-vv", - str(self.project.directory / self.settings.source_directory), - str(self.project.directory / tests_dir), - *[str(self.project.directory / path) for path in self.include_dirs.get()], - ] - command += flatten(["--ignore", str(self.project.directory / path)] for path in self.ignore_dirs.get()) + command = ["pytest", "-vv", *self.paths.get()] + command += flatten( + ["--ignore", str(self.project.directory / path)] + for path in self.ignore_dirs.get() + ) command += ["--log-cli-level", "INFO"] if self.coverage.is_filled(): coverage_file = f"coverage{self.coverage.get().get_suffix()}" @@ -86,18 +75,27 @@ def pytest( name: str = "pytest", group: str = "test", project: Project | None = None, - tests_dir: Path | str | None = None, - include_dirs: Sequence[Path | str] = (), + paths: Sequence[str] | None = None, + include_dirs: Sequence[str] = (), ignore_dirs: Sequence[Path | str] = (), allow_no_tests: bool = False, doctest_modules: bool = True, marker: str | None = None, coverage: CoverageFormat | None = None, ) -> PytestTask: + """Create a task for running Pytest. Note that Pytest must be installed in the Python virtual environment. + + Args: + paths: The paths that contain Pythen test files. If not specified, uses the test and source directories + from the project's `PythonSettings`. + """ + + if paths is None: + paths = python_settings(project).get_source_paths() + project = project or Project.current() task = project.task(name, PytestTask, group=group) - task.tests_dir = Path(tests_dir) if tests_dir is not None else None - task.include_dirs = list(map(Path, include_dirs)) + task.paths = [*paths, *include_dirs] task.ignore_dirs = list(map(Path, ignore_dirs)) task.allow_no_tests = allow_no_tests task.doctest_modules = doctest_modules diff --git a/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py b/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py index a0503043..c8f1906c 100644 --- a/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py @@ -30,7 +30,11 @@ def get_execute_command(self) -> list[str]: return self.run_pyupgrade(self.additional_files.get(), ("--exit-zero-even-if-changed",)) def run_pyupgrade(self, files: Iterable[Path], extra: Iterable[str]) -> list[str]: - command = [self.pyupgrade_bin.get(), f"--py{self.python_version.get_or('3').replace('.', '')}-plus", *extra] + command = [ + self.pyupgrade_bin.get(), + f"--py{self.python_version.get_or('3').replace('.', '')}-plus", + *extra, + ] if self.keep_runtime_typing.get(): command.append("--keep-runtime-typing") command.extend(str(f) for f in files) @@ -95,31 +99,33 @@ def pyupgrade( exclude_patterns: Collection[str] = (), keep_runtime_typing: bool = False, python_version: str = "3", - additional_files: Sequence[Path] = (), + paths: Sequence[Path] | None = None, version_spec: str | None = None, ) -> PyUpgradeTasks: """ - :param version_spec: If specified, the pyupgrade tool will be installed as a PEX and does not need to be installed - into the Python project's virtual env. + Args: + paths: A list of paths to pass to Pyupgrade. If not specified, the source and test directories from the + project's `PythonSettings` are used. + version_spec: If specified, the pyupgrade tool will be installed as a PEX and does not need to be installed + into the Python project's virtual env. """ project = project or Project.current() if version_spec is not None: pyupgrade_bin = pex_build( - "pyupgrade", requirements=[f"pyupgrade{version_spec}"], console_script="pyupgrade", project=project + "pyupgrade", + requirements=[f"pyupgrade{version_spec}"], + console_script="pyupgrade", + project=project, ).output_file.map(str) else: pyupgrade_bin = Supplier.of("pyupgrade") - settings = python_settings(project) + if paths is None: + paths = python_settings(project).get_source_paths() - directories = list(additional_files) - directories.append(project.directory / settings.source_directory) - test_directory = settings.get_tests_directory() - if test_directory is not None: - directories.append(project.directory / test_directory) - files = {f.resolve() for p in directories for f in Path(p).glob("**/*.py")} + files = {f.resolve() for p in paths for f in Path(p).glob("**/*.py")} exclude = [e.resolve() for e in exclude] filtered_files = [ f diff --git a/kraken-build/src/kraken/std/python/tasks/update_lockfile_task.py b/kraken-build/src/kraken/std/python/tasks/update_lockfile_task.py index b650129a..fa7af3a2 100644 --- a/kraken-build/src/kraken/std/python/tasks/update_lockfile_task.py +++ b/kraken-build/src/kraken/std/python/tasks/update_lockfile_task.py @@ -39,7 +39,8 @@ def update_lockfile_task( ) -> UpdateLockfileTask: """Creates an update task for the given project. - The update task relies on the build system configured in the Python project settings.""" + The update task relies on the build system configured in the Python project settings. + """ project = project or Project.current() task = project.task(name, UpdateLockfileTask, group=group) diff --git a/kraken-build/src/kraken/std/sccache.py b/kraken-build/src/kraken/std/sccache.py index 47b2a807..74228b33 100644 --- a/kraken-build/src/kraken/std/sccache.py +++ b/kraken-build/src/kraken/std/sccache.py @@ -89,7 +89,11 @@ def stop(self, show_stats: bool = False) -> None: env = {"SCCACHE_NO_DAEMON": "1"} command = [str(self.bin) if self.bin else "sccache", "--stop-server"] - sp.check_call(command, env={**os.environ, **env}, stdout=None if show_stats else sp.DEVNULL) + sp.check_call( + command, + env={**os.environ, **env}, + stdout=None if show_stats else sp.DEVNULL, + ) self._proc.wait(10) if self.is_running(): diff --git a/kraken-build/tests/resources.py b/kraken-build/tests/resources.py index 949da39c..766c524d 100644 --- a/kraken-build/tests/resources.py +++ b/kraken-build/tests/resources.py @@ -2,7 +2,6 @@ Helper functions to access test resources. """ - from pathlib import Path diff --git a/kraken-wrapper/pyproject.toml b/kraken-wrapper/pyproject.toml index 500ee5e0..b48d8785 100644 --- a/kraken-wrapper/pyproject.toml +++ b/kraken-wrapper/pyproject.toml @@ -1,16 +1,20 @@ [build-system] -requires = ["poetry-core"] +requires = [ + "poetry-core", +] build-backend = "poetry.core.masonry.api" [tool.poetry] name = "kraken-wrapper" version = "0.33.2" description = "" -authors = ["Niklas Rosenstein "] +authors = [ + "Niklas Rosenstein ", +] license = "MIT" readme = "README.md" packages = [ - {include = "kraken/wrapper", from = "src"}, + { include = "kraken/wrapper", from = "src" }, ] classifiers = [] keywords = [] @@ -35,33 +39,3 @@ krakenw = "kraken.wrapper.main:main" [tool.slap] typed = true - -# Linter/Formatter configuration -# ------------------------------ - -[tool.mypy] -explicit_package_bases = true -mypy_path = ["src"] -namespace_packages = true -pretty = true -python_version = "3.10" -show_error_codes = true -show_error_context = true -strict = true -warn_no_return = true -warn_redundant_casts = true -warn_unreachable = true -warn_unused_ignores = true - -[tool.isort] -combine_as_imports = true -line_length = 120 -profile = "black" - -[tool.black] -line-length = 120 - -[tool.pytest.ini_options] -markers = [ - "integration", -] From 14ecfa0d22437b8903dab17da1c745578a6e3891 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 2 Feb 2024 13:48:39 +0100 Subject: [PATCH 03/79] address nits --- .kraken.py | 1 - kraken-build/src/kraken/common/_fs.py | 6 +- kraken-build/src/kraken/common/_importlib.py | 6 +- kraken-build/src/kraken/common/_runner.py | 3 +- kraken-build/src/kraken/common/_text.py | 3 +- .../src/kraken/core/system/context.py | 6 +- .../kraken/core/system/executor/__init__.py | 30 ++++++---- .../kraken/core/system/executor/default.py | 6 +- .../src/kraken/core/system/project_test.py | 12 ++-- kraken-build/src/kraken/core/system/task.py | 6 +- .../src/kraken/std/python/buildsystem/pdm.py | 4 +- kraken-build/src/kraken/std/python/project.py | 56 +++++++++-------- .../src/kraken/std/python/settings.py | 3 +- .../src/kraken/std/python/tasks/black_task.py | 6 +- .../kraken/std/python/tasks/flake8_task.py | 5 +- .../src/kraken/std/python/tasks/isort_task.py | 2 +- .../src/kraken/std/python/tasks/mypy_task.py | 11 ++-- .../src/kraken/std/python/tasks/pycln_task.py | 7 ++- .../kraken/std/python/tasks/pytest_task.py | 5 +- .../kraken/std/python/tasks/pyupgrade_task.py | 2 +- kraken-wrapper/pyproject.toml | 32 ++++++++++ .../src/kraken/wrapper/_buildenv.py | 7 ++- .../src/kraken/wrapper/_buildenv_manager.py | 5 +- .../src/kraken/wrapper/_buildenv_venv.py | 39 +++++++++--- kraken-wrapper/src/kraken/wrapper/_config.py | 15 ++++- kraken-wrapper/src/kraken/wrapper/main.py | 60 ++++++++++++++++--- 26 files changed, 237 insertions(+), 101 deletions(-) diff --git a/.kraken.py b/.kraken.py index 8e172e25..8043de1d 100644 --- a/.kraken.py +++ b/.kraken.py @@ -91,7 +91,6 @@ def configure_project() -> None: from kraken.build import project -from kraken.std.python.project import python_project try: project.subproject("docs") diff --git a/kraken-build/src/kraken/common/_fs.py b/kraken-build/src/kraken/common/_fs.py index 56b9eb80..0655b91a 100644 --- a/kraken-build/src/kraken/common/_fs.py +++ b/kraken-build/src/kraken/common/_fs.py @@ -14,7 +14,8 @@ def atomic_file_swap( mode: Literal["w"], always_revert: bool = ..., create_dirs: bool = ..., -) -> ContextManager[TextIO]: ... +) -> ContextManager[TextIO]: + ... @overload @@ -23,7 +24,8 @@ def atomic_file_swap( mode: Literal["wb"], always_revert: bool = ..., create_dirs: bool = ..., -) -> ContextManager[BinaryIO]: ... +) -> ContextManager[BinaryIO]: + ... @contextlib.contextmanager # type: ignore diff --git a/kraken-build/src/kraken/common/_importlib.py b/kraken-build/src/kraken/common/_importlib.py index aa8363c6..59bf814d 100644 --- a/kraken-build/src/kraken/common/_importlib.py +++ b/kraken-build/src/kraken/common/_importlib.py @@ -8,11 +8,13 @@ @overload -def import_class(fqn: str) -> type: ... +def import_class(fqn: str) -> type: + ... @overload -def import_class(fqn: str, base_type: type[T]) -> type[T]: ... +def import_class(fqn: str, base_type: type[T]) -> type[T]: + ... def import_class(fqn: str, base_type: "Type[T] | None" = None) -> "Type[T] | type": diff --git a/kraken-build/src/kraken/common/_runner.py b/kraken-build/src/kraken/common/_runner.py index 0076ab13..5a2f0681 100644 --- a/kraken-build/src/kraken/common/_runner.py +++ b/kraken-build/src/kraken/common/_runner.py @@ -68,7 +68,8 @@ class ProjectFinder(ABC): """ @abstractmethod - def find_project(self, directory: Path) -> "ProjectInfo | None": ... + def find_project(self, directory: Path) -> "ProjectInfo | None": + ... ## diff --git a/kraken-build/src/kraken/common/_text.py b/kraken-build/src/kraken/common/_text.py index ff47df94..c456ee58 100644 --- a/kraken-build/src/kraken/common/_text.py +++ b/kraken-build/src/kraken/common/_text.py @@ -7,7 +7,8 @@ class SupportsLen(Protocol): - def __len__(self) -> int: ... + def __len__(self) -> int: + ... def pluralize(word: str, count: "int | SupportsLen") -> str: diff --git a/kraken-build/src/kraken/core/system/context.py b/kraken-build/src/kraken/core/system/context.py index 3ee33fc4..3bd99603 100644 --- a/kraken-build/src/kraken/core/system/context.py +++ b/kraken-build/src/kraken/core/system/context.py @@ -389,10 +389,12 @@ def execute(self, tasks: list[str | Task] | TaskGraph | None = None) -> None: @overload def listen( self, event_type: str | ContextEvent.Type - ) -> Callable[[ContextEvent.T_Listener], ContextEvent.T_Listener]: ... + ) -> Callable[[ContextEvent.T_Listener], ContextEvent.T_Listener]: + ... @overload - def listen(self, event_type: str | ContextEvent.Type, listener: ContextEvent.Listener) -> None: ... + def listen(self, event_type: str | ContextEvent.Type, listener: ContextEvent.Listener) -> None: + ... def listen(self, event_type: str | ContextEvent.Type, listener: ContextEvent.Listener | None = None) -> Any: """Registers a listener to the context for the given event type.""" diff --git a/kraken-build/src/kraken/core/system/executor/__init__.py b/kraken-build/src/kraken/core/system/executor/__init__.py index a73f3711..aa64b92d 100644 --- a/kraken-build/src/kraken/core/system/executor/__init__.py +++ b/kraken-build/src/kraken/core/system/executor/__init__.py @@ -56,25 +56,35 @@ def tasks( class GraphExecutorObserver(abc.ABC): """Observes events in a Kraken task executor.""" - def before_execute_graph(self, graph: Graph) -> None: ... + def before_execute_graph(self, graph: Graph) -> None: + ... - def before_prepare_task(self, task: Task) -> None: ... + def before_prepare_task(self, task: Task) -> None: + ... - def after_prepare_task(self, task: Task, status: TaskStatus) -> None: ... + def after_prepare_task(self, task: Task, status: TaskStatus) -> None: + ... - def before_execute_task(self, task: Task, status: TaskStatus) -> None: ... + def before_execute_task(self, task: Task, status: TaskStatus) -> None: + ... - def on_task_output(self, task: Task, chunk: bytes) -> None: ... + def on_task_output(self, task: Task, chunk: bytes) -> None: + ... - def after_execute_task(self, task: Task, status: TaskStatus) -> None: ... + def after_execute_task(self, task: Task, status: TaskStatus) -> None: + ... - def before_teardown_task(self, task: Task) -> None: ... + def before_teardown_task(self, task: Task) -> None: + ... - def after_teardown_task(self, task: Task, status: TaskStatus) -> None: ... + def after_teardown_task(self, task: Task, status: TaskStatus) -> None: + ... - def after_execute_graph(self, graph: Graph) -> None: ... + def after_execute_graph(self, graph: Graph) -> None: + ... class GraphExecutor(abc.ABC): @abc.abstractmethod - def execute_graph(self, graph: Graph, observer: GraphExecutorObserver) -> None: ... + def execute_graph(self, graph: Graph, observer: GraphExecutorObserver) -> None: + ... diff --git a/kraken-build/src/kraken/core/system/executor/default.py b/kraken-build/src/kraken/core/system/executor/default.py index 980a3dc7..926f38a7 100644 --- a/kraken-build/src/kraken/core/system/executor/default.py +++ b/kraken-build/src/kraken/core/system/executor/default.py @@ -18,10 +18,12 @@ class TaskExecutor(abc.ABC): @abc.abstractmethod - def execute_task(self, task: Task, done: Callable[[TaskStatus], None]) -> None: ... + def execute_task(self, task: Task, done: Callable[[TaskStatus], None]) -> None: + ... @abc.abstractmethod - def teardown_task(self, task: Task, done: Callable[[TaskStatus], None]) -> None: ... + def teardown_task(self, task: Task, done: Callable[[TaskStatus], None]) -> None: + ... class DefaultTaskExecutor(TaskExecutor): diff --git a/kraken-build/src/kraken/core/system/project_test.py b/kraken-build/src/kraken/core/system/project_test.py index 2f359365..7f17727e 100644 --- a/kraken-build/src/kraken/core/system/project_test.py +++ b/kraken-build/src/kraken/core/system/project_test.py @@ -21,7 +21,8 @@ def test__Project__resolve_outputs__can_find_dataclass_in_properties(kraken_proj class MyTask(Task): out_prop: Property[MyDescriptor] = Property.output() - def execute(self) -> None: ... + def execute(self) -> None: + ... task = kraken_project.task("carrier", MyTask) task.out_prop = MyDescriptor("foobar") @@ -32,7 +33,8 @@ def test__Project__resolve_outputs__can_not_find_input_property(kraken_project: class MyTask(Task): out_prop: Property[MyDescriptor] - def execute(self) -> None: ... + def execute(self) -> None: + ... task = kraken_project.task("carrier", MyTask) task.out_prop = MyDescriptor("foobar") @@ -43,7 +45,8 @@ def test__Project__resolve_outputs_supplier(kraken_project: Project) -> None: class MyTask(Task): out_prop: Property[MyDescriptor] = Property.output() - def execute(self) -> None: ... + def execute(self) -> None: + ... task = kraken_project.task("carrier", MyTask) task.out_prop = MyDescriptor("foobar") @@ -66,7 +69,8 @@ def test__Project__do__does_not_set_property_on_None_value(kraken_project: Proje class MyTask(Task): in_prop: Property[str] - def execute(self) -> None: ... + def execute(self) -> None: + ... kraken_project.task("carrier", MyTask) assert kraken_project.resolve_tasks(":carrier").select(str).supplier().get() == [] diff --git a/kraken-build/src/kraken/core/system/task.py b/kraken-build/src/kraken/core/system/task.py index 56275fac..ab4585d4 100644 --- a/kraken-build/src/kraken/core/system/task.py +++ b/kraken-build/src/kraken/core/system/task.py @@ -744,10 +744,12 @@ def __iter__(self) -> Iterable[str]: return iter(self._ptt) @overload - def __getitem__(self, partition: str) -> Collection[Task]: ... + def __getitem__(self, partition: str) -> Collection[Task]: + ... @overload - def __getitem__(self, partition: Task) -> Collection[str]: ... + def __getitem__(self, partition: Task) -> Collection[str]: + ... def __getitem__(self, partition: str | Task) -> Collection[str] | Collection[Task]: if isinstance(partition, str): diff --git a/kraken-build/src/kraken/std/python/buildsystem/pdm.py b/kraken-build/src/kraken/std/python/buildsystem/pdm.py index 8138370d..c09d2618 100644 --- a/kraken-build/src/kraken/std/python/buildsystem/pdm.py +++ b/kraken-build/src/kraken/std/python/buildsystem/pdm.py @@ -95,7 +95,9 @@ def set_package_indexes(self, indexes: Sequence[PackageIndex]) -> None: key=lambda x: ( 0 if x.priority == PackageIndex.Priority.default - else 1 if x.priority == PackageIndex.Priority.primary else 2 + else 1 + if x.priority == PackageIndex.Priority.primary + else 2 ), ) diff --git a/kraken-build/src/kraken/std/python/project.py b/kraken-build/src/kraken/std/python/project.py index b378d598..e4fc7bdc 100644 --- a/kraken-build/src/kraken/std/python/project.py +++ b/kraken-build/src/kraken/std/python/project.py @@ -1,15 +1,14 @@ """ New-style API and template for defining the tasks for an entire Python project.""" -from collections.abc import Sequence import logging -from pathlib import Path import re -from kraken.common.supplier import Supplier -from kraken.std.python.buildsystem import detect_build_system -from kraken.std.python.pyproject import PackageIndex -from kraken.std.python.tasks.mypy_task import MypyConfig +from collections.abc import Sequence +from pathlib import Path + from nr.stream import Optional +from kraken.std.python.buildsystem import detect_build_system +from kraken.std.python.pyproject import PackageIndex from kraken.std.python.tasks.pytest_task import CoverageFormat logger = logging.getLogger(__name__) @@ -119,20 +118,20 @@ def python_project( from kraken.build import project from kraken.common import not_none - from .settings import python_settings - from .tasks.login_task import login as login_task - from .tasks.update_lockfile_task import update_lockfile_task - from .tasks.update_pyproject_task import update_pyproject_task - from .tasks.install_task import install as install_task + + from .pyproject import Pyproject + from .tasks.black_task import BlackConfig, black as black_tasks + from .tasks.flake8_task import Flake8Config, flake8 as flake8_tasks from .tasks.info_task import info as info_task - from .tasks.pyupgrade_task import pyupgrade as pyupgrade_task + from .tasks.install_task import install as install_task + from .tasks.isort_task import IsortConfig, isort as isort_tasks + from .tasks.login_task import login as login_task + from .tasks.mypy_task import MypyConfig, mypy as mypy_task from .tasks.pycln_task import pycln as pycln_task - from .tasks.isort_task import isort as isort_tasks, IsortConfig - from .tasks.black_task import black as black_tasks, BlackConfig - from .tasks.flake8_task import flake8 as flake8_tasks, Flake8Config - from .tasks.mypy_task import mypy as mypy_task, MypyConfig from .tasks.pytest_task import pytest as pytest_task - from .pyproject import Pyproject + from .tasks.pyupgrade_task import pyupgrade as pyupgrade_task + from .tasks.update_lockfile_task import update_lockfile_task + from .tasks.update_pyproject_task import update_pyproject_task if additional_lint_directories is None: additional_lint_directories = [] @@ -144,11 +143,10 @@ def python_project( source_paths.insert(1, tests_directory) logger.info("Source paths for Python project %s: %s", project.address, source_paths) - settings = python_settings() build_system = not_none(detect_build_system(project.directory)) pyproject = Pyproject.read(project.directory / "pyproject.toml") handler = build_system.get_pyproject_reader(pyproject) - project_version = handler.get_version() + # project_version = handler.get_version() # NOTE(@niklas): This is not entirely correct, but good enough in practice. We assume that for a version range, # the lowest Python version comes first in the version spec. We also need to support Poetry-style semver @@ -162,21 +160,21 @@ def python_project( ) python_version = "3" - login = login_task() - update_lockfile = update_lockfile_task() - update_pyproject = update_pyproject_task() - install = install_task() - info = info_task(build_system=build_system) + login_task() + update_lockfile_task() + update_pyproject_task() + install_task() + info_task(build_system=build_system) - pyupgrade = pyupgrade_task( + pyupgrade_task( python_version=python_version, keep_runtime_typing=pyupgrade_keep_runtime_typing, exclude=[Path(x) for x in exclude_lint_directories], - paths=[Path(x) for x in source_paths], + paths=source_paths, version_spec=pyupgrade_version_spec, ) - pycln = pycln_task( + pycln_task( paths=source_paths, exclude_directories=exclude_lint_directories, remove_all_unused_imports=pycln_remove_all_unused_imports, @@ -208,7 +206,7 @@ def python_project( ) flake8.depends_on(black.format, isort.format, mode="order-only") - mypy = mypy_task( + mypy_task( paths=source_paths, config=MypyConfig( exclude_directories=exclude_lint_directories, @@ -226,7 +224,7 @@ def python_project( else: coverage = None - pytest = pytest_task( + pytest_task( paths=source_paths, ignore_dirs=pytest_ignore_dirs, coverage=coverage, diff --git a/kraken-build/src/kraken/std/python/settings.py b/kraken-build/src/kraken/std/python/settings.py index 978a0c42..a3212466 100644 --- a/kraken-build/src/kraken/std/python/settings.py +++ b/kraken-build/src/kraken/std/python/settings.py @@ -4,9 +4,10 @@ from dataclasses import dataclass, field from pathlib import Path +from nr.stream import Optional + from kraken.core import Project from kraken.std.python.pyproject import PackageIndex -from nr.stream import Optional from .buildsystem import PythonBuildSystem, detect_build_system diff --git a/kraken-build/src/kraken/std/python/tasks/black_task.py b/kraken-build/src/kraken/std/python/tasks/black_task.py index 91a1e1d2..a329a73e 100644 --- a/kraken-build/src/kraken/std/python/tasks/black_task.py +++ b/kraken-build/src/kraken/std/python/tasks/black_task.py @@ -1,9 +1,9 @@ from __future__ import annotations -from dataclasses import dataclass +import re from collections.abc import Sequence +from dataclasses import dataclass from pathlib import Path -import re from typing import Any import tomli_w @@ -84,7 +84,7 @@ def prepare(self) -> TaskStatus | None: config = self.config.get() config_file = self.config_file.get() if config is not None and config_file is not None: - raise RuntimeError(f"BlackTask.config and .config_file cannot be mixed") + raise RuntimeError("BlackTask.config and .config_file cannot be mixed") if config is not None: config_file = self.project.build_directory / self.name / "black.cfg" config_file.parent.mkdir(parents=True, exist_ok=True) diff --git a/kraken-build/src/kraken/std/python/tasks/flake8_task.py b/kraken-build/src/kraken/std/python/tasks/flake8_task.py index 08ae547c..5d71b444 100644 --- a/kraken-build/src/kraken/std/python/tasks/flake8_task.py +++ b/kraken-build/src/kraken/std/python/tasks/flake8_task.py @@ -1,16 +1,15 @@ from __future__ import annotations + +import os from collections.abc import Sequence from configparser import ConfigParser from dataclasses import dataclass -import os - from pathlib import Path from kraken.common import Supplier from kraken.core import Project, Property from kraken.core.system.task import TaskStatus from kraken.std.python.settings import python_settings -from kraken.std.python.tasks.isort_task import IsortConfig from kraken.std.python.tasks.pex_build_task import pex_build from .base_task import EnvironmentAwareDispatchTask diff --git a/kraken-build/src/kraken/std/python/tasks/isort_task.py b/kraken-build/src/kraken/std/python/tasks/isort_task.py index 155d6b0d..24c41d5e 100644 --- a/kraken-build/src/kraken/std/python/tasks/isort_task.py +++ b/kraken-build/src/kraken/std/python/tasks/isort_task.py @@ -70,7 +70,7 @@ def prepare(self) -> TaskStatus | None: config = self.config.get() config_file = self.config_file.get() if config is not None and config_file is not None: - raise RuntimeError(f"IsortTask.config and .config_file cannot be mixed") + raise RuntimeError("IsortTask.config and .config_file cannot be mixed") if config is not None: config_file = self.project.build_directory / self.name / "isort.ini" config_file.parent.mkdir(parents=True, exist_ok=True) diff --git a/kraken-build/src/kraken/std/python/tasks/mypy_task.py b/kraken-build/src/kraken/std/python/tasks/mypy_task.py index ecadd4fc..9d5d47c5 100644 --- a/kraken-build/src/kraken/std/python/tasks/mypy_task.py +++ b/kraken-build/src/kraken/std/python/tasks/mypy_task.py @@ -1,11 +1,10 @@ from __future__ import annotations -from collections.abc import MutableMapping, Sequence +import re +from collections.abc import Mapping, MutableMapping, Sequence from configparser import ConfigParser from dataclasses import dataclass, field from pathlib import Path -import re -from typing import Mapping from kraken.common import Supplier from kraken.core import Project, Property @@ -130,10 +129,10 @@ def get_execute_command_v2(self, env: MutableMapping[str, str]) -> list[str]: # Task def prepare(self) -> TaskStatus | None: - config_file = self.config.get() + config = self.config.get() config_file = self.config_file.get() - if config_file is not None and config_file is not None: - raise RuntimeError(f"IsortTask.config and .config_file cannot be mixed") + if config is not None and config_file is not None: + raise RuntimeError("MypyTask.config and .config_file cannot be mixed") if config_file is None: config = config_file or MypyConfig() config_file = self.project.build_directory / self.name / "isort.ini" diff --git a/kraken-build/src/kraken/std/python/tasks/pycln_task.py b/kraken-build/src/kraken/std/python/tasks/pycln_task.py index 4ec364b6..bb831458 100644 --- a/kraken-build/src/kraken/std/python/tasks/pycln_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pycln_task.py @@ -1,15 +1,16 @@ from __future__ import annotations import dataclasses -from pathlib import Path import re -from typing import Sequence +from collections.abc import Sequence +from pathlib import Path from kraken.common.supplier import Supplier from kraken.core import Project, Property +from kraken.std.python.settings import python_settings from kraken.std.python.tasks.pex_build_task import pex_build -from .base_task import EnvironmentAwareDispatchTask, python_settings +from .base_task import EnvironmentAwareDispatchTask class PyclnTask(EnvironmentAwareDispatchTask): diff --git a/kraken-build/src/kraken/std/python/tasks/pytest_task.py b/kraken-build/src/kraken/std/python/tasks/pytest_task.py index 083a6e58..fc7889fd 100644 --- a/kraken-build/src/kraken/std/python/tasks/pytest_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pytest_task.py @@ -42,10 +42,7 @@ class PytestTask(EnvironmentAwareDispatchTask): def get_execute_command(self) -> list[str] | TaskStatus: command = ["pytest", "-vv", *self.paths.get()] - command += flatten( - ["--ignore", str(self.project.directory / path)] - for path in self.ignore_dirs.get() - ) + command += flatten(["--ignore", str(self.project.directory / path)] for path in self.ignore_dirs.get()) command += ["--log-cli-level", "INFO"] if self.coverage.is_filled(): coverage_file = f"coverage{self.coverage.get().get_suffix()}" diff --git a/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py b/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py index c8f1906c..71efdb2a 100644 --- a/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py @@ -99,7 +99,7 @@ def pyupgrade( exclude_patterns: Collection[str] = (), keep_runtime_typing: bool = False, python_version: str = "3", - paths: Sequence[Path] | None = None, + paths: Sequence[str] | None = None, version_spec: str | None = None, ) -> PyUpgradeTasks: """ diff --git a/kraken-wrapper/pyproject.toml b/kraken-wrapper/pyproject.toml index b48d8785..d2974bd0 100644 --- a/kraken-wrapper/pyproject.toml +++ b/kraken-wrapper/pyproject.toml @@ -39,3 +39,35 @@ krakenw = "kraken.wrapper.main:main" [tool.slap] typed = true + + + +# Linter/Formatter configuration +# ------------------------------ + +[tool.mypy] +explicit_package_bases = true +mypy_path = ["src"] +namespace_packages = true +pretty = true +python_version = "3.10" +show_error_codes = true +show_error_context = true +strict = true +warn_no_return = true +warn_redundant_casts = true +warn_unreachable = true +warn_unused_ignores = true + +[tool.isort] +combine_as_imports = true +line_length = 120 +profile = "black" + +[tool.black] +line-length = 120 + +[tool.pytest.ini_options] +markers = [ + "integration", +] diff --git a/kraken-wrapper/src/kraken/wrapper/_buildenv.py b/kraken-wrapper/src/kraken/wrapper/_buildenv.py index 97ff0053..75391a8e 100644 --- a/kraken-wrapper/src/kraken/wrapper/_buildenv.py +++ b/kraken-wrapper/src/kraken/wrapper/_buildenv.py @@ -47,7 +47,8 @@ def build(self, requirements: RequirementSpec, transitive: bool) -> None: def dispatch_to_kraken_cli(self, argv: list[str]) -> NoReturn: """Dispatch the kraken cli command in *argv* to the build environment. - :param argv: The arguments to pass to the kraken cli (without the "kraken" command name itself).""" + :param argv: The arguments to pass to the kraken cli (without the "kraken" command name itself). + """ @dataclasses.dataclass(frozen=True) @@ -102,7 +103,9 @@ class BuildEnvError(Exception): """ -def general_get_installed_distributions(kraken_command_prefix: Sequence[str]) -> list[Distribution]: +def general_get_installed_distributions( + kraken_command_prefix: Sequence[str], +) -> list[Distribution]: command = [*kraken_command_prefix, "query", "env"] output = subprocess.check_output(command).decode() return [Distribution(x["name"], x["version"], x["requirements"], x["extras"]) for x in json.loads(output)] diff --git a/kraken-wrapper/src/kraken/wrapper/_buildenv_manager.py b/kraken-wrapper/src/kraken/wrapper/_buildenv_manager.py index c7e36a4f..9941a4f9 100644 --- a/kraken-wrapper/src/kraken/wrapper/_buildenv_manager.py +++ b/kraken-wrapper/src/kraken/wrapper/_buildenv_manager.py @@ -126,7 +126,10 @@ def set_locked(self, lockfile: Lockfile) -> None: def _get_environment_for_type( - environment_type: EnvironmentType, base_path: Path, incremental: bool, show_install_logs: bool + environment_type: EnvironmentType, + base_path: Path, + incremental: bool, + show_install_logs: bool, ) -> BuildEnv: platform_name = platform.system().lower() if environment_type == EnvironmentType.VENV: diff --git a/kraken-wrapper/src/kraken/wrapper/_buildenv_venv.py b/kraken-wrapper/src/kraken/wrapper/_buildenv_venv.py index 4e66b882..45df9221 100644 --- a/kraken-wrapper/src/kraken/wrapper/_buildenv_venv.py +++ b/kraken-wrapper/src/kraken/wrapper/_buildenv_venv.py @@ -44,7 +44,11 @@ def __init__(self, path: Path, incremental: bool = False, show_pip_logs: bool = self._show_pip_logs = show_pip_logs def _run_command( - self, command: list[str], operation_name: str, log_file: Path | None, mode: Literal["a", "w"] = "w" + self, + command: list[str], + operation_name: str, + log_file: Path | None, + mode: Literal["a", "w"] = "w", ) -> None: if log_file: log_file.parent.mkdir(parents=True, exist_ok=True) @@ -121,7 +125,10 @@ def build(self, requirements: RequirementSpec, transitive: bool) -> None: try: current_python_version = findpython.get_python_interpreter_version(python_bin) except (subprocess.CalledProcessError, RuntimeError) as e: - logger.warning("Could not determine the version of the current Python build environment: %s", e) + logger.warning( + "Could not determine the version of the current Python build environment: %s", + e, + ) logger.info("Destroying existing environment at %s", self._path) safe_rmpath(self._path) else: @@ -145,18 +152,32 @@ def build(self, requirements: RequirementSpec, transitive: bool) -> None: if not self._path.exists(): # Find a Python interpreter that matches the given interpreter constraint. if requirements.interpreter_constraint is not None: - logger.info("Using Python interpreter constraint: %s", requirements.interpreter_constraint) + logger.info( + "Using Python interpreter constraint: %s", + requirements.interpreter_constraint, + ) python_origin_bin = find_python_interpreter(requirements.interpreter_constraint) logger.info("Using Python interpreter at %s", python_origin_bin) else: logger.info( - "No interpreter constraint specified, using current Python interpreter (%s)", sys.executable + "No interpreter constraint specified, using current Python interpreter (%s)", + sys.executable, ) python_origin_bin = sys.executable - command = [python_origin_bin, "-m", "venv", str(self._path), "--upgrade-deps"] + command = [ + python_origin_bin, + "-m", + "venv", + str(self._path), + "--upgrade-deps", + ] logger.info("Creating virtual environment at %s", os.path.relpath(self._path)) - self._run_command(command, operation_name="Create virtual environment", log_file=create_log) + self._run_command( + command, + operation_name="Create virtual environment", + log_file=create_log, + ) success_flag.touch() else: @@ -182,7 +203,11 @@ def build(self, requirements: RequirementSpec, transitive: bool) -> None: self._run_command(command, operation_name="Install dependencies", log_file=install_log) # Make sure the pythonpath from the requirements is encoded into the enviroment. - command = [python_bin, "-c", "from sysconfig import get_path; print(get_path('purelib'))"] + command = [ + python_bin, + "-c", + "from sysconfig import get_path; print(get_path('purelib'))", + ] site_packages = Path(subprocess.check_output(command).decode().strip()) pth_file = site_packages / "krakenw.pth" if requirements.pythonpath: diff --git a/kraken-wrapper/src/kraken/wrapper/_config.py b/kraken-wrapper/src/kraken/wrapper/_config.py index d3a8de1b..62b09027 100644 --- a/kraken-wrapper/src/kraken/wrapper/_config.py +++ b/kraken-wrapper/src/kraken/wrapper/_config.py @@ -40,11 +40,17 @@ class CredentialCheck(NamedTuple): raw_result: str hint: str - def __init__(self, config: MutableMapping[str, Any], path: Path, use_keyring_if_available: bool) -> None: + def __init__( + self, + config: MutableMapping[str, Any], + path: Path, + use_keyring_if_available: bool, + ) -> None: self._config = config self._path = path self._has_keyring = use_keyring_if_available and not isinstance( - keyring.get_keyring(), (keyring.backends.fail.Keyring, keyring.backends.null.Keyring) + keyring.get_keyring(), + (keyring.backends.fail.Keyring, keyring.backends.null.Keyring), ) def get_credentials(self, host: str) -> Credentials | None: @@ -118,7 +124,10 @@ def check_credential(self, host: str, username: str, password: str) -> Credentia url_suffix = ( self._config.get("auth", {}) .get(host, {}) - .get("auth_check_url_suffix", "artifactory/api/pypi/python-all/simple/flask/") + .get( + "auth_check_url_suffix", + "artifactory/api/pypi/python-all/simple/flask/", + ) ) url = f"https://{host}/{url_suffix}" curl_command = f"curl --user '{username}:{password}' {url}" diff --git a/kraken-wrapper/src/kraken/wrapper/main.py b/kraken-wrapper/src/kraken/wrapper/main.py index de3e6d15..68da2b77 100644 --- a/kraken-wrapper/src/kraken/wrapper/main.py +++ b/kraken-wrapper/src/kraken/wrapper/main.py @@ -71,7 +71,12 @@ def _get_argument_parser() -> argparse.ArgumentParser: # NOTE (@NiklasRosenstein): If we combine "+" with remainder, we get options passed after the `cmd` # passed directly into `args` without argparse treating it like an option. This is not the case # when using `nargs=1` for `cmd`. - parser.add_argument("cmd", nargs="*", metavar="cmd", help="{auth,list-pythons,lock} or a kraken command") + parser.add_argument( + "cmd", + nargs="*", + metavar="cmd", + help="{auth,list-pythons,lock} or a kraken command", + ) parser.add_argument("args", nargs=argparse.REMAINDER, help="additional arguments") return parser @@ -106,13 +111,20 @@ def lock(prog: str, argv: list[str], manager: BuildEnvManager, project: Project) extra_distributions.discard("pip") # We'll always have that in a virtual env. if extra_distributions: - logger.warning("Found extra distributions in your Kraken build enviroment: %s", ", ".join(extra_distributions)) + logger.warning( + "Found extra distributions in your Kraken build enviroment: %s", + ", ".join(extra_distributions), + ) had_lockfile = project.lockfile_path.exists() lockfile.write_to(project.lockfile_path) manager.set_locked(lockfile) - logger.info("Lock file %s (%s)", "updated" if had_lockfile else "created", os.path.relpath(project.lockfile_path)) + logger.info( + "Lock file %s (%s)", + "updated" if had_lockfile else "created", + os.path.relpath(project.lockfile_path), + ) sys.exit(0) @@ -195,7 +207,11 @@ def auth_check(auth: AuthModel, args: AuthOptions, host: str, username: str, pas # If verbose, also display the CURL command that people can use plus the first part of the response if args.verbose: - logger.info("Checking auth for host %s with command: %s", host, credential_result.curl_command) + logger.info( + "Checking auth for host %s with command: %s", + host, + credential_result.curl_command, + ) logger.info( "First 10 lines of response (limited to 1000 chars): %s", ("\n".join(credential_result.raw_result.split("\n")[0:10])[0:1000]), @@ -225,11 +241,29 @@ def _print_env_status(manager: BuildEnvManager, project: Project) -> None: table = AsciiTable() table.headers = ["Key", "Source", "Value"] - table.rows.append(("Requirements", str(project.requirements_path), project.requirements.to_hash(hash_algorithm))) + table.rows.append( + ( + "Requirements", + str(project.requirements_path), + project.requirements.to_hash(hash_algorithm), + ) + ) if project.lockfile: table.rows.append(("Lockfile", str(project.lockfile_path), "-")) - table.rows.append((" Requirements hash", "", project.lockfile.requirements.to_hash(hash_algorithm))) - table.rows.append((" Pinned hash", "", project.lockfile.to_pinned_requirement_spec().to_hash(hash_algorithm))) + table.rows.append( + ( + " Requirements hash", + "", + project.lockfile.requirements.to_hash(hash_algorithm), + ) + ) + table.rows.append( + ( + " Pinned hash", + "", + project.lockfile.to_pinned_requirement_spec().to_hash(hash_algorithm), + ) + ) else: table.rows.append(("Lockfile", str(project.lockfile_path), "n/a")) if manager.exists(): @@ -404,7 +438,11 @@ def main() -> NoReturn: if cmd in ("a", "auth"): # The `auth` comand does not require any current project information, it can be used globally. - auth(f"{parser.prog} auth", argv, use_keyring_if_available=not env_options.no_keyring) + auth( + f"{parser.prog} auth", + argv, + use_keyring_if_available=not env_options.no_keyring, + ) if cmd in ("list-pythons",): list_pythons(f"{parser.prog} list-pythons", argv) @@ -415,7 +453,11 @@ def main() -> NoReturn: project = load_project(Path.cwd(), outdated_check=not env_options.upgrade) manager = BuildEnvManager( project.directory / BUILDENV_PATH, - AuthModel(config, DEFAULT_CONFIG_PATH, use_keyring_if_available=not env_options.no_keyring), + AuthModel( + config, + DEFAULT_CONFIG_PATH, + use_keyring_if_available=not env_options.no_keyring, + ), incremental=env_options.incremental, show_install_logs=env_options.show_install_logs, ) From 5fefdcf702ffac5d323ffa23d987a9d5c6b0b4eb Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 2 Feb 2024 13:52:54 +0100 Subject: [PATCH 04/79] fix --- kraken-build/src/kraken/std/python/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kraken-build/src/kraken/std/python/settings.py b/kraken-build/src/kraken/std/python/settings.py index a3212466..3d0aefb7 100644 --- a/kraken-build/src/kraken/std/python/settings.py +++ b/kraken-build/src/kraken/std/python/settings.py @@ -140,7 +140,7 @@ def get_source_paths(self) -> list[str]: return [ str(self.source_directory), *Optional(self.get_tests_directory()) - .map(lambda p: [str(p.relative_to(self.project.directory))]) + .map(lambda p: [str(p.relative_to(self.project.directory) if p.is_absolute() else p)]) .or_else([]), *map(str, self.lint_enforced_directories), ] From a2720112d59d83c3eb524b0988a90584a23ee85e Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 2 Feb 2024 13:54:08 +0100 Subject: [PATCH 05/79] fix example --- examples/pdm-project/.kraken.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/examples/pdm-project/.kraken.py b/examples/pdm-project/.kraken.py index 690ed9cb..90ab00c8 100644 --- a/examples/pdm-project/.kraken.py +++ b/examples/pdm-project/.kraken.py @@ -1,18 +1,23 @@ import os -from kraken.std.python.project import python_library, python_package_index +# from kraken.std.python.project import python_library, python_package_index -python_package_index( +# python_package_index( +# alias="local", +# index_url=os.environ["LOCAL_PACKAGE_INDEX"], +# is_package_source=False, +# credentials=(os.environ["LOCAL_USER"], os.environ["LOCAL_PASSWORD"]), +# ) + +# python_library() + +from kraken.std import python + +python.python_settings(always_use_managed_env=True).add_package_index( alias="local", index_url=os.environ["LOCAL_PACKAGE_INDEX"], - is_package_source=False, credentials=(os.environ["LOCAL_USER"], os.environ["LOCAL_PASSWORD"]), ) - -python_library() - -# python.python_settings(always_use_managed_env=True).add_package_index( -# ) -# python.install() -# python.mypy() -# python.publish(package_index="local", distributions=python.build(as_version="0.1.0").output_files) +python.install() +python.mypy() +python.publish(package_index="local", distributions=python.build(as_version="0.1.0").output_files) From dbe33bbe9e8a3210734255f4b18ffd31f596ffd6 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 2 Feb 2024 13:54:38 +0100 Subject: [PATCH 06/79] use `python_project()` for pdm-project example --- examples/pdm-project/.kraken.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/examples/pdm-project/.kraken.py b/examples/pdm-project/.kraken.py index 90ab00c8..ddf3a88a 100644 --- a/examples/pdm-project/.kraken.py +++ b/examples/pdm-project/.kraken.py @@ -1,23 +1,12 @@ import os -# from kraken.std.python.project import python_library, python_package_index +from kraken.std.python.project import python_project, python_package_index -# python_package_index( -# alias="local", -# index_url=os.environ["LOCAL_PACKAGE_INDEX"], -# is_package_source=False, -# credentials=(os.environ["LOCAL_USER"], os.environ["LOCAL_PASSWORD"]), -# ) - -# python_library() - -from kraken.std import python - -python.python_settings(always_use_managed_env=True).add_package_index( +python_package_index( alias="local", index_url=os.environ["LOCAL_PACKAGE_INDEX"], + is_package_source=False, credentials=(os.environ["LOCAL_USER"], os.environ["LOCAL_PASSWORD"]), ) -python.install() -python.mypy() -python.publish(package_index="local", distributions=python.build(as_version="0.1.0").output_files) + +python_project() From 05d3ef65c470c54999159cbd614afb39fa59aef9 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 2 Feb 2024 14:19:04 +0100 Subject: [PATCH 07/79] update --- kraken-build/src/kraken/std/python/tasks/mypy_task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kraken-build/src/kraken/std/python/tasks/mypy_task.py b/kraken-build/src/kraken/std/python/tasks/mypy_task.py index 9d5d47c5..c06af15c 100644 --- a/kraken-build/src/kraken/std/python/tasks/mypy_task.py +++ b/kraken-build/src/kraken/std/python/tasks/mypy_task.py @@ -48,7 +48,7 @@ class MypyConfig: def dump(self) -> ConfigParser: config = ConfigParser(inline_comment_prefixes="#;") - config.read(MYPY_BASE_CONFIG) + config.read_string(MYPY_BASE_CONFIG) if self.exclude_directories or self.exclude_patterns: exclude_patterns = [] @@ -135,7 +135,7 @@ def prepare(self) -> TaskStatus | None: raise RuntimeError("MypyTask.config and .config_file cannot be mixed") if config_file is None: config = config_file or MypyConfig() - config_file = self.project.build_directory / self.name / "isort.ini" + config_file = self.project.build_directory / self.name / "mypy.ini" config_file.parent.mkdir(parents=True, exist_ok=True) config.to_file(config_file) self.__config_file = config_file From e6b95f455d74fe5f19dae5573ab8b7799793a69a Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 2 Feb 2024 15:01:08 +0100 Subject: [PATCH 08/79] set mypy_path config --- kraken-build/src/kraken/std/python/project.py | 1 + kraken-build/src/kraken/std/python/tasks/mypy_task.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/kraken-build/src/kraken/std/python/project.py b/kraken-build/src/kraken/std/python/project.py index e4fc7bdc..54b53423 100644 --- a/kraken-build/src/kraken/std/python/project.py +++ b/kraken-build/src/kraken/std/python/project.py @@ -209,6 +209,7 @@ def python_project( mypy_task( paths=source_paths, config=MypyConfig( + mypy_path=source_directory, exclude_directories=exclude_lint_directories, global_overrides={}, module_overrides={}, diff --git a/kraken-build/src/kraken/std/python/tasks/mypy_task.py b/kraken-build/src/kraken/std/python/tasks/mypy_task.py index c06af15c..923bbfcd 100644 --- a/kraken-build/src/kraken/std/python/tasks/mypy_task.py +++ b/kraken-build/src/kraken/std/python/tasks/mypy_task.py @@ -34,6 +34,8 @@ @dataclass class MypyConfig: + mypy_path: Sequence[str] = () + exclude_directories: Sequence[str] = () """ A list of directories to exclude. """ @@ -65,7 +67,7 @@ def dump(self) -> ConfigParser: for key, value in options.items(): config.set(section, key, value if isinstance(value, str) else ",".join(value)) - # config.set("mypy", "mypy_path", str(source_directory)) + config.set("mypy", "mypy_path", ":".join(self.mypy_path)) return config From 84d599cafc74103423c92e1d1df708bba36f7a38 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 2 Feb 2024 15:05:28 +0100 Subject: [PATCH 09/79] fix changelog --- .changelog/0.33.1.toml | 21 +++++++++++++++++++++ kraken-build/.changelog/0.33.1.toml | 19 ------------------- 2 files changed, 21 insertions(+), 19 deletions(-) delete mode 100644 kraken-build/.changelog/0.33.1.toml diff --git a/.changelog/0.33.1.toml b/.changelog/0.33.1.toml index e870ea39..e07710f2 100644 --- a/.changelog/0.33.1.toml +++ b/.changelog/0.33.1.toml @@ -46,3 +46,24 @@ type = "improvement" description = "Use latest Pip version and resolver in Pex, and use `--venv prepend` for Novella" author = "niklas.rosenstein@helsing.ai" component = "kraken-build" + +[[entries]] +id = "2f2d584f-2a2a-49cc-b265-56e7b1ed3ce9" +type = "feature" +description = "Add `GetItemSupplier` and `Supplier.__getitem__()`" +author = "niklas.rosenstein@helsing.ai" +component = "kraken-build" + +[[entries]] +id = "ee333218-60d5-438d-bcec-bee90bf3c2d2" +type = "improvement" +description = "Add `PexBuildTask.output_scripts_dir` and `.output_scripts` properties" +author = "niklas.rosenstein@helsing.ai" +component = "kraken-build" + +[[entries]] +id = "c47a5861-45f1-4d42-b28a-ef725bf363f6" +type = "feature" +description = "add `NovellaTask()` and `novella()` factory function to build documentation with [Novella](https://github.com/NiklasRosenstein/novella/)" +author = "niklas.rosenstein@helsing.ai" +component = "kraken-build" diff --git a/kraken-build/.changelog/0.33.1.toml b/kraken-build/.changelog/0.33.1.toml deleted file mode 100644 index 6c35e23d..00000000 --- a/kraken-build/.changelog/0.33.1.toml +++ /dev/null @@ -1,19 +0,0 @@ -release-date = "2024-01-12" - -[[entries]] -id = "2f2d584f-2a2a-49cc-b265-56e7b1ed3ce9" -type = "feature" -description = "Add `GetItemSupplier` and `Supplier.__getitem__()`" -author = "niklas.rosenstein@helsing.ai" - -[[entries]] -id = "ee333218-60d5-438d-bcec-bee90bf3c2d2" -type = "improvement" -description = "Add `PexBuildTask.output_scripts_dir` and `.output_scripts` properties" -author = "niklas.rosenstein@helsing.ai" - -[[entries]] -id = "c47a5861-45f1-4d42-b28a-ef725bf363f6" -type = "feature" -description = "add `NovellaTask()` and `novella()` factory function to build documentation with [Novella](https://github.com/NiklasRosenstein/novella/)" -author = "niklas.rosenstein@helsing.ai" From 84291c4ca1e04db2998901abe30e9cc554e1ef48 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 2 Feb 2024 15:06:29 +0100 Subject: [PATCH 10/79] breaking change: Default project groups `lint` now only depends on `check` for order instead of strictly --- .changelog/_unreleased.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index 884beb4c..e11c89ec 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -3,3 +3,9 @@ id = "fb3fcc4a-a38a-445f-b124-571395b7ac86" type = "feature" description = "Introduce `kraken.std.python.project.python_project()` function which creates all tasks for a Python project." author = "niklas.rosenstein@helsing.ai" + +[[entries]] +id = "ae70e553-81c2-4eb2-a6f3-210ca6c24992" +type = "breaking change" +description = "Default project groups `lint` now only depends on `check` for order instead of strictly" +author = "niklas.rosenstein@helsing.ai" From 4bd6ff5ddc6e4c69c2d84695b2d6e4296cd7980d Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 2 Feb 2024 15:38:50 +0100 Subject: [PATCH 11/79] mypy fixes --- kraken-build/src/kraken/std/python/tasks/mypy_task.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/kraken-build/src/kraken/std/python/tasks/mypy_task.py b/kraken-build/src/kraken/std/python/tasks/mypy_task.py index 923bbfcd..fecb2d0b 100644 --- a/kraken-build/src/kraken/std/python/tasks/mypy_task.py +++ b/kraken-build/src/kraken/std/python/tasks/mypy_task.py @@ -67,7 +67,8 @@ def dump(self) -> ConfigParser: for key, value in options.items(): config.set(section, key, value if isinstance(value, str) else ",".join(value)) - config.set("mypy", "mypy_path", ":".join(self.mypy_path)) + if self.mypy_path: + config.set("mypy", "mypy_path", ":".join(self.mypy_path)) return config @@ -124,7 +125,6 @@ def get_execute_command_v2(self, env: MutableMapping[str, str]) -> list[str]: command += ["--python-version", self.python_version.get()] command += [*self.paths.get()] - command += [str(directory) for directory in self.settings.lint_enforced_directories] command += self.additional_args.get() return command @@ -136,7 +136,7 @@ def prepare(self) -> TaskStatus | None: if config is not None and config_file is not None: raise RuntimeError("MypyTask.config and .config_file cannot be mixed") if config_file is None: - config = config_file or MypyConfig() + config = config or MypyConfig() config_file = self.project.build_directory / self.name / "mypy.ini" config_file.parent.mkdir(parents=True, exist_ok=True) config.to_file(config_file) @@ -176,6 +176,8 @@ def mypy( if paths is None: paths = python_settings(project).get_source_paths() + if config is None: + config = MypyConfig(mypy_path=[str(python_settings(project).source_directory)]) task = project.task(name, MypyTask, group="lint") task.mypy_pex_bin = mypy_pex_bin From f79fe304bdc2f377c9a238c5e86af48d12b94416 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 2 Feb 2024 15:42:50 +0100 Subject: [PATCH 12/79] remove type ignore --- kraken-build/src/kraken/core/address/_address_test.py | 2 +- kraken-build/src/kraken/core/cli/serialize.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kraken-build/src/kraken/core/address/_address_test.py b/kraken-build/src/kraken/core/address/_address_test.py index a64e0214..859644d3 100644 --- a/kraken-build/src/kraken/core/address/_address_test.py +++ b/kraken-build/src/kraken/core/address/_address_test.py @@ -1,6 +1,6 @@ from __future__ import annotations -import dill # type: ignore[import-untyped] +import dill from pytest import raises from kraken.core.address import Address diff --git a/kraken-build/src/kraken/core/cli/serialize.py b/kraken-build/src/kraken/core/cli/serialize.py index 6be6bbb5..1afda09e 100644 --- a/kraken-build/src/kraken/core/cli/serialize.py +++ b/kraken-build/src/kraken/core/cli/serialize.py @@ -5,7 +5,7 @@ from collections.abc import Iterable from pathlib import Path -import dill # type: ignore[import-untyped] +import dill from kraken.common import pluralize from kraken.core import Context, TaskGraph From 656f2707c48cdbe8c81298f5c62ce48ca84a4464 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 2 Feb 2024 15:53:14 +0100 Subject: [PATCH 13/79] add import-untyped back --- kraken-build/src/kraken/core/address/_address_test.py | 2 +- kraken-build/src/kraken/core/cli/serialize.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kraken-build/src/kraken/core/address/_address_test.py b/kraken-build/src/kraken/core/address/_address_test.py index 859644d3..a64e0214 100644 --- a/kraken-build/src/kraken/core/address/_address_test.py +++ b/kraken-build/src/kraken/core/address/_address_test.py @@ -1,6 +1,6 @@ from __future__ import annotations -import dill +import dill # type: ignore[import-untyped] from pytest import raises from kraken.core.address import Address diff --git a/kraken-build/src/kraken/core/cli/serialize.py b/kraken-build/src/kraken/core/cli/serialize.py index 1afda09e..6be6bbb5 100644 --- a/kraken-build/src/kraken/core/cli/serialize.py +++ b/kraken-build/src/kraken/core/cli/serialize.py @@ -5,7 +5,7 @@ from collections.abc import Iterable from pathlib import Path -import dill +import dill # type: ignore[import-untyped] from kraken.common import pluralize from kraken.core import Context, TaskGraph From d4e38869d304ee5e53369c8f1bf1faf0a4518b9e Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 2 Feb 2024 16:47:31 +0100 Subject: [PATCH 14/79] update --- kraken-build/src/kraken/std/python/tasks/mypy_task.py | 3 +-- .../tests/kraken_std/integration/python/test_python.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kraken-build/src/kraken/std/python/tasks/mypy_task.py b/kraken-build/src/kraken/std/python/tasks/mypy_task.py index fecb2d0b..71f1503e 100644 --- a/kraken-build/src/kraken/std/python/tasks/mypy_task.py +++ b/kraken-build/src/kraken/std/python/tasks/mypy_task.py @@ -135,8 +135,7 @@ def prepare(self) -> TaskStatus | None: config_file = self.config_file.get() if config is not None and config_file is not None: raise RuntimeError("MypyTask.config and .config_file cannot be mixed") - if config_file is None: - config = config or MypyConfig() + if config_file is None and config is not None: config_file = self.project.build_directory / self.name / "mypy.ini" config_file.parent.mkdir(parents=True, exist_ok=True) config.to_file(config_file) diff --git a/kraken-build/tests/kraken_std/integration/python/test_python.py b/kraken-build/tests/kraken_std/integration/python/test_python.py index 796434af..48f9b3df 100644 --- a/kraken-build/tests/kraken_std/integration/python/test_python.py +++ b/kraken-build/tests/kraken_std/integration/python/test_python.py @@ -65,6 +65,7 @@ def pypiserver(docker_service_manager: DockerServiceManager) -> str: "pypiserver/pypiserver:latest", ["--passwords", "/.htpasswd", "-a", "update", "--hash-algo", "sha256"], ports=[f"{PYPISERVER_PORT}:8080"], + platform="linux/amd64", volumes=[f"{htpasswd.absolute()}:/.htpasswd"], detach=True, probe=("GET", index_url), From 5e41c7c1721abfd1aa55e4bb6bcd6ffa6b99845c Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 2 Feb 2024 17:05:08 +0100 Subject: [PATCH 15/79] now? --- kraken-build/pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kraken-build/pyproject.toml b/kraken-build/pyproject.toml index 2a8309ff..0a48eee4 100644 --- a/kraken-build/pyproject.toml +++ b/kraken-build/pyproject.toml @@ -100,6 +100,10 @@ ignore_errors = true ignore_missing_imports = true module = "networkx.*" +[[tool.mypy.overrides]] +warn_unused_ignores = false +module = "dill" + [tool.isort] combine_as_imports = true line_length = 120 From b9807d738827159e8f3ad4197e60e5732a2d0cfb Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 27 Feb 2024 15:23:37 +0100 Subject: [PATCH 16/79] add mising logging import after merge --- kraken-build/src/kraken/std/python/tasks/mypy_task.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kraken-build/src/kraken/std/python/tasks/mypy_task.py b/kraken-build/src/kraken/std/python/tasks/mypy_task.py index ca948506..d6a3b846 100644 --- a/kraken-build/src/kraken/std/python/tasks/mypy_task.py +++ b/kraken-build/src/kraken/std/python/tasks/mypy_task.py @@ -1,10 +1,11 @@ from __future__ import annotations +import logging import re +import sys from collections.abc import Mapping, MutableMapping, Sequence from configparser import ConfigParser from dataclasses import dataclass, field -import sys from pathlib import Path from kraken.common import Supplier From 6a2098a44212e024d54d81c9e61d24340122b383 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 27 Feb 2024 17:15:49 +0100 Subject: [PATCH 17/79] fix pyupgrade() factory function which was not considering source directory relative to the current project --- kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py b/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py index 71efdb2a..9e2095c8 100644 --- a/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py @@ -53,6 +53,7 @@ def execute(self) -> TaskStatus: # We copy the file because there is no way to make pyupgrade not edit the files old_dir = self.settings.project.directory.resolve() new_file_for_old_file = {} + with TemporaryDirectory() as new_dir: for file in self.additional_files.get(): new_file = new_dir / file.resolve().relative_to(old_dir) @@ -125,7 +126,7 @@ def pyupgrade( if paths is None: paths = python_settings(project).get_source_paths() - files = {f.resolve() for p in paths for f in Path(p).glob("**/*.py")} + files = {f.resolve() for p in paths for f in Path(project.directory / p).glob("**/*.py")} exclude = [e.resolve() for e in exclude] filtered_files = [ f From 0e444ccceafb8fe65d9b85e92cc843e90078df17 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 27 Feb 2024 18:05:57 +0100 Subject: [PATCH 18/79] implement missing publish logic --- examples/pdm-project/.kraken.py | 1 + examples/pdm-project/pdm.lock | 431 ++++++++++++++++++ kraken-build/src/kraken/std/git/version.py | 13 +- kraken-build/src/kraken/std/python/project.py | 53 ++- kraken-build/src/kraken/std/python/version.py | 2 +- .../integration/python/test_python.py | 2 - 6 files changed, 490 insertions(+), 12 deletions(-) create mode 100644 examples/pdm-project/pdm.lock diff --git a/examples/pdm-project/.kraken.py b/examples/pdm-project/.kraken.py index ddf3a88a..64c72a7a 100644 --- a/examples/pdm-project/.kraken.py +++ b/examples/pdm-project/.kraken.py @@ -6,6 +6,7 @@ alias="local", index_url=os.environ["LOCAL_PACKAGE_INDEX"], is_package_source=False, + publish=True, credentials=(os.environ["LOCAL_USER"], os.environ["LOCAL_PASSWORD"]), ) diff --git a/examples/pdm-project/pdm.lock b/examples/pdm-project/pdm.lock new file mode 100644 index 00000000..de683c1e --- /dev/null +++ b/examples/pdm-project/pdm.lock @@ -0,0 +1,431 @@ +# This file is @generated by PDM. +# It is not intended for manual editing. + +[metadata] +groups = ["default"] +strategy = ["cross_platform", "inherit_metadata"] +lock_version = "4.4.1" +content_hash = "sha256:bdbfbe1cbb98ff36db365f927c20019a0e0358ad0b3933de8e0a5fd1960e73f2" + +[[package]] +name = "black" +version = "24.2.0" +requires_python = ">=3.8" +summary = "The uncompromising code formatter." +groups = ["default"] +dependencies = [ + "click>=8.0.0", + "mypy-extensions>=0.4.3", + "packaging>=22.0", + "pathspec>=0.9.0", + "platformdirs>=2", + "tomli>=1.1.0; python_version < \"3.11\"", + "typing-extensions>=4.0.1; python_version < \"3.11\"", +] +files = [ + {file = "black-24.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6981eae48b3b33399c8757036c7f5d48a535b962a7c2310d19361edeef64ce29"}, + {file = "black-24.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d533d5e3259720fdbc1b37444491b024003e012c5173f7d06825a77508085430"}, + {file = "black-24.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61a0391772490ddfb8a693c067df1ef5227257e72b0e4108482b8d41b5aee13f"}, + {file = "black-24.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:992e451b04667116680cb88f63449267c13e1ad134f30087dec8527242e9862a"}, + {file = "black-24.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:163baf4ef40e6897a2a9b83890e59141cc8c2a98f2dda5080dc15c00ee1e62cd"}, + {file = "black-24.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e37c99f89929af50ffaf912454b3e3b47fd64109659026b678c091a4cd450fb2"}, + {file = "black-24.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9de21bafcba9683853f6c96c2d515e364aee631b178eaa5145fc1c61a3cc92"}, + {file = "black-24.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:9db528bccb9e8e20c08e716b3b09c6bdd64da0dd129b11e160bf082d4642ac23"}, + {file = "black-24.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d84f29eb3ee44859052073b7636533ec995bd0f64e2fb43aeceefc70090e752b"}, + {file = "black-24.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e08fb9a15c914b81dd734ddd7fb10513016e5ce7e6704bdd5e1251ceee51ac9"}, + {file = "black-24.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:810d445ae6069ce64030c78ff6127cd9cd178a9ac3361435708b907d8a04c693"}, + {file = "black-24.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:ba15742a13de85e9b8f3239c8f807723991fbfae24bad92d34a2b12e81904982"}, + {file = "black-24.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:057c3dc602eaa6fdc451069bd027a1b2635028b575a6c3acfd63193ced20d9c8"}, + {file = "black-24.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:08654d0797e65f2423f850fc8e16a0ce50925f9337fb4a4a176a7aa4026e63f8"}, + {file = "black-24.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca610d29415ee1a30a3f30fab7a8f4144e9d34c89a235d81292a1edb2b55f540"}, + {file = "black-24.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:4dd76e9468d5536abd40ffbc7a247f83b2324f0c050556d9c371c2b9a9a95e31"}, + {file = "black-24.2.0-py3-none-any.whl", hash = "sha256:e8a6ae970537e67830776488bca52000eaa37fa63b9988e8c487458d9cd5ace6"}, + {file = "black-24.2.0.tar.gz", hash = "sha256:bce4f25c27c3435e4dace4815bcb2008b87e167e3bf4ee47ccdc5ce906eb4894"}, +] + +[[package]] +name = "click" +version = "8.1.7" +requires_python = ">=3.7" +summary = "Composable command line interface toolkit" +groups = ["default"] +dependencies = [ + "colorama; platform_system == \"Windows\"", +] +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[[package]] +name = "codespell" +version = "2.2.6" +requires_python = ">=3.8" +summary = "Codespell" +groups = ["default"] +files = [ + {file = "codespell-2.2.6-py3-none-any.whl", hash = "sha256:9ee9a3e5df0990604013ac2a9f22fa8e57669c827124a2e961fe8a1da4cacc07"}, + {file = "codespell-2.2.6.tar.gz", hash = "sha256:a8c65d8eb3faa03deabab6b3bbe798bea72e1799c7e9e955d57eca4096abcff9"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +summary = "Cross-platform colored terminal text." +groups = ["default"] +marker = "sys_platform == \"win32\" or platform_system == \"Windows\"" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "coverage" +version = "7.4.3" +requires_python = ">=3.8" +summary = "Code coverage measurement for Python" +groups = ["default"] +files = [ + {file = "coverage-7.4.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8580b827d4746d47294c0e0b92854c85a92c2227927433998f0d3320ae8a71b6"}, + {file = "coverage-7.4.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:718187eeb9849fc6cc23e0d9b092bc2348821c5e1a901c9f8975df0bc785bfd4"}, + {file = "coverage-7.4.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:767b35c3a246bcb55b8044fd3a43b8cd553dd1f9f2c1eeb87a302b1f8daa0524"}, + {file = "coverage-7.4.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae7f19afe0cce50039e2c782bff379c7e347cba335429678450b8fe81c4ef96d"}, + {file = "coverage-7.4.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba3a8aaed13770e970b3df46980cb068d1c24af1a1968b7818b69af8c4347efb"}, + {file = "coverage-7.4.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ee866acc0861caebb4f2ab79f0b94dbfbdbfadc19f82e6e9c93930f74e11d7a0"}, + {file = "coverage-7.4.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:506edb1dd49e13a2d4cac6a5173317b82a23c9d6e8df63efb4f0380de0fbccbc"}, + {file = "coverage-7.4.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd6545d97c98a192c5ac995d21c894b581f1fd14cf389be90724d21808b657e2"}, + {file = "coverage-7.4.3-cp310-cp310-win32.whl", hash = "sha256:f6a09b360d67e589236a44f0c39218a8efba2593b6abdccc300a8862cffc2f94"}, + {file = "coverage-7.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:18d90523ce7553dd0b7e23cbb28865db23cddfd683a38fb224115f7826de78d0"}, + {file = "coverage-7.4.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cbbe5e739d45a52f3200a771c6d2c7acf89eb2524890a4a3aa1a7fa0695d2a47"}, + {file = "coverage-7.4.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:489763b2d037b164846ebac0cbd368b8a4ca56385c4090807ff9fad817de4113"}, + {file = "coverage-7.4.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:451f433ad901b3bb00184d83fd83d135fb682d780b38af7944c9faeecb1e0bfe"}, + {file = "coverage-7.4.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fcc66e222cf4c719fe7722a403888b1f5e1682d1679bd780e2b26c18bb648cdc"}, + {file = "coverage-7.4.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3ec74cfef2d985e145baae90d9b1b32f85e1741b04cd967aaf9cfa84c1334f3"}, + {file = "coverage-7.4.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:abbbd8093c5229c72d4c2926afaee0e6e3140de69d5dcd918b2921f2f0c8baba"}, + {file = "coverage-7.4.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:35eb581efdacf7b7422af677b92170da4ef34500467381e805944a3201df2079"}, + {file = "coverage-7.4.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8249b1c7334be8f8c3abcaaa996e1e4927b0e5a23b65f5bf6cfe3180d8ca7840"}, + {file = "coverage-7.4.3-cp311-cp311-win32.whl", hash = "sha256:cf30900aa1ba595312ae41978b95e256e419d8a823af79ce670835409fc02ad3"}, + {file = "coverage-7.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:18c7320695c949de11a351742ee001849912fd57e62a706d83dfc1581897fa2e"}, + {file = "coverage-7.4.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b51bfc348925e92a9bd9b2e48dad13431b57011fd1038f08316e6bf1df107d10"}, + {file = "coverage-7.4.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d6cdecaedea1ea9e033d8adf6a0ab11107b49571bbb9737175444cea6eb72328"}, + {file = "coverage-7.4.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b2eccb883368f9e972e216c7b4c7c06cabda925b5f06dde0650281cb7666a30"}, + {file = "coverage-7.4.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c00cdc8fa4e50e1cc1f941a7f2e3e0f26cb2a1233c9696f26963ff58445bac7"}, + {file = "coverage-7.4.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9a4a8dd3dcf4cbd3165737358e4d7dfbd9d59902ad11e3b15eebb6393b0446e"}, + {file = "coverage-7.4.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:062b0a75d9261e2f9c6d071753f7eef0fc9caf3a2c82d36d76667ba7b6470003"}, + {file = "coverage-7.4.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:ebe7c9e67a2d15fa97b77ea6571ce5e1e1f6b0db71d1d5e96f8d2bf134303c1d"}, + {file = "coverage-7.4.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c0a120238dd71c68484f02562f6d446d736adcc6ca0993712289b102705a9a3a"}, + {file = "coverage-7.4.3-cp312-cp312-win32.whl", hash = "sha256:37389611ba54fd6d278fde86eb2c013c8e50232e38f5c68235d09d0a3f8aa352"}, + {file = "coverage-7.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:d25b937a5d9ffa857d41be042b4238dd61db888533b53bc76dc082cb5a15e914"}, + {file = "coverage-7.4.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3b253094dbe1b431d3a4ac2f053b6d7ede2664ac559705a704f621742e034f1f"}, + {file = "coverage-7.4.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:77fbfc5720cceac9c200054b9fab50cb2a7d79660609200ab83f5db96162d20c"}, + {file = "coverage-7.4.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6679060424faa9c11808598504c3ab472de4531c571ab2befa32f4971835788e"}, + {file = "coverage-7.4.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4af154d617c875b52651dd8dd17a31270c495082f3d55f6128e7629658d63765"}, + {file = "coverage-7.4.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8640f1fde5e1b8e3439fe482cdc2b0bb6c329f4bb161927c28d2e8879c6029ee"}, + {file = "coverage-7.4.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:69b9f6f66c0af29642e73a520b6fed25ff9fd69a25975ebe6acb297234eda501"}, + {file = "coverage-7.4.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0842571634f39016a6c03e9d4aba502be652a6e4455fadb73cd3a3a49173e38f"}, + {file = "coverage-7.4.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a78ed23b08e8ab524551f52953a8a05d61c3a760781762aac49f8de6eede8c45"}, + {file = "coverage-7.4.3-cp39-cp39-win32.whl", hash = "sha256:c0524de3ff096e15fcbfe8f056fdb4ea0bf497d584454f344d59fce069d3e6e9"}, + {file = "coverage-7.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:0209a6369ccce576b43bb227dc8322d8ef9e323d089c6f3f26a597b09cb4d2aa"}, + {file = "coverage-7.4.3-pp38.pp39.pp310-none-any.whl", hash = "sha256:7cbde573904625509a3f37b6fecea974e363460b556a627c60dc2f47e2fffa51"}, + {file = "coverage-7.4.3.tar.gz", hash = "sha256:276f6077a5c61447a48d133ed13e759c09e62aff0dc84274a68dc18660104d52"}, +] + +[[package]] +name = "coverage" +version = "7.4.3" +extras = ["toml"] +requires_python = ">=3.8" +summary = "Code coverage measurement for Python" +groups = ["default"] +dependencies = [ + "coverage==7.4.3", + "tomli; python_full_version <= \"3.11.0a6\"", +] +files = [ + {file = "coverage-7.4.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8580b827d4746d47294c0e0b92854c85a92c2227927433998f0d3320ae8a71b6"}, + {file = "coverage-7.4.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:718187eeb9849fc6cc23e0d9b092bc2348821c5e1a901c9f8975df0bc785bfd4"}, + {file = "coverage-7.4.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:767b35c3a246bcb55b8044fd3a43b8cd553dd1f9f2c1eeb87a302b1f8daa0524"}, + {file = "coverage-7.4.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae7f19afe0cce50039e2c782bff379c7e347cba335429678450b8fe81c4ef96d"}, + {file = "coverage-7.4.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba3a8aaed13770e970b3df46980cb068d1c24af1a1968b7818b69af8c4347efb"}, + {file = "coverage-7.4.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ee866acc0861caebb4f2ab79f0b94dbfbdbfadc19f82e6e9c93930f74e11d7a0"}, + {file = "coverage-7.4.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:506edb1dd49e13a2d4cac6a5173317b82a23c9d6e8df63efb4f0380de0fbccbc"}, + {file = "coverage-7.4.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd6545d97c98a192c5ac995d21c894b581f1fd14cf389be90724d21808b657e2"}, + {file = "coverage-7.4.3-cp310-cp310-win32.whl", hash = "sha256:f6a09b360d67e589236a44f0c39218a8efba2593b6abdccc300a8862cffc2f94"}, + {file = "coverage-7.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:18d90523ce7553dd0b7e23cbb28865db23cddfd683a38fb224115f7826de78d0"}, + {file = "coverage-7.4.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cbbe5e739d45a52f3200a771c6d2c7acf89eb2524890a4a3aa1a7fa0695d2a47"}, + {file = "coverage-7.4.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:489763b2d037b164846ebac0cbd368b8a4ca56385c4090807ff9fad817de4113"}, + {file = "coverage-7.4.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:451f433ad901b3bb00184d83fd83d135fb682d780b38af7944c9faeecb1e0bfe"}, + {file = "coverage-7.4.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fcc66e222cf4c719fe7722a403888b1f5e1682d1679bd780e2b26c18bb648cdc"}, + {file = "coverage-7.4.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3ec74cfef2d985e145baae90d9b1b32f85e1741b04cd967aaf9cfa84c1334f3"}, + {file = "coverage-7.4.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:abbbd8093c5229c72d4c2926afaee0e6e3140de69d5dcd918b2921f2f0c8baba"}, + {file = "coverage-7.4.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:35eb581efdacf7b7422af677b92170da4ef34500467381e805944a3201df2079"}, + {file = "coverage-7.4.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8249b1c7334be8f8c3abcaaa996e1e4927b0e5a23b65f5bf6cfe3180d8ca7840"}, + {file = "coverage-7.4.3-cp311-cp311-win32.whl", hash = "sha256:cf30900aa1ba595312ae41978b95e256e419d8a823af79ce670835409fc02ad3"}, + {file = "coverage-7.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:18c7320695c949de11a351742ee001849912fd57e62a706d83dfc1581897fa2e"}, + {file = "coverage-7.4.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b51bfc348925e92a9bd9b2e48dad13431b57011fd1038f08316e6bf1df107d10"}, + {file = "coverage-7.4.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d6cdecaedea1ea9e033d8adf6a0ab11107b49571bbb9737175444cea6eb72328"}, + {file = "coverage-7.4.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b2eccb883368f9e972e216c7b4c7c06cabda925b5f06dde0650281cb7666a30"}, + {file = "coverage-7.4.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c00cdc8fa4e50e1cc1f941a7f2e3e0f26cb2a1233c9696f26963ff58445bac7"}, + {file = "coverage-7.4.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9a4a8dd3dcf4cbd3165737358e4d7dfbd9d59902ad11e3b15eebb6393b0446e"}, + {file = "coverage-7.4.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:062b0a75d9261e2f9c6d071753f7eef0fc9caf3a2c82d36d76667ba7b6470003"}, + {file = "coverage-7.4.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:ebe7c9e67a2d15fa97b77ea6571ce5e1e1f6b0db71d1d5e96f8d2bf134303c1d"}, + {file = "coverage-7.4.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c0a120238dd71c68484f02562f6d446d736adcc6ca0993712289b102705a9a3a"}, + {file = "coverage-7.4.3-cp312-cp312-win32.whl", hash = "sha256:37389611ba54fd6d278fde86eb2c013c8e50232e38f5c68235d09d0a3f8aa352"}, + {file = "coverage-7.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:d25b937a5d9ffa857d41be042b4238dd61db888533b53bc76dc082cb5a15e914"}, + {file = "coverage-7.4.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3b253094dbe1b431d3a4ac2f053b6d7ede2664ac559705a704f621742e034f1f"}, + {file = "coverage-7.4.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:77fbfc5720cceac9c200054b9fab50cb2a7d79660609200ab83f5db96162d20c"}, + {file = "coverage-7.4.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6679060424faa9c11808598504c3ab472de4531c571ab2befa32f4971835788e"}, + {file = "coverage-7.4.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4af154d617c875b52651dd8dd17a31270c495082f3d55f6128e7629658d63765"}, + {file = "coverage-7.4.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8640f1fde5e1b8e3439fe482cdc2b0bb6c329f4bb161927c28d2e8879c6029ee"}, + {file = "coverage-7.4.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:69b9f6f66c0af29642e73a520b6fed25ff9fd69a25975ebe6acb297234eda501"}, + {file = "coverage-7.4.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0842571634f39016a6c03e9d4aba502be652a6e4455fadb73cd3a3a49173e38f"}, + {file = "coverage-7.4.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a78ed23b08e8ab524551f52953a8a05d61c3a760781762aac49f8de6eede8c45"}, + {file = "coverage-7.4.3-cp39-cp39-win32.whl", hash = "sha256:c0524de3ff096e15fcbfe8f056fdb4ea0bf497d584454f344d59fce069d3e6e9"}, + {file = "coverage-7.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:0209a6369ccce576b43bb227dc8322d8ef9e323d089c6f3f26a597b09cb4d2aa"}, + {file = "coverage-7.4.3-pp38.pp39.pp310-none-any.whl", hash = "sha256:7cbde573904625509a3f37b6fecea974e363460b556a627c60dc2f47e2fffa51"}, + {file = "coverage-7.4.3.tar.gz", hash = "sha256:276f6077a5c61447a48d133ed13e759c09e62aff0dc84274a68dc18660104d52"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +requires_python = ">=3.7" +summary = "Backport of PEP 654 (exception groups)" +groups = ["default"] +marker = "python_version < \"3.11\"" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[[package]] +name = "flake8" +version = "7.0.0" +requires_python = ">=3.8.1" +summary = "the modular source code checker: pep8 pyflakes and co" +groups = ["default"] +dependencies = [ + "mccabe<0.8.0,>=0.7.0", + "pycodestyle<2.12.0,>=2.11.0", + "pyflakes<3.3.0,>=3.2.0", +] +files = [ + {file = "flake8-7.0.0-py2.py3-none-any.whl", hash = "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3"}, + {file = "flake8-7.0.0.tar.gz", hash = "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +requires_python = ">=3.7" +summary = "brain-dead simple config-ini parsing" +groups = ["default"] +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isort" +version = "5.13.2" +requires_python = ">=3.8.0" +summary = "A Python utility / library to sort Python imports." +groups = ["default"] +files = [ + {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, + {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, +] + +[[package]] +name = "mccabe" +version = "0.7.0" +requires_python = ">=3.6" +summary = "McCabe checker, plugin for flake8" +groups = ["default"] +files = [ + {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, + {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, +] + +[[package]] +name = "mypy" +version = "0.991" +requires_python = ">=3.7" +summary = "Optional static typing for Python" +groups = ["default"] +dependencies = [ + "mypy-extensions>=0.4.3", + "tomli>=1.1.0; python_version < \"3.11\"", + "typing-extensions>=3.10", +] +files = [ + {file = "mypy-0.991-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7d17e0a9707d0772f4a7b878f04b4fd11f6f5bcb9b3813975a9b13c9332153ab"}, + {file = "mypy-0.991-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0714258640194d75677e86c786e80ccf294972cc76885d3ebbb560f11db0003d"}, + {file = "mypy-0.991-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0c8f3be99e8a8bd403caa8c03be619544bc2c77a7093685dcf308c6b109426c6"}, + {file = "mypy-0.991-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc9ec663ed6c8f15f4ae9d3c04c989b744436c16d26580eaa760ae9dd5d662eb"}, + {file = "mypy-0.991-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4307270436fd7694b41f913eb09210faff27ea4979ecbcd849e57d2da2f65305"}, + {file = "mypy-0.991-cp310-cp310-win_amd64.whl", hash = "sha256:901c2c269c616e6cb0998b33d4adbb4a6af0ac4ce5cd078afd7bc95830e62c1c"}, + {file = "mypy-0.991-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d13674f3fb73805ba0c45eb6c0c3053d218aa1f7abead6e446d474529aafc372"}, + {file = "mypy-0.991-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c8cd4fb70e8584ca1ed5805cbc7c017a3d1a29fb450621089ffed3e99d1857f"}, + {file = "mypy-0.991-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:209ee89fbb0deed518605edddd234af80506aec932ad28d73c08f1400ef80a33"}, + {file = "mypy-0.991-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37bd02ebf9d10e05b00d71302d2c2e6ca333e6c2a8584a98c00e038db8121f05"}, + {file = "mypy-0.991-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:26efb2fcc6b67e4d5a55561f39176821d2adf88f2745ddc72751b7890f3194ad"}, + {file = "mypy-0.991-cp311-cp311-win_amd64.whl", hash = "sha256:3a700330b567114b673cf8ee7388e949f843b356a73b5ab22dd7cff4742a5297"}, + {file = "mypy-0.991-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:98e781cd35c0acf33eb0295e8b9c55cdbef64fcb35f6d3aa2186f289bed6e80d"}, + {file = "mypy-0.991-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6d7464bac72a85cb3491c7e92b5b62f3dcccb8af26826257760a552a5e244aa5"}, + {file = "mypy-0.991-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c9166b3f81a10cdf9b49f2d594b21b31adadb3d5e9db9b834866c3258b695be3"}, + {file = "mypy-0.991-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8472f736a5bfb159a5e36740847808f6f5b659960115ff29c7cecec1741c648"}, + {file = "mypy-0.991-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5e80e758243b97b618cdf22004beb09e8a2de1af481382e4d84bc52152d1c476"}, + {file = "mypy-0.991-cp39-cp39-win_amd64.whl", hash = "sha256:74e259b5c19f70d35fcc1ad3d56499065c601dfe94ff67ae48b85596b9ec1461"}, + {file = "mypy-0.991-py3-none-any.whl", hash = "sha256:de32edc9b0a7e67c2775e574cb061a537660e51210fbf6006b0b36ea695ae9bb"}, + {file = "mypy-0.991.tar.gz", hash = "sha256:3c0165ba8f354a6d9881809ef29f1a9318a236a6d81c690094c5df32107bde06"}, +] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +requires_python = ">=3.5" +summary = "Type system extensions for programs checked with the mypy type checker." +groups = ["default"] +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "packaging" +version = "23.2" +requires_python = ">=3.7" +summary = "Core utilities for Python packages" +groups = ["default"] +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +requires_python = ">=3.8" +summary = "Utility library for gitignore style pattern matching of file paths." +groups = ["default"] +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + +[[package]] +name = "platformdirs" +version = "4.2.0" +requires_python = ">=3.8" +summary = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +groups = ["default"] +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[[package]] +name = "pluggy" +version = "1.4.0" +requires_python = ">=3.8" +summary = "plugin and hook calling mechanisms for python" +groups = ["default"] +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[[package]] +name = "pycodestyle" +version = "2.11.1" +requires_python = ">=3.8" +summary = "Python style guide checker" +groups = ["default"] +files = [ + {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"}, + {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"}, +] + +[[package]] +name = "pyflakes" +version = "3.2.0" +requires_python = ">=3.8" +summary = "passive checker of Python programs" +groups = ["default"] +files = [ + {file = "pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a"}, + {file = "pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f"}, +] + +[[package]] +name = "pytest" +version = "8.0.2" +requires_python = ">=3.8" +summary = "pytest: simple powerful testing with Python" +groups = ["default"] +dependencies = [ + "colorama; sys_platform == \"win32\"", + "exceptiongroup>=1.0.0rc8; python_version < \"3.11\"", + "iniconfig", + "packaging", + "pluggy<2.0,>=1.3.0", + "tomli>=1.0.0; python_version < \"3.11\"", +] +files = [ + {file = "pytest-8.0.2-py3-none-any.whl", hash = "sha256:edfaaef32ce5172d5466b5127b42e0d6d35ebbe4453f0e3505d96afd93f6b096"}, + {file = "pytest-8.0.2.tar.gz", hash = "sha256:d4051d623a2e0b7e51960ba963193b09ce6daeb9759a451844a21e4ddedfc1bd"}, +] + +[[package]] +name = "pytest-cov" +version = "4.1.0" +requires_python = ">=3.7" +summary = "Pytest plugin for measuring coverage." +groups = ["default"] +dependencies = [ + "coverage[toml]>=5.2.1", + "pytest>=4.6", +] +files = [ + {file = "pytest-cov-4.1.0.tar.gz", hash = "sha256:3904b13dfbfec47f003b8e77fd5b589cd11904a21ddf1ab38a64f204d6a10ef6"}, + {file = "pytest_cov-4.1.0-py3-none-any.whl", hash = "sha256:6ba70b9e97e69fcc3fb45bfeab2d0a138fb65c4d0d6a41ef33983ad114be8c3a"}, +] + +[[package]] +name = "pytest-timeout" +version = "2.2.0" +requires_python = ">=3.7" +summary = "pytest plugin to abort hanging tests" +groups = ["default"] +dependencies = [ + "pytest>=5.0.0", +] +files = [ + {file = "pytest-timeout-2.2.0.tar.gz", hash = "sha256:3b0b95dabf3cb50bac9ef5ca912fa0cfc286526af17afc806824df20c2f72c90"}, + {file = "pytest_timeout-2.2.0-py3-none-any.whl", hash = "sha256:bde531e096466f49398a59f2dde76fa78429a09a12411466f88a07213e220de2"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +requires_python = ">=3.7" +summary = "A lil' TOML parser" +groups = ["default"] +marker = "python_version < \"3.11\"" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.10.0" +requires_python = ">=3.8" +summary = "Backported and Experimental Type Hints for Python 3.8+" +groups = ["default"] +files = [ + {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, + {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, +] diff --git a/kraken-build/src/kraken/std/git/version.py b/kraken-build/src/kraken/std/git/version.py index 73e5eb2f..15d97e91 100644 --- a/kraken-build/src/kraken/std/git/version.py +++ b/kraken-build/src/kraken/std/git/version.py @@ -7,13 +7,17 @@ from pathlib import Path +class NotAGitRepositoryError(Exception): + pass + + def git_describe(path: Path | None, tags: bool = True, dirty: bool = True) -> str: """Describe a repository with tags. :param path: The directory in which to describe. :param tags: Whether to include tags (adds the `--tags` flag). :param dirty: Whether to include if the directory tree is dirty (adds the `--dirty` flag). - :raise ValueError: If `git describe` failed. + :raise NotAGitRepositoryError: If the directory is not a git repository. :return: The Git head description. """ @@ -23,8 +27,11 @@ def git_describe(path: Path | None, tags: bool = True, dirty: bool = True) -> st if dirty: command.append("--dirty") try: - return sp.check_output(command, cwd=path).decode().strip() - except sp.CalledProcessError: + return sp.check_output(command, cwd=path, stderr=sp.PIPE).decode().strip() + except sp.CalledProcessError as exc: + stderr = exc.stderr.decode() + if "not a git repository" in stderr: + raise NotAGitRepositoryError(path) count = int(sp.check_output(["git", "rev-list", "HEAD", "--count"], cwd=path).decode().strip()) short_rev = sp.check_output(["git", "rev-parse", "--short", "HEAD"], cwd=path).decode().strip() return f"0.0.0-{count}-g{short_rev}" diff --git a/kraken-build/src/kraken/std/python/project.py b/kraken-build/src/kraken/std/python/project.py index 54b53423..d41f2b61 100644 --- a/kraken-build/src/kraken/std/python/project.py +++ b/kraken-build/src/kraken/std/python/project.py @@ -4,12 +4,16 @@ import re from collections.abc import Sequence from pathlib import Path +from typing import Literal from nr.stream import Optional +from kraken.std.git.version import GitVersion, NotAGitRepositoryError, git_describe from kraken.std.python.buildsystem import detect_build_system from kraken.std.python.pyproject import PackageIndex +from kraken.std.python.settings import python_settings from kraken.std.python.tasks.pytest_task import CoverageFormat +from kraken.std.python.version import git_version_to_python_version logger = logging.getLogger(__name__) @@ -66,6 +70,8 @@ def python_project( additional_lint_directories: Sequence[str] | None = None, exclude_lint_directories: Sequence[str] = (), line_length: int = 120, + enforce_project_version: str | None = None, + detect_git_version_build_type: Literal["release", "develop", "branch"] = "develop", pyupgrade_keep_runtime_typing: bool = False, pycln_remove_all_unused_imports: bool = False, pytest_ignore_dirs: Sequence[str] = (), @@ -105,11 +111,19 @@ def python_project( linted and formatted but would otherwise be included via *source_directory*, *tests_directory* and *additional_directories*. line_length: The line length to assume for all formatters and linters. + enforce_project_version: When set, enforces the specified version number for the project when building wheels + and publishing them. If not specified, the version number will be derived from the Git repository using + `git describe --tags`. + detect_git_version_build_type: When specified, this influences the version number that will be automatically + generated when `enforce_project_version` is not set. When set to `"release"`, the current commit in + the Git repository must have exactly one tag associated with it. When set to `"develop"` (the default), + the version number will be derived from the most recent tag and the distance to the current commit. If + the current commit is tagged, the version number will be the tag name anyway. When set to `"branch"`, the + version number will be derived from the distance to the most recent tag and include the SHA of the commit. pyupgrade_keep_runtime_typing: Whether to not replace `typing` type hints. This is required for example for projects using Typer as it does not support all modern type hints at runtime. pycln_remove_all_unused_imports: Remove all unused imports, including these with side effects. For reference, see https://hadialqattan.github.io/pycln/#/?id=-a-all-flag. - flake8_additional_requirements: Additional Python requirements to install alongside Flake8. This should be used to add Flake8 plugins. flake8_extend_ignore: Flake8 lints to ignore. The default ignores lints that would otherwise conflict with @@ -121,12 +135,14 @@ def python_project( from .pyproject import Pyproject from .tasks.black_task import BlackConfig, black as black_tasks + from .tasks.build_task import build as build_task from .tasks.flake8_task import Flake8Config, flake8 as flake8_tasks from .tasks.info_task import info as info_task from .tasks.install_task import install as install_task from .tasks.isort_task import IsortConfig, isort as isort_tasks from .tasks.login_task import login as login_task from .tasks.mypy_task import MypyConfig, mypy as mypy_task + from .tasks.publish_task import publish as publish_task from .tasks.pycln_task import pycln as pycln_task from .tasks.pytest_task import pytest as pytest_task from .tasks.pyupgrade_task import pyupgrade as pyupgrade_task @@ -233,10 +249,35 @@ def python_project( allow_no_tests=True, ) + if not enforce_project_version: + try: + git_version = GitVersion.parse(git_describe(project.directory)) + except NotAGitRepositoryError: + logger.info("No Git repository found in %s, not enforcing a project version", project.directory) + enforce_project_version = None + else: + match detect_git_version_build_type: + case "release" | "develop": + if ( + detect_git_version_build_type == "release" + and git_version.distance + and git_version.distance.value > 0 + ): + raise ValueError( + f"Git version for project {project.directory} is not a release version: {git_version}" + ) + enforce_project_version = git_version_to_python_version(git_version) + case "branch": + enforce_project_version = git_version_to_python_version(git_version, include_sha=True) + + if enforce_project_version: + logger.info("Enforcing version %s for project %s", enforce_project_version, project.directory) + + # Create publish tasks for any package index with publish=True. + build = build_task(as_version=enforce_project_version) + for index in python_settings().package_indexes.values(): + if index.publish: + publish_task(package_index=index.alias, distributions=build.output_files, interactive=False) + # TODO(@niklas): Support auto-detecting when Mypy stubtests need to be run or # accept arguments for stubtests. - - # version: str | None = None, - # version: The current version number of the package. If not specified, it will be automatically derived - # from Git using `git describe --tags`. Kraken will automatically bump the version number on-disk for - # you for relevant operations so you can keep the version number in your `pyproject.toml` at `0.0.0`. diff --git a/kraken-build/src/kraken/std/python/version.py b/kraken-build/src/kraken/std/python/version.py index 90a6b3f3..0d50dd86 100644 --- a/kraken-build/src/kraken/std/python/version.py +++ b/kraken-build/src/kraken/std/python/version.py @@ -9,7 +9,7 @@ } -def git_version_to_python_version(value: str | GitVersion, include_sha: bool) -> str: +def git_version_to_python_version(value: str | GitVersion, include_sha: bool = False) -> str: """Converts a Git version to a Python version. :param value: The Git version to convert. diff --git a/kraken-build/tests/kraken_std/integration/python/test_python.py b/kraken-build/tests/kraken_std/integration/python/test_python.py index 48f9b3df..caab66c2 100644 --- a/kraken-build/tests/kraken_std/integration/python/test_python.py +++ b/kraken-build/tests/kraken_std/integration/python/test_python.py @@ -105,10 +105,8 @@ def test__python_project_install_lint_and_publish( logger.info("Loading and executing Kraken project (%s)", tempdir / consumer_dir) Context.__init__(kraken_ctx, kraken_ctx.build_directory) kraken_ctx.load_project(directory=tempdir / consumer_dir) - # NOTE: The Slap project doesn't need an apply because we don't write the package index into the pyproject.toml. kraken_ctx.execute([":apply"]) - kraken_ctx.execute([":python.install"]) # TODO (@NiklasRosenstein): Test importing the consumer project. From 833c78d982c05da0900b894ad312fb15e1d094d4 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 27 Feb 2024 19:05:20 +0100 Subject: [PATCH 19/79] remove linter config from pyproject.toml --- kraken-build/pyproject.toml | 37 ----------------------------------- kraken-wrapper/pyproject.toml | 27 ------------------------- 2 files changed, 64 deletions(-) diff --git a/kraken-build/pyproject.toml b/kraken-build/pyproject.toml index e86b5cf4..439ae853 100644 --- a/kraken-build/pyproject.toml +++ b/kraken-build/pyproject.toml @@ -72,46 +72,9 @@ mkdocs-material = "*" novella = "0.2.6" pydoc-markdown = "^4.6.0" -# Slap configuration -# ------------------ - [tool.slap] typed = true -# Linter/Formatter configuration -# ------------------------------ - -[tool.mypy] -explicit_package_bases = true -mypy_path = ["src"] -namespace_packages = true -pretty = true -python_version = "3.10" -show_error_codes = true -show_error_context = true -strict = true -warn_no_return = true -warn_redundant_casts = true -warn_unreachable = true -warn_unused_ignores = true - -[[tool.mypy.overrides]] -ignore_errors = true -ignore_missing_imports = true -module = "networkx.*" - -[[tool.mypy.overrides]] -warn_unused_ignores = false -module = "dill" - -[tool.isort] -combine_as_imports = true -line_length = 120 -profile = "black" - -[tool.black] -line-length = 120 - [tool.pytest.ini_options] markers = [ "integration", diff --git a/kraken-wrapper/pyproject.toml b/kraken-wrapper/pyproject.toml index 91dbc597..e969b22b 100644 --- a/kraken-wrapper/pyproject.toml +++ b/kraken-wrapper/pyproject.toml @@ -41,33 +41,6 @@ krakenw = "kraken.wrapper.main:main" [tool.slap] typed = true - - -# Linter/Formatter configuration -# ------------------------------ - -[tool.mypy] -explicit_package_bases = true -mypy_path = ["src"] -namespace_packages = true -pretty = true -python_version = "3.10" -show_error_codes = true -show_error_context = true -strict = true -warn_no_return = true -warn_redundant_casts = true -warn_unreachable = true -warn_unused_ignores = true - -[tool.isort] -combine_as_imports = true -line_length = 120 -profile = "black" - -[tool.black] -line-length = 120 - [tool.pytest.ini_options] markers = [ "integration", From df57e6b6905e03c9b103de44d96b669f9710074d Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 27 Feb 2024 19:19:56 +0100 Subject: [PATCH 20/79] pass line length to flake and virtual-env/src paths to isort --- kraken-build/src/kraken/std/python/project.py | 4 +++- .../src/kraken/std/python/tasks/flake8_task.py | 2 ++ .../src/kraken/std/python/tasks/isort_task.py | 12 +++++++++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/kraken-build/src/kraken/std/python/project.py b/kraken-build/src/kraken/std/python/project.py index d41f2b61..b10cc047 100644 --- a/kraken-build/src/kraken/std/python/project.py +++ b/kraken-build/src/kraken/std/python/project.py @@ -216,7 +216,9 @@ def python_project( flake8 = flake8_tasks( paths=source_paths, - config=Flake8Config(extend_ignore=flake8_extend_ignore, exclude=exclude_lint_directories), + config=Flake8Config( + max_line_length=line_length, extend_ignore=flake8_extend_ignore, exclude=exclude_lint_directories + ), version_spec=flake8_version_spec, additional_requirements=flake8_additional_requirements, ) diff --git a/kraken-build/src/kraken/std/python/tasks/flake8_task.py b/kraken-build/src/kraken/std/python/tasks/flake8_task.py index 5d71b444..29598a47 100644 --- a/kraken-build/src/kraken/std/python/tasks/flake8_task.py +++ b/kraken-build/src/kraken/std/python/tasks/flake8_task.py @@ -17,6 +17,7 @@ @dataclass class Flake8Config: + max_line_length: int extend_ignore: Sequence[str] exclude: Sequence[str] @@ -38,6 +39,7 @@ def dump(self, project_root: Path, config_file_dir: Path) -> ConfigParser: config = ConfigParser() config.add_section(flake8_section) + config.set(flake8_section, "max-line-length", str(self.max_line_length)) config.set(flake8_section, "extend-ignore", ",".join(self.extend_ignore)) config.set(flake8_section, "exclude", ",".join(exclude)) diff --git a/kraken-build/src/kraken/std/python/tasks/isort_task.py b/kraken-build/src/kraken/std/python/tasks/isort_task.py index 24c41d5e..0a20775f 100644 --- a/kraken-build/src/kraken/std/python/tasks/isort_task.py +++ b/kraken-build/src/kraken/std/python/tasks/isort_task.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from collections.abc import Sequence from configparser import ConfigParser from dataclasses import dataclass @@ -56,6 +57,15 @@ def get_execute_command(self) -> list[str]: command += ["--check-only", "--diff"] if self.__config_file: command += ["--settings-file", str(self.__config_file)] + + # When running isort from a PEX binary, we have to point it explicitly at the virtual env + # and source directories to ensure it knows what imports are first, second and third party. + if venv := os.getenv("VIRTUAL_ENV"): + command += ["--virtual-env", venv] + settings = python_settings(project=self.project) + for path in filter(None, [settings.source_directory, settings.get_tests_directory()]): + command += ["--src", str(path)] + return command # Task @@ -72,7 +82,7 @@ def prepare(self) -> TaskStatus | None: if config is not None and config_file is not None: raise RuntimeError("IsortTask.config and .config_file cannot be mixed") if config is not None: - config_file = self.project.build_directory / self.name / "isort.ini" + config_file = self.project.build_directory.absolute() / self.name / "isort.ini" config_file.parent.mkdir(parents=True, exist_ok=True) config.to_file(config_file) self.__config_file = config_file From ea3c688036c33595cf63a0f2e594f260f78e28d3 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 27 Feb 2024 19:51:18 +0100 Subject: [PATCH 21/79] use python_project() for kraken itself --- .kraken.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.kraken.py b/.kraken.py index 8043de1d..8186475e 100644 --- a/.kraken.py +++ b/.kraken.py @@ -1,4 +1,5 @@ from kraken.common import buildscript +from kraken.std.python.project import python_project buildscript(requirements=["kraken-build>=0.33.2"]) @@ -99,4 +100,5 @@ def configure_project() -> None: for subproject in [project.subproject("kraken-build"), project.subproject("kraken-wrapper")]: with subproject.as_current(): - configure_project() + python_project(black_version_spec="==23.12.1", isort_version_spec="==5.13.2") + # configure_project() From 8a34fa8e0f1b6a61631e57eb8afa37b62a37beab Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 27 Feb 2024 20:54:29 +0100 Subject: [PATCH 22/79] fix mypy path --- kraken-build/src/kraken/core/address/_address_test.py | 2 +- kraken-build/src/kraken/core/cli/serialize.py | 2 +- kraken-build/src/kraken/std/python/project.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kraken-build/src/kraken/core/address/_address_test.py b/kraken-build/src/kraken/core/address/_address_test.py index a64e0214..859644d3 100644 --- a/kraken-build/src/kraken/core/address/_address_test.py +++ b/kraken-build/src/kraken/core/address/_address_test.py @@ -1,6 +1,6 @@ from __future__ import annotations -import dill # type: ignore[import-untyped] +import dill from pytest import raises from kraken.core.address import Address diff --git a/kraken-build/src/kraken/core/cli/serialize.py b/kraken-build/src/kraken/core/cli/serialize.py index 6be6bbb5..1afda09e 100644 --- a/kraken-build/src/kraken/core/cli/serialize.py +++ b/kraken-build/src/kraken/core/cli/serialize.py @@ -5,7 +5,7 @@ from collections.abc import Iterable from pathlib import Path -import dill # type: ignore[import-untyped] +import dill from kraken.common import pluralize from kraken.core import Context, TaskGraph diff --git a/kraken-build/src/kraken/std/python/project.py b/kraken-build/src/kraken/std/python/project.py index b10cc047..821e43e9 100644 --- a/kraken-build/src/kraken/std/python/project.py +++ b/kraken-build/src/kraken/std/python/project.py @@ -227,7 +227,7 @@ def python_project( mypy_task( paths=source_paths, config=MypyConfig( - mypy_path=source_directory, + mypy_path=[source_directory], exclude_directories=exclude_lint_directories, global_overrides={}, module_overrides={}, From d6271ef64328c02a097598ca6d64637450cc6cfa Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Wed, 6 Mar 2024 23:35:03 +0100 Subject: [PATCH 23/79] improvement: Raise `EmptyGitRepositoryError` and `NotAGitRepositoryError` respectively in `git_describe()` --- .changelog/_unreleased.toml | 6 ++++++ kraken-build/src/kraken/std/git/version.py | 15 ++++++++++++--- kraken-build/src/kraken/std/python/project.py | 5 ++++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index 4d4432aa..29cce565 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -55,3 +55,9 @@ type = "feature" description = "Add shellcheck capabilities" author = "niklas.rosenstein@helsing.ai" component = "kraken-build" + +[[entries]] +id = "4cb0fe80-d359-4302-8893-c0fd62bf0631" +type = "improvement" +description = "Raise `EmptyGitRepositoryError` and `NotAGitRepositoryError` respectively in `git_describe()`" +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/std/git/version.py b/kraken-build/src/kraken/std/git/version.py index 15d97e91..78abe53f 100644 --- a/kraken-build/src/kraken/std/git/version.py +++ b/kraken-build/src/kraken/std/git/version.py @@ -11,6 +11,10 @@ class NotAGitRepositoryError(Exception): pass +class EmptyGitRepositoryError(Exception): + pass + + def git_describe(path: Path | None, tags: bool = True, dirty: bool = True) -> str: """Describe a repository with tags. @@ -32,9 +36,14 @@ def git_describe(path: Path | None, tags: bool = True, dirty: bool = True) -> st stderr = exc.stderr.decode() if "not a git repository" in stderr: raise NotAGitRepositoryError(path) - count = int(sp.check_output(["git", "rev-list", "HEAD", "--count"], cwd=path).decode().strip()) - short_rev = sp.check_output(["git", "rev-parse", "--short", "HEAD"], cwd=path).decode().strip() - return f"0.0.0-{count}-g{short_rev}" + try: + count = int(sp.check_output(["git", "rev-list", "HEAD", "--count"], stderr=sp.PIPE, cwd=path).decode().strip()) + except sp.CalledProcessError as exc: + stderr = exc.stderr.decode() + if "unknown revision" in stderr: + raise EmptyGitRepositoryError(path) + short_rev = sp.check_output(["git", "rev-parse", "--short", "HEAD"], cwd=path).decode().strip() + return f"0.0.0-{count}-g{short_rev}" @dataclasses.dataclass diff --git a/kraken-build/src/kraken/std/python/project.py b/kraken-build/src/kraken/std/python/project.py index 821e43e9..494bfa57 100644 --- a/kraken-build/src/kraken/std/python/project.py +++ b/kraken-build/src/kraken/std/python/project.py @@ -8,7 +8,7 @@ from nr.stream import Optional -from kraken.std.git.version import GitVersion, NotAGitRepositoryError, git_describe +from kraken.std.git.version import EmptyGitRepositoryError, GitVersion, NotAGitRepositoryError, git_describe from kraken.std.python.buildsystem import detect_build_system from kraken.std.python.pyproject import PackageIndex from kraken.std.python.settings import python_settings @@ -257,6 +257,9 @@ def python_project( except NotAGitRepositoryError: logger.info("No Git repository found in %s, not enforcing a project version", project.directory) enforce_project_version = None + except EmptyGitRepositoryError: + logger.info("Empty Git repository found in %s, not enforcing a project version", project.directory) + enforce_project_version = None else: match detect_git_version_build_type: case "release" | "develop": From a1fe14fbdfa3a3d20172e32ae59013a2254f8a9c Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Wed, 6 Mar 2024 23:45:13 +0100 Subject: [PATCH 24/79] fix passing pyupgrade_bin to the format task --- kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py b/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py index 9e2095c8..59c58cb0 100644 --- a/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py @@ -141,7 +141,7 @@ def pyupgrade( check_task.python_version = python_version format_task = project.task(name, PyUpgradeTask, group="fmt") - check_task.pyupgrade_bin = pyupgrade_bin + format_task.pyupgrade_bin = pyupgrade_bin format_task.additional_files = filtered_files format_task.keep_runtime_typing = keep_runtime_typing format_task.python_version = python_version From 23917e11faa175c395853989fcb8917a23edc4d8 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Thu, 7 Mar 2024 00:33:05 +0100 Subject: [PATCH 25/79] improvement: Add order-only dependencies between check and format tasks created for Python projects --- .changelog/_unreleased.toml | 5 +++++ kraken-build/src/kraken/std/python/tasks/black_task.py | 3 +++ kraken-build/src/kraken/std/python/tasks/isort_task.py | 3 +++ kraken-build/src/kraken/std/python/tasks/pycln_task.py | 3 +++ kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py | 3 +++ 5 files changed, 17 insertions(+) create mode 100644 .changelog/_unreleased.toml diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml new file mode 100644 index 00000000..43e8b47d --- /dev/null +++ b/.changelog/_unreleased.toml @@ -0,0 +1,5 @@ +[[entries]] +id = "d9440e6f-b19b-4b49-bad4-dc7860228cf8" +type = "improvement" +description = "Add order-only dependencies between check and format tasks created for Python projects" +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/std/python/tasks/black_task.py b/kraken-build/src/kraken/std/python/tasks/black_task.py index a329a73e..8f903e6b 100644 --- a/kraken-build/src/kraken/std/python/tasks/black_task.py +++ b/kraken-build/src/kraken/std/python/tasks/black_task.py @@ -147,4 +147,7 @@ def black( format_task.additional_args = additional_args format_task.paths = paths + # When we run both, it makes no sense to run the check task before the format task. + check_task.depends_on(format_task, mode="order-only") + return BlackTasks(check_task, format_task) diff --git a/kraken-build/src/kraken/std/python/tasks/isort_task.py b/kraken-build/src/kraken/std/python/tasks/isort_task.py index 0a20775f..c62cf712 100644 --- a/kraken-build/src/kraken/std/python/tasks/isort_task.py +++ b/kraken-build/src/kraken/std/python/tasks/isort_task.py @@ -140,4 +140,7 @@ def isort( format_task.config_file = config_file format_task.paths = paths + # When we run both, it makes no sense to run the check task before the format task. + check_task.depends_on(format_task, mode="order-only") + return IsortTasks(check_task, format_task) diff --git a/kraken-build/src/kraken/std/python/tasks/pycln_task.py b/kraken-build/src/kraken/std/python/tasks/pycln_task.py index bb831458..b60309c9 100644 --- a/kraken-build/src/kraken/std/python/tasks/pycln_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pycln_task.py @@ -99,4 +99,7 @@ def pycln( format_task.pycln_bin = pycln_bin format_task.additional_args = additional_args + # When we run both, it makes no sense to run the check task before the format task. + check_task.depends_on(format_task, mode="order-only") + return PyclnTasks(check_task, format_task) diff --git a/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py b/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py index 59c58cb0..304b91d3 100644 --- a/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pyupgrade_task.py @@ -146,4 +146,7 @@ def pyupgrade( format_task.keep_runtime_typing = keep_runtime_typing format_task.python_version = python_version + # When we run both, it makes no sense to run the check task before the format task. + check_task.depends_on(format_task, mode="order-only") + return PyUpgradeTasks(check_task, format_task) From db84932abb9e579e9b28edd5df5cdf0ccb073858 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Sun, 10 Mar 2024 14:06:30 +0100 Subject: [PATCH 26/79] improvement: `build_docker_image(dockerfile)` now treats the path relative to the current project --- .changelog/_unreleased.toml | 6 ++++++ kraken-build/src/kraken/std/docker/__init__.py | 8 ++++++-- kraken-build/src/kraken/std/python/project.py | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index 43e8b47d..4dc27d4f 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -3,3 +3,9 @@ id = "d9440e6f-b19b-4b49-bad4-dc7860228cf8" type = "improvement" description = "Add order-only dependencies between check and format tasks created for Python projects" author = "@NiklasRosenstein" + +[[entries]] +id = "84e8fb46-7aef-4dfc-b4ec-5ad507c82544" +type = "improvement" +description = "`build_docker_image(dockerfile)` now treats the path relative to the current project" +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/std/docker/__init__.py b/kraken-build/src/kraken/std/docker/__init__.py index 1f966521..af4461b9 100644 --- a/kraken-build/src/kraken/std/docker/__init__.py +++ b/kraken-build/src/kraken/std/docker/__init__.py @@ -1,10 +1,11 @@ from __future__ import annotations from collections.abc import Sequence +from pathlib import Path from typing import Any from kraken.common import import_class -from kraken.core import Project, Task +from kraken.core import Project, Task, Property from kraken.std.docker.tasks.base_build_task import BaseBuildTask from kraken.std.docker.tasks.manifest_tool_push_task import ManifestToolPushTask @@ -23,12 +24,15 @@ def build_docker_image( name: str = "buildDocker", backend: str = DEFAULT_BUILD_BACKEND, project: Project | None = None, + dockerfile: str | Path | Property[Path] = "Dockerfile", **kwds: Any, ) -> BaseBuildTask: """Create a new task in the current project that builds a Docker image and eventually pushes it.""" + project = project or Project.current() task_class = import_class(BUILD_BACKENDS[backend], BaseBuildTask) # type: ignore[type-abstract] - return (project or Project.current()).do(name, task_class, **kwds) + dockerfile = project.directory / dockerfile + return project.do(name, task_class, dockerfile=dockerfile, **kwds) def manifest_tool( diff --git a/kraken-build/src/kraken/std/python/project.py b/kraken-build/src/kraken/std/python/project.py index 494bfa57..49871167 100644 --- a/kraken-build/src/kraken/std/python/project.py +++ b/kraken-build/src/kraken/std/python/project.py @@ -80,7 +80,7 @@ def python_project( black_version_spec: str = ">=24.1.1,<25.0.0", flake8_version_spec: str = ">=7.0.0,<8.0.0", flake8_additional_requirements: Sequence[str] = (), - flake8_extend_ignore: Sequence[str] = ("W503", "W504", "E203", "E704"), + flake8_extend_ignore: Sequence[str] = ("W503", "W504", "E203", "E704", "E701"), mypy_version_spec: str = ">=1.8.0,<2.0.0", pycln_version_spec: str = ">=2.4.0,<3.0.0", pyupgrade_version_spec: str = ">=3.15.0,<4.0.0", From 7c5462f5cc1bc882595f0a1211bf9f9030ad5a4b Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Sun, 10 Mar 2024 14:25:44 +0100 Subject: [PATCH 27/79] fix --- kraken-build/src/kraken/std/docker/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kraken-build/src/kraken/std/docker/__init__.py b/kraken-build/src/kraken/std/docker/__init__.py index af4461b9..2d9a56c4 100644 --- a/kraken-build/src/kraken/std/docker/__init__.py +++ b/kraken-build/src/kraken/std/docker/__init__.py @@ -4,12 +4,12 @@ from pathlib import Path from typing import Any -from kraken.common import import_class -from kraken.core import Project, Task, Property +from kraken.common import Supplier, import_class +from kraken.core import Project, Task from kraken.std.docker.tasks.base_build_task import BaseBuildTask from kraken.std.docker.tasks.manifest_tool_push_task import ManifestToolPushTask -__all__ = ["build_docker_image", "manifest_tool", "sidecar_container"] +__all__ = ["build_docker_image", "manifest_tool"] DEFAULT_BUILD_BACKEND = "native" BUILD_BACKENDS = { @@ -24,14 +24,14 @@ def build_docker_image( name: str = "buildDocker", backend: str = DEFAULT_BUILD_BACKEND, project: Project | None = None, - dockerfile: str | Path | Property[Path] = "Dockerfile", + dockerfile: str | Path | Supplier[Path] = "Dockerfile", **kwds: Any, ) -> BaseBuildTask: """Create a new task in the current project that builds a Docker image and eventually pushes it.""" project = project or Project.current() task_class = import_class(BUILD_BACKENDS[backend], BaseBuildTask) # type: ignore[type-abstract] - dockerfile = project.directory / dockerfile + dockerfile = Supplier.of(dockerfile).map(project.directory.joinpath) return project.do(name, task_class, dockerfile=dockerfile, **kwds) From 7e7ffddeb1229f18dbb326392076137743a56192 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Sun, 17 Mar 2024 05:08:05 +0100 Subject: [PATCH 28/79] update .kraken.py, fix mypy, black 24 fmt --- .kraken.py | 101 +----------------- kraken-build/.kraken.py | 2 + kraken-build/src/kraken/common/_fs.py | 6 +- kraken-build/src/kraken/common/_importlib.py | 6 +- kraken-build/src/kraken/common/_runner.py | 3 +- kraken-build/src/kraken/common/_text.py | 3 +- .../src/kraken/core/system/context.py | 6 +- .../kraken/core/system/executor/__init__.py | 30 ++---- .../kraken/core/system/executor/default.py | 6 +- .../src/kraken/core/system/project_test.py | 12 +-- kraken-build/src/kraken/core/system/task.py | 6 +- .../src/kraken/std/docker/__init__.py | 4 +- .../src/kraken/std/python/buildsystem/pdm.py | 4 +- .../kraken/std/python/tasks/flake8_task.py | 1 - kraken-wrapper/.kraken.py | 2 + .../src/kraken/wrapper/_buildenv_uv.py | 3 +- 16 files changed, 37 insertions(+), 158 deletions(-) create mode 100644 kraken-build/.kraken.py create mode 100644 kraken-wrapper/.kraken.py diff --git a/.kraken.py b/.kraken.py index 8186475e..65369249 100644 --- a/.kraken.py +++ b/.kraken.py @@ -1,104 +1,9 @@ from kraken.common import buildscript -from kraken.std.python.project import python_project buildscript(requirements=["kraken-build>=0.33.2"]) -import os - -from kraken.std import python -from kraken.std.git import git_describe - - -def configure_project() -> None: - from kraken.build import project - - python.pyupgrade(python_version="3.10", version_spec="==3.15.0") - python.pycln(version_spec="==2.4.0") - - python.black(additional_args=["--config", "pyproject.toml"], version_spec="==23.12.1") - python.flake8(version_spec="==6.1.0") - python.isort(version_spec="==5.13.2") - python.mypy(additional_args=["--exclude", "src/tests/integration/.*/data/.*"], version_spec="==1.8.0") - - if project.directory.joinpath("tests").is_dir(): - # Explicit list of test directories, Pytest skips the build directory if not specified explicitly. - python.pytest(ignore_dirs=["src/tests/integration"], include_dirs=["src/kraken/build"]) - - if project.directory.joinpath("tests/integration").is_dir(): - python.pytest( - name="pytestIntegration", - tests_dir="src/tests/integration", - ignore_dirs=["src/tests/integration/python/data"], - group="integrationTest", - ) - - python.install() - python.info() - - ( - python.python_settings() - .add_package_index( - "pypi", - credentials=(os.environ["PYPI_USER"], os.environ["PYPI_PASSWORD"]) if "PYPI_USER" in os.environ else None, - ) - .add_package_index( - "testpypi", - credentials=(os.environ["TESTPYPI_USER"], os.environ["TESTPYPI_PASSWORD"]) - if "TESTPYPI_USER" in os.environ - else None, - ) - ) - - do_publish = True - as_version: str | None = None - is_release = False - if "CI" in os.environ: - if os.environ["GITHUB_REF_TYPE"] == "tag": - # TODO (@NiklasRosenstein): It would be nice to add a test that checks if the version in the package - # is consistent (ie. run `slap release --validate `). - is_release = True - as_version = os.environ["GITHUB_REF_NAME"] - elif os.environ["GITHUB_REF_TYPE"] == "branch": - if os.environ["GITHUB_REF_NAME"] == "develop": - as_version = python.git_version_to_python(git_describe(project.directory), include_sha=False) - else: - # NOTE (@NiklasRosenstein): PyPI/TestPypi cannot use PEP 440 local versions (which the version with - # included SHA would qualify as), so we don't publish from branches at all. - do_publish = False - else: - raise EnvironmentError( - f"GITHUB_REF_TYPE={os.environ['GITHUB_REF_TYPE']}, GITHUB_REF_NAME={os.environ['GITHUB_REF_NAME']}" - ) - else: - do_publish = False - as_version = python.git_version_to_python(git_describe(project.directory), include_sha=False) - - build_task = python.build(as_version=as_version) - - if do_publish: - testpypi = python.publish( - name="publishToTestPypi", - package_index="testpypi", - distributions=build_task.output_files, - skip_existing=True, - ) - if is_release: - python.publish( - name="publishToPypi", - package_index="pypi", - distributions=build_task.output_files, - after=[testpypi], - ) - - from kraken.build import project -try: - project.subproject("docs") -except ImportError: - pass - -for subproject in [project.subproject("kraken-build"), project.subproject("kraken-wrapper")]: - with subproject.as_current(): - python_project(black_version_spec="==23.12.1", isort_version_spec="==5.13.2") - # configure_project() +project.subproject("docs") +project.subproject("kraken-build") +project.subproject("kraken-wrapper") diff --git a/kraken-build/.kraken.py b/kraken-build/.kraken.py new file mode 100644 index 00000000..60d6e165 --- /dev/null +++ b/kraken-build/.kraken.py @@ -0,0 +1,2 @@ +from kraken.std.python.project import python_project +python_project() diff --git a/kraken-build/src/kraken/common/_fs.py b/kraken-build/src/kraken/common/_fs.py index 0655b91a..56b9eb80 100644 --- a/kraken-build/src/kraken/common/_fs.py +++ b/kraken-build/src/kraken/common/_fs.py @@ -14,8 +14,7 @@ def atomic_file_swap( mode: Literal["w"], always_revert: bool = ..., create_dirs: bool = ..., -) -> ContextManager[TextIO]: - ... +) -> ContextManager[TextIO]: ... @overload @@ -24,8 +23,7 @@ def atomic_file_swap( mode: Literal["wb"], always_revert: bool = ..., create_dirs: bool = ..., -) -> ContextManager[BinaryIO]: - ... +) -> ContextManager[BinaryIO]: ... @contextlib.contextmanager # type: ignore diff --git a/kraken-build/src/kraken/common/_importlib.py b/kraken-build/src/kraken/common/_importlib.py index 59bf814d..aa8363c6 100644 --- a/kraken-build/src/kraken/common/_importlib.py +++ b/kraken-build/src/kraken/common/_importlib.py @@ -8,13 +8,11 @@ @overload -def import_class(fqn: str) -> type: - ... +def import_class(fqn: str) -> type: ... @overload -def import_class(fqn: str, base_type: type[T]) -> type[T]: - ... +def import_class(fqn: str, base_type: type[T]) -> type[T]: ... def import_class(fqn: str, base_type: "Type[T] | None" = None) -> "Type[T] | type": diff --git a/kraken-build/src/kraken/common/_runner.py b/kraken-build/src/kraken/common/_runner.py index 5a2f0681..0076ab13 100644 --- a/kraken-build/src/kraken/common/_runner.py +++ b/kraken-build/src/kraken/common/_runner.py @@ -68,8 +68,7 @@ class ProjectFinder(ABC): """ @abstractmethod - def find_project(self, directory: Path) -> "ProjectInfo | None": - ... + def find_project(self, directory: Path) -> "ProjectInfo | None": ... ## diff --git a/kraken-build/src/kraken/common/_text.py b/kraken-build/src/kraken/common/_text.py index c456ee58..ff47df94 100644 --- a/kraken-build/src/kraken/common/_text.py +++ b/kraken-build/src/kraken/common/_text.py @@ -7,8 +7,7 @@ class SupportsLen(Protocol): - def __len__(self) -> int: - ... + def __len__(self) -> int: ... def pluralize(word: str, count: "int | SupportsLen") -> str: diff --git a/kraken-build/src/kraken/core/system/context.py b/kraken-build/src/kraken/core/system/context.py index c02e8501..bd9435d5 100644 --- a/kraken-build/src/kraken/core/system/context.py +++ b/kraken-build/src/kraken/core/system/context.py @@ -384,12 +384,10 @@ def execute(self, tasks: list[str | Task] | TaskGraph | None = None) -> None: @overload def listen( self, event_type: str | ContextEvent.Type - ) -> Callable[[ContextEvent.T_Listener], ContextEvent.T_Listener]: - ... + ) -> Callable[[ContextEvent.T_Listener], ContextEvent.T_Listener]: ... @overload - def listen(self, event_type: str | ContextEvent.Type, listener: ContextEvent.Listener) -> None: - ... + def listen(self, event_type: str | ContextEvent.Type, listener: ContextEvent.Listener) -> None: ... def listen(self, event_type: str | ContextEvent.Type, listener: ContextEvent.Listener | None = None) -> Any: """Registers a listener to the context for the given event type.""" diff --git a/kraken-build/src/kraken/core/system/executor/__init__.py b/kraken-build/src/kraken/core/system/executor/__init__.py index aa64b92d..a73f3711 100644 --- a/kraken-build/src/kraken/core/system/executor/__init__.py +++ b/kraken-build/src/kraken/core/system/executor/__init__.py @@ -56,35 +56,25 @@ def tasks( class GraphExecutorObserver(abc.ABC): """Observes events in a Kraken task executor.""" - def before_execute_graph(self, graph: Graph) -> None: - ... + def before_execute_graph(self, graph: Graph) -> None: ... - def before_prepare_task(self, task: Task) -> None: - ... + def before_prepare_task(self, task: Task) -> None: ... - def after_prepare_task(self, task: Task, status: TaskStatus) -> None: - ... + def after_prepare_task(self, task: Task, status: TaskStatus) -> None: ... - def before_execute_task(self, task: Task, status: TaskStatus) -> None: - ... + def before_execute_task(self, task: Task, status: TaskStatus) -> None: ... - def on_task_output(self, task: Task, chunk: bytes) -> None: - ... + def on_task_output(self, task: Task, chunk: bytes) -> None: ... - def after_execute_task(self, task: Task, status: TaskStatus) -> None: - ... + def after_execute_task(self, task: Task, status: TaskStatus) -> None: ... - def before_teardown_task(self, task: Task) -> None: - ... + def before_teardown_task(self, task: Task) -> None: ... - def after_teardown_task(self, task: Task, status: TaskStatus) -> None: - ... + def after_teardown_task(self, task: Task, status: TaskStatus) -> None: ... - def after_execute_graph(self, graph: Graph) -> None: - ... + def after_execute_graph(self, graph: Graph) -> None: ... class GraphExecutor(abc.ABC): @abc.abstractmethod - def execute_graph(self, graph: Graph, observer: GraphExecutorObserver) -> None: - ... + def execute_graph(self, graph: Graph, observer: GraphExecutorObserver) -> None: ... diff --git a/kraken-build/src/kraken/core/system/executor/default.py b/kraken-build/src/kraken/core/system/executor/default.py index 926f38a7..980a3dc7 100644 --- a/kraken-build/src/kraken/core/system/executor/default.py +++ b/kraken-build/src/kraken/core/system/executor/default.py @@ -18,12 +18,10 @@ class TaskExecutor(abc.ABC): @abc.abstractmethod - def execute_task(self, task: Task, done: Callable[[TaskStatus], None]) -> None: - ... + def execute_task(self, task: Task, done: Callable[[TaskStatus], None]) -> None: ... @abc.abstractmethod - def teardown_task(self, task: Task, done: Callable[[TaskStatus], None]) -> None: - ... + def teardown_task(self, task: Task, done: Callable[[TaskStatus], None]) -> None: ... class DefaultTaskExecutor(TaskExecutor): diff --git a/kraken-build/src/kraken/core/system/project_test.py b/kraken-build/src/kraken/core/system/project_test.py index 7f17727e..2f359365 100644 --- a/kraken-build/src/kraken/core/system/project_test.py +++ b/kraken-build/src/kraken/core/system/project_test.py @@ -21,8 +21,7 @@ def test__Project__resolve_outputs__can_find_dataclass_in_properties(kraken_proj class MyTask(Task): out_prop: Property[MyDescriptor] = Property.output() - def execute(self) -> None: - ... + def execute(self) -> None: ... task = kraken_project.task("carrier", MyTask) task.out_prop = MyDescriptor("foobar") @@ -33,8 +32,7 @@ def test__Project__resolve_outputs__can_not_find_input_property(kraken_project: class MyTask(Task): out_prop: Property[MyDescriptor] - def execute(self) -> None: - ... + def execute(self) -> None: ... task = kraken_project.task("carrier", MyTask) task.out_prop = MyDescriptor("foobar") @@ -45,8 +43,7 @@ def test__Project__resolve_outputs_supplier(kraken_project: Project) -> None: class MyTask(Task): out_prop: Property[MyDescriptor] = Property.output() - def execute(self) -> None: - ... + def execute(self) -> None: ... task = kraken_project.task("carrier", MyTask) task.out_prop = MyDescriptor("foobar") @@ -69,8 +66,7 @@ def test__Project__do__does_not_set_property_on_None_value(kraken_project: Proje class MyTask(Task): in_prop: Property[str] - def execute(self) -> None: - ... + def execute(self) -> None: ... kraken_project.task("carrier", MyTask) assert kraken_project.resolve_tasks(":carrier").select(str).supplier().get() == [] diff --git a/kraken-build/src/kraken/core/system/task.py b/kraken-build/src/kraken/core/system/task.py index ab4585d4..56275fac 100644 --- a/kraken-build/src/kraken/core/system/task.py +++ b/kraken-build/src/kraken/core/system/task.py @@ -744,12 +744,10 @@ def __iter__(self) -> Iterable[str]: return iter(self._ptt) @overload - def __getitem__(self, partition: str) -> Collection[Task]: - ... + def __getitem__(self, partition: str) -> Collection[Task]: ... @overload - def __getitem__(self, partition: Task) -> Collection[str]: - ... + def __getitem__(self, partition: Task) -> Collection[str]: ... def __getitem__(self, partition: str | Task) -> Collection[str] | Collection[Task]: if isinstance(partition, str): diff --git a/kraken-build/src/kraken/std/docker/__init__.py b/kraken-build/src/kraken/std/docker/__init__.py index 2d9a56c4..394face5 100644 --- a/kraken-build/src/kraken/std/docker/__init__.py +++ b/kraken-build/src/kraken/std/docker/__init__.py @@ -2,7 +2,7 @@ from collections.abc import Sequence from pathlib import Path -from typing import Any +from typing import Any, cast from kraken.common import Supplier, import_class from kraken.core import Project, Task @@ -31,7 +31,7 @@ def build_docker_image( project = project or Project.current() task_class = import_class(BUILD_BACKENDS[backend], BaseBuildTask) # type: ignore[type-abstract] - dockerfile = Supplier.of(dockerfile).map(project.directory.joinpath) + dockerfile = cast(Supplier[Path | str], Supplier.of(dockerfile)).map(project.directory.joinpath) return project.do(name, task_class, dockerfile=dockerfile, **kwds) diff --git a/kraken-build/src/kraken/std/python/buildsystem/pdm.py b/kraken-build/src/kraken/std/python/buildsystem/pdm.py index 4c28ef28..620cd85a 100644 --- a/kraken-build/src/kraken/std/python/buildsystem/pdm.py +++ b/kraken-build/src/kraken/std/python/buildsystem/pdm.py @@ -95,9 +95,7 @@ def set_package_indexes(self, indexes: Sequence[PackageIndex]) -> None: key=lambda x: ( 0 if x.priority == PackageIndex.Priority.default - else 1 - if x.priority == PackageIndex.Priority.primary - else 2 + else 1 if x.priority == PackageIndex.Priority.primary else 2 ), ) diff --git a/kraken-build/src/kraken/std/python/tasks/flake8_task.py b/kraken-build/src/kraken/std/python/tasks/flake8_task.py index 679c9fc5..29598a47 100644 --- a/kraken-build/src/kraken/std/python/tasks/flake8_task.py +++ b/kraken-build/src/kraken/std/python/tasks/flake8_task.py @@ -4,7 +4,6 @@ from collections.abc import Sequence from configparser import ConfigParser from dataclasses import dataclass -from collections.abc import Sequence from pathlib import Path from kraken.common import Supplier diff --git a/kraken-wrapper/.kraken.py b/kraken-wrapper/.kraken.py new file mode 100644 index 00000000..60d6e165 --- /dev/null +++ b/kraken-wrapper/.kraken.py @@ -0,0 +1,2 @@ +from kraken.std.python.project import python_project +python_project() diff --git a/kraken-wrapper/src/kraken/wrapper/_buildenv_uv.py b/kraken-wrapper/src/kraken/wrapper/_buildenv_uv.py index 46e1da52..34890f68 100644 --- a/kraken-wrapper/src/kraken/wrapper/_buildenv_uv.py +++ b/kraken-wrapper/src/kraken/wrapper/_buildenv_uv.py @@ -7,8 +7,7 @@ if TYPE_CHECKING: - def find_uv_bin() -> str: - ... + def find_uv_bin() -> str: ... else: from uv.__main__ import find_uv_bin From 30a4d38b5b97ddd761ec2c598c6a9fe0cea10ac1 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Sun, 17 Mar 2024 05:09:10 +0100 Subject: [PATCH 29/79] disable krakenw-based tests for now and re-instate selftests --- .github/workflows/on-commit.yaml | 132 +++++++++++++++---------------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/.github/workflows/on-commit.yaml b/.github/workflows/on-commit.yaml index 0f60f4c2..307b25ce 100644 --- a/.github/workflows/on-commit.yaml +++ b/.github/workflows/on-commit.yaml @@ -68,96 +68,96 @@ jobs: # == Unit tests, linting, and type checking == - test: - runs-on: arc-amd64-small - strategy: - fail-fast: false - matrix: - python-version: ["3.10", "3.11", "3.12", "3.x"] - tasks: ["check lint", "test"] - steps: - - uses: actions/checkout@v4 - - uses: NiklasRosenstein/slap@gha/install/v1 - - uses: actions/setup-python@v5 - with: { python-version: "${{ matrix.python-version }}" } - - run: pip install pipx && pipx install poetry && pipx install pdm && pipx install kraken-wrapper==0.34.1 && krakenw config --installer=UV - - run: rustup update - - - name: Restore Kraken build cache - uses: actions/cache/restore@v4 - with: - path: build - key: build-cache:${{ runner.os }}:${{ hashFiles('.kraken.lock') }} - - - name: Restore Venv - uses: actions/cache/restore@v4 - with: - path: .venvs/ - key: build-cache:${{ runner.os }}:venv:${{ matrix.python-version }} - - # Explicitly mention python.install to ensure that Pip install is rerun. - - run: krakenw run python.install ${{ matrix.tasks }} -vv - - - name: Save Venv - uses: actions/cache/save@v4 - with: - path: .venvs/ - key: build-cache:${{ runner.os }}:venv:${{ matrix.python-version }} - - - name: Save Kraken build cache - uses: actions/cache/save@v4 - with: - path: build - key: build-cache:${{ runner.os }}:${{ hashFiles('.kraken.lock') }} - - # == Try running Kraken as defined in the kraken.yaml file == - - # selftest: + # test: # runs-on: arc-amd64-small # strategy: # fail-fast: false # matrix: # python-version: ["3.10", "3.11", "3.12", "3.x"] + # tasks: ["check lint", "test"] # steps: # - uses: actions/checkout@v4 # - uses: NiklasRosenstein/slap@gha/install/v1 # - uses: actions/setup-python@v5 # with: { python-version: "${{ matrix.python-version }}" } - # - run: pip install pipx && pipx install poetry && pipx install pdm + # - run: pip install pipx && pipx install poetry && pipx install pdm && pipx install kraken-wrapper==0.34.1 && krakenw config --installer=UV # - run: rustup update - # - name: Restore cache + # - name: Restore Kraken build cache + # uses: actions/cache/restore@v4 + # with: + # path: build + # key: build-cache:${{ runner.os }}:${{ hashFiles('.kraken.lock') }} + + # - name: Restore Venv # uses: actions/cache/restore@v4 # with: - # path: | - # build - # .venvs - # key: build-cache:${{ runner.os }}:selftest + # path: .venvs/ + # key: build-cache:${{ runner.os }}:venv:${{ matrix.python-version }} - # - run: slap config --venv-type=uv && slap install --link --no-venv-check ${{ matrix.only }} + # # Explicitly mention python.install to ensure that Pip install is rerun. + # - run: krakenw run python.install ${{ matrix.tasks }} -vv - # - run: kraken run python.install fmt lint test -vv - # - run: kraken q ls - # - run: kraken q tree - # - run: kraken q viz - # - run: kraken q d python.mypy + # - name: Save Venv + # uses: actions/cache/save@v4 + # with: + # path: .venvs/ + # key: build-cache:${{ runner.os }}:venv:${{ matrix.python-version }} - # - name: Save cache + # - name: Save Kraken build cache # uses: actions/cache/save@v4 # with: - # path: | - # build - # .venvs - # key: build-cache:${{ runner.os }}:selftest + # path: build + # key: build-cache:${{ runner.os }}:${{ hashFiles('.kraken.lock') }} - uv-installer: + # == Try running Kraken as defined in the kraken.yaml file == + + selftest: runs-on: arc-amd64-small + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.x"] steps: - uses: actions/checkout@v4 - uses: NiklasRosenstein/slap@gha/install/v1 - - run: slap config --venv-type=uv && slap install --link --no-venv-check - - run: krakenw --reinstall --use=UV - - run: krakenw run fmt lint + - uses: actions/setup-python@v5 + with: { python-version: "${{ matrix.python-version }}" } + - run: pip install pipx && pipx install poetry && pipx install pdm + - run: rustup update + + - name: Restore cache + uses: actions/cache/restore@v4 + with: + path: | + build + .venvs + key: build-cache:${{ runner.os }}:selftest + + - run: slap config --venv-type=uv && slap install --link --no-venv-check ${{ matrix.only }} + + - run: kraken run python.install fmt lint test -vv + - run: kraken q ls + - run: kraken q tree + - run: kraken q viz + - run: kraken q d python.mypy + + - name: Save cache + uses: actions/cache/save@v4 + with: + path: | + build + .venvs + key: build-cache:${{ runner.os }}:selftest + + # uv-installer: + # runs-on: arc-amd64-small + # steps: + # - uses: actions/checkout@v4 + # - uses: NiklasRosenstein/slap@gha/install/v1 + # - run: slap config --venv-type=uv && slap install --link --no-venv-check + # - run: krakenw --reinstall --use=UV + # - run: krakenw run fmt lint examples-docker-manual: runs-on: arc-amd64-small From 4f1ad4e2c9bae8e2dd9594a5c818ad0bcd17743b Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Sun, 17 Mar 2024 23:45:26 +0100 Subject: [PATCH 30/79] add mitmproxy back as dev dependency --- kraken-build/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/kraken-build/pyproject.toml b/kraken-build/pyproject.toml index d576f867..d9d13375 100644 --- a/kraken-build/pyproject.toml +++ b/kraken-build/pyproject.toml @@ -66,6 +66,7 @@ types-Deprecated = "^1.2.9" types-requests = "^2.28.0" types-termcolor = "^1.1.5" pytest-xdist = {version = "^3.5.0", extras = ["psutil"]} +mitmproxy = "^10.2.3" [tool.poetry.group.docs.dependencies] mkdocs = "*" From b2a6bf52a400cad66189d270042f4b234a58d49c Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Mon, 18 Mar 2024 00:08:07 +0100 Subject: [PATCH 31/79] fix slap-project and make it use python_project() --- examples/slap-project/.kraken.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/examples/slap-project/.kraken.py b/examples/slap-project/.kraken.py index 4161906d..77eb5c3e 100644 --- a/examples/slap-project/.kraken.py +++ b/examples/slap-project/.kraken.py @@ -1,16 +1,11 @@ import os -from kraken.std import python +from kraken.std.python.project import python_package_index, python_project -python.python_settings(always_use_managed_env=True).add_package_index( +python_package_index( alias="local", index_url=os.environ["LOCAL_PACKAGE_INDEX"], credentials=(os.environ["LOCAL_USER"], os.environ["LOCAL_PASSWORD"]), + publish=True, ) -python.install() -python.mypy() -python.flake8() -python.black() -python.isort() -python.pytest() -python.publish(package_index="local", distributions=python.build(as_version="0.1.0").output_files) +python_project() From e1560b3c5467f373a6fb9aa30475097b204902da Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Mon, 18 Mar 2024 00:10:43 +0100 Subject: [PATCH 32/79] add `python_project()` and `python_package_index()` to `kraken.std` as public API --- examples/slap-project/.kraken.py | 2 +- kraken-build/src/kraken/std/__init__.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/slap-project/.kraken.py b/examples/slap-project/.kraken.py index 77eb5c3e..62cca8fb 100644 --- a/examples/slap-project/.kraken.py +++ b/examples/slap-project/.kraken.py @@ -1,6 +1,6 @@ import os -from kraken.std.python.project import python_package_index, python_project +from kraken.std import python_package_index, python_project python_package_index( alias="local", diff --git a/kraken-build/src/kraken/std/__init__.py b/kraken-build/src/kraken/std/__init__.py index a51f1e97..33b1b52f 100644 --- a/kraken-build/src/kraken/std/__init__.py +++ b/kraken-build/src/kraken/std/__init__.py @@ -1 +1,5 @@ __version__ = "0.35.6" + +from kraken.std.python.project import python_package_index, python_project + +__all__ = ["python_package_index", "python_project"] From 7713dfa1b154826d1b7e57c9cd64f32bde1d4386 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Mon, 18 Mar 2024 00:14:55 +0100 Subject: [PATCH 33/79] use python_project() for all examples --- examples/pdm-project-consumer/.kraken.py | 8 +++----- examples/pdm-project/.kraken.py | 8 +++----- examples/poetry-project-consumer/.kraken.py | 8 +++----- examples/poetry-project/.kraken.py | 9 ++++----- examples/rust-pdm-project-consumer/.kraken.py | 8 +++----- examples/rust-pdm-project/.kraken.py | 13 +++++++------ examples/rust-poetry-project-consumer/.kraken.py | 8 +++----- examples/rust-poetry-project/.kraken.py | 13 +++++++------ examples/slap-project-consumer/.kraken.py | 11 +++-------- examples/slap-project/.kraken.py | 2 +- 10 files changed, 37 insertions(+), 51 deletions(-) diff --git a/examples/pdm-project-consumer/.kraken.py b/examples/pdm-project-consumer/.kraken.py index 1bb1a4b4..5eafa744 100644 --- a/examples/pdm-project-consumer/.kraken.py +++ b/examples/pdm-project-consumer/.kraken.py @@ -1,12 +1,10 @@ import os -from kraken.std import python +from kraken.std import python_package_index, python_project -python.python_settings(always_use_managed_env=True).add_package_index( +python_package_index( alias="local", index_url=os.environ["LOCAL_PACKAGE_INDEX"], credentials=(os.environ["LOCAL_USER"], os.environ["LOCAL_PASSWORD"]), ) -python.install() -python.mypy() -python.update_pyproject_task() +python_project(enforce_project_version="0.1.0") diff --git a/examples/pdm-project/.kraken.py b/examples/pdm-project/.kraken.py index 64c72a7a..850f2ebb 100644 --- a/examples/pdm-project/.kraken.py +++ b/examples/pdm-project/.kraken.py @@ -1,13 +1,11 @@ import os -from kraken.std.python.project import python_project, python_package_index +from kraken.std import python_package_index, python_project python_package_index( alias="local", index_url=os.environ["LOCAL_PACKAGE_INDEX"], - is_package_source=False, - publish=True, credentials=(os.environ["LOCAL_USER"], os.environ["LOCAL_PASSWORD"]), + publish=True, ) - -python_project() +python_project(enforce_project_version="0.1.0") diff --git a/examples/poetry-project-consumer/.kraken.py b/examples/poetry-project-consumer/.kraken.py index 1bb1a4b4..5eafa744 100644 --- a/examples/poetry-project-consumer/.kraken.py +++ b/examples/poetry-project-consumer/.kraken.py @@ -1,12 +1,10 @@ import os -from kraken.std import python +from kraken.std import python_package_index, python_project -python.python_settings(always_use_managed_env=True).add_package_index( +python_package_index( alias="local", index_url=os.environ["LOCAL_PACKAGE_INDEX"], credentials=(os.environ["LOCAL_USER"], os.environ["LOCAL_PASSWORD"]), ) -python.install() -python.mypy() -python.update_pyproject_task() +python_project(enforce_project_version="0.1.0") diff --git a/examples/poetry-project/.kraken.py b/examples/poetry-project/.kraken.py index 2eadfdcc..850f2ebb 100644 --- a/examples/poetry-project/.kraken.py +++ b/examples/poetry-project/.kraken.py @@ -1,12 +1,11 @@ import os -from kraken.std import python +from kraken.std import python_package_index, python_project -python.python_settings(always_use_managed_env=True).add_package_index( +python_package_index( alias="local", index_url=os.environ["LOCAL_PACKAGE_INDEX"], credentials=(os.environ["LOCAL_USER"], os.environ["LOCAL_PASSWORD"]), + publish=True, ) -python.install() -python.mypy() -python.publish(package_index="local", distributions=python.build(as_version="0.1.0").output_files) +python_project(enforce_project_version="0.1.0") diff --git a/examples/rust-pdm-project-consumer/.kraken.py b/examples/rust-pdm-project-consumer/.kraken.py index 1bb1a4b4..5eafa744 100644 --- a/examples/rust-pdm-project-consumer/.kraken.py +++ b/examples/rust-pdm-project-consumer/.kraken.py @@ -1,12 +1,10 @@ import os -from kraken.std import python +from kraken.std import python_package_index, python_project -python.python_settings(always_use_managed_env=True).add_package_index( +python_package_index( alias="local", index_url=os.environ["LOCAL_PACKAGE_INDEX"], credentials=(os.environ["LOCAL_USER"], os.environ["LOCAL_PASSWORD"]), ) -python.install() -python.mypy() -python.update_pyproject_task() +python_project(enforce_project_version="0.1.0") diff --git a/examples/rust-pdm-project/.kraken.py b/examples/rust-pdm-project/.kraken.py index 8a7f8c62..c4709eae 100644 --- a/examples/rust-pdm-project/.kraken.py +++ b/examples/rust-pdm-project/.kraken.py @@ -1,13 +1,14 @@ import os -from kraken.std import python +from kraken.std import python_package_index, python_project +from kraken.std.python import mypy_stubtest -python.python_settings(always_use_managed_env=True).add_package_index( +python_package_index( alias="local", index_url=os.environ["LOCAL_PACKAGE_INDEX"], credentials=(os.environ["LOCAL_USER"], os.environ["LOCAL_PASSWORD"]), + publish=True, ) -python.install() -python.mypy() -python.mypy_stubtest(package="rust_pdm_project", ignore_missing_stubs=True) -python.publish(package_index="local", distributions=python.build(as_version="0.1.0").output_files) +python_project(enforce_project_version="0.1.0") + +mypy_stubtest(package="rust_pdm_project", ignore_missing_stubs=True) diff --git a/examples/rust-poetry-project-consumer/.kraken.py b/examples/rust-poetry-project-consumer/.kraken.py index 1bb1a4b4..5eafa744 100644 --- a/examples/rust-poetry-project-consumer/.kraken.py +++ b/examples/rust-poetry-project-consumer/.kraken.py @@ -1,12 +1,10 @@ import os -from kraken.std import python +from kraken.std import python_package_index, python_project -python.python_settings(always_use_managed_env=True).add_package_index( +python_package_index( alias="local", index_url=os.environ["LOCAL_PACKAGE_INDEX"], credentials=(os.environ["LOCAL_USER"], os.environ["LOCAL_PASSWORD"]), ) -python.install() -python.mypy() -python.update_pyproject_task() +python_project(enforce_project_version="0.1.0") diff --git a/examples/rust-poetry-project/.kraken.py b/examples/rust-poetry-project/.kraken.py index 746c6f85..49c335c3 100644 --- a/examples/rust-poetry-project/.kraken.py +++ b/examples/rust-poetry-project/.kraken.py @@ -1,13 +1,14 @@ import os -from kraken.std import python +from kraken.std import python_package_index, python_project +from kraken.std.python import mypy_stubtest -python.python_settings(always_use_managed_env=True).add_package_index( +python_package_index( alias="local", index_url=os.environ["LOCAL_PACKAGE_INDEX"], credentials=(os.environ["LOCAL_USER"], os.environ["LOCAL_PASSWORD"]), + publish=True, ) -python.install() -python.mypy() -python.mypy_stubtest(package="rust_poetry_project", ignore_missing_stubs=True) -python.publish(package_index="local", distributions=python.build(as_version="0.1.0").output_files) +python_project(enforce_project_version="0.1.0") + +mypy_stubtest(package="rust_poetry_project", ignore_missing_stubs=True) diff --git a/examples/slap-project-consumer/.kraken.py b/examples/slap-project-consumer/.kraken.py index 601aedfc..5eafa744 100644 --- a/examples/slap-project-consumer/.kraken.py +++ b/examples/slap-project-consumer/.kraken.py @@ -1,15 +1,10 @@ import os -from kraken.std import python +from kraken.std import python_package_index, python_project -python.python_settings(always_use_managed_env=True).add_package_index( +python_package_index( alias="local", index_url=os.environ["LOCAL_PACKAGE_INDEX"], credentials=(os.environ["LOCAL_USER"], os.environ["LOCAL_PASSWORD"]), ) -python.install() -python.mypy() -python.flake8() -python.black() -python.isort() -python.pytest() +python_project(enforce_project_version="0.1.0") diff --git a/examples/slap-project/.kraken.py b/examples/slap-project/.kraken.py index 62cca8fb..850f2ebb 100644 --- a/examples/slap-project/.kraken.py +++ b/examples/slap-project/.kraken.py @@ -8,4 +8,4 @@ credentials=(os.environ["LOCAL_USER"], os.environ["LOCAL_PASSWORD"]), publish=True, ) -python_project() +python_project(enforce_project_version="0.1.0") From ce301b23b7694bbef905842b6eb71546d4bf0d15 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Mon, 18 Mar 2024 00:35:36 +0100 Subject: [PATCH 34/79] bump python versions to 3.10 (Mypy will complain in its latest version that 3.7 is no longer supported) --- examples/lint-enforced-directories-project/pyproject.toml | 2 +- examples/poetry-project-consumer/pyproject.toml | 2 +- examples/poetry-project/pyproject.toml | 2 +- examples/rust-poetry-project-consumer/pyproject.toml | 2 +- examples/rust-poetry-project/pyproject.toml | 2 +- examples/version-project/pyproject.toml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/lint-enforced-directories-project/pyproject.toml b/examples/lint-enforced-directories-project/pyproject.toml index 7aea6846..00709b68 100644 --- a/examples/lint-enforced-directories-project/pyproject.toml +++ b/examples/lint-enforced-directories-project/pyproject.toml @@ -5,7 +5,7 @@ description = "" authors = ["Josh Cowling "] [tool.poetry.dependencies] -python = "^3.7" +python = "^3.10" [tool.poetry.dev-dependencies] black = "*" diff --git a/examples/poetry-project-consumer/pyproject.toml b/examples/poetry-project-consumer/pyproject.toml index b7e24cd0..988d797f 100644 --- a/examples/poetry-project-consumer/pyproject.toml +++ b/examples/poetry-project-consumer/pyproject.toml @@ -5,7 +5,7 @@ description = "" authors = ["Niklas Rosenstein "] [tool.poetry.dependencies] -python = "^3.7" +python = "^3.10" poetry-project = { version = "0.1.0", source = "local" } [tool.poetry.dev-dependencies] diff --git a/examples/poetry-project/pyproject.toml b/examples/poetry-project/pyproject.toml index f34ce79d..579dc1f7 100644 --- a/examples/poetry-project/pyproject.toml +++ b/examples/poetry-project/pyproject.toml @@ -5,7 +5,7 @@ description = "" authors = ["Niklas Rosenstein "] [tool.poetry.dependencies] -python = "^3.7" +python = "^3.10" [tool.poetry.dev-dependencies] mypy = "^0.971" diff --git a/examples/rust-poetry-project-consumer/pyproject.toml b/examples/rust-poetry-project-consumer/pyproject.toml index 53331e13..349d9a97 100644 --- a/examples/rust-poetry-project-consumer/pyproject.toml +++ b/examples/rust-poetry-project-consumer/pyproject.toml @@ -5,7 +5,7 @@ description = "" authors = [] [tool.poetry.dependencies] -python = "^3.7" +python = "^3.10" rust-poetry-project = { version = "0.1.0", source = "local" } [tool.poetry.dev-dependencies] diff --git a/examples/rust-poetry-project/pyproject.toml b/examples/rust-poetry-project/pyproject.toml index e6b33194..2b8e985c 100644 --- a/examples/rust-poetry-project/pyproject.toml +++ b/examples/rust-poetry-project/pyproject.toml @@ -6,7 +6,7 @@ authors = [] include = ["target/"] [tool.poetry.dependencies] -python = "^3.7" +python = "^3.10" [tool.poetry.dev-dependencies] mypy = "^1.0" diff --git a/examples/version-project/pyproject.toml b/examples/version-project/pyproject.toml index 9f8b6f08..bb1323fa 100644 --- a/examples/version-project/pyproject.toml +++ b/examples/version-project/pyproject.toml @@ -10,7 +10,7 @@ packages = [ ] [tool.poetry.dependencies] -python = "^3.7" +python = "^3.10" [build-system] requires = [ From 9b90ee93ac1b5e900c82d6cb468edd3b79f5b6b8 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Mon, 18 Mar 2024 00:50:53 +0100 Subject: [PATCH 35/79] update python version requirement in more example projects --- examples/rust-pdm-project-consumer/pyproject.toml | 2 +- examples/rust-pdm-project/pyproject.toml | 2 +- examples/slap-project/pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/rust-pdm-project-consumer/pyproject.toml b/examples/rust-pdm-project-consumer/pyproject.toml index e8f0ee8b..8935c865 100644 --- a/examples/rust-pdm-project-consumer/pyproject.toml +++ b/examples/rust-pdm-project-consumer/pyproject.toml @@ -4,7 +4,7 @@ version = "0.1.0" dependencies = [ "rust-pdm-project" ] -requires-python = "~=3.7" +requires-python = "~=3.10" [build-system] requires = ["pdm-backend"] diff --git a/examples/rust-pdm-project/pyproject.toml b/examples/rust-pdm-project/pyproject.toml index 860bd01b..c17e90ae 100644 --- a/examples/rust-pdm-project/pyproject.toml +++ b/examples/rust-pdm-project/pyproject.toml @@ -3,7 +3,7 @@ name = "rust-pdm-project" version = "0.1.0" description = "" authors = [] -requires-python = ">=3.7" +requires-python = ">=3.10" [tool.pdm.dev-dependencies] build = ["maturin~=1.0", "pip~=23.0"] diff --git a/examples/slap-project/pyproject.toml b/examples/slap-project/pyproject.toml index f52380e7..ba43aded 100644 --- a/examples/slap-project/pyproject.toml +++ b/examples/slap-project/pyproject.toml @@ -20,7 +20,7 @@ keywords = [] # Repository = "" [tool.poetry.dependencies] -python = "^3.6" +python = "^3.10" [tool.poetry.dev-dependencies] pytest = "*" From 7e15977dd9fb22144e9276eb06efb507584d62eb Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Mon, 18 Mar 2024 00:51:53 +0100 Subject: [PATCH 36/79] improvement: Override `MaturinPoetryPyprojectHandler.get_python_version_constraint()` to fall back on `[project.version]` key if `[tool.poetry.version]` is not set --- .changelog/_unreleased.toml | 6 ++++++ kraken-build/src/kraken/std/python/buildsystem/maturin.py | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index 4dc27d4f..17cce1f0 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -9,3 +9,9 @@ id = "84e8fb46-7aef-4dfc-b4ec-5ad507c82544" type = "improvement" description = "`build_docker_image(dockerfile)` now treats the path relative to the current project" author = "@NiklasRosenstein" + +[[entries]] +id = "788398a7-320e-45b2-9590-d977387c2ed1" +type = "improvement" +description = "Override `MaturinPoetryPyprojectHandler.get_python_version_constraint()` to fall back on `[project.version]` key if `[tool.poetry.version]` is not set" +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/std/python/buildsystem/maturin.py b/kraken-build/src/kraken/std/python/buildsystem/maturin.py index cae298dd..fb43cd4e 100644 --- a/kraken-build/src/kraken/std/python/buildsystem/maturin.py +++ b/kraken-build/src/kraken/std/python/buildsystem/maturin.py @@ -152,6 +152,11 @@ def synchronize_project_section_to_poetry_state(self) -> None: else: project_section[field_name] = poetry_value + def get_python_version_constraint(self) -> str | None: + return PoetryPyprojectHandler.get_python_version_constraint( + self + ) or PyprojectHandler.get_python_version_constraint(self) + class MaturinPoetryPythonBuildSystem(PoetryPythonBuildSystem): """A maturin-backed version of the Poetry build system, that invokes the maturin build-backend. From 1e2a456f182979e14f55c91b202111fd9a07eb76 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Mon, 18 Mar 2024 00:52:45 +0100 Subject: [PATCH 37/79] feature: Add `pex_set_global_store_path()` function --- .changelog/_unreleased.toml | 6 ++++ .../kraken/std/python/tasks/pex_build_task.py | 35 ++++++++++++------- kraken-build/tests/kraken_std/conftest.py | 5 +++ 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index 17cce1f0..b27222c1 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -15,3 +15,9 @@ id = "788398a7-320e-45b2-9590-d977387c2ed1" type = "improvement" description = "Override `MaturinPoetryPyprojectHandler.get_python_version_constraint()` to fall back on `[project.version]` key if `[tool.poetry.version]` is not set" author = "@NiklasRosenstein" + +[[entries]] +id = "ef21ce61-3e39-4177-9182-72c67e80be63" +type = "feature" +description = "Add `pex_set_global_store_path()` function" +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/std/python/tasks/pex_build_task.py b/kraken-build/src/kraken/std/python/tasks/pex_build_task.py index 061d4848..493d513c 100644 --- a/kraken-build/src/kraken/std/python/tasks/pex_build_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pex_build_task.py @@ -13,7 +13,8 @@ from kraken.std.util.url import inject_url_credentials, redact_url_password logger = logging.getLogger(__name__) -default_index_url: str | None = None +_default_index_url: str | None = None +_global_store_path: Path | None = None class PexBuildTask(Task): @@ -48,16 +49,18 @@ def _get_output_file_path(self) -> Path: ] ).encode() ).hexdigest() - return ( - self.project.context.build_directory - / ".store" - / f"{hashsum}-{self.binary_name.get()}" - / self.binary_name.get() - ).with_suffix(".pex") + + if _global_store_path: + store_path = _global_store_path + else: + store_path = self.project.context.build_directory / ".store" + + return (store_path / f"{hashsum}-{self.binary_name.get()}" / self.binary_name.get()).with_suffix(".pex") def prepare(self) -> TaskStatus | None: - self.output_file = self._get_output_file_path().absolute() - if self.output_file.get().exists(): + if not self.output_file.is_set(): + self.output_file = self._get_output_file_path().absolute() + if not self.always_rebuild.get() and self.output_file.get().exists(): return TaskStatus.skipped(f"PEX `{self.binary_name.get()}` already exists ({self.output_file.get()})") return TaskStatus.pending() @@ -185,8 +188,16 @@ def pex_build( def pex_set_default_index_url(url: str) -> None: """Set the default index URL for Pex globally.""" - global default_index_url - default_index_url = url + global _default_index_url + _default_index_url = url + + +def pex_set_global_store_path(path: Path | None) -> None: + """Set the global Pex store path. This can be used to override the default Pex store directory, which is + inside the current context's build directory.""" + + global _global_store_path + _global_store_path = path def _get_default_index_url(project: Project | None) -> str | None: @@ -201,7 +212,7 @@ def _get_default_index_url(project: Project | None) -> str | None: if idx.priority == PackageIndex.Priority.default: break else: - return default_index_url + return _default_index_url if idx.credentials: return inject_url_credentials(idx.index_url, *idx.credentials) diff --git a/kraken-build/tests/kraken_std/conftest.py b/kraken-build/tests/kraken_std/conftest.py index 93444bc3..0e03c688 100644 --- a/kraken-build/tests/kraken_std/conftest.py +++ b/kraken-build/tests/kraken_std/conftest.py @@ -6,6 +6,7 @@ import pytest +from kraken.std.python.tasks.pex_build_task import pex_set_global_store_path from tests.kraken_std.util.docker import DockerServiceManager @@ -29,3 +30,7 @@ def chdir_context(path: Path) -> Iterator[None]: yield finally: os.chdir(cwd) + + +# Speed up building PEX's in test. +pex_set_global_store_path(Path(__file__).parent.parent.parent / "build/.store") From 57ac00b69757f8ca4a9e5cb4265f83747bc6a738 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Mon, 18 Mar 2024 00:53:09 +0100 Subject: [PATCH 38/79] also lint consumer project in python integration test --- kraken-build/tests/kraken_std/integration/python/test_python.py | 1 + 1 file changed, 1 insertion(+) diff --git a/kraken-build/tests/kraken_std/integration/python/test_python.py b/kraken-build/tests/kraken_std/integration/python/test_python.py index 9222d6d6..8a496448 100644 --- a/kraken-build/tests/kraken_std/integration/python/test_python.py +++ b/kraken-build/tests/kraken_std/integration/python/test_python.py @@ -121,6 +121,7 @@ def test__python_project_install_lint_and_publish( print() kraken_ctx.execute([":python.install"]) + kraken_ctx.execute([":lint"]) # TODO (@NiklasRosenstein): Test importing the consumer project. From f441c7b1a688ea1f83601f2454b370eaf19c5f3f Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Mon, 18 Mar 2024 00:58:51 +0100 Subject: [PATCH 39/79] feature: Add `python_app()` function to build a PEX from your Python project. --- .changelog/_unreleased.toml | 12 +++++ kraken-build/src/kraken/std/python/project.py | 45 +++++++++++++++++++ .../kraken/std/python/tasks/pex_build_task.py | 7 +++ 3 files changed, 64 insertions(+) diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index b27222c1..026ddc05 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -21,3 +21,15 @@ id = "ef21ce61-3e39-4177-9182-72c67e80be63" type = "feature" description = "Add `pex_set_global_store_path()` function" author = "@NiklasRosenstein" + +[[entries]] +id = "79e9748a-4d81-4636-be2e-3772b150c976" +type = "feature" +description = "Add `python_app()` function to build a PEX from your Python project." +author = "@NiklasRosenstein" + +[[entries]] +id = "897d94dc-79e7-45bf-abba-bd346a3fb7d5" +type = "feature" +description = "Add `PexBuildTask.always_rebuild` property and add `pex_build(always_rebuild, output_file)` parameters" +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/std/python/project.py b/kraken-build/src/kraken/std/python/project.py index 49871167..0ca44835 100644 --- a/kraken-build/src/kraken/std/python/project.py +++ b/kraken-build/src/kraken/std/python/project.py @@ -286,3 +286,48 @@ def python_project( # TODO(@niklas): Support auto-detecting when Mypy stubtests need to be run or # accept arguments for stubtests. + + +def python_app( + *, + app_name: str, + entry_point: str | None = None, + console_script: str | None = None, + interpreter_constraint: str | None = None, + venv_mode: Literal["append", "prepend"] | None = None, + name: str = "build-pex", +) -> None: + """Build a PEX binary from a Python application. + + This function must be called after `python_project()`. + + Args: + app_name: The name of the applicaiton. This will be used as the binary output filename. The output PEX + file will be written into the build directory. + entry_point: The Python entrypoint to run when the PEX application is invoked. + console_script: The console script to run when the PEX application is invoked. + interpreter_constraint: A Python version specifier that informs the version of Python that the PEX + is built against. If this is not set, the constraint will be deduced from the project (e.g. via + a Python version specifier in `pyproject.toml`). + venv_mode: Whether the virtual env environment variables should be appended/prepended when the PEX runs. + name: Override the default task name. + + Note that `entry_point` and `console_script` are mutually exclusive. + """ + + from kraken.build import project + from kraken.std.python.tasks.pex_build_task import pex_build + + output_file = project.build_directory / "pex" / app_name + + pex_build( + binary_name=app_name, + requirements=[str(project.directory.absolute())], + entry_point=entry_point, + console_script=console_script, + interpreter_constraint=interpreter_constraint, + venv=venv_mode, + always_rebuild=True, + output_file=output_file, + task_name=name, + ) diff --git a/kraken-build/src/kraken/std/python/tasks/pex_build_task.py b/kraken-build/src/kraken/std/python/tasks/pex_build_task.py index 493d513c..355fa3a9 100644 --- a/kraken-build/src/kraken/std/python/tasks/pex_build_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pex_build_task.py @@ -30,6 +30,7 @@ class PexBuildTask(Task): pex_binary: Property[Path | None] = Property.default(None) python: Property[Path | None] = Property.default(None) index_url: Property[str | None] = Property.default(None) + always_rebuild: Property[bool] = Property.default(False) #: The path to the built PEX file will be written to this property. output_file: Property[Path] = Property.output() @@ -153,6 +154,8 @@ def pex_build( interpreter_constraint: str | None = None, venv: Literal["prepend", "append"] | None = None, index_url: str | None = None, + always_rebuild: bool = False, + output_file: Path | None = None, task_name: str | None = None, project: Project | None = None, ) -> PexBuildTask: @@ -171,6 +174,8 @@ def pex_build( and existing_task.interpreter_constraint.get() == interpreter_constraint and existing_task.venv.get() == venv and existing_task.index_url.get() == index_url + and existing_task.always_rebuild.get() == always_rebuild + and existing_task.output_file.get_or(None) == output_file ): return existing_task @@ -182,6 +187,8 @@ def pex_build( task.interpreter_constraint = interpreter_constraint task.venv = venv task.index_url = index_url + task.always_rebuild = always_rebuild + task.output_file = output_file return task From a5ecc69f78dcc620ce19fda4534b705dd7b6aee5 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Mon, 18 Mar 2024 01:20:17 +0100 Subject: [PATCH 40/79] update --- examples/pdm-project/pyproject.toml | 2 +- examples/slap-project-consumer/pyproject.toml | 2 +- .../integration/python/test_python.py | 27 ++++++++++--------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/examples/pdm-project/pyproject.toml b/examples/pdm-project/pyproject.toml index 4f6ef694..7fc207db 100644 --- a/examples/pdm-project/pyproject.toml +++ b/examples/pdm-project/pyproject.toml @@ -9,7 +9,7 @@ dependencies = [ "pytest", "pytest-cov", ] -requires-python = ">=3.9" +requires-python = ">=3.10" readme = "README.md" license = {text = "MIT"} diff --git a/examples/slap-project-consumer/pyproject.toml b/examples/slap-project-consumer/pyproject.toml index 335af784..4921a13d 100644 --- a/examples/slap-project-consumer/pyproject.toml +++ b/examples/slap-project-consumer/pyproject.toml @@ -20,7 +20,7 @@ keywords = [] # Repository = "" [tool.poetry.dependencies] -python = "^3.6" +python = "^3.10" slap-project = { version = "0.1.0", source = "local" } [tool.poetry.dev-dependencies] diff --git a/kraken-build/tests/kraken_std/integration/python/test_python.py b/kraken-build/tests/kraken_std/integration/python/test_python.py index 8a496448..0ac6298f 100644 --- a/kraken-build/tests/kraken_std/integration/python/test_python.py +++ b/kraken-build/tests/kraken_std/integration/python/test_python.py @@ -7,7 +7,6 @@ import unittest.mock from collections.abc import Iterator from pathlib import Path -from typing import TypeVar from unittest.mock import patch import httpx @@ -19,7 +18,7 @@ from kraken.std.python.buildsystem.maturin import MaturinPoetryPyprojectHandler from kraken.std.python.buildsystem.pdm import PdmPyprojectHandler from kraken.std.python.buildsystem.poetry import PoetryPyprojectHandler -from kraken.std.python.pyproject import Pyproject +from kraken.std.python.pyproject import Pyproject, PyprojectHandler from tests.kraken_std.util.docker import DockerServiceManager from tests.resources import example_dir @@ -168,22 +167,20 @@ def test__python_project_upgrade_python_version_string( assert build_as_version == tomli.loads(conf_file.read().decode("UTF-8"))["tool"]["poetry"]["version"] -M = TypeVar("M", PdmPyprojectHandler, PoetryPyprojectHandler) - - @pytest.mark.parametrize( "project_dir, reader, expected_python_version", [ - ("poetry-project", PoetryPyprojectHandler, "^3.7"), - ("slap-project", PoetryPyprojectHandler, "^3.6"), - ("pdm-project", PdmPyprojectHandler, ">=3.9"), - ("rust-poetry-project", MaturinPoetryPyprojectHandler, "^3.7"), + ("poetry-project", PoetryPyprojectHandler, "^3.10"), + ("slap-project", PoetryPyprojectHandler, "^3.10"), + ("pdm-project", PdmPyprojectHandler, ">=3.10"), + ("rust-poetry-project", MaturinPoetryPyprojectHandler, "^3.10"), + ("rust-pdm-project", PdmPyprojectHandler, ">=3.10"), ], ) @unittest.mock.patch.dict(os.environ, {}) def test__python_pyproject_reads_correct_data( project_dir: str, - reader: type[M], + reader: type[PyprojectHandler], expected_python_version: str, kraken_project: Project, ) -> None: @@ -194,13 +191,17 @@ def test__python_pyproject_reads_correct_data( pyproject = Pyproject.read(new_dir / "pyproject.toml") local_build_system = python.buildsystem.detect_build_system(new_dir) assert local_build_system is not None - assert local_build_system.get_pyproject_reader(pyproject) is not None - assert local_build_system.get_pyproject_reader(pyproject).get_name() == project_dir + local = local_build_system.get_pyproject_reader(pyproject) + assert local is not None + + if isinstance(local, PoetryPyprojectHandler): + assert local_build_system.get_pyproject_reader(pyproject).get_name() == project_dir assert local_build_system.get_pyproject_reader(pyproject).get_python_version_constraint() == expected_python_version spec = reader(pyproject) - assert spec.get_name() == project_dir + if isinstance(spec, PoetryPyprojectHandler): + assert spec.get_name() == project_dir assert spec.get_python_version_constraint() == expected_python_version From 6510ee295fcfe26582adcc5217348ddb96992298 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Mon, 18 Mar 2024 01:33:50 +0100 Subject: [PATCH 41/79] fix --- examples/pdm-project-consumer/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pdm-project-consumer/pyproject.toml b/examples/pdm-project-consumer/pyproject.toml index ffb0aff0..2fde03ff 100644 --- a/examples/pdm-project-consumer/pyproject.toml +++ b/examples/pdm-project-consumer/pyproject.toml @@ -8,7 +8,7 @@ authors = [ dependencies = [ "pdm-project" ] -requires-python = ">=3.9" +requires-python = ">=3.10" readme = "README.md" license = {text = "MIT"} From f0dc429a57dbd40ce5d41940de0aca01f99ad1a4 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 19 Mar 2024 22:47:11 +0100 Subject: [PATCH 42/79] Protobuf --- .changelog/_unreleased.toml | 12 +++ .../src/kraken/std/protobuf/__init__.py | 102 ++++++++++++++++-- 2 files changed, 103 insertions(+), 11 deletions(-) diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index 026ddc05..7bfc9752 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -33,3 +33,15 @@ id = "897d94dc-79e7-45bf-abba-bd346a3fb7d5" type = "feature" description = "Add `PexBuildTask.always_rebuild` property and add `pex_build(always_rebuild, output_file)` parameters" author = "@NiklasRosenstein" + +[[entries]] +id = "1ae440c5-cd7a-4fdc-a6a9-da10165fa260" +type = "feature" +description = "Add `ProtocTask` and `BufInstallTask` to `kraken.std.protobuf`" +author = "@NiklasRosenstein" + +[[entries]] +id = "691608ae-68bd-45d6-abbb-bb6a1b7200eb" +type = "breaking change" +description = "Remove `buf_lint()` and `buf_format()` helper functions from `kraken.std.protobuf`" +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/std/protobuf/__init__.py b/kraken-build/src/kraken/std/protobuf/__init__.py index 603ce569..cd111c24 100644 --- a/kraken-build/src/kraken/std/protobuf/__init__.py +++ b/kraken-build/src/kraken/std/protobuf/__init__.py @@ -3,17 +3,75 @@ from __future__ import annotations import subprocess as sp +import sys +from collections.abc import Sequence +from pathlib import Path +from platform import machine + +import httpx + +from kraken.core import Property, Task, TaskStatus + + +class BufInstallTask(Task): + """Installs [buf](https://github.com/bufbuild/buf) from GitHub.""" + + description = "Install buf." + version: Property[str] = Property.default("1.30.0", help="The version of `buf` to install.") + output_file: Property[Path] = Property.output(help="The path to the installed `buf` binary.") + + def _get_dist_url(self) -> str: + version = self.version.get() + suffix = "" + match sys.platform: + case "linux": + platform = "Linux" + case "darwin": + platform = "Darwin" + case "win32": + platform = "Windows" + suffix = ".exe" + case _: + raise NotImplementedError(f"Platform {sys.platform} is not supported by `buf`.") + match machine(): + case "x86_64": + arch = "x86_64" + case "aarch64": + arch = "aarch64" if platform == "Linux" else "arm64" + case _: + raise NotImplementedError(f"Architecture {machine()} is not supported by `buf`.") + return f"https://github.com/bufbuild/buf/releases/download/v{version}/buf-{platform}-{arch}{suffix}" + + def _get_output_file(self) -> Path: + filename = f"buf-v{self.version.get()}{'.exe' if sys.platform == 'win32' else ''}" + return self.project.context.build_directory / filename + + def prepare(self) -> TaskStatus | None: + if self._get_output_file().is_file(): + return TaskStatus.skipped("buf is already installed.") + return None -from kraken.core import Project, Task, TaskStatus + def execute(self) -> TaskStatus | None: + dist_url = self._get_dist_url() + output_file = self._get_output_file() + output_file.parent.mkdir(parents=True, exist_ok=True) + + response = httpx.get(dist_url, timeout=10) + response = response.raise_for_status() + output_file.write_bytes(response.content) + + self.output_file = output_file + return TaskStatus.succeeded(f"Installed buf to {output_file}.") class BufFormatTask(Task): """Format Protobuf files with `buf`.""" description = "Format Protobuf files with buf." + buf_bin: Property[str] = Property.default("buf", help="Path to `buf` binary.") def execute(self) -> TaskStatus | None: - command = ["buf", "format", "-w"] + command = [self.buf_bin.get(), "format", "-w"] result = sp.call(command, cwd=self.project.directory / "proto") return TaskStatus.from_exit_code(command, result) @@ -23,21 +81,43 @@ class BufLintTask(Task): """Lint Protobuf files with `buf`.""" description = "Lint Protobuf files with buf." + buf_bin: Property[str] = Property.default("buf", help="Path to `buf` binary.") def execute(self) -> TaskStatus | None: - command = ["buf", "lint"] + command = [self.buf_bin.get(), "lint"] result = sp.call(command, cwd=self.project.directory / "proto") return TaskStatus.from_exit_code(command, result) -def buf_format(*, name: str = "buf.format", project: Project | None = None) -> BufFormatTask: - """Format Protobuf files with `buf`.""" - project = project or Project.current() - return project.task(name, BufFormatTask, group="fmt") +class ProtocTask(Task): + """Generate code with `protoc`.""" + protoc_bin: Property[str] = Property.default("protoc", help="Path to `protoc` binary.") + proto_dir: Property[Sequence[Path | str]] = Property.required(help="The directories containing the .proto files.") + generators: Property[Sequence[tuple[str, Path]]] = Property.required( + help="The code generators to use. Each tuple contains the language name and the output directory." + ) + create_gitignore: Property[bool] = Property.default(True, help="Create a .gitignore file in the output directory.") -def buf_lint(*, name: str = "buf.lint", project: Project | None = None) -> BufLintTask: - """Lint Protobuf files with `buf`.""" - project = project or Project.current() - return project.task(name, BufLintTask, group="lint") + def generate(self, language: str, output_dir: Path) -> None: + """Helper to specify a code generator.""" + self.generators.setdefault(()) + self.generators.setmap(lambda v: [*v, (language, output_dir)]) + + def execute(self) -> TaskStatus | None: + print(">>", self.generators, self.generators._value) + command = [self.protoc_bin.get()] + for proto_dir in self.proto_dir.get(): + command += [f"--proto_path={self.project.directory / proto_dir}"] + for language, output_dir in self.generators.get(): + output_dir.mkdir(parents=True, exist_ok=True) + if self.create_gitignore.get(): + output_dir.joinpath(".gitignore").write_text("*\n") + command += [f"--{language}_out={output_dir}"] + command += [str(p) for p in Path(self.project.directory).rglob("*.proto")] + + return TaskStatus.from_exit_code( + command, + sp.call(command, cwd=self.project.directory), + ) From 33f11fec681ee57f1b81ed58a0ec58a347623358 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 19 Mar 2024 22:47:49 +0100 Subject: [PATCH 43/79] Add Protobuf support for `python_project()` and add `codegen` parameter to it and `python_app()` --- kraken-build/src/kraken/std/python/project.py | 111 +++++++++++++++--- 1 file changed, 97 insertions(+), 14 deletions(-) diff --git a/kraken-build/src/kraken/std/python/project.py b/kraken-build/src/kraken/std/python/project.py index 0ca44835..be1fbb0a 100644 --- a/kraken-build/src/kraken/std/python/project.py +++ b/kraken-build/src/kraken/std/python/project.py @@ -2,22 +2,30 @@ import logging import re -from collections.abc import Sequence +from collections.abc import Iterable, Sequence from pathlib import Path -from typing import Literal +from typing import Literal, TypeVar from nr.stream import Optional +from kraken.core.system.task import Task from kraken.std.git.version import EmptyGitRepositoryError, GitVersion, NotAGitRepositoryError, git_describe +from kraken.std.protobuf import BufFormatTask, BufInstallTask, BufLintTask, ProtocTask from kraken.std.python.buildsystem import detect_build_system from kraken.std.python.pyproject import PackageIndex from kraken.std.python.settings import python_settings +from kraken.std.python.tasks.pex_build_task import PexBuildTask, pex_build from kraken.std.python.tasks.pytest_task import CoverageFormat from kraken.std.python.version import git_version_to_python_version +T = TypeVar("T") logger = logging.getLogger(__name__) +def concat(*args: Iterable[T]) -> list[T]: + return [x for arg in args for x in arg] + + def python_package_index( *, alias: str, @@ -69,6 +77,7 @@ def python_project( tests_directory: str = "tests", additional_lint_directories: Sequence[str] | None = None, exclude_lint_directories: Sequence[str] = (), + exclude_format_directories: Sequence[str] = (), line_length: int = 120, enforce_project_version: str | None = None, detect_git_version_build_type: Literal["release", "develop", "branch"] = "develop", @@ -84,6 +93,12 @@ def python_project( mypy_version_spec: str = ">=1.8.0,<2.0.0", pycln_version_spec: str = ">=2.4.0,<3.0.0", pyupgrade_version_spec: str = ">=3.15.0,<4.0.0", + protobuf_enabled: bool = True, + protobuf_output_dir: str | None = None, + grpcio_tools_version_spec: str = ">=1.62.1,<2.0.0", + mypy_protobuf_version_spec: str = ">=3.5.0,<4.0.0", + buf_version: str = "1.30.0", + codegen: Sequence[Task | str] = (), ) -> None: """ Use this function in a Python project. @@ -110,6 +125,8 @@ def python_project( exclude_lint_directories: Directories in the project that contain Python sourec code that should not be linted and formatted but would otherwise be included via *source_directory*, *tests_directory* and *additional_directories*. + exclude_format_directories: Similar to *exclude_lint_directories*, but the paths will only be excluded by + formatters (e.g. Black, Isort) and Flake8 but will still be linted by Mypy. line_length: The line length to assume for all formatters and linters. enforce_project_version: When set, enforces the specified version number for the project when building wheels and publishing them. If not specified, the version number will be derived from the Git repository using @@ -128,6 +145,16 @@ def python_project( be used to add Flake8 plugins. flake8_extend_ignore: Flake8 lints to ignore. The default ignores lints that would otherwise conflict with the way Black formats code. + protobuf_enabled: Enable Protobuf code generation tasks if a `proto/` directory. Is a no-op when the directory + does not exist. Creates tasks for linting, formatting and generating code from Protobuf files. + protobuf_output_dir: The output directory for generated Python code from Protobuf files. If not specified, the + default is the _only_ directory in the project's source directory plus `/proto`. If there is more than one + directory in the source directory, this must be specified, otherwise an error will be raised. + grpcio_tools_version_spec: The version specifier for the `grpcio-tools` package to use when generating code + from Protobuf files. + buf_version: The version specifier for the `buf` tool to use when linting and formatting Protobuf files. + codegen: A list of code generation tasks that should be executed before the project is built. This can be + used to generate code from Protobuf files, for example. """ from kraken.build import project @@ -168,7 +195,12 @@ def python_project( # the lowest Python version comes first in the version spec. We also need to support Poetry-style semver # range selectors here. if python_version := handler.get_python_version_constraint(): - python_version = Optional(re.search(r"[\d\.]+", python_version)).map(lambda m: m.group(0)).or_else(None) + python_version = ( + Optional(re.search(r"[\d\.]+", python_version)) + .map(lambda m: m.group(0)) + .map(lambda s: s.rstrip(".")) + .or_else(None) + ) if not python_version: logger.warning( "Unable to determine minimum Python version for project %s, fallback to '3'", @@ -176,30 +208,74 @@ def python_project( ) python_version = "3" - login_task() + login = login_task() update_lockfile_task() update_pyproject_task() - install_task() info_task(build_system=build_system) + install_task().depends_on(login) + + # === Protobuf + + if protobuf_enabled and project.directory.joinpath("proto").is_dir(): + + if not protobuf_output_dir: + srcdir = project.directory.joinpath(source_directory) + source_dirs = [d for d in srcdir.iterdir() if not d.name.endswith(".egg-info")] + if len(source_dirs) != 1: + raise ValueError( + f"Multiple source directories found in {srcdir}; `protobuf_output_dir` must be specified" + ) + protobuf_output_dir = str(source_dirs[0] / "proto") + + buf_binary = project.task("buf.install", BufInstallTask) + buf_binary.version = buf_version + buf_format = project.task("buf.format", BufFormatTask, group="fmt") + buf_format.buf_bin = buf_binary.output_file.map(lambda p: str(p.absolute())) + buf_lint = project.task("buf.lint", BufLintTask, group="lint") + buf_lint.buf_bin = buf_binary.output_file.map(lambda p: str(p.absolute())) + + protoc_bin = pex_build( + binary_name="protoc", + requirements=[f"grpcio-tools{grpcio_tools_version_spec}", f"mypy-protobuf{mypy_protobuf_version_spec}"], + entry_point="grpc_tools.protoc", + venv="prepend", + ).output_file.map(lambda p: str(p.absolute())) + + protoc = project.task("protoc-python", ProtocTask) + protoc.protoc_bin = protoc_bin + protoc.proto_dir = ["proto"] + protoc.generate("python", Path(protobuf_output_dir)) + protoc.generate("mypy", Path(protobuf_output_dir)) + + codegen = [*codegen, protoc] + exclude_format_directories = [ + *exclude_format_directories, + # Ensure that the generated Protobuf code is not linted or formatted + str((project.directory / protobuf_output_dir).relative_to(project.directory)), + ] + + # === Python tooling pyupgrade_task( python_version=python_version, keep_runtime_typing=pyupgrade_keep_runtime_typing, - exclude=[Path(x) for x in exclude_lint_directories], + exclude=[Path(x) for x in concat(exclude_lint_directories, exclude_format_directories)], paths=source_paths, version_spec=pyupgrade_version_spec, ) pycln_task( paths=source_paths, - exclude_directories=exclude_lint_directories, + exclude_directories=concat(exclude_lint_directories, exclude_format_directories), remove_all_unused_imports=pycln_remove_all_unused_imports, version_spec=pycln_version_spec, ) black = black_tasks( paths=source_paths, - config=BlackConfig(line_length=line_length, exclude_directories=exclude_lint_directories), + config=BlackConfig( + line_length=line_length, exclude_directories=concat(exclude_lint_directories, exclude_format_directories) + ), version_spec=black_version_spec, ) @@ -208,7 +284,7 @@ def python_project( config=IsortConfig( profile="black", line_length=line_length, - extend_skip=exclude_lint_directories, + extend_skip=concat(exclude_lint_directories, exclude_format_directories), ), version_spec=isort_version_spec, ) @@ -217,7 +293,9 @@ def python_project( flake8 = flake8_tasks( paths=source_paths, config=Flake8Config( - max_line_length=line_length, extend_ignore=flake8_extend_ignore, exclude=exclude_lint_directories + max_line_length=line_length, + extend_ignore=flake8_extend_ignore, + exclude=concat(exclude_lint_directories, exclude_format_directories), ), version_spec=flake8_version_spec, additional_requirements=flake8_additional_requirements, @@ -235,7 +313,7 @@ def python_project( version_spec=mypy_version_spec, python_version=python_version, use_daemon=mypy_use_daemon, - ) + ).depends_on(*codegen) # TODO(@niklas): Improve this heuristic to check whether Coverage reporting should be enabled. if "pytest-cov" in str(dict(pyproject)): @@ -249,7 +327,7 @@ def python_project( coverage=coverage, doctest_modules=True, allow_no_tests=True, - ) + ).depends_on(*codegen) if not enforce_project_version: try: @@ -280,6 +358,7 @@ def python_project( # Create publish tasks for any package index with publish=True. build = build_task(as_version=enforce_project_version) + build.depends_on(*codegen) for index in python_settings().package_indexes.values(): if index.publish: publish_task(package_index=index.alias, distributions=build.output_files, interactive=False) @@ -296,7 +375,8 @@ def python_app( interpreter_constraint: str | None = None, venv_mode: Literal["append", "prepend"] | None = None, name: str = "build-pex", -) -> None: + dependencies: Sequence[Task | str] = (), +) -> PexBuildTask: """Build a PEX binary from a Python application. This function must be called after `python_project()`. @@ -320,7 +400,7 @@ def python_app( output_file = project.build_directory / "pex" / app_name - pex_build( + task = pex_build( binary_name=app_name, requirements=[str(project.directory.absolute())], entry_point=entry_point, @@ -331,3 +411,6 @@ def python_app( output_file=output_file, task_name=name, ) + task.depends_on(*dependencies) + + return task From 5e9fffdfb181f42a2abdb643bc29e9456b847b54 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 19 Mar 2024 22:54:21 +0100 Subject: [PATCH 44/79] make `python_app()` automatically depend on codegen produced by `python_project()` if called after --- kraken-build/src/kraken/std/python/project.py | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/kraken-build/src/kraken/std/python/project.py b/kraken-build/src/kraken/std/python/project.py index be1fbb0a..c1f810a1 100644 --- a/kraken-build/src/kraken/std/python/project.py +++ b/kraken-build/src/kraken/std/python/project.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Literal, TypeVar +from attr import dataclass from nr.stream import Optional from kraken.core.system.task import Task @@ -99,7 +100,7 @@ def python_project( mypy_protobuf_version_spec: str = ">=3.5.0,<4.0.0", buf_version: str = "1.30.0", codegen: Sequence[Task | str] = (), -) -> None: +) -> "PythonProject": """ Use this function in a Python project. @@ -366,6 +367,10 @@ def python_project( # TODO(@niklas): Support auto-detecting when Mypy stubtests need to be run or # accept arguments for stubtests. + data = PythonProject(codegen=codegen) + project.metadata.append(data) + return data + def python_app( *, @@ -379,7 +384,8 @@ def python_app( ) -> PexBuildTask: """Build a PEX binary from a Python application. - This function must be called after `python_project()`. + This function should be called after `python_project()`. If any Python code generation is employed by the project, + the generated PEX build task will depend on it. Args: app_name: The name of the applicaiton. This will be used as the binary output filename. The output PEX @@ -398,6 +404,11 @@ def python_app( from kraken.build import project from kraken.std.python.tasks.pex_build_task import pex_build + # If there's `python_project()` metadata, we depend on it's codegen. + project_metadata = project.find_metadata(PythonProject) + if project_metadata: + dependencies = [*dependencies, *project_metadata.codegen] + output_file = project.build_directory / "pex" / app_name task = pex_build( @@ -414,3 +425,10 @@ def python_app( task.depends_on(*dependencies) return task + + +@dataclass +class PythonProject: + """Result metadata from calling `python_project()`.""" + + codegen: Sequence[Task | str] From 28f1be7748952183ce862dc6b8c2b8604f770cde Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 19 Mar 2024 23:23:07 +0100 Subject: [PATCH 45/79] delete contents of proto output directory before regenerating --- kraken-build/src/kraken/std/protobuf/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kraken-build/src/kraken/std/protobuf/__init__.py b/kraken-build/src/kraken/std/protobuf/__init__.py index cd111c24..dae651c7 100644 --- a/kraken-build/src/kraken/std/protobuf/__init__.py +++ b/kraken-build/src/kraken/std/protobuf/__init__.py @@ -7,6 +7,7 @@ from collections.abc import Sequence from pathlib import Path from platform import machine +from shutil import rmtree import httpx @@ -101,16 +102,19 @@ class ProtocTask(Task): create_gitignore: Property[bool] = Property.default(True, help="Create a .gitignore file in the output directory.") def generate(self, language: str, output_dir: Path) -> None: - """Helper to specify a code generator.""" + """Helper to specify a code generator. + + IMPORTANT: The contents of *output_dir* will be deleted before running `protoc`.""" + self.generators.setdefault(()) self.generators.setmap(lambda v: [*v, (language, output_dir)]) def execute(self) -> TaskStatus | None: - print(">>", self.generators, self.generators._value) command = [self.protoc_bin.get()] for proto_dir in self.proto_dir.get(): command += [f"--proto_path={self.project.directory / proto_dir}"] for language, output_dir in self.generators.get(): + rmtree(output_dir, ignore_errors=True) output_dir.mkdir(parents=True, exist_ok=True) if self.create_gitignore.get(): output_dir.joinpath(".gitignore").write_text("*\n") From 2a2f7b26d69bcb3c8f239bee2406ed08568c0289 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 19 Mar 2024 23:40:47 +0100 Subject: [PATCH 46/79] also generate grpc code in python projects --- kraken-build/src/kraken/std/python/project.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kraken-build/src/kraken/std/python/project.py b/kraken-build/src/kraken/std/python/project.py index c1f810a1..a342f28a 100644 --- a/kraken-build/src/kraken/std/python/project.py +++ b/kraken-build/src/kraken/std/python/project.py @@ -246,7 +246,11 @@ def python_project( protoc.protoc_bin = protoc_bin protoc.proto_dir = ["proto"] protoc.generate("python", Path(protobuf_output_dir)) + protoc.generate("grpc_python", Path(protobuf_output_dir)) protoc.generate("mypy", Path(protobuf_output_dir)) + protoc.generate("mypy_grpc", Path(protobuf_output_dir)) + # TODO(@niklas): Seems the standard GRPCio tools can already generate .pyi files, but not for the grpc stubs? + # protoc.generate("pyi", Path(protobuf_output_dir)) codegen = [*codegen, protoc] exclude_format_directories = [ From c48d749475df501c683b8c6be6cb397fef555137 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Wed, 20 Mar 2024 00:27:38 +0100 Subject: [PATCH 47/79] update Protobuf docs --- docs/docs/lang/protobuf.md | 42 +++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/docs/docs/lang/protobuf.md b/docs/docs/lang/protobuf.md index c6cd346f..862810a3 100644 --- a/docs/docs/lang/protobuf.md +++ b/docs/docs/lang/protobuf.md @@ -2,22 +2,40 @@ [Buf]: https://buf.build/docs/ -Format and lint Proto files using [buf][]. +We currently support automatic linting, formatting and code generation from Protobuf files using [buf][] +and [protoc][] when using the `python_project()` function. -__Quickstart__ +No local development tools are needed as both `protoc` and `buf` are fetched for you by Kraken. -```py -# .kraken.py -from kraken.core import Project -from kraken.std.protobuf import BufFormatTask, BufLintTask +### Project layout +``` +my-project/ + proto/ + my_project/ + service.proto + src/ + my_project/ + __init__.py +``` + +Kraken will then generate the following files via the `protoc-python` task: -project = Project.current() -project.task(name, BufLintTask, group="lint") -project.task(name, BufFormatTask, group="fmt") ``` +my-proect/ + src/ + my_project/ + .gitignore + service_pb2.py + service_pb2_grpc.py +``` + +The `.gitignore` file contains all the generated files. + +### Tasks -## Requirements +The following tasks are created by `python_project()` when a `proto/` directory exists: -- The buf lint task will only succeed when executed in a `/proto` directory -- The buf format task can be executed in the root of the project directory and will format inplace all of the proto files that exist in the repo +* `protoc-python` - Generates Python code from the `.proto` files, including Mypy stub files. +* `buf.lint` - Lints the `.proto` files using [buf][] (group: `lint`) +* `buf.format` - Formats the `.proto` files using [buf][] (group: `fmt`) From d32474490cd8eca677629f36db359199520ddf1e Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Thu, 21 Mar 2024 00:14:10 +0100 Subject: [PATCH 48/79] update protobuf docs; --- docs/docs/lang/protobuf.md | 185 ++++++++++++++++++++++++++++++++----- docs/mkdocs.yml | 8 +- 2 files changed, 168 insertions(+), 25 deletions(-) diff --git a/docs/docs/lang/protobuf.md b/docs/docs/lang/protobuf.md index 862810a3..6eacb010 100644 --- a/docs/docs/lang/protobuf.md +++ b/docs/docs/lang/protobuf.md @@ -1,36 +1,175 @@ # Protobuf [Buf]: https://buf.build/docs/ + [Buffrs]: https://github.com/helsing-ai/buffrs + [Helsing]: https://helsing.ai/ -We currently support automatic linting, formatting and code generation from Protobuf files using [buf][] -and [protoc][] when using the `python_project()` function. +Kraken implements an opinionated workflow for working with Protobuf files, which is currently implemented only for +Python projects using the `python_project()` function. All required tools will be installed for you automatically +by Kraken. -No local development tools are needed as both `protoc` and `buf` are fetched for you by Kraken. +* `buf` via the [Buf] [GitHub releases](https://github.com/bufbuild/buf/releases) +* `buffrs` via its [GitHub releases](https://github.com/helsing-ai/buffrs/releases) +* `protoc` via [grpcio-tools](https://pypi.org/project/grpcio-tools/) (as Pex) -### Project layout +There is currently no opinionated automation for -``` -my-project/ - proto/ - my_project/ - service.proto - src/ - my_project/ - __init__.py -``` +* Automatically authenticating with a Buffrs registry +* Publishing Buffrs packages -Kraken will then generate the following files via the `protoc-python` task: +## Buffrs -``` -my-proect/ - src/ - my_project/ - .gitignore - service_pb2.py - service_pb2_grpc.py -``` +[Buffrs][] is an opinionated Protobuf dependency manager developed at [Helsing]. It strongly advocates for code +generation in the immediate project that consumes Protobuf APIs instead of publishing pre-generated code to a package +repository. Buffrs is used to manage the Protobuf dependencies in your project. -The `.gitignore` file contains all the generated files. +There are three types of Buffrs projects: + +* __Libraries__, which can be depended on by any other Buffrs project and published as Buffrs packages. +* __APIs__, which can be published as Buffrs packages but can _not_ be depended upon by another Library or API. +* __Applications__, which can depend on any Buffrs package, but cannot be published. Typically these only depend on +APIs and do not have any Protobuf files of their own. + +When a `Proto.toml` file is detected in a project, Kraken will use `buffrs` to run `buffrs install` to install +dependencies into the `proto/vendor` folder which will be considered the canonical source for Protobuf files to generate code from. + +## Projects without Buffrs + +Projects without a `Proto.toml` can still be used with Kraken, they simply won't be able to make use of any +kind of dependency management for Protobuf files. In this case, the `proto` directory in your project is considered +the canonical source for Protobuf files to generate code from. + +## Python code generation + +Kraken will generate Python code in such a way that all APIs can be imported from a `proto` namespace package. + +??? note "Packaging Python projects with Protobuf files" + When packaging a Python project using Protobuf for distribution as a Python package, you have to ensure manually + that your package manager includes the generated Protobuf files in the package. This is not done automatically by + Kraken. + + For example, in a Poetry project, you need to amend the packages configuration in `pyproject.toml`: + + ```toml + [tool.poetry] + packages = [ + { include = "proto/my_project", from = "src" }, + { include = "src/my_project", from = "src" }, + ] + ``` + +=== "With Buffrs" + + __Project structure__ + + Given the following project structure: + + ``` + +- Proto.toml + +- proto/ + | +- my_project/ + | | +- service.proto + +- src/ + | +- my_project/ + | | +- __init__.py + ``` + + And the following `Proto.toml` file: + + ```toml + edition = "0.8" + + [dependencies] + another_service = { version = "^0.1.0" } + ``` + + Running `buffrs install`, which Kraken will do for you automatically, you might end up with the following + `proto` directory. Note how Buffrs copies your _own_ Protobuf files into `proto/vendor` alongside the dependencies + to establish a consistent import system. + + ``` + +- proto/ + | +- vendor/ + | | +- another_service/ + | | | +- service.proto + | | +- my_project/ + | | | +- service.proto + | +- my_project/ + | | +- service.proto + ``` + + __Generated code__ + + Kraken will generate the following Python files: + + ``` + +- src/ + | +- proto/ + | | +- .gitignore + | | +- my_project/ + | | | +- __init__.py + | | | +- service_pb2.py + | | | +- service_pb2_grpc.py + | | | +- service_pb2.pyi + | | | +- service_pb2_grpc.pyi + | | +- another_service/ + | | | +- __init__.py + | | | +- service_pb2.py + | | | +- service_pb2.pyi + | | | +- service_pb2_grpc.py + | | | +- service_pb2_grpc.pyi + ``` + + __Imports__ + + Allowing you to import the generated code like so from your own code in `src/my_project/*`: + + ```python + from proto.my_project.service_pb2 import MyMessage + from proto.another_service.service_pb2_grpc import AnotherServiceStub + # ... + ``` + +=== "Without Buffrs" + + __Project structure__ + + Given the following project structure: + + ``` + +- proto/ + | +- my_project/ + | | +- service.proto + +- src/ + | +- my_project/ + | | +- __init__.py + ``` + + __Generated code__ + + Kraken will generate the following Python files: + + ``` + +- src/ + | +- proto/ + | | +- .gitignore + | | +- my_project/ + | | | +- __init__.py + | | | +- service_pb2.py + | | | +- service_pb2_grpc.py + | | | +- service_pb2.pyi + | | | +- service_pb2_grpc.pyi + ``` + + __Imports__ + + Allowing you to import the generated code like so from your own code in `src/my_project/*`: + + ```python + from proto.my_project.service_pb2 import MyMessage + from proto.my_project.service_pb2_grpc import MyServiceStub + # ... + ``` ### Tasks diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 79de04e5..4bc01844 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -6,8 +6,8 @@ theme: palette: primary: blue accent: green - # features: - # - navigation.tabs + features: + - navigation.sections hooks: - mksync-hook.py @@ -38,6 +38,9 @@ markdown_extensions: - pymdownx.inlinehilite - pymdownx.snippets - pymdownx.superfences + - pymdownx.details + - pymdownx.tabbed: + alternate_style: true nav: - index.md @@ -66,6 +69,7 @@ nav: - api/kraken.std.mitm.md - api/kraken.std.protobuf.md - api/kraken.std.python.md + - api/kraken.std.shellcheck.md - api/kraken.std.sccache.md - api/kraken.std.util.md - Command-line: From 69323c0e144802f4cf36ddfdfbf0651404932dd9 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Thu, 21 Mar 2024 00:14:56 +0100 Subject: [PATCH 49/79] add examples/python-protobuf --- examples/python-protobuf/.gitignore | 162 +++++++++++++++++++ examples/python-protobuf/.kraken.py | 3 + examples/python-protobuf/README.md | 1 + examples/python-protobuf/proto/service.proto | 4 + examples/python-protobuf/pyproject.toml | 12 ++ examples/python-protobuf/src/myserver.py | 3 + examples/python-protobuf/tests/__init__.py | 0 7 files changed, 185 insertions(+) create mode 100644 examples/python-protobuf/.gitignore create mode 100644 examples/python-protobuf/.kraken.py create mode 100644 examples/python-protobuf/README.md create mode 100644 examples/python-protobuf/proto/service.proto create mode 100644 examples/python-protobuf/pyproject.toml create mode 100644 examples/python-protobuf/src/myserver.py create mode 100644 examples/python-protobuf/tests/__init__.py diff --git a/examples/python-protobuf/.gitignore b/examples/python-protobuf/.gitignore new file mode 100644 index 00000000..3a8816c9 --- /dev/null +++ b/examples/python-protobuf/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm-project.org/#use-with-ide +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/examples/python-protobuf/.kraken.py b/examples/python-protobuf/.kraken.py new file mode 100644 index 00000000..9f59fe27 --- /dev/null +++ b/examples/python-protobuf/.kraken.py @@ -0,0 +1,3 @@ +from kraken.std.python.project import python_project + +python_project() diff --git a/examples/python-protobuf/README.md b/examples/python-protobuf/README.md new file mode 100644 index 00000000..8f28477f --- /dev/null +++ b/examples/python-protobuf/README.md @@ -0,0 +1 @@ +# python-protobuf diff --git a/examples/python-protobuf/proto/service.proto b/examples/python-protobuf/proto/service.proto new file mode 100644 index 00000000..4ea1143d --- /dev/null +++ b/examples/python-protobuf/proto/service.proto @@ -0,0 +1,4 @@ +syntax = "proto3"; +package myservice; +message MyMessage {} +service MyService {} diff --git a/examples/python-protobuf/pyproject.toml b/examples/python-protobuf/pyproject.toml new file mode 100644 index 00000000..06c07782 --- /dev/null +++ b/examples/python-protobuf/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "python-protobuf" +version = "0.1.0" +description = "Default template for PDM package" +authors = [{name = "Niklas Rosenstein", email = "rosensteinniklas@gmail.com"}] +dependencies = [] +requires-python = "==3.10.*" +readme = "README.md" +license = {text = "MIT"} + +[tool.pdm] +distribution = false diff --git a/examples/python-protobuf/src/myserver.py b/examples/python-protobuf/src/myserver.py new file mode 100644 index 00000000..4b46e23e --- /dev/null +++ b/examples/python-protobuf/src/myserver.py @@ -0,0 +1,3 @@ + +from proto.service_pb2 import MyMessage +from proto.service_pb2_grpc import MyServiceStub diff --git a/examples/python-protobuf/tests/__init__.py b/examples/python-protobuf/tests/__init__.py new file mode 100644 index 00000000..e69de29b From 09e25838a3d57302ad4c812a3e10822e95591dde Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Thu, 21 Mar 2024 00:15:13 +0100 Subject: [PATCH 50/79] feature: Add `kraken.std.util.fetch_tarball()` --- .changelog/_unreleased.toml | 6 ++ .../src/kraken/std/buffrs/__init__.py | 54 ++++++++++++- kraken-build/src/kraken/std/buffrs/tasks.py | 9 ++- .../src/kraken/std/protobuf/__init__.py | 12 ++- kraken-build/src/kraken/std/python/project.py | 61 ++++++++------- kraken-build/src/kraken/std/util/__init__.py | 3 +- .../src/kraken/std/util/fetch_tarball_task.py | 75 +++++++++++++++++++ 7 files changed, 184 insertions(+), 36 deletions(-) create mode 100644 kraken-build/src/kraken/std/util/fetch_tarball_task.py diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index 7bfc9752..02698a47 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -45,3 +45,9 @@ id = "691608ae-68bd-45d6-abbb-bb6a1b7200eb" type = "breaking change" description = "Remove `buf_lint()` and `buf_format()` helper functions from `kraken.std.protobuf`" author = "@NiklasRosenstein" + +[[entries]] +id = "07fb65a6-fe5a-420a-ad96-36a50714dee1" +type = "feature" +description = "Add `kraken.std.util.fetch_tarball()`" +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/std/buffrs/__init__.py b/kraken-build/src/kraken/std/buffrs/__init__.py index cfee36fc..b711baf7 100644 --- a/kraken-build/src/kraken/std/buffrs/__init__.py +++ b/kraken-build/src/kraken/std/buffrs/__init__.py @@ -1,11 +1,15 @@ from __future__ import annotations import logging +import platform as _platform +from pathlib import Path from typing import cast +from kraken.common.supplier import Supplier from kraken.core import Project from .tasks import BuffrsInstallTask, BuffrsLoginTask, BuffrsPublishTask +from kraken.std.util import fetch_tarball logger = logging.getLogger(__name__) @@ -19,6 +23,7 @@ def buffrs_login( project: Project | None = None, registry: str, token: str, + buffrs_bin: Supplier[str] | None = None, ) -> BuffrsLoginTask: """Create a task to log into an Artifactory registry with Buffrs. The task is created in the root project regardless from where it is called. Note that currently we only support a single registry to push to, because @@ -35,15 +40,20 @@ def buffrs_login( task = root_project.task("buffrsLogin", BuffrsLoginTask) task.registry = registry task.token = token + task.buffrs_bin = buffrs_bin or buffrs_fetch_binary().map(str) return task -def buffrs_install(*, project: Project | None = None) -> BuffrsInstallTask: +def buffrs_install(*, project: Project | None = None, + buffrs_bin: Supplier[str] | None = None,) -> BuffrsInstallTask: """Installs buffrs dependencies defined in the `Proto.toml`""" project = project or Project.current() - return project.task("buffrsInstall", BuffrsInstallTask) + task = project.task("buffrsInstall", BuffrsInstallTask) + task.buffrs_bin = buffrs_bin or buffrs_fetch_binary().map(str) + + return task def buffrs_publish( @@ -52,6 +62,7 @@ def buffrs_publish( registry: str, repository: str, version: str | None = None, + buffrs_bin: Supplier[str] | None = None, ) -> BuffrsPublishTask: """Publishes the buffrs package to the repository of the project.""" @@ -61,4 +72,43 @@ def buffrs_publish( task.registry = registry task.repository = repository task.version = version + task.buffrs_bin = buffrs_bin or buffrs_fetch_binary().map(str) return task + + +def buffrs_fetch_binary( + version: str = "0.8.0", + target_triplet: str | None = None, +) -> Supplier[Path]: + """Fetches the path to the Buffrs binary and returns a supplier for the Path to it. + + The binary will be fetched from the [GitHub releases](https://github.com/helsing-ai/buffrs/releases). + """ + + target_triplet = target_triplet or get_buffrs_triplet() + suffix = ".zip" if "-windows" in target_triplet else ".tar.gz" + binary = "buffrs.exe" if "-windows" in target_triplet else "buffrs" + name = f"buffrs-v{version}-{target_triplet}" + url = f"https://github.com/helsing-ai/buffrs/releases/download/v{version}/{name}{suffix}" + + return fetch_tarball(name="buffrs", url=url).out.map(lambda p: p.absolute() / name / binary) + + +def get_buffrs_triplet() -> str: + """ Returns the Buffrs target triplet for the current platform.""" + + match (_platform.machine(), _platform.system()): + case ("x86_64", "Linux"): + return "x86_64-unknown-linux-gnu" + case ("x86_64", "Darwin"): + return "x86_64-apple-darwin" + case ("x86_64", "Windows"): + return "x86_64-pc-windows-msvc" + case ("aarch64", "Linux"): + return "arm-unknown-linux-gnueabihf" + case ("aarch64", "Darwin"): + return "aarch64-apple-darwin" + case ("aarch64", "Windows"): + return "i686-pc-windows-msvc" + case _: + raise NotImplementedError(f"Platform {_platform.machine()} is not supported by Buffrs.") diff --git a/kraken-build/src/kraken/std/buffrs/tasks.py b/kraken-build/src/kraken/std/buffrs/tasks.py index 97a54704..73112b82 100644 --- a/kraken-build/src/kraken/std/buffrs/tasks.py +++ b/kraken-build/src/kraken/std/buffrs/tasks.py @@ -16,9 +16,10 @@ class BuffrsLoginTask(Task): help="The Artifactory URL to publish to (e.g. `https:///artifactory`)." ) token: Property[str] = Property.required(help="The token for the registry.") + buffrs_bin: Property[str] = Property.default("buffrs", help="The path to the buffrs binary.") def execute(self) -> TaskStatus: - command = ["buffrs", "login", "--registry", self.registry.get()] + command = [self.buffrs_bin.get(), "login", "--registry", self.registry.get()] return TaskStatus.from_exit_code( command, sp.run(command, cwd=self.project.directory, input=self.token.get(), text=True).returncode, @@ -29,9 +30,10 @@ class BuffrsInstallTask(Task): """Install dependencies defined in `Proto.toml`""" description = "Runs `buffrs install` to download protobuf dependencies" + buffrs_bin: Property[str] = Property.default("buffrs", help="The path to the buffrs binary.") def execute(self) -> TaskStatus: - command = ["buffrs", "install"] + command = [self.buffrs_bin.get(), "install"] return TaskStatus.from_exit_code( command, sp.call(command, cwd=self.project.directory), @@ -52,10 +54,11 @@ class BuffrsPublishTask(Task): help="The Artifactory repository to publish to (this should be a Generic repository)." ) version: Property[str | None] = Property.default(None, help="Override the version from the manifest.") + buffrs_bin: Property[str] = Property.default("buffrs", help="The path to the buffrs binary.") def execute(self) -> TaskStatus: command = [ - "buffrs", + self.buffrs_bin.get(), "publish", "--registry", self.registry.get(), diff --git a/kraken-build/src/kraken/std/protobuf/__init__.py b/kraken-build/src/kraken/std/protobuf/__init__.py index dae651c7..716d277f 100644 --- a/kraken-build/src/kraken/std/protobuf/__init__.py +++ b/kraken-build/src/kraken/std/protobuf/__init__.py @@ -57,7 +57,7 @@ def execute(self) -> TaskStatus | None: output_file = self._get_output_file() output_file.parent.mkdir(parents=True, exist_ok=True) - response = httpx.get(dist_url, timeout=10) + response = httpx.get(dist_url, timeout=10, follow_redirects=True) response = response.raise_for_status() output_file.write_bytes(response.content) @@ -95,7 +95,7 @@ class ProtocTask(Task): """Generate code with `protoc`.""" protoc_bin: Property[str] = Property.default("protoc", help="Path to `protoc` binary.") - proto_dir: Property[Sequence[Path | str]] = Property.required(help="The directories containing the .proto files.") + proto_dir: Property[Path | str] = Property.required(help="The directories containing the .proto files.") generators: Property[Sequence[tuple[str, Path]]] = Property.required( help="The code generators to use. Each tuple contains the language name and the output directory." ) @@ -110,9 +110,13 @@ def generate(self, language: str, output_dir: Path) -> None: self.generators.setmap(lambda v: [*v, (language, output_dir)]) def execute(self) -> TaskStatus | None: + + # TODO: Re-organize proto_dir to be prefixed with a `proto/` directory that is not contained + # in the `--proto_path` argument. This is necessary to ensure we generate imports in + # a `proto/` namespace package. + command = [self.protoc_bin.get()] - for proto_dir in self.proto_dir.get(): - command += [f"--proto_path={self.project.directory / proto_dir}"] + command += [f"--proto_path={self.project.directory / self.proto_dir.get()}"] for language, output_dir in self.generators.get(): rmtree(output_dir, ignore_errors=True) output_dir.mkdir(parents=True, exist_ok=True) diff --git a/kraken-build/src/kraken/std/python/project.py b/kraken-build/src/kraken/std/python/project.py index a342f28a..0a0a936e 100644 --- a/kraken-build/src/kraken/std/python/project.py +++ b/kraken-build/src/kraken/std/python/project.py @@ -11,7 +11,6 @@ from kraken.core.system.task import Task from kraken.std.git.version import EmptyGitRepositoryError, GitVersion, NotAGitRepositoryError, git_describe -from kraken.std.protobuf import BufFormatTask, BufInstallTask, BufLintTask, ProtocTask from kraken.std.python.buildsystem import detect_build_system from kraken.std.python.pyproject import PackageIndex from kraken.std.python.settings import python_settings @@ -95,7 +94,6 @@ def python_project( pycln_version_spec: str = ">=2.4.0,<3.0.0", pyupgrade_version_spec: str = ">=3.15.0,<4.0.0", protobuf_enabled: bool = True, - protobuf_output_dir: str | None = None, grpcio_tools_version_spec: str = ">=1.62.1,<2.0.0", mypy_protobuf_version_spec: str = ">=3.5.0,<4.0.0", buf_version: str = "1.30.0", @@ -147,10 +145,8 @@ def python_project( flake8_extend_ignore: Flake8 lints to ignore. The default ignores lints that would otherwise conflict with the way Black formats code. protobuf_enabled: Enable Protobuf code generation tasks if a `proto/` directory. Is a no-op when the directory - does not exist. Creates tasks for linting, formatting and generating code from Protobuf files. - protobuf_output_dir: The output directory for generated Python code from Protobuf files. If not specified, the - default is the _only_ directory in the project's source directory plus `/proto`. If there is more than one - directory in the source directory, this must be specified, otherwise an error will be raised. + does not exist. Creates tasks for linting, formatting and generating code from Protobuf files. The + generated code will be placed into the `source_directory`. grpcio_tools_version_spec: The version specifier for the `grpcio-tools` package to use when generating code from Protobuf files. buf_version: The version specifier for the `buf` tool to use when linting and formatting Protobuf files. @@ -219,21 +215,30 @@ def python_project( if protobuf_enabled and project.directory.joinpath("proto").is_dir(): - if not protobuf_output_dir: - srcdir = project.directory.joinpath(source_directory) - source_dirs = [d for d in srcdir.iterdir() if not d.name.endswith(".egg-info")] - if len(source_dirs) != 1: - raise ValueError( - f"Multiple source directories found in {srcdir}; `protobuf_output_dir` must be specified" - ) - protobuf_output_dir = str(source_dirs[0] / "proto") - - buf_binary = project.task("buf.install", BufInstallTask) - buf_binary.version = buf_version + from kraken.std.protobuf import BufFormatTask, BufInstallTask, BufLintTask, ProtocTask + from kraken.std.buffrs import buffrs_install as buffrs_install_task + from kraken.std.util import fetch_tarball + + if project.directory.joinpath("Proto.toml").is_file(): + buffrs_install = buffrs_install_task() + else: + buffrs_install = None + + # TODO(@niklas): This is a temporary solution to fetch the Buf binary until we have a proper way to + buf_binary = fetch_tarball( + name="buf", + url=f"https://github.com/bufbuild/buf/releases/download/v{buf_version}/buf-Linux-aarch64.tar.gz", + ).out.map(lambda p: p.absolute() / "buf" / "bin" / "buf").map(str) + buf_format = project.task("buf.format", BufFormatTask, group="fmt") - buf_format.buf_bin = buf_binary.output_file.map(lambda p: str(p.absolute())) + buf_format.buf_bin = buf_binary + buf_lint = project.task("buf.lint", BufLintTask, group="lint") - buf_lint.buf_bin = buf_binary.output_file.map(lambda p: str(p.absolute())) + buf_lint.buf_bin = buf_binary + + if buffrs_install is not None: + buf_lint.depends_on(buffrs_install) + buf_format.depends_on(buffrs_install) protoc_bin = pex_build( binary_name="protoc", @@ -244,19 +249,23 @@ def python_project( protoc = project.task("protoc-python", ProtocTask) protoc.protoc_bin = protoc_bin - protoc.proto_dir = ["proto"] - protoc.generate("python", Path(protobuf_output_dir)) - protoc.generate("grpc_python", Path(protobuf_output_dir)) - protoc.generate("mypy", Path(protobuf_output_dir)) - protoc.generate("mypy_grpc", Path(protobuf_output_dir)) + protoc.proto_dir = "proto" + protoc.generate("python", Path(source_directory)) + protoc.generate("grpc_python", Path(source_directory)) + protoc.generate("mypy", Path(source_directory)) + protoc.generate("mypy_grpc", Path(source_directory)) # TODO(@niklas): Seems the standard GRPCio tools can already generate .pyi files, but not for the grpc stubs? - # protoc.generate("pyi", Path(protobuf_output_dir)) + # protoc.generate("pyi", Path(source_directory)) + + if buffrs_install is not None: + protoc.depends_on(buffrs_install) + protoc.proto_dir = "proto/vendor" codegen = [*codegen, protoc] exclude_format_directories = [ *exclude_format_directories, # Ensure that the generated Protobuf code is not linted or formatted - str((project.directory / protobuf_output_dir).relative_to(project.directory)), + str((project.directory / source_directory / "proto").relative_to(project.directory)), ] # === Python tooling diff --git a/kraken-build/src/kraken/std/util/__init__.py b/kraken-build/src/kraken/std/util/__init__.py index 18c7f74e..a25847b9 100644 --- a/kraken-build/src/kraken/std/util/__init__.py +++ b/kraken-build/src/kraken/std/util/__init__.py @@ -3,5 +3,6 @@ from __future__ import annotations from .copyright_task import check_and_format_copyright +from .fetch_tarball_task import fetch_tarball -__all__ = ["check_file_exists_and_is_committed", "check_valid_readme_exists", "check_and_format_copyright"] +__all__ = ["check_and_format_copyright", "fetch_tarball"] diff --git a/kraken-build/src/kraken/std/util/fetch_tarball_task.py b/kraken-build/src/kraken/std/util/fetch_tarball_task.py new file mode 100644 index 00000000..adc89497 --- /dev/null +++ b/kraken-build/src/kraken/std/util/fetch_tarball_task.py @@ -0,0 +1,75 @@ + + +import contextlib +import hashlib +from pathlib import Path +import shutil +import tempfile +from typing import Literal, cast + +import httpx +from kraken.core.system.property import Property +from kraken.core.system.task import Task, TaskStatus + + +class FetchTarballTask(Task): + """ Fetches a tarball from a URL and unpacks it. May also be used to fetch ZIP files. """ + + name_: Property[str] = Property.required(help="The name of the task.") + url: Property[str] = Property.required(help="The URL to fetch the tarball from.") + format: Property[Literal["tar", "zip"] | None] = Property.default("tar", help="The format of the tarball. If not set, will be inferred from the file extension in the URL.") + out: Property[Path] = Property.output(help="The path to the unpacked tarball.") + + # TODO(@niklas): SHA256 checksum verification + + def _unpack(self, archive: Path, store_path: Path) -> None: + """ Unpacks the archive at the given path to the store path. """ + + if (format_ := self.format.get()) is None: + format_ = "zip" if archive.suffix == ".zip" else "tar" + + shutil.unpack_archive(archive, store_path, format_) + + # Task + + def prepare(self) -> TaskStatus | None: + store_path = self.out.get() + if store_path.exists(): + return TaskStatus.skipped(f"{store_path} already exists.") + return None + + def execute(self) -> TaskStatus | None: + + print(f"Downloading {self.url.get()} ...") + with tempfile.TemporaryDirectory() as tmp, Path(tmp).joinpath("archive").open("wb") as f: + with httpx.stream("GET", self.url.get(), follow_redirects=True, timeout=60) as response: + response.raise_for_status() + for chunk in response.iter_bytes(): + f.write(chunk) + + store_path = self.out.get() + print(f"Unpacking to {store_path} ...") + self._unpack(Path(tmp) / "archive", store_path) + + return TaskStatus.succeeded(f"Fetched tarball from {self.url.get()} to {store_path}.") + + +def fetch_tarball(*, name: str, url: str, format: Literal["tar", "zip"] | None = None) -> FetchTarballTask: + """ Fetches a tarball from a URL and unpacks it. May also be used to fetch ZIP files. """ + + from kraken.build import context + + urlhash = hashlib.md5(url.encode()).hexdigest() + dest = context.build_directory / ".store" / f"{urlhash}-{name}" + task_name = f"{name}-{urlhash}" + + if task_name in context.root_project.tasks(): + task = cast(FetchTarballTask, context.root_project.task(task_name)) + else: + task = context.root_project.task(task_name, FetchTarballTask) + task.name_ = name + task.url = url + task.format = format + task.out = dest + + return task From f0b9da6c1ac37fcdd77e0dd165224a687127530b Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Thu, 21 Mar 2024 22:09:14 +0100 Subject: [PATCH 51/79] breaking change: Add `enable_error_code = "ignore-without-code, possibly-undefined"` to default Mypy settings --- .changelog/_unreleased.toml | 6 ++++++ kraken-build/src/kraken/std/python/tasks/mypy_task.py | 1 + 2 files changed, 7 insertions(+) diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index 02698a47..d59edf6e 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -51,3 +51,9 @@ id = "07fb65a6-fe5a-420a-ad96-36a50714dee1" type = "feature" description = "Add `kraken.std.util.fetch_tarball()`" author = "@NiklasRosenstein" + +[[entries]] +id = "ec6bf736-6a6e-4625-9703-89bff7094d56" +type = "breaking change" +description = "Add `enable_error_code = \"ignore-without-code, possibly-undefined\"` to default Mypy settings" +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/std/python/tasks/mypy_task.py b/kraken-build/src/kraken/std/python/tasks/mypy_task.py index d6a3b846..c6f240ef 100644 --- a/kraken-build/src/kraken/std/python/tasks/mypy_task.py +++ b/kraken-build/src/kraken/std/python/tasks/mypy_task.py @@ -33,6 +33,7 @@ warn_unreachable = true warn_unused_configs = true warn_unused_ignores = true +enable_error_code = "ignore-without-code, possibly-undefined" """ From d9826aa5c7cdbb84a3a3f1e8f0458d7e0be0c4ed Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Thu, 21 Mar 2024 23:58:01 +0100 Subject: [PATCH 52/79] improvement: `Supplier.of()` and related functions now take an `Iterable[Supplier[Any]]` as the `derived_from` parameter instead of a `Sequence` as it is converted to a `tuple` later anyway. --- .changelog/_unreleased.toml | 6 ++++++ kraken-build/src/kraken/common/supplier.py | 14 +++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index d59edf6e..13cb1f18 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -57,3 +57,9 @@ id = "ec6bf736-6a6e-4625-9703-89bff7094d56" type = "breaking change" description = "Add `enable_error_code = \"ignore-without-code, possibly-undefined\"` to default Mypy settings" author = "@NiklasRosenstein" + +[[entries]] +id = "33f61844-e82b-40da-aba9-f552142d6abd" +type = "improvement" +description = "`Supplier.of()` and related functions now take an `Iterable[Supplier[Any]]` as the `derived_from` parameter instead of a `Sequence` as it is converted to a `tuple` later anyway." +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/common/supplier.py b/kraken-build/src/kraken/common/supplier.py index 7e20334b..db96acf1 100644 --- a/kraken-build/src/kraken/common/supplier.py +++ b/kraken-build/src/kraken/common/supplier.py @@ -95,17 +95,17 @@ def lineage(self) -> Iterable[tuple["Supplier[Any]", list["Supplier[Any]"]]]: stack += derived_from @staticmethod - def of(value: "T | Supplier[T]", derived_from: Sequence["Supplier[Any]"] = ()) -> "Supplier[T]": + def of(value: "T | Supplier[T]", derived_from: Iterable["Supplier[Any]"] = ()) -> "Supplier[T]": if isinstance(value, Supplier): return value return OfSupplier(value, derived_from) @staticmethod - def of_callable(func: Callable[[], T], derived_from: Sequence["Supplier[Any]"] = ()) -> "Supplier[T]": + def of_callable(func: Callable[[], T], derived_from: Iterable["Supplier[Any]"] = ()) -> "Supplier[T]": return OfCallableSupplier(func, derived_from) @staticmethod - def void(from_exc: "Exception | None" = None, derived_from: Sequence["Supplier[Any]"] = ()) -> "Supplier[T]": + def void(from_exc: "Exception | None" = None, derived_from: Iterable["Supplier[Any]"] = ()) -> "Supplier[T]": """Returns a supplier that always raises :class:`Empty`.""" return VoidSupplier(from_exc, derived_from) @@ -201,9 +201,9 @@ def __eq__(self, other: object) -> bool: class OfCallableSupplier(Supplier[T]): - def __init__(self, func: Callable[[], T], derived_from: Sequence[Supplier[Any]]) -> None: + def __init__(self, func: Callable[[], T], derived_from: Iterable[Supplier[Any]]) -> None: self._func = func - self._derived_from = derived_from + self._derived_from = tuple(derived_from) def derived_from(self) -> Iterable[Supplier[Any]]: return self._derived_from @@ -222,7 +222,7 @@ def __eq__(self, other: object) -> bool: class OfSupplier(Supplier[T]): - def __init__(self, value: T, derived_from: Sequence[Supplier[Any]]) -> None: + def __init__(self, value: T, derived_from: Iterable[Supplier[Any]]) -> None: self._value = value self._derived_from = tuple(derived_from) @@ -243,7 +243,7 @@ def __eq__(self, other: object) -> bool: class VoidSupplier(Supplier[T]): - def __init__(self, from_exc: "Exception | None", derived_from: Sequence[Supplier[Any]]) -> None: + def __init__(self, from_exc: "Exception | None", derived_from: Iterable[Supplier[Any]]) -> None: self._from_exc = from_exc self._derived_from = tuple(derived_from) From d8dd16326463279f4cc126a6388f9839c7287c32 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Thu, 21 Mar 2024 23:58:52 +0100 Subject: [PATCH 53/79] feature: Add `kraken.build.experimental.functional.v1` API with common tasks for operations in a functional manner (fetching a file, tarball, writing a file and executing a shell command) --- .changelog/_unreleased.toml | 6 ++ .../experimental/functional/v1/__init__.py | 13 ++++ .../functional/v1/fetch_file_task.py | 61 ++++++++++++++++ .../functional/v1/fetch_tarball_task.py | 70 +++++++++++++++++++ .../functional/v1/shell_cmd_task.py | 40 +++++++++++ .../functional/v1/write_file_task.py | 54 ++++++++++++++ 6 files changed, 244 insertions(+) create mode 100644 kraken-build/src/kraken/build/experimental/functional/v1/__init__.py create mode 100644 kraken-build/src/kraken/build/experimental/functional/v1/fetch_file_task.py create mode 100644 kraken-build/src/kraken/build/experimental/functional/v1/fetch_tarball_task.py create mode 100644 kraken-build/src/kraken/build/experimental/functional/v1/shell_cmd_task.py create mode 100644 kraken-build/src/kraken/build/experimental/functional/v1/write_file_task.py diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index 13cb1f18..20c6b21e 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -63,3 +63,9 @@ id = "33f61844-e82b-40da-aba9-f552142d6abd" type = "improvement" description = "`Supplier.of()` and related functions now take an `Iterable[Supplier[Any]]` as the `derived_from` parameter instead of a `Sequence` as it is converted to a `tuple` later anyway." author = "@NiklasRosenstein" + +[[entries]] +id = "6bddf45d-7195-42d9-a490-2a567371d7f4" +type = "feature" +description = "Add `kraken.build.experimental.functional.v1` API with common tasks for operations in a functional manner (fetching a file, tarball, writing a file and executing a shell command)" +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/build/experimental/functional/v1/__init__.py b/kraken-build/src/kraken/build/experimental/functional/v1/__init__.py new file mode 100644 index 00000000..d24957c2 --- /dev/null +++ b/kraken-build/src/kraken/build/experimental/functional/v1/__init__.py @@ -0,0 +1,13 @@ +""" This module provides an experimental new functional API for defining Kraken tasks. """ + +from .write_file_task import write_file +from .fetch_file_task import fetch_file +from .fetch_tarball_task import fetch_tarball +from .shell_cmd_task import shell_cmd + +__all__ = [ + "write_file", + "fetch_file", + "fetch_tarball", + "shell_cmd", +] diff --git a/kraken-build/src/kraken/build/experimental/functional/v1/fetch_file_task.py b/kraken-build/src/kraken/build/experimental/functional/v1/fetch_file_task.py new file mode 100644 index 00000000..a8fd2687 --- /dev/null +++ b/kraken-build/src/kraken/build/experimental/functional/v1/fetch_file_task.py @@ -0,0 +1,61 @@ +import hashlib +from pathlib import Path +from typing import cast + +import httpx +from kraken.core.system.property import Property +from kraken.core.system.task import Task, TaskStatus + + +class FetchFileTask(Task): + """ Fetches a tarball from a URL and unpacks it. May also be used to fetch ZIP files. """ + + url: Property[str] = Property.required(help="The URL to fetch the tarball from.") + chmod: Property[int | None] = Property.default(None, help="The file mode to set on the downloaded file.") + out: Property[Path] = Property.output(help="The path to the unpacked tarball.") + + # TODO(@niklas): SHA256 checksum verification + + # Task + + def prepare(self) -> TaskStatus | None: + store_path = self.out.get() + if store_path.exists(): + return TaskStatus.skipped(f"{store_path} already exists.") + return None + + def execute(self) -> TaskStatus | None: + store_path = self.out.get() + url = self.url.get() + + print(f"Downloading {url} ...") + + with store_path.open("wb") as f: + with httpx.stream("GET", url, follow_redirects=True, timeout=60) as response: + response.raise_for_status() + for chunk in response.iter_bytes(): + f.write(chunk) + if (chmod := self.chmod.get()) is not None: + store_path.chmod(chmod) + + return TaskStatus.succeeded(f"Fetched file from {url} to {store_path}.") + + +def fetch_file(*, name: str, url: str, chmod: int | None = None, suffix: str = "") -> FetchFileTask: + """ Fetches a tarball from a URL and unpacks it. May also be used to fetch ZIP files. """ + + from kraken.build import context + + urlhash = hashlib.md5(url.encode()).hexdigest() + dest = context.build_directory / ".store" / f"{urlhash}-{name}{suffix}" + task_name = f"{name}-{urlhash}" + + if task_name in context.root_project.tasks(): + task = cast(FetchFileTask, context.root_project.task(task_name)) + else: + task = context.root_project.task(task_name, FetchFileTask) + task.url = url + task.chmod = chmod + task.out = dest + + return task diff --git a/kraken-build/src/kraken/build/experimental/functional/v1/fetch_tarball_task.py b/kraken-build/src/kraken/build/experimental/functional/v1/fetch_tarball_task.py new file mode 100644 index 00000000..ab340902 --- /dev/null +++ b/kraken-build/src/kraken/build/experimental/functional/v1/fetch_tarball_task.py @@ -0,0 +1,70 @@ +import hashlib +from pathlib import Path +import shutil +import tempfile +from typing import Literal, cast + +import httpx +from kraken.core.system.property import Property +from kraken.core.system.task import Task, TaskStatus + + +class FetchTarballTask(Task): + """ Fetches a tarball from a URL and unpacks it. May also be used to fetch ZIP files. """ + + url: Property[str] = Property.required(help="The URL to fetch the tarball from.") + format: Property[Literal["tar", "zip"] | None] = Property.default("tar", help="The format of the tarball. If not set, will be inferred from the file extension in the URL.") + out: Property[Path] = Property.output(help="The path to the unpacked tarball.") + + # TODO(@niklas): SHA256 checksum verification + + def _unpack(self, archive: Path, store_path: Path) -> None: + """ Unpacks the archive at the given path to the store path. """ + + if (format_ := self.format.get()) is None: + format_ = "zip" if archive.suffix == ".zip" else "tar" + + shutil.unpack_archive(archive, store_path, format_) + + # Task + + def prepare(self) -> TaskStatus | None: + store_path = self.out.get() + if store_path.exists(): + return TaskStatus.skipped(f"{store_path} already exists.") + return None + + def execute(self) -> TaskStatus | None: + + print(f"Downloading {self.url.get()} ...") + with tempfile.TemporaryDirectory() as tmp, Path(tmp).joinpath("archive").open("wb") as f: + with httpx.stream("GET", self.url.get(), follow_redirects=True, timeout=60) as response: + response.raise_for_status() + for chunk in response.iter_bytes(): + f.write(chunk) + + store_path = self.out.get() + print(f"Unpacking to {store_path} ...") + self._unpack(Path(tmp) / "archive", store_path) + + return TaskStatus.succeeded(f"Fetched tarball from {self.url.get()} to {store_path}.") + + +def fetch_tarball(*, name: str, url: str, format: Literal["tar", "zip"] | None = None) -> FetchTarballTask: + """ Fetches a tarball from a URL and unpacks it. May also be used to fetch ZIP files. """ + + from kraken.build import context + + urlhash = hashlib.md5(url.encode()).hexdigest() + dest = context.build_directory / ".store" / f"{urlhash}-{name}" + task_name = f"{name}-{urlhash}" + + if task_name in context.root_project.tasks(): + task = cast(FetchTarballTask, context.root_project.task(task_name)) + else: + task = context.root_project.task(task_name, FetchTarballTask) + task.url = url + task.format = format + task.out = dest + + return task diff --git a/kraken-build/src/kraken/build/experimental/functional/v1/shell_cmd_task.py b/kraken-build/src/kraken/build/experimental/functional/v1/shell_cmd_task.py new file mode 100644 index 00000000..0997fd04 --- /dev/null +++ b/kraken-build/src/kraken/build/experimental/functional/v1/shell_cmd_task.py @@ -0,0 +1,40 @@ +import os +import subprocess +from typing import Any +from kraken.common.supplier import Supplier +from kraken.core import Task, Property, TaskStatus + + +class ShellCmdTask(Task): + """ Executes a shell command. """ + + shell: Property[str | None] = Property.default(None, help="The shell command to execute. Defaults to the $SHELL variable.") + script: Property[str] = Property.required(help="The script to execute.") + cwd: Property[str] = Property.default("", help="The working directory to execute the command in.") + env: Property[dict[str, str]] = Property.default({}, help="The environment variables to set for the command.") + + def execute(self) -> TaskStatus | None: + shell = self.shell.get() or os.getenv("SHELL") or "sh" + script = self.script.get() + cwd = self.cwd.get() or self.project.directory + env = {**os.environ, **self.env.get()} + command = [shell, "-c", script] + return TaskStatus.from_exit_code( + command, + subprocess.call(command, env=env, cwd=cwd, stdin=subprocess.DEVNULL), + ) + + +def shell_cmd(*, name: str, template: str, shell: str | None = None, **kwargs: str | Supplier[str]) -> ShellCmdTask: + """ Create a task that runs a shell command. The *template* may contain `{key}` placeholders that will be + replaced with the corresponding value from *kwargs*. """ + + from kraken.build import project + + kwargs_suppliers = {k: Supplier.of(v) for k, v in kwargs.items()} + script = Supplier.of(template, kwargs_suppliers.values()).map(lambda s: s.format(**{k: v.get() for k, v in kwargs_suppliers.items()})) + + task = project.task(name, ShellCmdTask) + task.shell = shell + task.script = script + return task diff --git a/kraken-build/src/kraken/build/experimental/functional/v1/write_file_task.py b/kraken-build/src/kraken/build/experimental/functional/v1/write_file_task.py new file mode 100644 index 00000000..c72b8399 --- /dev/null +++ b/kraken-build/src/kraken/build/experimental/functional/v1/write_file_task.py @@ -0,0 +1,54 @@ +import hashlib +from pathlib import Path +from textwrap import dedent + +from kraken.common.supplier import Supplier +from kraken.core.system.property import Property +from kraken.core.system.task import Task, TaskStatus + + +class WriteFile(Task): + """ Fetches a file from a URL. """ + + content: Property[str] = Property.required(help="The content of the file.") + out: Property[Path] = Property.output(help="The path to the output file.") + + # Task + + def prepare(self) -> TaskStatus | None: + store_path = self.out.get() + if store_path.exists() and store_path.read_text() == self.content.get(): + return TaskStatus.skipped(f"{store_path} already exists.") + return None + + def execute(self) -> TaskStatus | None: + store_path = self.out.get() + content = self.content.get() + store_path.parent.mkdir(parents=True, exist_ok=True) + store_path.write_text(content) + return TaskStatus.succeeded(f"Wrote {len(content)} {store_path}") + + +def write_file(*, name: str, content: str | Supplier[str] | None = None, content_dedent: str | Supplier[str] | None = None) -> Supplier[Path]: + """ Writes a file to the store. + + NOTE: Because task names must be known before hand but the content hash can only be known at a later time, the + task name is fixed as specified with *name* and thus may conflict if the same name is reused. + """ + + from kraken.build import context + + if content_dedent is not None: + content = Supplier.of(content_dedent).map(dedent) + elif content is not None: + content = Supplier.of(content) + else: + raise ValueError("Either content or content_dedent must be set.") + + store_dir =context.build_directory / ".store" + dest = content.map(lambda c: hashlib.md5(c.encode()).hexdigest()).map(lambda h: store_dir / f"{h}-{name}") + + task = context.root_project.task(name, WriteFile) + task.content =content + task.out = dest + return task.out From 72dc73d9ab41bf58fcece09f3f804364c9bc338b Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 22 Mar 2024 00:25:21 +0100 Subject: [PATCH 54/79] feature: Install Buffrs CLI from GitHub releases --- .changelog/_unreleased.toml | 6 +++++ .../src/kraken/std/buffrs/__init__.py | 27 ++++++++++--------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index 20c6b21e..d52ae3c1 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -69,3 +69,9 @@ id = "6bddf45d-7195-42d9-a490-2a567371d7f4" type = "feature" description = "Add `kraken.build.experimental.functional.v1` API with common tasks for operations in a functional manner (fetching a file, tarball, writing a file and executing a shell command)" author = "@NiklasRosenstein" + +[[entries]] +id = "640c9b5a-e459-429c-8aa1-b92430cb5065" +type = "feature" +description = "Install Buffrs CLI from GitHub releases" +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/std/buffrs/__init__.py b/kraken-build/src/kraken/std/buffrs/__init__.py index b711baf7..5945ef00 100644 --- a/kraken-build/src/kraken/std/buffrs/__init__.py +++ b/kraken-build/src/kraken/std/buffrs/__init__.py @@ -5,11 +5,11 @@ from pathlib import Path from typing import cast +from kraken.build.experimental.functional.v1 import fetch_tarball from kraken.common.supplier import Supplier from kraken.core import Project from .tasks import BuffrsInstallTask, BuffrsLoginTask, BuffrsPublishTask -from kraken.std.util import fetch_tarball logger = logging.getLogger(__name__) @@ -40,18 +40,21 @@ def buffrs_login( task = root_project.task("buffrsLogin", BuffrsLoginTask) task.registry = registry task.token = token - task.buffrs_bin = buffrs_bin or buffrs_fetch_binary().map(str) + task.buffrs_bin = buffrs_bin or get_buffrs_binary().map(str) return task -def buffrs_install(*, project: Project | None = None, - buffrs_bin: Supplier[str] | None = None,) -> BuffrsInstallTask: +def buffrs_install( + *, + project: Project | None = None, + buffrs_bin: Supplier[str] | None = None, +) -> BuffrsInstallTask: """Installs buffrs dependencies defined in the `Proto.toml`""" project = project or Project.current() task = project.task("buffrsInstall", BuffrsInstallTask) - task.buffrs_bin = buffrs_bin or buffrs_fetch_binary().map(str) + task.buffrs_bin = buffrs_bin or get_buffrs_binary().map(str) return task @@ -72,11 +75,11 @@ def buffrs_publish( task.registry = registry task.repository = repository task.version = version - task.buffrs_bin = buffrs_bin or buffrs_fetch_binary().map(str) + task.buffrs_bin = buffrs_bin or get_buffrs_binary().map(str) return task -def buffrs_fetch_binary( +def get_buffrs_binary( version: str = "0.8.0", target_triplet: str | None = None, ) -> Supplier[Path]: @@ -95,20 +98,20 @@ def buffrs_fetch_binary( def get_buffrs_triplet() -> str: - """ Returns the Buffrs target triplet for the current platform.""" + """Returns the Buffrs target triplet for the current platform.""" match (_platform.machine(), _platform.system()): case ("x86_64", "Linux"): return "x86_64-unknown-linux-gnu" case ("x86_64", "Darwin"): return "x86_64-apple-darwin" - case ("x86_64", "Windows"): + case ("AMD64", "Windows"): return "x86_64-pc-windows-msvc" case ("aarch64", "Linux"): return "arm-unknown-linux-gnueabihf" - case ("aarch64", "Darwin"): + case ("arm64", "Darwin"): return "aarch64-apple-darwin" - case ("aarch64", "Windows"): - return "i686-pc-windows-msvc" + # case ("aarch64", "Windows"): + # return "i686-pc-windows-msvc" case _: raise NotImplementedError(f"Platform {_platform.machine()} is not supported by Buffrs.") From ee5334fe3a8a790ee02c7919782877ad3a66d0dc Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 22 Mar 2024 00:25:51 +0100 Subject: [PATCH 55/79] improvement: `buffrs.install` task now produces a `.gitignore` file in the `proto/vendor` directory to exclude it --- .changelog/_unreleased.toml | 6 ++++++ kraken-build/src/kraken/std/buffrs/tasks.py | 11 ++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index d52ae3c1..7f52a0bf 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -75,3 +75,9 @@ id = "640c9b5a-e459-429c-8aa1-b92430cb5065" type = "feature" description = "Install Buffrs CLI from GitHub releases" author = "@NiklasRosenstein" + +[[entries]] +id = "6cd1582c-d975-4888-809c-89cd7d66073c" +type = "improvement" +description = "`buffrs.install` task now produces a `.gitignore` file in the `proto/vendor` directory to exclude it" +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/std/buffrs/tasks.py b/kraken-build/src/kraken/std/buffrs/tasks.py index 73112b82..330b334d 100644 --- a/kraken-build/src/kraken/std/buffrs/tasks.py +++ b/kraken-build/src/kraken/std/buffrs/tasks.py @@ -34,11 +34,20 @@ class BuffrsInstallTask(Task): def execute(self) -> TaskStatus: command = [self.buffrs_bin.get(), "install"] - return TaskStatus.from_exit_code( + + result = TaskStatus.from_exit_code( command, sp.call(command, cwd=self.project.directory), ) + if result.is_succeeded(): + # Create a .gitignore file in the proto/vendor directory to ensure it does not get committed. + vendor_dir = self.project.directory / "proto" / "vendor" + vendor_dir.mkdir(exist_ok=True, parents=True) + vendor_dir.joinpath(".gitignore").write_text("*\n") + + return result + class BuffrsPublishTask(Task): """This task uses buffrs to publish a new release of the buffrs package. From 92d62e5aedf3a4dd7630440eff41828996a1539c Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 22 Mar 2024 00:28:54 +0100 Subject: [PATCH 56/79] improvement: Pass a default `buf.yaml` configuration to `buf lint` to disable `PACKAGE_VERSION_SUFFIX` and `PACKAGE_DIRECTORY_MATCH` lint rules for compatibility with Buffrs. --- .changelog/_unreleased.toml | 14 +- .../src/kraken/std/protobuf/__init__.py | 161 +++++++++--------- kraken-build/src/kraken/std/python/project.py | 23 +-- 3 files changed, 99 insertions(+), 99 deletions(-) diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index 7f52a0bf..ef109e10 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -37,7 +37,7 @@ author = "@NiklasRosenstein" [[entries]] id = "1ae440c5-cd7a-4fdc-a6a9-da10165fa260" type = "feature" -description = "Add `ProtocTask` and `BufInstallTask` to `kraken.std.protobuf`" +description = "Add `ProtocTask` to `kraken.std.protobuf`" author = "@NiklasRosenstein" [[entries]] @@ -81,3 +81,15 @@ id = "6cd1582c-d975-4888-809c-89cd7d66073c" type = "improvement" description = "`buffrs.install` task now produces a `.gitignore` file in the `proto/vendor` directory to exclude it" author = "@NiklasRosenstein" + +[[entries]] +id = "f8a491fb-daea-4279-b5d6-eb15f73a847b" +type = "breaking change" +description = "Remove `BufLintTask` and `BufInstallTask` in favor of `buf()` function that uses our new `kraken.build.experimental.functional.v1` API to implement the same functionality more streamlined" +author = "@NiklasRosenstein" + +[[entries]] +id = "7d264f42-d0c1-4077-9fbb-c869ab093b46" +type = "improvement" +description = "Pass a default `buf.yaml` configuration to `buf lint` to disable `PACKAGE_VERSION_SUFFIX` and `PACKAGE_DIRECTORY_MATCH` lint rules for compatibility with Buffrs." +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/std/protobuf/__init__.py b/kraken-build/src/kraken/std/protobuf/__init__.py index 716d277f..07445b15 100644 --- a/kraken-build/src/kraken/std/protobuf/__init__.py +++ b/kraken-build/src/kraken/std/protobuf/__init__.py @@ -2,95 +2,17 @@ from __future__ import annotations +import platform as _platform import subprocess as sp -import sys from collections.abc import Sequence from pathlib import Path -from platform import machine from shutil import rmtree -import httpx - +from kraken.build.experimental.functional.v1 import fetch_file, fetch_tarball, shell_cmd, write_file +from kraken.common.supplier import Supplier from kraken.core import Property, Task, TaskStatus -class BufInstallTask(Task): - """Installs [buf](https://github.com/bufbuild/buf) from GitHub.""" - - description = "Install buf." - version: Property[str] = Property.default("1.30.0", help="The version of `buf` to install.") - output_file: Property[Path] = Property.output(help="The path to the installed `buf` binary.") - - def _get_dist_url(self) -> str: - version = self.version.get() - suffix = "" - match sys.platform: - case "linux": - platform = "Linux" - case "darwin": - platform = "Darwin" - case "win32": - platform = "Windows" - suffix = ".exe" - case _: - raise NotImplementedError(f"Platform {sys.platform} is not supported by `buf`.") - match machine(): - case "x86_64": - arch = "x86_64" - case "aarch64": - arch = "aarch64" if platform == "Linux" else "arm64" - case _: - raise NotImplementedError(f"Architecture {machine()} is not supported by `buf`.") - return f"https://github.com/bufbuild/buf/releases/download/v{version}/buf-{platform}-{arch}{suffix}" - - def _get_output_file(self) -> Path: - filename = f"buf-v{self.version.get()}{'.exe' if sys.platform == 'win32' else ''}" - return self.project.context.build_directory / filename - - def prepare(self) -> TaskStatus | None: - if self._get_output_file().is_file(): - return TaskStatus.skipped("buf is already installed.") - return None - - def execute(self) -> TaskStatus | None: - dist_url = self._get_dist_url() - output_file = self._get_output_file() - output_file.parent.mkdir(parents=True, exist_ok=True) - - response = httpx.get(dist_url, timeout=10, follow_redirects=True) - response = response.raise_for_status() - output_file.write_bytes(response.content) - - self.output_file = output_file - return TaskStatus.succeeded(f"Installed buf to {output_file}.") - - -class BufFormatTask(Task): - """Format Protobuf files with `buf`.""" - - description = "Format Protobuf files with buf." - buf_bin: Property[str] = Property.default("buf", help="Path to `buf` binary.") - - def execute(self) -> TaskStatus | None: - command = [self.buf_bin.get(), "format", "-w"] - result = sp.call(command, cwd=self.project.directory / "proto") - - return TaskStatus.from_exit_code(command, result) - - -class BufLintTask(Task): - """Lint Protobuf files with `buf`.""" - - description = "Lint Protobuf files with buf." - buf_bin: Property[str] = Property.default("buf", help="Path to `buf` binary.") - - def execute(self) -> TaskStatus | None: - command = [self.buf_bin.get(), "lint"] - result = sp.call(command, cwd=self.project.directory / "proto") - - return TaskStatus.from_exit_code(command, result) - - class ProtocTask(Task): """Generate code with `protoc`.""" @@ -129,3 +51,80 @@ def execute(self) -> TaskStatus | None: command, sp.call(command, cwd=self.project.directory), ) + + +def get_buf_binary(version: str, target_triplet: str | None = None) -> Supplier[Path]: + """Fetches the `buf` binary from GitHub.""" + + target_triplet = target_triplet or get_buf_triplet() + + if "Windows" in target_triplet: + url = f"https://github.com/bufbuild/buf/releases/download/v{version}/buf-{target_triplet}.exe" + return fetch_file(name="buf", url=url, chmod=0o777, suffix=".exe").out.map(lambda p: p.absolute()) + else: + url = f"https://github.com/bufbuild/buf/releases/download/v{version}/buf-{target_triplet}.tar.gz" + return fetch_tarball(name="buf", url=url).out.map(lambda p: p.absolute() / "buf" / "bin" / "buf") + + +def get_buf_triplet() -> str: + match (_platform.machine(), _platform.system()): + case (machine, "Linux"): + return f"Linux-{machine}" + case (machine, "Darwin"): + return f"Darwin-{machine}" + case ("AMD64", "Windows"): + return "Windows-x86_64" + case other: + raise NotImplementedError(f"Platform {other} is not supported by `buf`.") + + +def buf( + *, + buf_version: str = "1.30.0", + path: str = "proto", + exclude_path: Sequence[str] = (), + dependencies: Sequence[Task] = (), +) -> tuple[Task, Task]: + from shlex import quote + + buf_bin = get_buf_binary(buf_version).map(str) + + # Configure buf; see https://buf.build/docs/lint/rules + buf_config = write_file( + name="buf.yaml", + content_dedent=""" + version: v1 + lint: + use: + - DEFAULT + except: + - PACKAGE_VERSION_SUFFIX + # NOTE(@niklas): We only ignore this rule because Buffrs uses hyphens in place of underscores for + # the generated directories, but the `package` directive in Protobuf can't contain hyphens. + - PACKAGE_DIRECTORY_MATCH + """, + ).map(str) + + exclude_args = " ".join(f"--exclude-path {quote(path)}" for path in exclude_path) + + buf_format = shell_cmd( + name="buf.format", + template='"{buf}" format -w --config "{config}" "{path}" {exclude_args}', + buf=buf_bin, + path=path, + config=buf_config, + exclude_args=exclude_args, + ) + buf_format.depends_on(*dependencies) + + buf_lint = shell_cmd( + name="buf.lint", + template='"{buf}" lint --config "{config}" "{path}" {exclude_args}', + buf=buf_bin, + path=path, + config=buf_config, + exclude_args=exclude_args, + ) + buf_lint.depends_on(*dependencies) + + return (buf_format, buf_lint) diff --git a/kraken-build/src/kraken/std/python/project.py b/kraken-build/src/kraken/std/python/project.py index 0a0a936e..73554900 100644 --- a/kraken-build/src/kraken/std/python/project.py +++ b/kraken-build/src/kraken/std/python/project.py @@ -215,30 +215,19 @@ def python_project( if protobuf_enabled and project.directory.joinpath("proto").is_dir(): - from kraken.std.protobuf import BufFormatTask, BufInstallTask, BufLintTask, ProtocTask from kraken.std.buffrs import buffrs_install as buffrs_install_task - from kraken.std.util import fetch_tarball + from kraken.std.protobuf import ProtocTask, buf_apply if project.directory.joinpath("Proto.toml").is_file(): buffrs_install = buffrs_install_task() else: buffrs_install = None - # TODO(@niklas): This is a temporary solution to fetch the Buf binary until we have a proper way to - buf_binary = fetch_tarball( - name="buf", - url=f"https://github.com/bufbuild/buf/releases/download/v{buf_version}/buf-Linux-aarch64.tar.gz", - ).out.map(lambda p: p.absolute() / "buf" / "bin" / "buf").map(str) - - buf_format = project.task("buf.format", BufFormatTask, group="fmt") - buf_format.buf_bin = buf_binary - - buf_lint = project.task("buf.lint", BufLintTask, group="lint") - buf_lint.buf_bin = buf_binary - - if buffrs_install is not None: - buf_lint.depends_on(buffrs_install) - buf_format.depends_on(buffrs_install) + buf_apply( + buf_version=buf_version, + path="proto/vendor" if buffrs_install else "proto", + dependencies=[buffrs_install] if buffrs_install else [], + ) protoc_bin = pex_build( binary_name="protoc", From 55655ab5d394bfb428d190eda2f691b62b442eed Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 22 Mar 2024 00:29:12 +0100 Subject: [PATCH 57/79] we moved fetch_tarball() --- kraken-build/src/kraken/std/util/__init__.py | 3 +- .../src/kraken/std/util/fetch_tarball_task.py | 75 ------------------- 2 files changed, 1 insertion(+), 77 deletions(-) delete mode 100644 kraken-build/src/kraken/std/util/fetch_tarball_task.py diff --git a/kraken-build/src/kraken/std/util/__init__.py b/kraken-build/src/kraken/std/util/__init__.py index a25847b9..eb53d438 100644 --- a/kraken-build/src/kraken/std/util/__init__.py +++ b/kraken-build/src/kraken/std/util/__init__.py @@ -3,6 +3,5 @@ from __future__ import annotations from .copyright_task import check_and_format_copyright -from .fetch_tarball_task import fetch_tarball -__all__ = ["check_and_format_copyright", "fetch_tarball"] +__all__ = ["check_and_format_copyright"] diff --git a/kraken-build/src/kraken/std/util/fetch_tarball_task.py b/kraken-build/src/kraken/std/util/fetch_tarball_task.py deleted file mode 100644 index adc89497..00000000 --- a/kraken-build/src/kraken/std/util/fetch_tarball_task.py +++ /dev/null @@ -1,75 +0,0 @@ - - -import contextlib -import hashlib -from pathlib import Path -import shutil -import tempfile -from typing import Literal, cast - -import httpx -from kraken.core.system.property import Property -from kraken.core.system.task import Task, TaskStatus - - -class FetchTarballTask(Task): - """ Fetches a tarball from a URL and unpacks it. May also be used to fetch ZIP files. """ - - name_: Property[str] = Property.required(help="The name of the task.") - url: Property[str] = Property.required(help="The URL to fetch the tarball from.") - format: Property[Literal["tar", "zip"] | None] = Property.default("tar", help="The format of the tarball. If not set, will be inferred from the file extension in the URL.") - out: Property[Path] = Property.output(help="The path to the unpacked tarball.") - - # TODO(@niklas): SHA256 checksum verification - - def _unpack(self, archive: Path, store_path: Path) -> None: - """ Unpacks the archive at the given path to the store path. """ - - if (format_ := self.format.get()) is None: - format_ = "zip" if archive.suffix == ".zip" else "tar" - - shutil.unpack_archive(archive, store_path, format_) - - # Task - - def prepare(self) -> TaskStatus | None: - store_path = self.out.get() - if store_path.exists(): - return TaskStatus.skipped(f"{store_path} already exists.") - return None - - def execute(self) -> TaskStatus | None: - - print(f"Downloading {self.url.get()} ...") - with tempfile.TemporaryDirectory() as tmp, Path(tmp).joinpath("archive").open("wb") as f: - with httpx.stream("GET", self.url.get(), follow_redirects=True, timeout=60) as response: - response.raise_for_status() - for chunk in response.iter_bytes(): - f.write(chunk) - - store_path = self.out.get() - print(f"Unpacking to {store_path} ...") - self._unpack(Path(tmp) / "archive", store_path) - - return TaskStatus.succeeded(f"Fetched tarball from {self.url.get()} to {store_path}.") - - -def fetch_tarball(*, name: str, url: str, format: Literal["tar", "zip"] | None = None) -> FetchTarballTask: - """ Fetches a tarball from a URL and unpacks it. May also be used to fetch ZIP files. """ - - from kraken.build import context - - urlhash = hashlib.md5(url.encode()).hexdigest() - dest = context.build_directory / ".store" / f"{urlhash}-{name}" - task_name = f"{name}-{urlhash}" - - if task_name in context.root_project.tasks(): - task = cast(FetchTarballTask, context.root_project.task(task_name)) - else: - task = context.root_project.task(task_name, FetchTarballTask) - task.name_ = name - task.url = url - task.format = format - task.out = dest - - return task From 7233e1aa51284fd408e34624f1091cd24015fed9 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 22 Mar 2024 00:29:47 +0100 Subject: [PATCH 58/79] fix buf import --- kraken-build/src/kraken/std/python/project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kraken-build/src/kraken/std/python/project.py b/kraken-build/src/kraken/std/python/project.py index 73554900..a8411092 100644 --- a/kraken-build/src/kraken/std/python/project.py +++ b/kraken-build/src/kraken/std/python/project.py @@ -216,14 +216,14 @@ def python_project( if protobuf_enabled and project.directory.joinpath("proto").is_dir(): from kraken.std.buffrs import buffrs_install as buffrs_install_task - from kraken.std.protobuf import ProtocTask, buf_apply + from kraken.std.protobuf import ProtocTask, buf if project.directory.joinpath("Proto.toml").is_file(): buffrs_install = buffrs_install_task() else: buffrs_install = None - buf_apply( + buf( buf_version=buf_version, path="proto/vendor" if buffrs_install else "proto", dependencies=[buffrs_install] if buffrs_install else [], From 760c38b4796d7ed7d34e1a75f7676960868aeeb1 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 22 Mar 2024 00:45:33 +0100 Subject: [PATCH 59/79] fix: Remove default files that should be excluded for `isort`, `pycln` and `black` so that a `build/` directory in the `src/` directory also gets checked. We can do this safely because we expect the `src/` folder in a Python project to contain only files that should be checked unless otherwise explicitly specified --- .changelog/_unreleased.toml | 6 ++++++ kraken-build/src/kraken/std/python/tasks/black_task.py | 4 +++- kraken-build/src/kraken/std/python/tasks/isort_task.py | 1 + kraken-build/src/kraken/std/python/tasks/pycln_task.py | 3 +-- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index ef109e10..8ee26f23 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -93,3 +93,9 @@ id = "7d264f42-d0c1-4077-9fbb-c869ab093b46" type = "improvement" description = "Pass a default `buf.yaml` configuration to `buf lint` to disable `PACKAGE_VERSION_SUFFIX` and `PACKAGE_DIRECTORY_MATCH` lint rules for compatibility with Buffrs." author = "@NiklasRosenstein" + +[[entries]] +id = "a8814dcb-b28b-42ed-aa0b-92297a63877d" +type = "fix" +description = "Remove default files that should be excluded for `isort`, `pycln` and `black` so that a `build/` directory in the `src/` directory also gets checked. We can do this safely because we expect the `src/` folder in a Python project to contain only files that should be checked unless otherwise explicitly specified" +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/std/python/tasks/black_task.py b/kraken-build/src/kraken/std/python/tasks/black_task.py index 8f903e6b..3d42cfab 100644 --- a/kraken-build/src/kraken/std/python/tasks/black_task.py +++ b/kraken-build/src/kraken/std/python/tasks/black_task.py @@ -37,8 +37,10 @@ def dump(self) -> dict[str, Any]: exclude_patterns.append("^/" + re.escape(dirname.strip("/")) + "/.*$") exclude_regex = "(" + "|".join(exclude_patterns) + ")" config["exclude"] = exclude_regex + else: + config["exclude"] = "" - return config + return {"tool": {"black": config}} def to_file(self, path: Path) -> None: path.write_text(tomli_w.dumps(self.dump())) diff --git a/kraken-build/src/kraken/std/python/tasks/isort_task.py b/kraken-build/src/kraken/std/python/tasks/isort_task.py index c62cf712..b86d2539 100644 --- a/kraken-build/src/kraken/std/python/tasks/isort_task.py +++ b/kraken-build/src/kraken/std/python/tasks/isort_task.py @@ -29,6 +29,7 @@ def to_config(self) -> ConfigParser: config.set(section_name, "profile", self.profile) config.set(section_name, "line_length", str(self.line_length)) config.set(section_name, "combine_as_imports", str(self.combine_as_imports).lower()) + config.set(section_name, "skip", "") config.set(section_name, "extend_skip", ",".join(self.extend_skip)) return config diff --git a/kraken-build/src/kraken/std/python/tasks/pycln_task.py b/kraken-build/src/kraken/std/python/tasks/pycln_task.py index b60309c9..c2f2d359 100644 --- a/kraken-build/src/kraken/std/python/tasks/pycln_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pycln_task.py @@ -27,8 +27,7 @@ class PyclnTask(EnvironmentAwareDispatchTask): # EnvironmentAwareDispatchTask def get_execute_command(self) -> list[str]: - command = [self.pycln_bin.get(), str(self.settings.source_directory)] - command += self.settings.get_tests_directory_as_args() + command = [self.pycln_bin.get(), "--exclude", "^$"] command += [str(directory) for directory in self.settings.lint_enforced_directories] command += [str(p) for p in self.additional_files.get()] if self.check_only.get(): From 203040bc73a58d69d8f419aa3ad950a61b79b179 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 22 Mar 2024 00:46:23 +0100 Subject: [PATCH 60/79] fmt on kraken-build/src/kraken/build/ --- kraken-build/src/kraken/build/__init__.py | 1 + .../experimental/functional/v1/__init__.py | 2 +- .../functional/v1/fetch_file_task.py | 5 +++-- .../functional/v1/fetch_tarball_task.py | 13 ++++++++----- .../functional/v1/shell_cmd_task.py | 18 +++++++++++------- .../functional/v1/write_file_task.py | 12 +++++++----- .../src/kraken/build/tests/test_import.py | 6 ++++-- .../src/kraken/build/utils/import_helper.py | 7 ++++--- 8 files changed, 39 insertions(+), 25 deletions(-) diff --git a/kraken-build/src/kraken/build/__init__.py b/kraken-build/src/kraken/build/__init__.py index a42eadf4..f159e6e9 100644 --- a/kraken-build/src/kraken/build/__init__.py +++ b/kraken-build/src/kraken/build/__init__.py @@ -1,4 +1,5 @@ from kraken.core import Context, Project + from .utils.import_helper import _KrakenBuildModuleWrapper # Install a wrapper around the module object to allow build-scripts to always import the current (i.e. their own) diff --git a/kraken-build/src/kraken/build/experimental/functional/v1/__init__.py b/kraken-build/src/kraken/build/experimental/functional/v1/__init__.py index d24957c2..df300b4f 100644 --- a/kraken-build/src/kraken/build/experimental/functional/v1/__init__.py +++ b/kraken-build/src/kraken/build/experimental/functional/v1/__init__.py @@ -1,9 +1,9 @@ """ This module provides an experimental new functional API for defining Kraken tasks. """ -from .write_file_task import write_file from .fetch_file_task import fetch_file from .fetch_tarball_task import fetch_tarball from .shell_cmd_task import shell_cmd +from .write_file_task import write_file __all__ = [ "write_file", diff --git a/kraken-build/src/kraken/build/experimental/functional/v1/fetch_file_task.py b/kraken-build/src/kraken/build/experimental/functional/v1/fetch_file_task.py index a8fd2687..4cf9b455 100644 --- a/kraken-build/src/kraken/build/experimental/functional/v1/fetch_file_task.py +++ b/kraken-build/src/kraken/build/experimental/functional/v1/fetch_file_task.py @@ -3,12 +3,13 @@ from typing import cast import httpx + from kraken.core.system.property import Property from kraken.core.system.task import Task, TaskStatus class FetchFileTask(Task): - """ Fetches a tarball from a URL and unpacks it. May also be used to fetch ZIP files. """ + """Fetches a tarball from a URL and unpacks it. May also be used to fetch ZIP files.""" url: Property[str] = Property.required(help="The URL to fetch the tarball from.") chmod: Property[int | None] = Property.default(None, help="The file mode to set on the downloaded file.") @@ -42,7 +43,7 @@ def execute(self) -> TaskStatus | None: def fetch_file(*, name: str, url: str, chmod: int | None = None, suffix: str = "") -> FetchFileTask: - """ Fetches a tarball from a URL and unpacks it. May also be used to fetch ZIP files. """ + """Fetches a tarball from a URL and unpacks it. May also be used to fetch ZIP files.""" from kraken.build import context diff --git a/kraken-build/src/kraken/build/experimental/functional/v1/fetch_tarball_task.py b/kraken-build/src/kraken/build/experimental/functional/v1/fetch_tarball_task.py index ab340902..5397b715 100644 --- a/kraken-build/src/kraken/build/experimental/functional/v1/fetch_tarball_task.py +++ b/kraken-build/src/kraken/build/experimental/functional/v1/fetch_tarball_task.py @@ -1,25 +1,28 @@ import hashlib -from pathlib import Path import shutil import tempfile +from pathlib import Path from typing import Literal, cast import httpx + from kraken.core.system.property import Property from kraken.core.system.task import Task, TaskStatus class FetchTarballTask(Task): - """ Fetches a tarball from a URL and unpacks it. May also be used to fetch ZIP files. """ + """Fetches a tarball from a URL and unpacks it. May also be used to fetch ZIP files.""" url: Property[str] = Property.required(help="The URL to fetch the tarball from.") - format: Property[Literal["tar", "zip"] | None] = Property.default("tar", help="The format of the tarball. If not set, will be inferred from the file extension in the URL.") + format: Property[Literal["tar", "zip"] | None] = Property.default( + "tar", help="The format of the tarball. If not set, will be inferred from the file extension in the URL." + ) out: Property[Path] = Property.output(help="The path to the unpacked tarball.") # TODO(@niklas): SHA256 checksum verification def _unpack(self, archive: Path, store_path: Path) -> None: - """ Unpacks the archive at the given path to the store path. """ + """Unpacks the archive at the given path to the store path.""" if (format_ := self.format.get()) is None: format_ = "zip" if archive.suffix == ".zip" else "tar" @@ -51,7 +54,7 @@ def execute(self) -> TaskStatus | None: def fetch_tarball(*, name: str, url: str, format: Literal["tar", "zip"] | None = None) -> FetchTarballTask: - """ Fetches a tarball from a URL and unpacks it. May also be used to fetch ZIP files. """ + """Fetches a tarball from a URL and unpacks it. May also be used to fetch ZIP files.""" from kraken.build import context diff --git a/kraken-build/src/kraken/build/experimental/functional/v1/shell_cmd_task.py b/kraken-build/src/kraken/build/experimental/functional/v1/shell_cmd_task.py index 0997fd04..69387f5f 100644 --- a/kraken-build/src/kraken/build/experimental/functional/v1/shell_cmd_task.py +++ b/kraken-build/src/kraken/build/experimental/functional/v1/shell_cmd_task.py @@ -1,14 +1,16 @@ import os import subprocess -from typing import Any + from kraken.common.supplier import Supplier -from kraken.core import Task, Property, TaskStatus +from kraken.core import Property, Task, TaskStatus class ShellCmdTask(Task): - """ Executes a shell command. """ + """Executes a shell command.""" - shell: Property[str | None] = Property.default(None, help="The shell command to execute. Defaults to the $SHELL variable.") + shell: Property[str | None] = Property.default( + None, help="The shell command to execute. Defaults to the $SHELL variable." + ) script: Property[str] = Property.required(help="The script to execute.") cwd: Property[str] = Property.default("", help="The working directory to execute the command in.") env: Property[dict[str, str]] = Property.default({}, help="The environment variables to set for the command.") @@ -26,13 +28,15 @@ def execute(self) -> TaskStatus | None: def shell_cmd(*, name: str, template: str, shell: str | None = None, **kwargs: str | Supplier[str]) -> ShellCmdTask: - """ Create a task that runs a shell command. The *template* may contain `{key}` placeholders that will be - replaced with the corresponding value from *kwargs*. """ + """Create a task that runs a shell command. The *template* may contain `{key}` placeholders that will be + replaced with the corresponding value from *kwargs*.""" from kraken.build import project kwargs_suppliers = {k: Supplier.of(v) for k, v in kwargs.items()} - script = Supplier.of(template, kwargs_suppliers.values()).map(lambda s: s.format(**{k: v.get() for k, v in kwargs_suppliers.items()})) + script = Supplier.of(template, kwargs_suppliers.values()).map( + lambda s: s.format(**{k: v.get() for k, v in kwargs_suppliers.items()}) + ) task = project.task(name, ShellCmdTask) task.shell = shell diff --git a/kraken-build/src/kraken/build/experimental/functional/v1/write_file_task.py b/kraken-build/src/kraken/build/experimental/functional/v1/write_file_task.py index c72b8399..41110c4c 100644 --- a/kraken-build/src/kraken/build/experimental/functional/v1/write_file_task.py +++ b/kraken-build/src/kraken/build/experimental/functional/v1/write_file_task.py @@ -8,7 +8,7 @@ class WriteFile(Task): - """ Fetches a file from a URL. """ + """Fetches a file from a URL.""" content: Property[str] = Property.required(help="The content of the file.") out: Property[Path] = Property.output(help="The path to the output file.") @@ -29,8 +29,10 @@ def execute(self) -> TaskStatus | None: return TaskStatus.succeeded(f"Wrote {len(content)} {store_path}") -def write_file(*, name: str, content: str | Supplier[str] | None = None, content_dedent: str | Supplier[str] | None = None) -> Supplier[Path]: - """ Writes a file to the store. +def write_file( + *, name: str, content: str | Supplier[str] | None = None, content_dedent: str | Supplier[str] | None = None +) -> Supplier[Path]: + """Writes a file to the store. NOTE: Because task names must be known before hand but the content hash can only be known at a later time, the task name is fixed as specified with *name* and thus may conflict if the same name is reused. @@ -45,10 +47,10 @@ def write_file(*, name: str, content: str | Supplier[str] | None = None, content else: raise ValueError("Either content or content_dedent must be set.") - store_dir =context.build_directory / ".store" + store_dir = context.build_directory / ".store" dest = content.map(lambda c: hashlib.md5(c.encode()).hexdigest()).map(lambda h: store_dir / f"{h}-{name}") task = context.root_project.task(name, WriteFile) - task.content =content + task.content = content task.out = dest return task.out diff --git a/kraken-build/src/kraken/build/tests/test_import.py b/kraken-build/src/kraken/build/tests/test_import.py index b0db6ec1..3d648026 100644 --- a/kraken-build/src/kraken/build/tests/test_import.py +++ b/kraken-build/src/kraken/build/tests/test_import.py @@ -1,10 +1,12 @@ +from pathlib import Path + import pytest + from kraken.core import Context, Project -from pathlib import Path def test_import_current_context_and_project_from_kraken_build() -> None: - """ Test that you can import the current Kraken build context and project from `kraken.build`. """ + """Test that you can import the current Kraken build context and project from `kraken.build`.""" with pytest.raises(RuntimeError): from kraken.build import context diff --git a/kraken-build/src/kraken/build/utils/import_helper.py b/kraken-build/src/kraken/build/utils/import_helper.py index a5313577..8d88a626 100644 --- a/kraken-build/src/kraken/build/utils/import_helper.py +++ b/kraken-build/src/kraken/build/utils/import_helper.py @@ -1,12 +1,13 @@ import sys import types from typing import Any + from kraken.core import Context, Project class _KrakenBuildModuleWrapper: - """ A wrapper for the `kraken.build` module to allow build-scripts to always import the current (i.e. their own) - project and the Kraken build context. """ + """A wrapper for the `kraken.build` module to allow build-scripts to always import the current (i.e. their own) + project and the Kraken build context.""" def __init__(self, module: types.ModuleType) -> None: self.__module = module @@ -21,5 +22,5 @@ def __getattr__(self, name: str) -> Any: @staticmethod def install(module_name: str) -> None: - """ Install the wrapper around the given module. """ + """Install the wrapper around the given module.""" sys.modules[module_name] = _KrakenBuildModuleWrapper(sys.modules[module_name]) # type: ignore[assignment] From 303549dd63ae718cf9bf0fc4ffc6bf5ed2e6ada3 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 22 Mar 2024 21:38:32 +0100 Subject: [PATCH 61/79] move new python_project() etc. to `kraken.build.python.v1alpha1` and the functional utils to `kraken.build.functional.v1alpha1` --- .../api/kraken.build.functional.v1alpha1.md | 5 +++ docs/docs/api/kraken.build.python.v1alpha1.md | 5 +++ kraken-build/.kraken.py | 2 +- .../v1 => functional/v1alpha1}/__init__.py | 4 ++- .../v1alpha1}/fetch_file_task.py | 0 .../v1alpha1}/fetch_tarball_task.py | 0 .../v1alpha1}/shell_cmd_task.py | 0 .../v1alpha1}/write_file_task.py | 0 .../kraken/build/python/v1alpha1/__init__.py | 7 ++++ .../python/v1alpha1}/project.py | 33 +++++++++---------- kraken-build/src/kraken/common/supplier.py | 2 +- kraken-build/src/kraken/std/__init__.py | 4 --- .../src/kraken/std/buffrs/__init__.py | 2 +- .../src/kraken/std/protobuf/__init__.py | 2 +- kraken-wrapper/.kraken.py | 2 +- 15 files changed, 41 insertions(+), 27 deletions(-) create mode 100644 docs/docs/api/kraken.build.functional.v1alpha1.md create mode 100644 docs/docs/api/kraken.build.python.v1alpha1.md rename kraken-build/src/kraken/build/{experimental/functional/v1 => functional/v1alpha1}/__init__.py (77%) rename kraken-build/src/kraken/build/{experimental/functional/v1 => functional/v1alpha1}/fetch_file_task.py (100%) rename kraken-build/src/kraken/build/{experimental/functional/v1 => functional/v1alpha1}/fetch_tarball_task.py (100%) rename kraken-build/src/kraken/build/{experimental/functional/v1 => functional/v1alpha1}/shell_cmd_task.py (100%) rename kraken-build/src/kraken/build/{experimental/functional/v1 => functional/v1alpha1}/write_file_task.py (100%) create mode 100644 kraken-build/src/kraken/build/python/v1alpha1/__init__.py rename kraken-build/src/kraken/{std/python => build/python/v1alpha1}/project.py (94%) diff --git a/docs/docs/api/kraken.build.functional.v1alpha1.md b/docs/docs/api/kraken.build.functional.v1alpha1.md new file mode 100644 index 00000000..a1127761 --- /dev/null +++ b/docs/docs/api/kraken.build.functional.v1alpha1.md @@ -0,0 +1,5 @@ +--- +title: kraken.build.functional.v1alpha1 +--- + +::: kraken.build.functional.v1alpha1 diff --git a/docs/docs/api/kraken.build.python.v1alpha1.md b/docs/docs/api/kraken.build.python.v1alpha1.md new file mode 100644 index 00000000..6f3a0199 --- /dev/null +++ b/docs/docs/api/kraken.build.python.v1alpha1.md @@ -0,0 +1,5 @@ +--- +title: kraken.build.python.v1alpha1 +--- + +::: kraken.build.python.v1alpha1 diff --git a/kraken-build/.kraken.py b/kraken-build/.kraken.py index 60d6e165..8e6595bf 100644 --- a/kraken-build/.kraken.py +++ b/kraken-build/.kraken.py @@ -1,2 +1,2 @@ -from kraken.std.python.project import python_project +from kraken.build.python.v1alpha1 import python_project python_project() diff --git a/kraken-build/src/kraken/build/experimental/functional/v1/__init__.py b/kraken-build/src/kraken/build/functional/v1alpha1/__init__.py similarity index 77% rename from kraken-build/src/kraken/build/experimental/functional/v1/__init__.py rename to kraken-build/src/kraken/build/functional/v1alpha1/__init__.py index df300b4f..3443f0e5 100644 --- a/kraken-build/src/kraken/build/experimental/functional/v1/__init__.py +++ b/kraken-build/src/kraken/build/functional/v1alpha1/__init__.py @@ -1,4 +1,6 @@ -""" This module provides an experimental new functional API for defining Kraken tasks. """ +""" This module provides an experimental new functional API for defining Kraken tasks. + +This API is marked as `v1alpha1` and should be used with caution. """ from .fetch_file_task import fetch_file from .fetch_tarball_task import fetch_tarball diff --git a/kraken-build/src/kraken/build/experimental/functional/v1/fetch_file_task.py b/kraken-build/src/kraken/build/functional/v1alpha1/fetch_file_task.py similarity index 100% rename from kraken-build/src/kraken/build/experimental/functional/v1/fetch_file_task.py rename to kraken-build/src/kraken/build/functional/v1alpha1/fetch_file_task.py diff --git a/kraken-build/src/kraken/build/experimental/functional/v1/fetch_tarball_task.py b/kraken-build/src/kraken/build/functional/v1alpha1/fetch_tarball_task.py similarity index 100% rename from kraken-build/src/kraken/build/experimental/functional/v1/fetch_tarball_task.py rename to kraken-build/src/kraken/build/functional/v1alpha1/fetch_tarball_task.py diff --git a/kraken-build/src/kraken/build/experimental/functional/v1/shell_cmd_task.py b/kraken-build/src/kraken/build/functional/v1alpha1/shell_cmd_task.py similarity index 100% rename from kraken-build/src/kraken/build/experimental/functional/v1/shell_cmd_task.py rename to kraken-build/src/kraken/build/functional/v1alpha1/shell_cmd_task.py diff --git a/kraken-build/src/kraken/build/experimental/functional/v1/write_file_task.py b/kraken-build/src/kraken/build/functional/v1alpha1/write_file_task.py similarity index 100% rename from kraken-build/src/kraken/build/experimental/functional/v1/write_file_task.py rename to kraken-build/src/kraken/build/functional/v1alpha1/write_file_task.py diff --git a/kraken-build/src/kraken/build/python/v1alpha1/__init__.py b/kraken-build/src/kraken/build/python/v1alpha1/__init__.py new file mode 100644 index 00000000..36d4128c --- /dev/null +++ b/kraken-build/src/kraken/build/python/v1alpha1/__init__.py @@ -0,0 +1,7 @@ +""" This module provides facilities for Kraken-infused Python projects. + +This API is marked as `v1alpha1` and should be used with caution. """ + +from .project import python_app, python_package_index, python_project + +__all__ = ["python_package_index", "python_project", "python_app"] diff --git a/kraken-build/src/kraken/std/python/project.py b/kraken-build/src/kraken/build/python/v1alpha1/project.py similarity index 94% rename from kraken-build/src/kraken/std/python/project.py rename to kraken-build/src/kraken/build/python/v1alpha1/project.py index a8411092..e9142492 100644 --- a/kraken-build/src/kraken/std/python/project.py +++ b/kraken-build/src/kraken/build/python/v1alpha1/project.py @@ -58,7 +58,7 @@ def python_package_index( # NOTE(@niklas): Currently this function is a simple wrapper, but it may replace the wrapped method eventually. - from .settings import python_settings + from .....std.python.settings import python_settings python_settings().add_package_index( alias=alias, @@ -156,22 +156,21 @@ def python_project( from kraken.build import project from kraken.common import not_none - - from .pyproject import Pyproject - from .tasks.black_task import BlackConfig, black as black_tasks - from .tasks.build_task import build as build_task - from .tasks.flake8_task import Flake8Config, flake8 as flake8_tasks - from .tasks.info_task import info as info_task - from .tasks.install_task import install as install_task - from .tasks.isort_task import IsortConfig, isort as isort_tasks - from .tasks.login_task import login as login_task - from .tasks.mypy_task import MypyConfig, mypy as mypy_task - from .tasks.publish_task import publish as publish_task - from .tasks.pycln_task import pycln as pycln_task - from .tasks.pytest_task import pytest as pytest_task - from .tasks.pyupgrade_task import pyupgrade as pyupgrade_task - from .tasks.update_lockfile_task import update_lockfile_task - from .tasks.update_pyproject_task import update_pyproject_task + from kraken.std.python.pyproject import Pyproject + from kraken.std.python.tasks.black_task import BlackConfig, black as black_tasks + from kraken.std.python.tasks.build_task import build as build_task + from kraken.std.python.tasks.flake8_task import Flake8Config, flake8 as flake8_tasks + from kraken.std.python.tasks.info_task import info as info_task + from kraken.std.python.tasks.install_task import install as install_task + from kraken.std.python.tasks.isort_task import IsortConfig, isort as isort_tasks + from kraken.std.python.tasks.login_task import login as login_task + from kraken.std.python.tasks.mypy_task import MypyConfig, mypy as mypy_task + from kraken.std.python.tasks.publish_task import publish as publish_task + from kraken.std.python.tasks.pycln_task import pycln as pycln_task + from kraken.std.python.tasks.pytest_task import pytest as pytest_task + from kraken.std.python.tasks.pyupgrade_task import pyupgrade as pyupgrade_task + from kraken.std.python.tasks.update_lockfile_task import update_lockfile_task + from kraken.std.python.tasks.update_pyproject_task import update_pyproject_task if additional_lint_directories is None: additional_lint_directories = [] diff --git a/kraken-build/src/kraken/common/supplier.py b/kraken-build/src/kraken/common/supplier.py index db96acf1..c0fd94c8 100644 --- a/kraken-build/src/kraken/common/supplier.py +++ b/kraken-build/src/kraken/common/supplier.py @@ -2,7 +2,7 @@ calculated lazily and track provenance of such computations. """ import abc -from collections.abc import Callable, Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Mapping from typing import Any, Generic, TypeVar from ._generic import NotSet diff --git a/kraken-build/src/kraken/std/__init__.py b/kraken-build/src/kraken/std/__init__.py index 8c0c63f9..ef5853c9 100644 --- a/kraken-build/src/kraken/std/__init__.py +++ b/kraken-build/src/kraken/std/__init__.py @@ -1,5 +1 @@ __version__ = "0.36.1" - -from kraken.std.python.project import python_package_index, python_project - -__all__ = ["python_package_index", "python_project"] diff --git a/kraken-build/src/kraken/std/buffrs/__init__.py b/kraken-build/src/kraken/std/buffrs/__init__.py index 5945ef00..70249e88 100644 --- a/kraken-build/src/kraken/std/buffrs/__init__.py +++ b/kraken-build/src/kraken/std/buffrs/__init__.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import cast -from kraken.build.experimental.functional.v1 import fetch_tarball +from kraken.build.functional.v1alpha1 import fetch_tarball from kraken.common.supplier import Supplier from kraken.core import Project diff --git a/kraken-build/src/kraken/std/protobuf/__init__.py b/kraken-build/src/kraken/std/protobuf/__init__.py index 07445b15..45bd0851 100644 --- a/kraken-build/src/kraken/std/protobuf/__init__.py +++ b/kraken-build/src/kraken/std/protobuf/__init__.py @@ -8,7 +8,7 @@ from pathlib import Path from shutil import rmtree -from kraken.build.experimental.functional.v1 import fetch_file, fetch_tarball, shell_cmd, write_file +from kraken.build.functional.v1alpha1 import fetch_file, fetch_tarball, shell_cmd, write_file from kraken.common.supplier import Supplier from kraken.core import Property, Task, TaskStatus diff --git a/kraken-wrapper/.kraken.py b/kraken-wrapper/.kraken.py index 60d6e165..8e6595bf 100644 --- a/kraken-wrapper/.kraken.py +++ b/kraken-wrapper/.kraken.py @@ -1,2 +1,2 @@ -from kraken.std.python.project import python_project +from kraken.build.python.v1alpha1 import python_project python_project() From 889138030bb51f837ad4ddf8d5afd6e40d0f8ac0 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 22 Mar 2024 21:39:33 +0100 Subject: [PATCH 62/79] rename functional to utils --- docs/docs/api/kraken.build.functional.v1alpha1.md | 5 ----- docs/docs/api/kraken.build.utils.v1alpha1.md | 5 +++++ .../kraken/build/{functional => utils}/v1alpha1/__init__.py | 0 .../build/{functional => utils}/v1alpha1/fetch_file_task.py | 0 .../{functional => utils}/v1alpha1/fetch_tarball_task.py | 0 .../build/{functional => utils}/v1alpha1/shell_cmd_task.py | 0 .../build/{functional => utils}/v1alpha1/write_file_task.py | 0 kraken-build/src/kraken/std/buffrs/__init__.py | 2 +- kraken-build/src/kraken/std/protobuf/__init__.py | 2 +- 9 files changed, 7 insertions(+), 7 deletions(-) delete mode 100644 docs/docs/api/kraken.build.functional.v1alpha1.md create mode 100644 docs/docs/api/kraken.build.utils.v1alpha1.md rename kraken-build/src/kraken/build/{functional => utils}/v1alpha1/__init__.py (100%) rename kraken-build/src/kraken/build/{functional => utils}/v1alpha1/fetch_file_task.py (100%) rename kraken-build/src/kraken/build/{functional => utils}/v1alpha1/fetch_tarball_task.py (100%) rename kraken-build/src/kraken/build/{functional => utils}/v1alpha1/shell_cmd_task.py (100%) rename kraken-build/src/kraken/build/{functional => utils}/v1alpha1/write_file_task.py (100%) diff --git a/docs/docs/api/kraken.build.functional.v1alpha1.md b/docs/docs/api/kraken.build.functional.v1alpha1.md deleted file mode 100644 index a1127761..00000000 --- a/docs/docs/api/kraken.build.functional.v1alpha1.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: kraken.build.functional.v1alpha1 ---- - -::: kraken.build.functional.v1alpha1 diff --git a/docs/docs/api/kraken.build.utils.v1alpha1.md b/docs/docs/api/kraken.build.utils.v1alpha1.md new file mode 100644 index 00000000..053ae686 --- /dev/null +++ b/docs/docs/api/kraken.build.utils.v1alpha1.md @@ -0,0 +1,5 @@ +--- +title: kraken.build.utils.v1alpha1 +--- + +::: kraken.build.utils.v1alpha1 diff --git a/kraken-build/src/kraken/build/functional/v1alpha1/__init__.py b/kraken-build/src/kraken/build/utils/v1alpha1/__init__.py similarity index 100% rename from kraken-build/src/kraken/build/functional/v1alpha1/__init__.py rename to kraken-build/src/kraken/build/utils/v1alpha1/__init__.py diff --git a/kraken-build/src/kraken/build/functional/v1alpha1/fetch_file_task.py b/kraken-build/src/kraken/build/utils/v1alpha1/fetch_file_task.py similarity index 100% rename from kraken-build/src/kraken/build/functional/v1alpha1/fetch_file_task.py rename to kraken-build/src/kraken/build/utils/v1alpha1/fetch_file_task.py diff --git a/kraken-build/src/kraken/build/functional/v1alpha1/fetch_tarball_task.py b/kraken-build/src/kraken/build/utils/v1alpha1/fetch_tarball_task.py similarity index 100% rename from kraken-build/src/kraken/build/functional/v1alpha1/fetch_tarball_task.py rename to kraken-build/src/kraken/build/utils/v1alpha1/fetch_tarball_task.py diff --git a/kraken-build/src/kraken/build/functional/v1alpha1/shell_cmd_task.py b/kraken-build/src/kraken/build/utils/v1alpha1/shell_cmd_task.py similarity index 100% rename from kraken-build/src/kraken/build/functional/v1alpha1/shell_cmd_task.py rename to kraken-build/src/kraken/build/utils/v1alpha1/shell_cmd_task.py diff --git a/kraken-build/src/kraken/build/functional/v1alpha1/write_file_task.py b/kraken-build/src/kraken/build/utils/v1alpha1/write_file_task.py similarity index 100% rename from kraken-build/src/kraken/build/functional/v1alpha1/write_file_task.py rename to kraken-build/src/kraken/build/utils/v1alpha1/write_file_task.py diff --git a/kraken-build/src/kraken/std/buffrs/__init__.py b/kraken-build/src/kraken/std/buffrs/__init__.py index 70249e88..2a7ebfa8 100644 --- a/kraken-build/src/kraken/std/buffrs/__init__.py +++ b/kraken-build/src/kraken/std/buffrs/__init__.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import cast -from kraken.build.functional.v1alpha1 import fetch_tarball +from kraken.build.utils.v1alpha1 import fetch_tarball from kraken.common.supplier import Supplier from kraken.core import Project diff --git a/kraken-build/src/kraken/std/protobuf/__init__.py b/kraken-build/src/kraken/std/protobuf/__init__.py index 45bd0851..45791dae 100644 --- a/kraken-build/src/kraken/std/protobuf/__init__.py +++ b/kraken-build/src/kraken/std/protobuf/__init__.py @@ -8,7 +8,7 @@ from pathlib import Path from shutil import rmtree -from kraken.build.functional.v1alpha1 import fetch_file, fetch_tarball, shell_cmd, write_file +from kraken.build.utils.v1alpha1 import fetch_file, fetch_tarball, shell_cmd, write_file from kraken.common.supplier import Supplier from kraken.core import Property, Task, TaskStatus From 73940aa84eea2d070922a4de4f7eb1d7c80628ec Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 22 Mar 2024 21:40:59 +0100 Subject: [PATCH 63/79] fix additional error code for mypy --- kraken-build/src/kraken/std/git/version.py | 2 ++ kraken-build/src/kraken/std/python/tasks/mypy_task.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/kraken-build/src/kraken/std/git/version.py b/kraken-build/src/kraken/std/git/version.py index 78abe53f..59928559 100644 --- a/kraken-build/src/kraken/std/git/version.py +++ b/kraken-build/src/kraken/std/git/version.py @@ -42,6 +42,8 @@ def git_describe(path: Path | None, tags: bool = True, dirty: bool = True) -> st stderr = exc.stderr.decode() if "unknown revision" in stderr: raise EmptyGitRepositoryError(path) + count = 0 + short_rev = sp.check_output(["git", "rev-parse", "--short", "HEAD"], cwd=path).decode().strip() return f"0.0.0-{count}-g{short_rev}" diff --git a/kraken-build/src/kraken/std/python/tasks/mypy_task.py b/kraken-build/src/kraken/std/python/tasks/mypy_task.py index c6f240ef..8a4a8a0f 100644 --- a/kraken-build/src/kraken/std/python/tasks/mypy_task.py +++ b/kraken-build/src/kraken/std/python/tasks/mypy_task.py @@ -33,7 +33,7 @@ warn_unreachable = true warn_unused_configs = true warn_unused_ignores = true -enable_error_code = "ignore-without-code, possibly-undefined" +enable_error_code = ignore-without-code, possibly-undefined """ From 127506269acf7ad098e8ce499008b743e0c2087e Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 22 Mar 2024 21:55:38 +0100 Subject: [PATCH 64/79] update docs --- docs/mkdocs.yml | 2 ++ .../src/kraken/build/python/__init__.py | 0 .../kraken/build/python/v1alpha1/__init__.py | 25 ++++++++++++++++++- .../kraken/build/python/v1alpha1/project.py | 16 +++++++----- .../src/kraken/build/utils/__init__.py | 0 .../kraken/build/utils/v1alpha1/__init__.py | 4 ++- 6 files changed, 39 insertions(+), 8 deletions(-) create mode 100644 kraken-build/src/kraken/build/python/__init__.py create mode 100644 kraken-build/src/kraken/build/utils/__init__.py diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 4bc01844..be2b2e69 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -57,6 +57,8 @@ nav: - lang/python.md - API Documentation: - api/kraken.build.md + - api/kraken.build.python.v1alpha1.md + - api/kraken.build.utils.v1alpha1.md - api/kraken.common.md - api/kraken.core.md - api/kraken.std.buffrs.md diff --git a/kraken-build/src/kraken/build/python/__init__.py b/kraken-build/src/kraken/build/python/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/kraken-build/src/kraken/build/python/v1alpha1/__init__.py b/kraken-build/src/kraken/build/python/v1alpha1/__init__.py index 36d4128c..e72bcbe4 100644 --- a/kraken-build/src/kraken/build/python/v1alpha1/__init__.py +++ b/kraken-build/src/kraken/build/python/v1alpha1/__init__.py @@ -1,6 +1,29 @@ """ This module provides facilities for Kraken-infused Python projects. -This API is marked as `v1alpha1` and should be used with caution. """ +__Features__ + +* Consume from and publish packages to alternative Python package indexes. +* Standardized Python project configuration (`src/` directory, `tests/` directory, tests with Pytest incl. doctests, + Mypy, Black, isort, Pycln, Pyupgrade, Flake8). +* Supports [Slap][], [Poetry][] and [PDM][]-managed Python projects. +* Built-in Protobuf support (dependency management using [Buffrs][], linting with [Buf][] and code generation with + [grpcio-tools][]). +* Produce a PEX application from your Python project using [`python_app()`](kraken.build.python.v1alpha1.python_app). + +!!! note "Tools" + Note that except for Pytest (which needs to be a development dependency to your Python project), all tools are + installed for you automatically by Kraken. + +[Slap]: https://github.com/NiklasRosenstein/slap +[Poetry]: https://python-poetry.org/docs +[PDM]: https://pdm-project.org/latest/ +[Buffrs]: https://github.com/helsing-ai/buffrs +[Buf]: https://buf.build/ +[grpcio-tools]: https://pypi.org/project/grpcio-tools/ + +!!! warning "Unstable API" + This API is unstable and should be used with caution. +""" from .project import python_app, python_package_index, python_project diff --git a/kraken-build/src/kraken/build/python/v1alpha1/project.py b/kraken-build/src/kraken/build/python/v1alpha1/project.py index e9142492..44f7d2f7 100644 --- a/kraken-build/src/kraken/build/python/v1alpha1/project.py +++ b/kraken-build/src/kraken/build/python/v1alpha1/project.py @@ -58,7 +58,7 @@ def python_package_index( # NOTE(@niklas): Currently this function is a simple wrapper, but it may replace the wrapped method eventually. - from .....std.python.settings import python_settings + from kraken.std.python.settings import python_settings python_settings().add_package_index( alias=alias, @@ -104,9 +104,13 @@ def python_project( The Python build system used for the library is automatically detected. Supported build systems are: - * [Slap](https://github.com/NiklasRosenstein/slap) - * [Poetry](https://python-poetry.org/docs) - * [PDM](https://pdm-project.org/latest/) + * [Slap][] + * [Poetry][] + * [PDM][] + + [Slap]: https://github.com/NiklasRosenstein/slap + [Poetry]: https://python-poetry.org/docs + [PDM]: https://pdm-project.org/latest/ Note: Pytest dependencies Your project should have the `pytest` dependency in it's development dependencies. Kraken does not currently @@ -385,8 +389,8 @@ def python_app( ) -> PexBuildTask: """Build a PEX binary from a Python application. - This function should be called after `python_project()`. If any Python code generation is employed by the project, - the generated PEX build task will depend on it. + This function should be called after [`python_project()`][kraken.build.python.v1alpha1.project.python_project]. + If any Python code generation is employed by the project, the generated PEX build task will depend on it. Args: app_name: The name of the applicaiton. This will be used as the binary output filename. The output PEX diff --git a/kraken-build/src/kraken/build/utils/__init__.py b/kraken-build/src/kraken/build/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/kraken-build/src/kraken/build/utils/v1alpha1/__init__.py b/kraken-build/src/kraken/build/utils/v1alpha1/__init__.py index 3443f0e5..7da4e1b9 100644 --- a/kraken-build/src/kraken/build/utils/v1alpha1/__init__.py +++ b/kraken-build/src/kraken/build/utils/v1alpha1/__init__.py @@ -1,6 +1,8 @@ """ This module provides an experimental new functional API for defining Kraken tasks. -This API is marked as `v1alpha1` and should be used with caution. """ +!!! warning + This API is unstable and should be used with caution. +""" from .fetch_file_task import fetch_file from .fetch_tarball_task import fetch_tarball From bf06379dd9427de995c2ebe430fdbdc8466e7e51 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 22 Mar 2024 21:57:09 +0100 Subject: [PATCH 65/79] update examples; --- examples/pdm-project-consumer/.kraken.py | 2 +- examples/pdm-project/.kraken.py | 2 +- examples/poetry-project-consumer/.kraken.py | 2 +- examples/poetry-project/.kraken.py | 2 +- examples/rust-pdm-project-consumer/.kraken.py | 2 +- examples/rust-pdm-project/.kraken.py | 2 +- examples/rust-poetry-project-consumer/.kraken.py | 2 +- examples/rust-poetry-project/.kraken.py | 2 +- examples/slap-project-consumer/.kraken.py | 2 +- examples/slap-project/.kraken.py | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/pdm-project-consumer/.kraken.py b/examples/pdm-project-consumer/.kraken.py index 5eafa744..01a11e9d 100644 --- a/examples/pdm-project-consumer/.kraken.py +++ b/examples/pdm-project-consumer/.kraken.py @@ -1,6 +1,6 @@ import os -from kraken.std import python_package_index, python_project +from kraken.build.python.v1alpha1 import python_package_index, python_project python_package_index( alias="local", diff --git a/examples/pdm-project/.kraken.py b/examples/pdm-project/.kraken.py index 850f2ebb..95d25607 100644 --- a/examples/pdm-project/.kraken.py +++ b/examples/pdm-project/.kraken.py @@ -1,6 +1,6 @@ import os -from kraken.std import python_package_index, python_project +from kraken.build.python.v1alpha1 import python_package_index, python_project python_package_index( alias="local", diff --git a/examples/poetry-project-consumer/.kraken.py b/examples/poetry-project-consumer/.kraken.py index 5eafa744..01a11e9d 100644 --- a/examples/poetry-project-consumer/.kraken.py +++ b/examples/poetry-project-consumer/.kraken.py @@ -1,6 +1,6 @@ import os -from kraken.std import python_package_index, python_project +from kraken.build.python.v1alpha1 import python_package_index, python_project python_package_index( alias="local", diff --git a/examples/poetry-project/.kraken.py b/examples/poetry-project/.kraken.py index 850f2ebb..95d25607 100644 --- a/examples/poetry-project/.kraken.py +++ b/examples/poetry-project/.kraken.py @@ -1,6 +1,6 @@ import os -from kraken.std import python_package_index, python_project +from kraken.build.python.v1alpha1 import python_package_index, python_project python_package_index( alias="local", diff --git a/examples/rust-pdm-project-consumer/.kraken.py b/examples/rust-pdm-project-consumer/.kraken.py index 5eafa744..01a11e9d 100644 --- a/examples/rust-pdm-project-consumer/.kraken.py +++ b/examples/rust-pdm-project-consumer/.kraken.py @@ -1,6 +1,6 @@ import os -from kraken.std import python_package_index, python_project +from kraken.build.python.v1alpha1 import python_package_index, python_project python_package_index( alias="local", diff --git a/examples/rust-pdm-project/.kraken.py b/examples/rust-pdm-project/.kraken.py index c4709eae..37412cfe 100644 --- a/examples/rust-pdm-project/.kraken.py +++ b/examples/rust-pdm-project/.kraken.py @@ -1,6 +1,6 @@ import os -from kraken.std import python_package_index, python_project +from kraken.build.python.v1alpha1 import python_package_index, python_project from kraken.std.python import mypy_stubtest python_package_index( diff --git a/examples/rust-poetry-project-consumer/.kraken.py b/examples/rust-poetry-project-consumer/.kraken.py index 5eafa744..01a11e9d 100644 --- a/examples/rust-poetry-project-consumer/.kraken.py +++ b/examples/rust-poetry-project-consumer/.kraken.py @@ -1,6 +1,6 @@ import os -from kraken.std import python_package_index, python_project +from kraken.build.python.v1alpha1 import python_package_index, python_project python_package_index( alias="local", diff --git a/examples/rust-poetry-project/.kraken.py b/examples/rust-poetry-project/.kraken.py index 49c335c3..63dfecf0 100644 --- a/examples/rust-poetry-project/.kraken.py +++ b/examples/rust-poetry-project/.kraken.py @@ -1,6 +1,6 @@ import os -from kraken.std import python_package_index, python_project +from kraken.build.python.v1alpha1 import python_package_index, python_project from kraken.std.python import mypy_stubtest python_package_index( diff --git a/examples/slap-project-consumer/.kraken.py b/examples/slap-project-consumer/.kraken.py index 5eafa744..01a11e9d 100644 --- a/examples/slap-project-consumer/.kraken.py +++ b/examples/slap-project-consumer/.kraken.py @@ -1,6 +1,6 @@ import os -from kraken.std import python_package_index, python_project +from kraken.build.python.v1alpha1 import python_package_index, python_project python_package_index( alias="local", diff --git a/examples/slap-project/.kraken.py b/examples/slap-project/.kraken.py index 850f2ebb..95d25607 100644 --- a/examples/slap-project/.kraken.py +++ b/examples/slap-project/.kraken.py @@ -1,6 +1,6 @@ import os -from kraken.std import python_package_index, python_project +from kraken.build.python.v1alpha1 import python_package_index, python_project python_package_index( alias="local", From f082237fb9a22cd760775efef14a3b01a99fc35d Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 22 Mar 2024 22:25:39 +0100 Subject: [PATCH 66/79] add back calling do --- kraken-build/src/kraken/core/system/project_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kraken-build/src/kraken/core/system/project_test.py b/kraken-build/src/kraken/core/system/project_test.py index 36900f2f..2f359365 100644 --- a/kraken-build/src/kraken/core/system/project_test.py +++ b/kraken-build/src/kraken/core/system/project_test.py @@ -52,7 +52,7 @@ def execute(self) -> None: ... def test__Project__do_normalizes_taskname_backwards_compatibility_pre_0_12_0(kraken_project: Project) -> None: with pytest.warns(DeprecationWarning) as warninfo: - task = kraken_project.task("this is a :test task", VoidTask) + task = kraken_project.do("this is a :test task", VoidTask) assert task.name == "this-is-a-test-task" assert str(warninfo.list[0].message) == ("Call to deprecated method do. (Use Project.task() instead)") assert str(warninfo.list[1].message) == ( From 88857ad99aa6bf1795bf6e2472f29008e3b549ee Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Sat, 23 Mar 2024 00:25:43 +0100 Subject: [PATCH 67/79] feature: Add `Supplier.agg()` --- .changelog/_unreleased.toml | 6 ++++++ kraken-build/src/kraken/common/supplier.py | 12 +++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index 8ee26f23..5df5decb 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -99,3 +99,9 @@ id = "a8814dcb-b28b-42ed-aa0b-92297a63877d" type = "fix" description = "Remove default files that should be excluded for `isort`, `pycln` and `black` so that a `build/` directory in the `src/` directory also gets checked. We can do this safely because we expect the `src/` folder in a Python project to contain only files that should be checked unless otherwise explicitly specified" author = "@NiklasRosenstein" + +[[entries]] +id = "863c7431-cfb0-4491-a6c1-7209a6e2cf59" +type = "feature" +description = "Add `Supplier.agg()`" +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/common/supplier.py b/kraken-build/src/kraken/common/supplier.py index c0fd94c8..77465b1d 100644 --- a/kraken-build/src/kraken/common/supplier.py +++ b/kraken-build/src/kraken/common/supplier.py @@ -3,7 +3,8 @@ import abc from collections.abc import Callable, Iterable, Mapping -from typing import Any, Generic, TypeVar +from functools import partial +from typing import Any, Generic, ParamSpec, TypeVar from ._generic import NotSet @@ -11,6 +12,7 @@ U = TypeVar("U") K = TypeVar("K") V = TypeVar("V") +P = ParamSpec("P") class Supplier(Generic[T], abc.ABC): @@ -110,6 +112,14 @@ def void(from_exc: "Exception | None" = None, derived_from: Iterable["Supplier[A return VoidSupplier(from_exc, derived_from) + @staticmethod + def agg(fn: Callable[P, "U"], *args: P.args, **kwargs: P.kwargs) -> "Supplier[U]": + derived_from = [ + *(arg for arg in args if isinstance(arg, Supplier)), + *(arg for arg in kwargs.values() if isinstance(arg, Supplier)), + ] + return OfCallableSupplier(partial(fn, *args, **kwargs), derived_from) + def __repr__(self) -> str: try: value = self.get_or(NotSet.Value) From 1803e0361a52c3ee1d2705b2967d30e3b151e7d6 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Sat, 23 Mar 2024 00:26:59 +0100 Subject: [PATCH 68/79] feature: Add `kraken.build.protobuf.v1alpha1.protobuf_project()` --- .changelog/_unreleased.toml | 6 + .../build/protobuf/v1alpha1/__init__.py | 14 ++ .../kraken/build/protobuf/v1alpha1/project.py | 128 ++++++++++++++++++ .../kraken/build/python/v1alpha1/__init__.py | 6 - .../kraken/build/python/v1alpha1/project.py | 105 ++++++-------- 5 files changed, 193 insertions(+), 66 deletions(-) create mode 100644 kraken-build/src/kraken/build/protobuf/v1alpha1/__init__.py create mode 100644 kraken-build/src/kraken/build/protobuf/v1alpha1/project.py diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index 5df5decb..59b31555 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -105,3 +105,9 @@ id = "863c7431-cfb0-4491-a6c1-7209a6e2cf59" type = "feature" description = "Add `Supplier.agg()`" author = "@NiklasRosenstein" + +[[entries]] +id = "16b2de57-29c2-410f-816a-018e9812871b" +type = "feature" +description = "Add `kraken.build.protobuf.v1alpha1.protobuf_project()`" +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/build/protobuf/v1alpha1/__init__.py b/kraken-build/src/kraken/build/protobuf/v1alpha1/__init__.py new file mode 100644 index 00000000..6c96f8c6 --- /dev/null +++ b/kraken-build/src/kraken/build/protobuf/v1alpha1/__init__.py @@ -0,0 +1,14 @@ +""" Provides an API for defining a Protobuf project that uses [Buffrs][] for dependency management, [Buf][] for +linting and code-generation capabilities with [grpcio-tools][]. + +[Buffrs]: https://github.com/helsing-ai/buffrs +[Buf]: https://buf.build/ +[grpcio-tools]: https://pypi.org/project/grpcio-tools/ + +!!! warning "Unstable API" + This API is unstable and should be consumed with caution. +""" + +from .project import protobuf_project, ProtobufProject + +__all__ = ["protobuf_project", "ProtobufProject"] diff --git a/kraken-build/src/kraken/build/protobuf/v1alpha1/project.py b/kraken-build/src/kraken/build/protobuf/v1alpha1/project.py new file mode 100644 index 00000000..c5bda93c --- /dev/null +++ b/kraken-build/src/kraken/build/protobuf/v1alpha1/project.py @@ -0,0 +1,128 @@ + +from collections.abc import Sequence +from dataclasses import dataclass +import logging +from pathlib import Path +from kraken.common.supplier import Supplier +from kraken.core.system.project import Project +from kraken.core.system.task import Task +from kraken.std.buffrs import buffrs_install as buffrs_install_task +from kraken.std.protobuf import ProtocTask, buf +from kraken.std.python.tasks.pex_build_task import pex_build + +logger = logging.getLogger(__name__) + + +def protobuf_project( + *, + buf_version: str = "1.30.0", + buffrs_version: str = "0.8.0", +) -> "ProtobufProject": + """ Defines tasks for a Protobuf project. + + * If a `Proto.toml` exists, [Buffrs][] will be be used to install dependencies. Buffrs will produce a `proto/vendor` + directory that will be used as the source directory for the remaining Protobuf tasks. If the `Proto.toml` does + not exist, the `proto` directory will be used as the source directory and no dependency management will be + performed. + * [Buf][] will be used to lint the Protobuf files (either `proto/vendor` excluding any Protobuf files that are + vendored in through dependencies, or the `proto` folder if Buffrs is not used). + + The returned [kraken.build.protobuf.v1alpah1.ProtobufProject][] instance can be used to create a code generator + for supported languages from the Protobuf files. + + Args: + buf_version: The version of [Buf][] to install from GitHub releases. + buffrs_version: The version of [Buffrs][] to install from GitHub releases. + """ + + from kraken.build import project + + if project.directory.joinpath("Proto.toml").is_file(): + logger.debug("[%s] Detected Proto.toml, using Buffrs for dependency management.", project) + buffrs_install = buffrs_install_task() + proto_dir = "proto/vendor" + else: + logger.debug("[%s] No Proto.toml detected, not using Buffrs for dependency management.", project) + buffrs_install = None + proto_dir = "proto" + + buf( + buf_version=buf_version, + path=proto_dir, + dependencies=[buffrs_install] if buffrs_install else [], + ) + + return ProtobufProject( + project=project, + proto_dir=proto_dir, + dependencies=[buffrs_install] if buffrs_install else [], + ) + + +@dataclass +class ProtobufProject: + """ Represents a Protobuf project, and allows creating code generators for supported versions. """ + + project: Project + """ The Kraken project. """ + + proto_dir: str + """ The directory where the Protobuf files are located (relative to the project directory). """ + + dependencies: Sequence[Task] + """ A list of tasks that protobuf generation should depend on.""" + + grpcio_tools_version_spec: str = ">=1.62.1,<2.0.0" + """ The version of grpcio-tools to use. """ + + mypy_protobuf_version_spec: str = ">=3.5.0,<4.0.0" + """ The version of mypy-protobuf to use. """ + + @property + def protoc(self) -> Supplier[str]: + """ Returns the ProtocTask for the project. """ + + return pex_build( + binary_name="protoc", + requirements=[f"grpcio-tools{self.grpcio_tools_version_spec}", f"mypy-protobuf{self.mypy_protobuf_version_spec}"], + entry_point="grpc_tools.protoc", + venv="prepend", + ).output_file.map(lambda p: str(p.absolute())) + + def python(self, source_dir: Path, ) -> tuple[Task, Supplier[Sequence[Path]]]: + """ Create a code generator for Python code from the Protobuf files. + + The Python code will be generated in a `proto` namespace package that will be placed into your projects + source directory. Before invoking the `protoc` compiler, the contents of the `proto` directory (the one + that contains the Protobuf source code) will be copied and wrapped into a temporary `proto` parent directory + to ensure that imports are generated correctly. + + Args: + name: The name of the task to create. + source_dir: The directory where Python source files should be generated. + Returns: + 1. The task that generates the Python code. + 2. The supplier for the generated Python files that should be excluded from formatting checks and linting. + """ + + protoc = self.project.task("protoc.python", ProtocTask) + protoc.protoc_bin = self.protoc + protoc.proto_dir = self.proto_dir + protoc.generate("python", Path(source_dir)) + protoc.generate("grpc_python", Path(source_dir)) + protoc.generate("mypy", Path(source_dir)) + protoc.generate("mypy_grpc", Path(source_dir)) + + # TODO(@niklas): The `protoc` command seems to have a `pyi_out` option, but it only generates .pyi files for + # the Protobuf messages, not the gRPC stubs. Am I missing something? Until then, we keep using mypy-protobuf. + # protoc.generate("pyi", Path(source_directory)) + + protoc.depends_on(*self.dependencies) + + out_dir = self.project.directory / source_dir / "proto" + out_files = Supplier.of_callable( + lambda: [*out_dir.rglob("*.py"), *out_dir.rglob("*.pyi")], + derived_from=[protoc.proto_dir], # Just some property of the task ensure lineage + ) + + return (protoc, out_files) diff --git a/kraken-build/src/kraken/build/python/v1alpha1/__init__.py b/kraken-build/src/kraken/build/python/v1alpha1/__init__.py index e72bcbe4..5254b0fc 100644 --- a/kraken-build/src/kraken/build/python/v1alpha1/__init__.py +++ b/kraken-build/src/kraken/build/python/v1alpha1/__init__.py @@ -6,8 +6,6 @@ * Standardized Python project configuration (`src/` directory, `tests/` directory, tests with Pytest incl. doctests, Mypy, Black, isort, Pycln, Pyupgrade, Flake8). * Supports [Slap][], [Poetry][] and [PDM][]-managed Python projects. -* Built-in Protobuf support (dependency management using [Buffrs][], linting with [Buf][] and code generation with - [grpcio-tools][]). * Produce a PEX application from your Python project using [`python_app()`](kraken.build.python.v1alpha1.python_app). !!! note "Tools" @@ -17,10 +15,6 @@ [Slap]: https://github.com/NiklasRosenstein/slap [Poetry]: https://python-poetry.org/docs [PDM]: https://pdm-project.org/latest/ -[Buffrs]: https://github.com/helsing-ai/buffrs -[Buf]: https://buf.build/ -[grpcio-tools]: https://pypi.org/project/grpcio-tools/ - !!! warning "Unstable API" This API is unstable and should be used with caution. """ diff --git a/kraken-build/src/kraken/build/python/v1alpha1/project.py b/kraken-build/src/kraken/build/python/v1alpha1/project.py index 44f7d2f7..b29f74d4 100644 --- a/kraken-build/src/kraken/build/python/v1alpha1/project.py +++ b/kraken-build/src/kraken/build/python/v1alpha1/project.py @@ -4,11 +4,12 @@ import re from collections.abc import Iterable, Sequence from pathlib import Path -from typing import Literal, TypeVar +from typing import Literal, Protocol, TypeVar, cast from attr import dataclass from nr.stream import Optional +from kraken.common.supplier import Supplier from kraken.core.system.task import Task from kraken.std.git.version import EmptyGitRepositoryError, GitVersion, NotAGitRepositoryError, git_describe from kraken.std.python.buildsystem import detect_build_system @@ -22,6 +23,18 @@ logger = logging.getLogger(__name__) +class PythonCodegen(Protocol): + """ Protocol for functions that produce a task to generate Python code.""" + + def __call__(self, source_dir: Path) -> tuple[Task, Supplier[Sequence[Path]]]: + """ + Returns: + 1. The code generation task. + 2. A supplier that provides the generated files. These are used to exclude them from linting and formatting. + """ + ... + + def concat(*args: Iterable[T]) -> list[T]: return [x for arg in args for x in arg] @@ -93,11 +106,7 @@ def python_project( mypy_version_spec: str = ">=1.8.0,<2.0.0", pycln_version_spec: str = ">=2.4.0,<3.0.0", pyupgrade_version_spec: str = ">=3.15.0,<4.0.0", - protobuf_enabled: bool = True, - grpcio_tools_version_spec: str = ">=1.62.1,<2.0.0", - mypy_protobuf_version_spec: str = ">=3.5.0,<4.0.0", - buf_version: str = "1.30.0", - codegen: Sequence[Task | str] = (), + codegen: Sequence[PythonCodegen] = (), ) -> "PythonProject": """ Use this function in a Python project. @@ -148,14 +157,9 @@ def python_project( be used to add Flake8 plugins. flake8_extend_ignore: Flake8 lints to ignore. The default ignores lints that would otherwise conflict with the way Black formats code. - protobuf_enabled: Enable Protobuf code generation tasks if a `proto/` directory. Is a no-op when the directory - does not exist. Creates tasks for linting, formatting and generating code from Protobuf files. The - generated code will be placed into the `source_directory`. - grpcio_tools_version_spec: The version specifier for the `grpcio-tools` package to use when generating code - from Protobuf files. - buf_version: The version specifier for the `buf` tool to use when linting and formatting Protobuf files. - codegen: A list of code generation tasks that should be executed before the project is built. This can be - used to generate code from Protobuf files, for example. + codegen: A list of code generators that should be run before the project is built. This is typically the + a function, such as [`protobuf_project()`][kraken.build.protobuf.v1alpha1.protobuf_project][`.python`][ + kraken.build.protobuf.v1alpha1.ProtobufProject.python]. """ from kraken.build import project @@ -214,51 +218,32 @@ def python_project( info_task(build_system=build_system) install_task().depends_on(login) - # === Protobuf - - if protobuf_enabled and project.directory.joinpath("proto").is_dir(): - - from kraken.std.buffrs import buffrs_install as buffrs_install_task - from kraken.std.protobuf import ProtocTask, buf - - if project.directory.joinpath("Proto.toml").is_file(): - buffrs_install = buffrs_install_task() - else: - buffrs_install = None + # === Codegen invokation + + codegen_tasks, codegen_files = [], [] + for codegen_fn in codegen: + task, files = codegen_fn(source_dir=project.directory / source_directory) + codegen_tasks.append(task) + codegen_files.append(files) + + print("### DEBUG") + print(codegen_files) + # Concatenate the excluded directories and the generated files from the codegen tasks. + new_exclude_format_paths: Supplier[Sequence[Path]] = Supplier.agg( + (lambda a, *b: [*a.get(), *(x for y in b for x in y.get())]), + Supplier.of(exclude_format_directories), + *codegen_files, + ) + print(new_exclude_format_paths.get()) - buf( - buf_version=buf_version, - path="proto/vendor" if buffrs_install else "proto", - dependencies=[buffrs_install] if buffrs_install else [], - ) + from typing import TYPE_CHECKING + if TYPE_CHECKING: + reveal_type(exclude_format_directories) - protoc_bin = pex_build( - binary_name="protoc", - requirements=[f"grpcio-tools{grpcio_tools_version_spec}", f"mypy-protobuf{mypy_protobuf_version_spec}"], - entry_point="grpc_tools.protoc", - venv="prepend", - ).output_file.map(lambda p: str(p.absolute())) - - protoc = project.task("protoc-python", ProtocTask) - protoc.protoc_bin = protoc_bin - protoc.proto_dir = "proto" - protoc.generate("python", Path(source_directory)) - protoc.generate("grpc_python", Path(source_directory)) - protoc.generate("mypy", Path(source_directory)) - protoc.generate("mypy_grpc", Path(source_directory)) - # TODO(@niklas): Seems the standard GRPCio tools can already generate .pyi files, but not for the grpc stubs? - # protoc.generate("pyi", Path(source_directory)) - - if buffrs_install is not None: - protoc.depends_on(buffrs_install) - protoc.proto_dir = "proto/vendor" - - codegen = [*codegen, protoc] - exclude_format_directories = [ - *exclude_format_directories, - # Ensure that the generated Protobuf code is not linted or formatted - str((project.directory / source_directory / "proto").relative_to(project.directory)), - ] + # *exclude_format_directories, + # # Ensure that the generated Protobuf code is not linted or formatted + # str((project.directory / source_directory / "proto").relative_to(project.directory)), + # ] # === Python tooling @@ -319,7 +304,7 @@ def python_project( version_spec=mypy_version_spec, python_version=python_version, use_daemon=mypy_use_daemon, - ).depends_on(*codegen) + ).depends_on(*codegen_tasks) # TODO(@niklas): Improve this heuristic to check whether Coverage reporting should be enabled. if "pytest-cov" in str(dict(pyproject)): @@ -333,7 +318,7 @@ def python_project( coverage=coverage, doctest_modules=True, allow_no_tests=True, - ).depends_on(*codegen) + ).depends_on(*codegen_tasks) if not enforce_project_version: try: @@ -364,7 +349,7 @@ def python_project( # Create publish tasks for any package index with publish=True. build = build_task(as_version=enforce_project_version) - build.depends_on(*codegen) + build.depends_on(*codegen_tasks) for index in python_settings().package_indexes.values(): if index.publish: publish_task(package_index=index.alias, distributions=build.output_files, interactive=False) From 6b483fbc62aca0f2d71a2950323dd0c23cba222b Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Sat, 23 Mar 2024 00:27:21 +0100 Subject: [PATCH 69/79] upodate example --- examples/python-protobuf/.kraken.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/python-protobuf/.kraken.py b/examples/python-protobuf/.kraken.py index 9f59fe27..5800a7f3 100644 --- a/examples/python-protobuf/.kraken.py +++ b/examples/python-protobuf/.kraken.py @@ -1,3 +1,5 @@ -from kraken.std.python.project import python_project +from kraken.build.python.v1alpha1 import python_project +from kraken.build.protobuf.v1alpha1 import protobuf_project -python_project() +proto = protobuf_project() +python_project(codegen = [proto.python]) From 1c882d10567cc695b3b8f78dd220c973f7a809f4 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Sat, 30 Mar 2024 18:46:53 +0100 Subject: [PATCH 70/79] fmt --- .../build/protobuf/v1alpha1/__init__.py | 2 +- .../kraken/build/protobuf/v1alpha1/project.py | 22 ++++++++++++------- .../kraken/build/python/v1alpha1/project.py | 9 ++++---- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/kraken-build/src/kraken/build/protobuf/v1alpha1/__init__.py b/kraken-build/src/kraken/build/protobuf/v1alpha1/__init__.py index 6c96f8c6..55742f01 100644 --- a/kraken-build/src/kraken/build/protobuf/v1alpha1/__init__.py +++ b/kraken-build/src/kraken/build/protobuf/v1alpha1/__init__.py @@ -9,6 +9,6 @@ This API is unstable and should be consumed with caution. """ -from .project import protobuf_project, ProtobufProject +from .project import ProtobufProject, protobuf_project __all__ = ["protobuf_project", "ProtobufProject"] diff --git a/kraken-build/src/kraken/build/protobuf/v1alpha1/project.py b/kraken-build/src/kraken/build/protobuf/v1alpha1/project.py index c5bda93c..b8b5138f 100644 --- a/kraken-build/src/kraken/build/protobuf/v1alpha1/project.py +++ b/kraken-build/src/kraken/build/protobuf/v1alpha1/project.py @@ -1,8 +1,8 @@ - +import logging from collections.abc import Sequence from dataclasses import dataclass -import logging from pathlib import Path + from kraken.common.supplier import Supplier from kraken.core.system.project import Project from kraken.core.system.task import Task @@ -18,7 +18,7 @@ def protobuf_project( buf_version: str = "1.30.0", buffrs_version: str = "0.8.0", ) -> "ProtobufProject": - """ Defines tasks for a Protobuf project. + """Defines tasks for a Protobuf project. * If a `Proto.toml` exists, [Buffrs][] will be be used to install dependencies. Buffrs will produce a `proto/vendor` directory that will be used as the source directory for the remaining Protobuf tasks. If the `Proto.toml` does @@ -61,7 +61,7 @@ def protobuf_project( @dataclass class ProtobufProject: - """ Represents a Protobuf project, and allows creating code generators for supported versions. """ + """Represents a Protobuf project, and allows creating code generators for supported versions.""" project: Project """ The Kraken project. """ @@ -80,17 +80,23 @@ class ProtobufProject: @property def protoc(self) -> Supplier[str]: - """ Returns the ProtocTask for the project. """ + """Returns the ProtocTask for the project.""" return pex_build( binary_name="protoc", - requirements=[f"grpcio-tools{self.grpcio_tools_version_spec}", f"mypy-protobuf{self.mypy_protobuf_version_spec}"], + requirements=[ + f"grpcio-tools{self.grpcio_tools_version_spec}", + f"mypy-protobuf{self.mypy_protobuf_version_spec}", + ], entry_point="grpc_tools.protoc", venv="prepend", ).output_file.map(lambda p: str(p.absolute())) - def python(self, source_dir: Path, ) -> tuple[Task, Supplier[Sequence[Path]]]: - """ Create a code generator for Python code from the Protobuf files. + def python( + self, + source_dir: Path, + ) -> tuple[Task, Supplier[Sequence[Path]]]: + """Create a code generator for Python code from the Protobuf files. The Python code will be generated in a `proto` namespace package that will be placed into your projects source directory. Before invoking the `protoc` compiler, the contents of the `proto` directory (the one diff --git a/kraken-build/src/kraken/build/python/v1alpha1/project.py b/kraken-build/src/kraken/build/python/v1alpha1/project.py index b29f74d4..a89fb8fb 100644 --- a/kraken-build/src/kraken/build/python/v1alpha1/project.py +++ b/kraken-build/src/kraken/build/python/v1alpha1/project.py @@ -3,10 +3,10 @@ import logging import re from collections.abc import Iterable, Sequence +from dataclasses import dataclass from pathlib import Path -from typing import Literal, Protocol, TypeVar, cast +from typing import Literal, Protocol, TypeVar -from attr import dataclass from nr.stream import Optional from kraken.common.supplier import Supplier @@ -15,7 +15,7 @@ from kraken.std.python.buildsystem import detect_build_system from kraken.std.python.pyproject import PackageIndex from kraken.std.python.settings import python_settings -from kraken.std.python.tasks.pex_build_task import PexBuildTask, pex_build +from kraken.std.python.tasks.pex_build_task import PexBuildTask from kraken.std.python.tasks.pytest_task import CoverageFormat from kraken.std.python.version import git_version_to_python_version @@ -24,7 +24,7 @@ class PythonCodegen(Protocol): - """ Protocol for functions that produce a task to generate Python code.""" + """Protocol for functions that produce a task to generate Python code.""" def __call__(self, source_dir: Path) -> tuple[Task, Supplier[Sequence[Path]]]: """ @@ -237,6 +237,7 @@ def python_project( print(new_exclude_format_paths.get()) from typing import TYPE_CHECKING + if TYPE_CHECKING: reveal_type(exclude_format_directories) From 03f54257a3cd1359f440fec8337553dd42daa96e Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Wed, 3 Apr 2024 21:13:51 +0200 Subject: [PATCH 71/79] feature: add `pex_build(python_shebang)` parameter --- .changelog/_unreleased.toml | 6 ++++++ .../src/kraken/std/python/tasks/pex_build_task.py | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index 59b31555..aec1e7a8 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -111,3 +111,9 @@ id = "16b2de57-29c2-410f-816a-018e9812871b" type = "feature" description = "Add `kraken.build.protobuf.v1alpha1.protobuf_project()`" author = "@NiklasRosenstein" + +[[entries]] +id = "54f6e294-e60f-402a-97ae-17cbce9e8bc6" +type = "feature" +description = "add `pex_build(python_shebang)` parameter" +author = "niklas.rosenstein@helsing.ai" diff --git a/kraken-build/src/kraken/std/python/tasks/pex_build_task.py b/kraken-build/src/kraken/std/python/tasks/pex_build_task.py index 32dfffd8..a67749ad 100644 --- a/kraken-build/src/kraken/std/python/tasks/pex_build_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pex_build_task.py @@ -31,6 +31,7 @@ class PexBuildTask(Task): python: Property[Path | None] = Property.default(None) index_url: Property[str | None] = Property.default(None) always_rebuild: Property[bool] = Property.default(False) + python_shebang: Property[str | None] = Property.default(None) #: The path to the built PEX file will be written to this property. output_file: Property[Path] = Property.output() @@ -47,6 +48,7 @@ def _get_output_file_path(self) -> Path: self.venv.get() or "", self.pex_binary.map(str).get() or "", self.python.map(str).get() or "", + self.python_shebang.get() or "", ] ).encode() ).hexdigest() @@ -78,6 +80,7 @@ def execute(self) -> TaskStatus | None: pex_binary=self.pex_binary.get(), python=self.python.get(), index_url=self.index_url.get() or _get_default_index_url(self.project), + python_shebang=self.python_shebang.get(), ) except subprocess.CalledProcessError as exc: return TaskStatus.from_exit_code(exc.cmd, exc.returncode) @@ -97,6 +100,7 @@ def _build_pex( pex_binary: Path | None = None, python: Path | None = None, index_url: str | None = None, + python_shebang: str | None = None, log: logging.Logger | None = None, ) -> None: """Invokes the `pex` CLI to build a PEX file and write it to :param:`output_file`. @@ -109,6 +113,9 @@ def _build_pex( :param pex_binary: Path to the `pex` binary to execute. If not specified, `python -m pex` will be used taking into account the :param:`python` parameter. :param python: The Python executable to run `python -m pex` with. If not set, defaults to :data:`sys.executable`. + :param python_shebang: The shebang for the generated PEX. This may need to be set to ensure that it works in + all target environemnts, otherwise it will default to a compatible Python interpreter as specified with the + *interpreter_constraint* option, which may be too specific. """ if pex_binary is not None: @@ -135,6 +142,8 @@ def _build_pex( command += ["--venv", venv] for key, value in (inject_env or {}).items(): command += ["--inject-env", f"{key}={value}"] + if python_shebang is not None: + command += ["--python-shebang", python_shebang] safe_command = list(command) if index_url is not None: @@ -155,6 +164,7 @@ def pex_build( venv: Literal["prepend", "append"] | None = None, index_url: str | None = None, always_rebuild: bool = False, + python_shebang: str | None = None, output_file: Path | None = None, task_name: str | None = None, project: Project | None = None, @@ -175,6 +185,7 @@ def pex_build( and existing_task.venv.get() == venv and existing_task.index_url.get() == index_url and existing_task.always_rebuild.get() == always_rebuild + and existing_task.python_shebang.get() == python_shebang and existing_task.output_file.get_or(None) == output_file ): return existing_task @@ -188,6 +199,7 @@ def pex_build( task.venv = venv task.index_url = index_url task.always_rebuild = always_rebuild + task.python_shebang = python_shebang task.output_file = output_file return task From 0cd6dc52c4a5ebd223c048fac7ef50c411af0996 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Wed, 3 Apr 2024 21:14:10 +0200 Subject: [PATCH 72/79] add `python_app(python_shebang)` parameter --- kraken-build/src/kraken/build/python/v1alpha1/project.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kraken-build/src/kraken/build/python/v1alpha1/project.py b/kraken-build/src/kraken/build/python/v1alpha1/project.py index a89fb8fb..a3db2e19 100644 --- a/kraken-build/src/kraken/build/python/v1alpha1/project.py +++ b/kraken-build/src/kraken/build/python/v1alpha1/project.py @@ -369,6 +369,7 @@ def python_app( entry_point: str | None = None, console_script: str | None = None, interpreter_constraint: str | None = None, + python_shebang: str | None = None, venv_mode: Literal["append", "prepend"] | None = None, name: str = "build-pex", dependencies: Sequence[Task | str] = (), @@ -410,6 +411,7 @@ def python_app( interpreter_constraint=interpreter_constraint, venv=venv_mode, always_rebuild=True, + python_shebang=python_shebang, output_file=output_file, task_name=name, ) From 075930af14be57cd2ac50b9dff21c0d3bd8c8e62 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Wed, 3 Apr 2024 23:22:41 +0200 Subject: [PATCH 73/79] feature: Add `pex_build(python_versions)` --- .changelog/_unreleased.toml | 6 ++ .../kraken/build/python/v1alpha1/project.py | 9 +-- .../kraken/std/python/tasks/pex_build_task.py | 58 +++++++++++++++++++ 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index aec1e7a8..f69874d2 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -117,3 +117,9 @@ id = "54f6e294-e60f-402a-97ae-17cbce9e8bc6" type = "feature" description = "add `pex_build(python_shebang)` parameter" author = "niklas.rosenstein@helsing.ai" + +[[entries]] +id = "9ac5c38b-cb7c-4aa2-bd01-5cf3601dc953" +type = "feature" +description = "Add `pex_build(python_versions)`" +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/build/python/v1alpha1/project.py b/kraken-build/src/kraken/build/python/v1alpha1/project.py index a3db2e19..4f297437 100644 --- a/kraken-build/src/kraken/build/python/v1alpha1/project.py +++ b/kraken-build/src/kraken/build/python/v1alpha1/project.py @@ -236,11 +236,6 @@ def python_project( ) print(new_exclude_format_paths.get()) - from typing import TYPE_CHECKING - - if TYPE_CHECKING: - reveal_type(exclude_format_directories) - # *exclude_format_directories, # # Ensure that the generated Protobuf code is not linted or formatted # str((project.directory / source_directory / "proto").relative_to(project.directory)), @@ -369,8 +364,9 @@ def python_app( entry_point: str | None = None, console_script: str | None = None, interpreter_constraint: str | None = None, - python_shebang: str | None = None, venv_mode: Literal["append", "prepend"] | None = None, + python_shebang: str | None = None, + python_versions: Sequence[str] = (), name: str = "build-pex", dependencies: Sequence[Task | str] = (), ) -> PexBuildTask: @@ -412,6 +408,7 @@ def python_app( venv=venv_mode, always_rebuild=True, python_shebang=python_shebang, + python_versions=python_versions, output_file=output_file, task_name=name, ) diff --git a/kraken-build/src/kraken/std/python/tasks/pex_build_task.py b/kraken-build/src/kraken/std/python/tasks/pex_build_task.py index a67749ad..3036fea1 100644 --- a/kraken-build/src/kraken/std/python/tasks/pex_build_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pex_build_task.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Literal +from kraken.common import findpython from kraken.core.system.project import Project from kraken.core.system.property import Property from kraken.core.system.task import Task, TaskStatus @@ -32,6 +33,14 @@ class PexBuildTask(Task): index_url: Property[str | None] = Property.default(None) always_rebuild: Property[bool] = Property.default(False) python_shebang: Property[str | None] = Property.default(None) + sh_boot: Property[bool] = Property.default(True) + + #: This is not a PEX option, but we use it to ensure that we find at least the specified Python versions + #: and use them all to build a multi-version PEX. The Python interpreters will be searched on the current + #: system PATH using the same method as the kraken-wrapper uses to find the Python interpreter (using the + #: `findpython` module). If any of the given Python versions is not available, it will cause an error unless + #: it is trailed by a question mark. Example: `["3.10", "3.11", "3.12?"]` + python_versions: Property[Sequence[str]] = Property.default_factory(list) #: The path to the built PEX file will be written to this property. output_file: Property[Path] = Property.output() @@ -49,6 +58,8 @@ def _get_output_file_path(self) -> Path: self.pex_binary.map(str).get() or "", self.python.map(str).get() or "", self.python_shebang.get() or "", + str(self.sh_boot.get()), + ":".join(self.python_versions.get()), ] ).encode() ).hexdigest() @@ -81,6 +92,8 @@ def execute(self) -> TaskStatus | None: python=self.python.get(), index_url=self.index_url.get() or _get_default_index_url(self.project), python_shebang=self.python_shebang.get(), + sh_boot=self.sh_boot.get(), + python_versions=self.python_versions.get(), ) except subprocess.CalledProcessError as exc: return TaskStatus.from_exit_code(exc.cmd, exc.returncode) @@ -101,6 +114,8 @@ def _build_pex( python: Path | None = None, index_url: str | None = None, python_shebang: str | None = None, + sh_boot: bool = True, + python_versions: Sequence[str] = (), log: logging.Logger | None = None, ) -> None: """Invokes the `pex` CLI to build a PEX file and write it to :param:`output_file`. @@ -116,6 +131,9 @@ def _build_pex( :param python_shebang: The shebang for the generated PEX. This may need to be set to ensure that it works in all target environemnts, otherwise it will default to a compatible Python interpreter as specified with the *interpreter_constraint* option, which may be too specific. + :param sh_boot: If True, the PEX will be generated with a shell bootstrapper that will run the PEX with the + correct Python interpreter. This is usually preferred (see PEX documentation). + :param python_versions: See :class:`PexBuildTask.python_versions`. """ if pex_binary is not None: @@ -144,6 +162,43 @@ def _build_pex( command += ["--inject-env", f"{key}={value}"] if python_shebang is not None: command += ["--python-shebang", python_shebang] + command += ["--sh-boot" if sh_boot else "--no-sh-boot"] + + # Find compatible Python versions. + if python_versions: + logger.info( + "Finding Python versions %s under interpreter_constraint %s", python_versions, interpreter_constraint + ) + python_versions_to_find = {version.rstrip("?"): version.endswith("?") for version in python_versions} + for candidate in findpython.get_candidates(): + if not python_versions_to_find: + break + if (candidate_version := candidate.get("exact_version")) is None: + continue + if interpreter_constraint is not None and not findpython.match_version_constraint( + interpreter_constraint, candidate_version + ): + continue + + for version, optional in python_versions_to_find.items(): + if candidate_version == version or candidate_version.startswith(version + "."): + logger.info( + "Found candidate Python version %s (%s) for requested version %s", + candidate_version, + candidate["path"], + version, + ) + python_versions_to_find.pop(version) + command += ["--python", candidate["path"]] + break + + # Drop optional versions that were not found. + missing_optional = {version for version, optional in python_versions_to_find.items() if optional} + if missing_optional: + logger.warning("Could not find optional Python versions %s", missing_optional) + missing_required = {version for version, optional in python_versions_to_find.items() if not optional} + if missing_required: + raise ValueError(f"Could not find required Python versions {missing_required}") safe_command = list(command) if index_url is not None: @@ -165,6 +220,7 @@ def pex_build( index_url: str | None = None, always_rebuild: bool = False, python_shebang: str | None = None, + python_versions: Sequence[str] = (), output_file: Path | None = None, task_name: str | None = None, project: Project | None = None, @@ -186,6 +242,7 @@ def pex_build( and existing_task.index_url.get() == index_url and existing_task.always_rebuild.get() == always_rebuild and existing_task.python_shebang.get() == python_shebang + and list(existing_task.python_versions.get()) == list(python_versions) and existing_task.output_file.get_or(None) == output_file ): return existing_task @@ -200,6 +257,7 @@ def pex_build( task.index_url = index_url task.always_rebuild = always_rebuild task.python_shebang = python_shebang + task.python_versions = python_versions task.output_file = output_file return task From a3b58093eac70fb6704adbff4bd7728f3fc77b18 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Thu, 4 Apr 2024 02:00:41 +0200 Subject: [PATCH 74/79] improvement: Improve the `--cache-to` argument produced by the `BuildxBuildTask.cache_repo` property to include `image-manifest=true,oci-mediatypes=true` options for better OCI registry support (see https://aws.amazon.com/blogs/containers/announcing-remote-cache-support-in-amazon-ecr-for-buildkit-clients/) --- .changelog/_unreleased.toml | 6 ++++++ .../src/kraken/std/docker/tasks/buildx_build_task.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index f69874d2..660fff82 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -123,3 +123,9 @@ id = "9ac5c38b-cb7c-4aa2-bd01-5cf3601dc953" type = "feature" description = "Add `pex_build(python_versions)`" author = "@NiklasRosenstein" + +[[entries]] +id = "2b5d4590-5c9b-4fd0-a15f-87d93adb6e7e" +type = "improvement" +description = "Improve the `--cache-to` argument produced by the `BuildxBuildTask.cache_repo` property to include `image-manifest=true,oci-mediatypes=true` options for better OCI registry support (see https://aws.amazon.com/blogs/containers/announcing-remote-cache-support-in-amazon-ecr-for-buildkit-clients/)" +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/std/docker/tasks/buildx_build_task.py b/kraken-build/src/kraken/std/docker/tasks/buildx_build_task.py index 86c177af..ee790e42 100644 --- a/kraken-build/src/kraken/std/docker/tasks/buildx_build_task.py +++ b/kraken-build/src/kraken/std/docker/tasks/buildx_build_task.py @@ -59,7 +59,7 @@ def execute(self) -> TaskStatus: if self.cache_repo.get(): # NOTE (@NiklasRosenstein): Buildx does not allow leading underscores, while Kaniko and Artifactory do. command += ["--cache-from", f"type=registry,ref={not_none(self.cache_repo.get())}"] - command += ["--cache-to", f"type=registry,ref={not_none(self.cache_repo.get())},mode=max,ignore-error=true"] + command += ["--cache-to", f"type=registry,ref={not_none(self.cache_repo.get())},mode=max,ignore-error=true,image-manifest=true,oci-mediatypes=true"] if not self.cache.get(): command += ["--no-cache"] if cache_from := self.cache_from.get(): From fa254914954fc628d182a4d394edb17ae7582262 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 3 Sep 2024 02:27:24 +0200 Subject: [PATCH 75/79] fmt and fix most lints post merge develop --- kraken-build/src/kraken/common/graphviz.py | 3 +-- kraken-build/src/kraken/common/iter.py | 4 ++-- kraken-build/src/kraken/core/cli/main.py | 4 +--- kraken-build/src/kraken/core/system/graph.py | 2 +- kraken-build/src/kraken/core/system/project.py | 1 - .../src/kraken/core/system/project_test.py | 2 +- kraken-build/src/kraken/core/system/task.py | 2 +- kraken-build/src/kraken/core/system/task_test.py | 1 + kraken-build/src/kraken/std/dist_test.py | 5 +++-- .../kraken/std/docker/tasks/buildx_build_task.py | 5 ++++- kraken-build/src/kraken/std/git/__init__.py | 2 +- .../src/kraken/std/python/buildsystem/pdm.py | 2 +- .../src/kraken/std/python/buildsystem/uv.py | 16 ++++++++-------- .../src/kraken/std/python/tasks/base_task.py | 1 - .../src/kraken/std/python/tasks/black_task.py | 4 +--- .../src/kraken/std/python/tasks/flake8_task.py | 3 +-- .../src/kraken/std/python/tasks/isort_task.py | 4 +--- .../src/kraken/std/python/tasks/pycln_task.py | 5 ++--- .../src/kraken/std/python/tasks/pylint_task.py | 3 +-- .../src/kraken/std/python/tasks/pytest_task.py | 7 +++---- .../kraken_std/integration/python/test_python.py | 2 +- .../kraken_std/python/tasks/test_pytest_task.py | 11 ++--------- kraken-wrapper/tests/iss-263/test_main.py | 6 +++--- 23 files changed, 40 insertions(+), 55 deletions(-) diff --git a/kraken-build/src/kraken/common/graphviz.py b/kraken-build/src/kraken/common/graphviz.py index a76344a6..be26974c 100644 --- a/kraken-build/src/kraken/common/graphviz.py +++ b/kraken-build/src/kraken/common/graphviz.py @@ -4,12 +4,11 @@ import logging import subprocess as sp import tempfile +import typing as t import webbrowser from pathlib import Path from typing import overload -import typing as t - logger = logging.getLogger(__name__) diff --git a/kraken-build/src/kraken/common/iter.py b/kraken-build/src/kraken/common/iter.py index 45a81e17..99454531 100644 --- a/kraken-build/src/kraken/common/iter.py +++ b/kraken-build/src/kraken/common/iter.py @@ -1,6 +1,6 @@ -from collections.abc import Iterable +from collections.abc import Callable, Iterable from itertools import filterfalse, tee -from typing import Callable, TypeVar +from typing import TypeVar T_co = TypeVar("T_co", covariant=True) diff --git a/kraken-build/src/kraken/core/cli/main.py b/kraken-build/src/kraken/core/cli/main.py index e270b674..6bf2cb3d 100644 --- a/kraken-build/src/kraken/core/cli/main.py +++ b/kraken-build/src/kraken/core/cli/main.py @@ -14,9 +14,6 @@ from pathlib import Path from typing import Any, NoReturn -from kraken.common.graphviz import render_to_browser -from kraken.common.graphviz import GraphvizWriter - from kraken.common import ( BuildscriptMetadata, ColorOptions, @@ -29,6 +26,7 @@ not_none, propagate_argparse_formatter_to_subparser, ) +from kraken.common.graphviz import GraphvizWriter, render_to_browser from kraken.common.pyenv import get_distributions from kraken.core.address import Address, AddressResolutionError from kraken.core.cli import serialize diff --git a/kraken-build/src/kraken/core/system/graph.py b/kraken-build/src/kraken/core/system/graph.py index 790e7840..5ac9a958 100644 --- a/kraken-build/src/kraken/core/system/graph.py +++ b/kraken-build/src/kraken/core/system/graph.py @@ -226,7 +226,7 @@ def set_non_strict_edge_for_removal(u: Address, v: Address) -> None: else: set_non_strict_edge_for_removal(failed_task_path, out_task_path) - return cast("DiGraph[Address]", restricted_view(self._digraph, self._ok_tasks, removable_edges)) # type: ignore[no-untyped-call] + return cast("DiGraph[Address]", restricted_view(self._digraph, self._ok_tasks, removable_edges)) # type: ignore[no-untyped-call] # noqa: E501 # Public API diff --git a/kraken-build/src/kraken/core/system/project.py b/kraken-build/src/kraken/core/system/project.py index 65e0ce17..0e19e722 100644 --- a/kraken-build/src/kraken/core/system/project.py +++ b/kraken-build/src/kraken/core/system/project.py @@ -5,7 +5,6 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, TypeVar, overload - from kraken.core.base import Currentable, MetadataContainer from kraken.core.system.kraken_object import KrakenObject from kraken.core.system.task import GroupTask, Task diff --git a/kraken-build/src/kraken/core/system/project_test.py b/kraken-build/src/kraken/core/system/project_test.py index 96ad679c..63b4f308 100644 --- a/kraken-build/src/kraken/core/system/project_test.py +++ b/kraken-build/src/kraken/core/system/project_test.py @@ -6,7 +6,7 @@ def test__Project__do_normalizes_taskname_backwards_compatibility_pre_0_12_0(kraken_project: Project) -> None: with pytest.warns(DeprecationWarning) as warninfo: - task = kraken_project.do("this is a :test task", VoidTask) + task = kraken_project.task("this is a :test task", VoidTask) assert task.name == "this-is-a-test-task" assert str(warninfo.list[0].message) == ("Call to deprecated method do. (Use Project.task() instead)") assert str(warninfo.list[1].message) == ( diff --git a/kraken-build/src/kraken/core/system/task.py b/kraken-build/src/kraken/core/system/task.py index af3eed20..32ff52b0 100644 --- a/kraken-build/src/kraken/core/system/task.py +++ b/kraken-build/src/kraken/core/system/task.py @@ -619,7 +619,7 @@ def __len__(self) -> int: return len(self._tasks) def __repr__(self) -> str: - return f"TaskSet(length={len(self._tasks)}, pttm={self._partition_to_task_map}, ttpm={self._task_to_partition_map})" + return f"TaskSet(length={len(self._tasks)}, pttm={self._partition_to_task_map}, ttpm={self._task_to_partition_map})" # noqa: E501 def __contains__(self, __x: object) -> bool: return __x in self._tasks diff --git a/kraken-build/src/kraken/core/system/task_test.py b/kraken-build/src/kraken/core/system/task_test.py index d32dc3c0..6f644211 100644 --- a/kraken-build/src/kraken/core/system/task_test.py +++ b/kraken-build/src/kraken/core/system/task_test.py @@ -1,4 +1,5 @@ from dataclasses import dataclass + from pytest import raises from kraken.core.system.project import Project diff --git a/kraken-build/src/kraken/std/dist_test.py b/kraken-build/src/kraken/std/dist_test.py index d1738b1b..0daba845 100644 --- a/kraken-build/src/kraken/std/dist_test.py +++ b/kraken-build/src/kraken/std/dist_test.py @@ -1,8 +1,9 @@ import tarfile -from kraken.core import Project, Task, Property + +from kraken.core import Project, Property, Task from kraken.core.system.task import TaskStatus -from kraken.std.dist import dist from kraken.std.descriptors.resource import Resource +from kraken.std.dist import dist def test_dist(kraken_project: Project) -> None: diff --git a/kraken-build/src/kraken/std/docker/tasks/buildx_build_task.py b/kraken-build/src/kraken/std/docker/tasks/buildx_build_task.py index ee790e42..c74202f1 100644 --- a/kraken-build/src/kraken/std/docker/tasks/buildx_build_task.py +++ b/kraken-build/src/kraken/std/docker/tasks/buildx_build_task.py @@ -59,7 +59,10 @@ def execute(self) -> TaskStatus: if self.cache_repo.get(): # NOTE (@NiklasRosenstein): Buildx does not allow leading underscores, while Kaniko and Artifactory do. command += ["--cache-from", f"type=registry,ref={not_none(self.cache_repo.get())}"] - command += ["--cache-to", f"type=registry,ref={not_none(self.cache_repo.get())},mode=max,ignore-error=true,image-manifest=true,oci-mediatypes=true"] + command += [ + "--cache-to", + f"type=registry,ref={not_none(self.cache_repo.get())},mode=max,ignore-error=true,image-manifest=true,oci-mediatypes=true", # noqa: E501 + ] if not self.cache.get(): command += ["--no-cache"] if cache_from := self.cache_from.get(): diff --git a/kraken-build/src/kraken/std/git/__init__.py b/kraken-build/src/kraken/std/git/__init__.py index 5ce49d68..5a9b7930 100644 --- a/kraken-build/src/kraken/std/git/__init__.py +++ b/kraken-build/src/kraken/std/git/__init__.py @@ -2,10 +2,10 @@ from __future__ import annotations +import warnings from collections.abc import Sequence from pathlib import Path from typing import Literal -import warnings from kraken.core import Project from kraken.std.util.check_file_contents_task import CheckFileContentsTask diff --git a/kraken-build/src/kraken/std/python/buildsystem/pdm.py b/kraken-build/src/kraken/std/python/buildsystem/pdm.py index e39abded..f6a2b851 100644 --- a/kraken-build/src/kraken/std/python/buildsystem/pdm.py +++ b/kraken-build/src/kraken/std/python/buildsystem/pdm.py @@ -6,9 +6,9 @@ import os import shutil import subprocess as sp +import sys from collections.abc import Sequence from pathlib import Path -import sys from typing import Any from kraken.common import NotSet diff --git a/kraken-build/src/kraken/std/python/buildsystem/uv.py b/kraken-build/src/kraken/std/python/buildsystem/uv.py index 4c5a5ce5..2663e790 100644 --- a/kraken-build/src/kraken/std/python/buildsystem/uv.py +++ b/kraken-build/src/kraken/std/python/buildsystem/uv.py @@ -6,17 +6,17 @@ from __future__ import annotations -from dataclasses import dataclass -from hashlib import md5 import logging -from os import fsdecode import os import shutil import subprocess as sp -from collections.abc import Sequence -from pathlib import Path import tempfile -from typing import TYPE_CHECKING, Annotated, Any, Iterable, MutableMapping, TypeVar +from collections.abc import Iterable, MutableMapping, Sequence +from dataclasses import dataclass +from hashlib import md5 +from os import fsdecode +from pathlib import Path +from typing import TYPE_CHECKING, Annotated, Any, TypeVar from urllib.parse import urlparse from kraken.common.sanitize import sanitize_http_basic_auth @@ -62,7 +62,7 @@ def unsafe_url(self) -> str: return self.url @staticmethod - def of(index: PackageIndex) -> "PipIndex": + def of(index: PackageIndex) -> PipIndex: credentials = index.credentials if isinstance(index, PythonSettings._PackageIndex) else None return PipIndex(index.index_url, credentials) @@ -73,7 +73,7 @@ class PipIndexes: supplemental: list[PipIndex] @staticmethod - def from_package_indexes(indexes: Iterable[T_PackageIndex]) -> "PipIndexes": + def from_package_indexes(indexes: Iterable[T_PackageIndex]) -> PipIndexes: default_index = next((idx for idx in indexes if idx.priority == PackageIndex.Priority.default), None) primary_index = next((idx for idx in indexes if idx.priority == PackageIndex.Priority.primary), None) remainder = [idx for idx in indexes if idx not in (default_index, primary_index)] diff --git a/kraken-build/src/kraken/std/python/tasks/base_task.py b/kraken-build/src/kraken/std/python/tasks/base_task.py index cd491f01..654e4625 100644 --- a/kraken-build/src/kraken/std/python/tasks/base_task.py +++ b/kraken-build/src/kraken/std/python/tasks/base_task.py @@ -7,7 +7,6 @@ import sys from collections.abc import Iterable, MutableMapping - from kraken.common.pyenv import VirtualEnvInfo, get_current_venv from kraken.core import Project, Task, TaskRelationship, TaskStatus from kraken.std.python.buildsystem import ManagedEnvironment diff --git a/kraken-build/src/kraken/std/python/tasks/black_task.py b/kraken-build/src/kraken/std/python/tasks/black_task.py index 44a2c002..2748c261 100644 --- a/kraken-build/src/kraken/std/python/tasks/black_task.py +++ b/kraken-build/src/kraken/std/python/tasks/black_task.py @@ -1,10 +1,8 @@ from __future__ import annotations import re -from collections.abc import Sequence -from dataclasses import dataclass -import dataclasses from collections.abc import MutableMapping, Sequence +from dataclasses import dataclass from pathlib import Path from typing import Any diff --git a/kraken-build/src/kraken/std/python/tasks/flake8_task.py b/kraken-build/src/kraken/std/python/tasks/flake8_task.py index 8763656f..a55051aa 100644 --- a/kraken-build/src/kraken/std/python/tasks/flake8_task.py +++ b/kraken-build/src/kraken/std/python/tasks/flake8_task.py @@ -1,10 +1,9 @@ from __future__ import annotations import os -from collections.abc import Sequence +from collections.abc import MutableMapping, Sequence from configparser import ConfigParser from dataclasses import dataclass -from collections.abc import MutableMapping, Sequence from pathlib import Path from kraken.common import Supplier diff --git a/kraken-build/src/kraken/std/python/tasks/isort_task.py b/kraken-build/src/kraken/std/python/tasks/isort_task.py index 5bc60687..0f0e29b1 100644 --- a/kraken-build/src/kraken/std/python/tasks/isort_task.py +++ b/kraken-build/src/kraken/std/python/tasks/isort_task.py @@ -1,11 +1,9 @@ from __future__ import annotations import os -from collections.abc import Sequence +from collections.abc import MutableMapping, Sequence from configparser import ConfigParser from dataclasses import dataclass -import dataclasses -from collections.abc import MutableMapping, Sequence from pathlib import Path from kraken.common import Supplier diff --git a/kraken-build/src/kraken/std/python/tasks/pycln_task.py b/kraken-build/src/kraken/std/python/tasks/pycln_task.py index 61246acf..b4056c29 100644 --- a/kraken-build/src/kraken/std/python/tasks/pycln_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pycln_task.py @@ -2,14 +2,13 @@ import dataclasses import re -from collections.abc import Sequence +from collections.abc import MutableMapping, Sequence from pathlib import Path -from typing import MutableMapping from kraken.common.supplier import Supplier from kraken.core import Project, Property -from kraken.std.python.settings import python_settings from kraken.core.system.task import TaskStatus +from kraken.std.python.settings import python_settings from kraken.std.python.tasks.pex_build_task import pex_build from .base_task import EnvironmentAwareDispatchTask diff --git a/kraken-build/src/kraken/std/python/tasks/pylint_task.py b/kraken-build/src/kraken/std/python/tasks/pylint_task.py index 35522c5a..36953898 100644 --- a/kraken-build/src/kraken/std/python/tasks/pylint_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pylint_task.py @@ -1,8 +1,7 @@ from __future__ import annotations -from collections.abc import Sequence +from collections.abc import MutableMapping, Sequence from pathlib import Path -from typing import MutableMapping from kraken.common import Supplier from kraken.core import Project, Property diff --git a/kraken-build/src/kraken/std/python/tasks/pytest_task.py b/kraken-build/src/kraken/std/python/tasks/pytest_task.py index a0beeaa3..9ddecd56 100644 --- a/kraken-build/src/kraken/std/python/tasks/pytest_task.py +++ b/kraken-build/src/kraken/std/python/tasks/pytest_task.py @@ -5,7 +5,6 @@ import shlex from collections.abc import MutableMapping, Sequence from pathlib import Path -import warnings from kraken.common import flatten from kraken.core import Project, Property, TaskStatus @@ -32,7 +31,7 @@ class PytestTask(EnvironmentAwareDispatchTask): description = "Run unit tests using Pytest." python_dependencies = ["pytest"] - paths: Property[Sequence[str]] + paths: Property[Sequence[str | Path]] ignore_dirs: Property[Sequence[Path]] = Property.default_factory(list) allow_no_tests: Property[bool] = Property.default(False) doctest_modules: Property[bool] = Property.default(True) @@ -42,7 +41,7 @@ class PytestTask(EnvironmentAwareDispatchTask): # EnvironmentAwareDispatchTask def get_execute_command_v2(self, env: MutableMapping[str, str]) -> list[str] | TaskStatus: - command = ["pytest", "-vv", *self.paths.get()] + command = ["pytest", "-vv", *map(str, self.paths.get())] command += flatten(["--ignore", str(self.project.directory / path)] for path in self.ignore_dirs.get()) command += ["--log-cli-level", "INFO"] if self.coverage.is_filled(): @@ -73,7 +72,7 @@ def pytest( name: str = "pytest", group: str = "test", project: Project | None = None, - paths: Sequence[str] | None = None, + paths: Sequence[str | Path] | None = None, ignore_dirs: Sequence[Path | str] = (), allow_no_tests: bool = False, doctest_modules: bool = True, diff --git a/kraken-build/tests/kraken_std/integration/python/test_python.py b/kraken-build/tests/kraken_std/integration/python/test_python.py index 3f3cb28f..a16bceb5 100644 --- a/kraken-build/tests/kraken_std/integration/python/test_python.py +++ b/kraken-build/tests/kraken_std/integration/python/test_python.py @@ -19,8 +19,8 @@ from kraken.std.python.buildsystem.maturin import MaturinPoetryPyprojectHandler from kraken.std.python.buildsystem.pdm import PdmPyprojectHandler from kraken.std.python.buildsystem.poetry import PoetryPyprojectHandler -from kraken.std.python.pyproject import Pyproject, PyprojectHandler from kraken.std.python.buildsystem.uv import UvPyprojectHandler +from kraken.std.python.pyproject import PyprojectHandler from kraken.std.util.http import http_probe from tests.kraken_std.util.docker import DockerServiceManager from tests.resources import example_dir diff --git a/kraken-build/tests/kraken_std/python/tasks/test_pytest_task.py b/kraken-build/tests/kraken_std/python/tasks/test_pytest_task.py index 7abc94ed..77a858d9 100644 --- a/kraken-build/tests/kraken_std/python/tasks/test_pytest_task.py +++ b/kraken-build/tests/kraken_std/python/tasks/test_pytest_task.py @@ -1,5 +1,4 @@ from kraken.core import Project - from kraken.std.python import pytest as pytest_task FAKE_TEST = """ @@ -17,10 +16,7 @@ def test__pytest_single_path(kraken_project: Project) -> None: test_test_dir.mkdir() (test_test_dir / "test_mock.py").write_text(FAKE_TEST) - task = pytest_task(tests_dir=test_test_dir) - - assert not task.is_skippable() - + pytest_task(paths=[test_test_dir]) kraken_project.context.execute([":test"]) @@ -34,8 +30,5 @@ def test__pytest_multiple_paths(kraken_project: Project) -> None: (src_test_dir / "test_mock.py").write_text(FAKE_TEST) (test_test_dir / "test_mock2.py").write_text(FAKE_TEST) - task = pytest_task(tests_dir=[src_test_dir, test_test_dir]) - - assert not task.is_skippable() - + pytest_task(paths=[src_test_dir, test_test_dir]) kraken_project.context.execute([":test"]) diff --git a/kraken-wrapper/tests/iss-263/test_main.py b/kraken-wrapper/tests/iss-263/test_main.py index 9fa88cfa..b242e9ae 100644 --- a/kraken-wrapper/tests/iss-263/test_main.py +++ b/kraken-wrapper/tests/iss-263/test_main.py @@ -4,16 +4,16 @@ [263]: https://github.com/kraken-build/kraken/issues/263 """ +import os +import shutil from collections.abc import Iterator from contextlib import contextmanager -import os from pathlib import Path -import shutil from tempfile import TemporaryDirectory import pytest -from kraken.wrapper.main import main +from kraken.wrapper.main import main EXAMPLE_PROJECT = Path(__file__).parent / "example_project" DEPENDENCY = Path(__file__).parent / "dependency" From dfaf3b296767cb0615955890271130c182390f2a Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 3 Sep 2024 02:29:20 +0200 Subject: [PATCH 76/79] move changelog entries --- .changelog/0.35.0.toml | 18 ------------------ .changelog/_unreleased.toml | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.changelog/0.35.0.toml b/.changelog/0.35.0.toml index 4782980a..80348032 100644 --- a/.changelog/0.35.0.toml +++ b/.changelog/0.35.0.toml @@ -32,18 +32,6 @@ type = "improvement" description = "Add task to login in all cargo registries" author = "quentin.santos@helsing.ai" -[[entries]] -id = "fb3fcc4a-a38a-445f-b124-571395b7ac86" -type = "feature" -description = "Introduce `kraken.std.python.project.python_project()` function which creates all tasks for a Python project." -author = "niklas.rosenstein@helsing.ai" - -[[entries]] -id = "ae70e553-81c2-4eb2-a6f3-210ca6c24992" -type = "breaking change" -description = "Default project groups `lint` now only depends on `check` for order instead of strictly" -author = "niklas.rosenstein@helsing.ai" - [[entries]] id = "bc4f2e14-e52b-48ba-b0f3-c1b6210e1682" type = "fix" @@ -57,9 +45,3 @@ type = "feature" description = "Add shellcheck capabilities" author = "niklas.rosenstein@helsing.ai" component = "kraken-build" - -[[entries]] -id = "4cb0fe80-d359-4302-8893-c0fd62bf0631" -type = "improvement" -description = "Raise `EmptyGitRepositoryError` and `NotAGitRepositoryError` respectively in `git_describe()`" -author = "@NiklasRosenstein" diff --git a/.changelog/_unreleased.toml b/.changelog/_unreleased.toml index 1b60ce36..03c2eb8a 100644 --- a/.changelog/_unreleased.toml +++ b/.changelog/_unreleased.toml @@ -141,3 +141,21 @@ id = "4560750d-452d-43f0-af08-d791aaf5134a" type = "improvement" description = "Upgrade keyring to v25" author = "@NiklasRosenstein" + +[[entries]] +id = "fb3fcc4a-a38a-445f-b124-571395b7ac86" +type = "feature" +description = "Introduce `kraken.std.python.project.python_project()` function which creates all tasks for a Python project." +author = "niklas.rosenstein@helsing.ai" + +[[entries]] +id = "ae70e553-81c2-4eb2-a6f3-210ca6c24992" +type = "breaking change" +description = "Default project groups `lint` now only depends on `check` for order instead of strictly" +author = "niklas.rosenstein@helsing.ai" + +[[entries]] +id = "4cb0fe80-d359-4302-8893-c0fd62bf0631" +type = "improvement" +description = "Raise `EmptyGitRepositoryError` and `NotAGitRepositoryError` respectively in `git_describe()`" +author = "@NiklasRosenstein" From 9cc0d378ba1cd482274a1fc77521705fda2b89bf Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 3 Sep 2024 23:10:07 +0200 Subject: [PATCH 77/79] Add `python_project(enable)` option to select the tools and features --- .../kraken/build/python/v1alpha1/project.py | 152 +++++++++++------- 1 file changed, 90 insertions(+), 62 deletions(-) diff --git a/kraken-build/src/kraken/build/python/v1alpha1/project.py b/kraken-build/src/kraken/build/python/v1alpha1/project.py index 4f297437..a25e8dfa 100644 --- a/kraken-build/src/kraken/build/python/v1alpha1/project.py +++ b/kraken-build/src/kraken/build/python/v1alpha1/project.py @@ -22,6 +22,8 @@ T = TypeVar("T") logger = logging.getLogger(__name__) +PythonTool = Literal["ruff", "mypy", "pylint", "black", "isort", "flake8", "pycln", "pyupgrade", "pytest"] + class PythonCodegen(Protocol): """Protocol for functions that produce a task to generate Python code.""" @@ -94,6 +96,7 @@ def python_project( line_length: int = 120, enforce_project_version: str | None = None, detect_git_version_build_type: Literal["release", "develop", "branch"] = "develop", + enable: Sequence[PythonTool] = ("ruff", "mypy", "pytest"), pyupgrade_keep_runtime_typing: bool = False, pycln_remove_all_unused_imports: bool = False, pytest_ignore_dirs: Sequence[str] = (), @@ -106,6 +109,7 @@ def python_project( mypy_version_spec: str = ">=1.8.0,<2.0.0", pycln_version_spec: str = ">=2.4.0,<3.0.0", pyupgrade_version_spec: str = ">=3.15.0,<4.0.0", + ruff_version_spec: str = ">=0.6.0,<0.7.0", codegen: Sequence[PythonCodegen] = (), ) -> "PythonProject": """ @@ -149,6 +153,7 @@ def python_project( the version number will be derived from the most recent tag and the distance to the current commit. If the current commit is tagged, the version number will be the tag name anyway. When set to `"branch"`, the version number will be derived from the distance to the most recent tag and include the SHA of the commit. + enable: A list of tools to enable for the project. The default is `["ruff", "mypy", "pytest"]`. pyupgrade_keep_runtime_typing: Whether to not replace `typing` type hints. This is required for example for projects using Typer as it does not support all modern type hints at runtime. pycln_remove_all_unused_imports: Remove all unused imports, including these with side effects. @@ -179,6 +184,7 @@ def python_project( from kraken.std.python.tasks.pyupgrade_task import pyupgrade as pyupgrade_task from kraken.std.python.tasks.update_lockfile_task import update_lockfile_task from kraken.std.python.tasks.update_pyproject_task import update_pyproject_task + from kraken.std.python.tasks.ruff_task import ruff as ruff_task if additional_lint_directories is None: additional_lint_directories = [] @@ -243,64 +249,85 @@ def python_project( # === Python tooling - pyupgrade_task( - python_version=python_version, - keep_runtime_typing=pyupgrade_keep_runtime_typing, - exclude=[Path(x) for x in concat(exclude_lint_directories, exclude_format_directories)], - paths=source_paths, - version_spec=pyupgrade_version_spec, - ) + if "pyupgrade" in enable: + pyupgrade_task( + python_version=python_version, + keep_runtime_typing=pyupgrade_keep_runtime_typing, + exclude=[Path(x) for x in concat(exclude_lint_directories, exclude_format_directories)], + paths=source_paths, + version_spec=pyupgrade_version_spec, + ) - pycln_task( - paths=source_paths, - exclude_directories=concat(exclude_lint_directories, exclude_format_directories), - remove_all_unused_imports=pycln_remove_all_unused_imports, - version_spec=pycln_version_spec, - ) + if "pycln" in enable: + pycln_task( + paths=source_paths, + exclude_directories=concat(exclude_lint_directories, exclude_format_directories), + remove_all_unused_imports=pycln_remove_all_unused_imports, + version_spec=pycln_version_spec, + ) - black = black_tasks( - paths=source_paths, - config=BlackConfig( - line_length=line_length, exclude_directories=concat(exclude_lint_directories, exclude_format_directories) - ), - version_spec=black_version_spec, - ) + if "black" in enable: + black = black_tasks( + paths=source_paths, + config=BlackConfig( + line_length=line_length, exclude_directories=concat(exclude_lint_directories, exclude_format_directories) + ), + version_spec=black_version_spec, + ) + else: + black = None + + if "isort" in enable: + isort = isort_tasks( + paths=source_paths, + config=IsortConfig( + profile="black", + line_length=line_length, + extend_skip=concat(exclude_lint_directories, exclude_format_directories), + ), + version_spec=isort_version_spec, + ) + if "black" in enable: + assert black is not None + isort.format.depends_on(black.format, mode="order-only") + + if "flake8" in enable: + flake8 = flake8_tasks( + paths=source_paths, + config=Flake8Config( + max_line_length=line_length, + extend_ignore=flake8_extend_ignore, + exclude=concat(exclude_lint_directories, exclude_format_directories), + ), + version_spec=flake8_version_spec, + additional_requirements=flake8_additional_requirements, + ) + if "black" in enable: + assert black is not None + flake8.depends_on(black.format, mode="order-only") + if "isort" in enable: + assert isort is not None + flake8.depends_on(isort.format, mode="order-only") + + if "ruff" in enable: + ruff_task( + additional_args=[f"--line-length={line_length}"], + version_spec=ruff_version_spec, + ) - isort = isort_tasks( - paths=source_paths, - config=IsortConfig( - profile="black", - line_length=line_length, - extend_skip=concat(exclude_lint_directories, exclude_format_directories), - ), - version_spec=isort_version_spec, - ) - isort.format.depends_on(black.format) - - flake8 = flake8_tasks( - paths=source_paths, - config=Flake8Config( - max_line_length=line_length, - extend_ignore=flake8_extend_ignore, - exclude=concat(exclude_lint_directories, exclude_format_directories), - ), - version_spec=flake8_version_spec, - additional_requirements=flake8_additional_requirements, - ) - flake8.depends_on(black.format, isort.format, mode="order-only") - - mypy_task( - paths=source_paths, - config=MypyConfig( - mypy_path=[source_directory], - exclude_directories=exclude_lint_directories, - global_overrides={}, - module_overrides={}, - ), - version_spec=mypy_version_spec, - python_version=python_version, - use_daemon=mypy_use_daemon, - ).depends_on(*codegen_tasks) + if "mypy" in enable: + mypy_task( + paths=source_paths, + config=MypyConfig( + mypy_path=[source_directory], + exclude_directories=exclude_lint_directories, + global_overrides={}, + module_overrides={}, + ), + version_spec=mypy_version_spec, + python_version=python_version, + use_daemon=mypy_use_daemon, + ).depends_on(*codegen_tasks) # TODO(@niklas): Improve this heuristic to check whether Coverage reporting should be enabled. if "pytest-cov" in str(dict(pyproject)): @@ -308,13 +335,14 @@ def python_project( else: coverage = None - pytest_task( - paths=source_paths, - ignore_dirs=pytest_ignore_dirs, - coverage=coverage, - doctest_modules=True, - allow_no_tests=True, - ).depends_on(*codegen_tasks) + if "pytest" in enable: + pytest_task( + paths=source_paths, + ignore_dirs=pytest_ignore_dirs, + coverage=coverage, + doctest_modules=True, + allow_no_tests=True, + ).depends_on(*codegen_tasks) if not enforce_project_version: try: From f63f8e45232cbd692976e3bd5f1d216dd54857f2 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 3 Sep 2024 23:10:14 +0200 Subject: [PATCH 78/79] fmt --- kraken-build/src/kraken/build/protobuf/v1alpha1/__init__.py | 2 +- kraken-build/src/kraken/build/python/v1alpha1/__init__.py | 2 +- kraken-build/src/kraken/build/python/v1alpha1/project.py | 5 +++-- kraken-build/src/kraken/build/utils/v1alpha1/__init__.py | 2 +- .../src/kraken/build/utils/v1alpha1/fetch_tarball_task.py | 1 - kraken-build/src/kraken/std/protobuf/__init__.py | 1 - kraken-build/src/kraken/std/python/buildsystem/pdm.py | 4 +++- 7 files changed, 9 insertions(+), 8 deletions(-) diff --git a/kraken-build/src/kraken/build/protobuf/v1alpha1/__init__.py b/kraken-build/src/kraken/build/protobuf/v1alpha1/__init__.py index 55742f01..5a77a872 100644 --- a/kraken-build/src/kraken/build/protobuf/v1alpha1/__init__.py +++ b/kraken-build/src/kraken/build/protobuf/v1alpha1/__init__.py @@ -1,4 +1,4 @@ -""" Provides an API for defining a Protobuf project that uses [Buffrs][] for dependency management, [Buf][] for +"""Provides an API for defining a Protobuf project that uses [Buffrs][] for dependency management, [Buf][] for linting and code-generation capabilities with [grpcio-tools][]. [Buffrs]: https://github.com/helsing-ai/buffrs diff --git a/kraken-build/src/kraken/build/python/v1alpha1/__init__.py b/kraken-build/src/kraken/build/python/v1alpha1/__init__.py index 5254b0fc..a48cbad3 100644 --- a/kraken-build/src/kraken/build/python/v1alpha1/__init__.py +++ b/kraken-build/src/kraken/build/python/v1alpha1/__init__.py @@ -1,4 +1,4 @@ -""" This module provides facilities for Kraken-infused Python projects. +"""This module provides facilities for Kraken-infused Python projects. __Features__ diff --git a/kraken-build/src/kraken/build/python/v1alpha1/project.py b/kraken-build/src/kraken/build/python/v1alpha1/project.py index a25e8dfa..78816285 100644 --- a/kraken-build/src/kraken/build/python/v1alpha1/project.py +++ b/kraken-build/src/kraken/build/python/v1alpha1/project.py @@ -1,4 +1,4 @@ -""" New-style API and template for defining the tasks for an entire Python project.""" +"""New-style API and template for defining the tasks for an entire Python project.""" import logging import re @@ -270,7 +270,8 @@ def python_project( black = black_tasks( paths=source_paths, config=BlackConfig( - line_length=line_length, exclude_directories=concat(exclude_lint_directories, exclude_format_directories) + line_length=line_length, + exclude_directories=concat(exclude_lint_directories, exclude_format_directories), ), version_spec=black_version_spec, ) diff --git a/kraken-build/src/kraken/build/utils/v1alpha1/__init__.py b/kraken-build/src/kraken/build/utils/v1alpha1/__init__.py index 7da4e1b9..8e3e88a0 100644 --- a/kraken-build/src/kraken/build/utils/v1alpha1/__init__.py +++ b/kraken-build/src/kraken/build/utils/v1alpha1/__init__.py @@ -1,4 +1,4 @@ -""" This module provides an experimental new functional API for defining Kraken tasks. +"""This module provides an experimental new functional API for defining Kraken tasks. !!! warning This API is unstable and should be used with caution. diff --git a/kraken-build/src/kraken/build/utils/v1alpha1/fetch_tarball_task.py b/kraken-build/src/kraken/build/utils/v1alpha1/fetch_tarball_task.py index 5397b715..f55cd89b 100644 --- a/kraken-build/src/kraken/build/utils/v1alpha1/fetch_tarball_task.py +++ b/kraken-build/src/kraken/build/utils/v1alpha1/fetch_tarball_task.py @@ -38,7 +38,6 @@ def prepare(self) -> TaskStatus | None: return None def execute(self) -> TaskStatus | None: - print(f"Downloading {self.url.get()} ...") with tempfile.TemporaryDirectory() as tmp, Path(tmp).joinpath("archive").open("wb") as f: with httpx.stream("GET", self.url.get(), follow_redirects=True, timeout=60) as response: diff --git a/kraken-build/src/kraken/std/protobuf/__init__.py b/kraken-build/src/kraken/std/protobuf/__init__.py index 2b9762be..36534d93 100644 --- a/kraken-build/src/kraken/std/protobuf/__init__.py +++ b/kraken-build/src/kraken/std/protobuf/__init__.py @@ -32,7 +32,6 @@ def generate(self, language: str, output_dir: Path) -> None: self.generators.setmap(lambda v: [*v, (language, output_dir)]) def execute(self) -> TaskStatus | None: - # TODO: Re-organize proto_dir to be prefixed with a `proto/` directory that is not contained # in the `--proto_path` argument. This is necessary to ensure we generate imports in # a `proto/` namespace package. diff --git a/kraken-build/src/kraken/std/python/buildsystem/pdm.py b/kraken-build/src/kraken/std/python/buildsystem/pdm.py index f6a2b851..cc0649e3 100644 --- a/kraken-build/src/kraken/std/python/buildsystem/pdm.py +++ b/kraken-build/src/kraken/std/python/buildsystem/pdm.py @@ -99,7 +99,9 @@ def set_package_indexes(self, indexes: Sequence[PackageIndex]) -> None: key=lambda x: ( 0 if x.priority == PackageIndex.Priority.default - else 1 if x.priority == PackageIndex.Priority.primary else 2 + else 1 + if x.priority == PackageIndex.Priority.primary + else 2 ), ) From 1ced3cc7502db4ebc547a2614aec534ea62a0393 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Sat, 7 Sep 2024 12:19:48 +0200 Subject: [PATCH 79/79] fix: dont include +dirty in Python version metadata derived from Git version --- kraken-build/src/kraken/build/python/v1alpha1/project.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kraken-build/src/kraken/build/python/v1alpha1/project.py b/kraken-build/src/kraken/build/python/v1alpha1/project.py index 78816285..393b91ed 100644 --- a/kraken-build/src/kraken/build/python/v1alpha1/project.py +++ b/kraken-build/src/kraken/build/python/v1alpha1/project.py @@ -355,6 +355,10 @@ def python_project( logger.info("Empty Git repository found in %s, not enforcing a project version", project.directory) enforce_project_version = None else: + if git_version.dirty: + # Remove the dirty flag as that will propagate into the version metadata. + logger.warning("Git repository for project %s has uncommitted changes.", project.directory) + git_version.dirty = False match detect_git_version_build_type: case "release" | "develop": if (