From 663795e7263898ce6af90cbbe996f7c731144995 Mon Sep 17 00:00:00 2001 From: Markus Binsteiner Date: Mon, 12 Feb 2024 17:29:22 +0100 Subject: [PATCH] feat: archive details --- .gitignore | 2 + pyproject.toml | 1 + src/kiara/interfaces/cli/archive/__init__.py | 0 src/kiara/interfaces/cli/archive/commands.py | 39 +++++++++++ src/kiara/interfaces/cli/data/commands.py | 8 +-- src/kiara/interfaces/python_api/__init__.py | 20 +++++- src/kiara/models/archives.py | 68 +++++++++++++++++-- src/kiara/registries/__init__.py | 9 +-- src/kiara/registries/aliases/__init__.py | 8 ++- .../data/data_store/filesystem_store.py | 10 ++- .../data/data_store/sqlite_store.py | 12 +++- .../jobs/job_store/filesystem_store.py | 2 +- 12 files changed, 156 insertions(+), 23 deletions(-) create mode 100644 src/kiara/interfaces/cli/archive/__init__.py create mode 100644 src/kiara/interfaces/cli/archive/commands.py diff --git a/.gitignore b/.gitignore index a7a92a1e9..cd642c305 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,5 @@ environment.yaml .pixi dev.py store.sqlite +**.kiarchive +**.kontext diff --git a/pyproject.toml b/pyproject.toml index bb4dc6f37..c902a89fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,6 +125,7 @@ sqlite_workflow_store = "kiara.registries.workflows.sqlite_store:SqliteWorkflowS run = "kiara.interfaces.cli.run:run" info = "kiara.interfaces.cli.info.commands:info" context = "kiara.interfaces.cli.context.commands:context" +archive = "kiara.interfaces.cli.archive.commands:context" data = "kiara.interfaces.cli.data.commands:data" module = "kiara.interfaces.cli.module.commands:module" operation = "kiara.interfaces.cli.operation.commands:operation" diff --git a/src/kiara/interfaces/cli/archive/__init__.py b/src/kiara/interfaces/cli/archive/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/kiara/interfaces/cli/archive/commands.py b/src/kiara/interfaces/cli/archive/commands.py new file mode 100644 index 000000000..d09c534c7 --- /dev/null +++ b/src/kiara/interfaces/cli/archive/commands.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2021, Markus Binsteiner +# +# Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) +from typing import TYPE_CHECKING + +import rich_click as click + +from kiara.utils.cli import ( + output_format_option, +) + +if TYPE_CHECKING: + pass + + +@click.group("archive") +@click.pass_context +def context(ctx): + """Kiara archive related sub-commands.""" + + +@context.command("explain") +@click.argument("archive", nargs=1, required=True) +@output_format_option() +@click.pass_context +def explain_archive( + ctx, + format: str, + archive: str, +): + """Print details of an archive file.""" + + from kiara.api import KiaraAPI + + kiara_api: KiaraAPI = ctx.obj.kiara_api + + info = kiara_api.get_archive_info(archive) diff --git a/src/kiara/interfaces/cli/data/commands.py b/src/kiara/interfaces/cli/data/commands.py index f14a564bf..4539d249c 100644 --- a/src/kiara/interfaces/cli/data/commands.py +++ b/src/kiara/interfaces/cli/data/commands.py @@ -521,7 +521,7 @@ def filter_value( @click.option( "--archive-alias", "-a", - help="The alias to use for the exported archive. If not provided, the first alias will be used.", + help="The alias to use for the exported archive. If not provided, the first alias will be used. This is used as default in the stored archive, if not overwritten by a user.", required=False, ) @click.option( @@ -544,7 +544,7 @@ def filter_value( @click.pass_context def export_data_store( ctx, - aliases: str, + aliases: Tuple[str], archive_alias: Union[None, str], path: Union[str, None], compression: str, @@ -618,7 +618,7 @@ def export_data_store( data_store_alias = kiara_api.context.data_registry.register_data_archive(data_store) # type: ignore alias_store_alias = kiara_api.context.alias_registry.register_archive(archive_store) # type: ignore - terminal_print("Exporting value into new data_store...") + terminal_print("Exporting value(s) into new data_store...") no_default_value = False @@ -639,7 +639,7 @@ def export_data_store( key = f"value_{idx}" values_to_store[key] = value if value_alias: - alias_map[key] = value_alias + alias_map[key] = [value_alias] try: diff --git a/src/kiara/interfaces/python_api/__init__.py b/src/kiara/interfaces/python_api/__init__.py index 28bc215f7..578dfd571 100644 --- a/src/kiara/interfaces/python_api/__init__.py +++ b/src/kiara/interfaces/python_api/__init__.py @@ -95,10 +95,10 @@ WorkflowsMap, ) from kiara.interfaces.python_api.workflow import Workflow + from kiara.models.archives import ArchiveInfo from kiara.models.module.pipeline import PipelineConfig, PipelineStructure from kiara.models.module.pipeline.pipeline import PipelineGroupInfo, PipelineInfo - logger = structlog.getLogger() yaml = YAML(typ="safe") @@ -448,6 +448,24 @@ def set_active_context(self, context_name: str, create: bool = False) -> None: ) self._current_context_alias = context_name + # ================================================================================================================== + # methods for archives + + def get_archive_info(self, archive_file: str) -> "ArchiveInfo": + + from kiara.context.config import KiaraArchiveReference + from kiara.models.archives import ArchiveInfo + + archive_ref = KiaraArchiveReference.load_existing_archive(archive_file) + + for archive in archive_ref.archives: + info = ArchiveInfo.create_from_instance( + kiara=self.context, instance=archive + ) + dbg(info.create_renderable()) + + return None + # ================================================================================================================== # methods for data_types diff --git a/src/kiara/models/archives.py b/src/kiara/models/archives.py index 14e24251a..70f042723 100644 --- a/src/kiara/models/archives.py +++ b/src/kiara/models/archives.py @@ -27,12 +27,13 @@ TypeInfoItemGroup, ) from kiara.models.documentation import ( + AuthorModel, AuthorsMetadataModel, ContextMetadataModel, DocumentationMetadataModel, ) from kiara.models.python_class import PythonClass -from kiara.registries import ArchiveDetails, KiaraArchive +from kiara.registries import ArchiveDetails, ArchiveMetadata, KiaraArchive from kiara.utils.json import orjson_dumps if TYPE_CHECKING: @@ -96,7 +97,7 @@ def create_renderable(self, **config: Any) -> RenderableType: table.add_row("Python class", self.python_class.create_renderable()) - table.add_row("is_writeable", "yes" if self.is_writable else "no") + # table.add_row("is_writeable", "yes" if self.is_writable else "no") table.add_row( "supported_item_types", ", ".join(sorted(self.supported_item_types)) ) @@ -136,6 +137,26 @@ def create_from_archive( archive_aliases: Union[Iterable[str], None] = None, ): + doc_str = archive.archive_metadata.get("description", None) + doc = DocumentationMetadataModel.create(doc_str) + + authors_raw = archive.archive_metadata.get("authors", []) + _authors = [] + for author in authors_raw: + author = AuthorModel(**author) + _authors.append(author) + authors = AuthorsMetadataModel(authors=_authors) + + tags = archive.archive_metadata.get("tags", []) + labels = archive.archive_metadata.get("labels", {}) + + references = archive.archive_metadata.get("references", {}) + # TODO: add references model + + context = ContextMetadataModel(tags=tags, labels=labels, references=references) + + # archive_types = list(archive.supported_item_types()) + archive_type_info = ArchiveTypeInfo.create_from_type_class( archive.__class__, kiara=kiara ) @@ -145,12 +166,15 @@ def create_from_archive( archive_aliases = list(archive_aliases) return ArchiveInfo( archive_type_info=archive_type_info, - type_name=str(archive.archive_id), - documentation=archive_type_info.documentation, - authors=archive_type_info.authors, - context=archive_type_info.context, + archive_alias=archive.archive_alias, archive_id=archive.archive_id, + type_name=str(archive.archive_id), + documentation=doc, + authors=authors, + context=context, + # archive_types=archive_types, details=archive.get_archive_details(), + metadata=archive.archive_metadata, config=archive.config.model_dump(), aliases=archive_aliases, ) @@ -160,17 +184,47 @@ def category_name(cls) -> str: return "info.archive" archive_id: uuid.UUID = Field(description="The (globally unique) archive id.") + archive_alias: str = Field(description="The archive alias.") + archive_type_info: ArchiveTypeInfo = Field( description="Information about this archives' type." ) + # archive_types: List[Literal["data", "alias", "job_record", "workflow"]] = Field(description="The archive type.") + config: Mapping[str, Any] = Field(description="The configuration of this archive.") details: ArchiveDetails = Field( description="Type dependent (runtime) details for this archive." ) + metadata: ArchiveMetadata = Field(description="Metadata for this archive.") aliases: List[str] = Field( description="Aliases for this archive.", default_factory=list ) + def create_renderable(self, **config: Any) -> RenderableType: + from kiara.utils.output import extract_renderable + + table = Table(show_header=False, box=box.SIMPLE) + table.add_column("property", style="i") + table.add_column("value") + + details = extract_renderable(self.details, render_config=config) + metadata = extract_renderable(self.metadata, render_config=config) + type_info = self.archive_type_info.create_renderable(**config) + # table.add_row("archive id", str(self.archive_id)) + # table.add_row("archive alias", self.archive_alias) + # table.add_row("archive type(s)", ", ".join(self.archive_types)) + table.add_row("details", details) + table.add_row("metadata", metadata) + table.add_row("type info", type_info) + if self.documentation.is_set: + table.add_row("doc", self.documentation.create_renderable(**config)) + if self.authors.authors: + table.add_row("author(s)", self.authors.create_renderable(**config)) + if self.context.labels or self.context.tags or self.context.references: + table.add_row("context", self.context.create_renderable(**config)) + + return table + class ArchiveGroupInfo(InfoItemGroup): @@ -211,7 +265,7 @@ def combined_size(self) -> int: if archive_info.archive_id in archive_ids: continue archive_ids.add(archive_info.archive_id) - size = archive_info.details.size + size = archive_info.details.root.get("size", 0) if size and size > 0: combined = combined + size diff --git a/src/kiara/registries/__init__.py b/src/kiara/registries/__init__.py index 5a8e9ac88..4253ccf4e 100644 --- a/src/kiara/registries/__init__.py +++ b/src/kiara/registries/__init__.py @@ -63,11 +63,8 @@ def create_new_store_config(cls, store_base_path: str, **kwargs) -> Self: logger = structlog.getLogger() -class ArchiveDetails(BaseModel): - - size: Union[int, None] = Field( - description="The size of the stored archive.", default=None - ) +class ArchiveDetails(RootModel): + root: Dict[str, Any] class ArchiveMetadata(RootModel): @@ -93,7 +90,7 @@ def get(self, key, default=None): # ) -NON_ARCHIVE_DETAILS = ArchiveDetails() +NON_ARCHIVE_DETAILS = ArchiveDetails(root={}) class KiaraArchive(abc.ABC, Generic[ARCHIVE_CONFIG_CLS]): diff --git a/src/kiara/registries/aliases/__init__.py b/src/kiara/registries/aliases/__init__.py index a65cb521b..b3ad85457 100644 --- a/src/kiara/registries/aliases/__init__.py +++ b/src/kiara/registries/aliases/__init__.py @@ -23,7 +23,7 @@ from kiara.defaults import INVALID_ALIAS_NAMES from kiara.exceptions import KiaraException from kiara.models.events.alias_registry import AliasArchiveAddedEvent -from kiara.registries import BaseArchive +from kiara.registries import ArchiveDetails, BaseArchive from kiara.registries.data import ValueLink if TYPE_CHECKING: @@ -58,6 +58,12 @@ def find_value_id_for_alias(self, alias: str) -> Union[uuid.UUID, None]: def find_aliases_for_value_id(self, value_id: uuid.UUID) -> Union[Set[str], None]: pass + def get_archive_details(self) -> ArchiveDetails: + all_aliases = self.retrieve_all_aliases() + return ArchiveDetails( + root={"no_aliases": len(all_aliases), "aliases": sorted(all_aliases.keys())} + ) + class AliasStore(AliasArchive): @abc.abstractmethod diff --git a/src/kiara/registries/data/data_store/filesystem_store.py b/src/kiara/registries/data/data_store/filesystem_store.py index de7d3028e..d8eaca86c 100644 --- a/src/kiara/registries/data/data_store/filesystem_store.py +++ b/src/kiara/registries/data/data_store/filesystem_store.py @@ -115,7 +115,15 @@ def get_archive_details(self) -> ArchiveDetails: size = sum( f.stat().st_size for f in self.data_store_path.glob("**/*") if f.is_file() ) - return ArchiveDetails(size=size) + all_values = self.value_ids + num_values = len(all_values) + return ArchiveDetails( + root={ + "size": size, + "no_values": num_values, + "value_ids": sorted((str(x) for x in all_values)), + } + ) @property def data_store_path(self) -> Path: diff --git a/src/kiara/registries/data/data_store/sqlite_store.py b/src/kiara/registries/data/data_store/sqlite_store.py index af8a783b7..fcdcbd6a2 100644 --- a/src/kiara/registries/data/data_store/sqlite_store.py +++ b/src/kiara/registries/data/data_store/sqlite_store.py @@ -34,7 +34,6 @@ from kiara.registries.data.data_store import BaseDataStore from kiara.utils.hashfs import shard from kiara.utils.json import orjson_dumps -from kiara.utils.windows import fix_windows_longpath if TYPE_CHECKING: from multiformats import CID @@ -460,7 +459,16 @@ def _delete_archive(self): def get_archive_details(self) -> ArchiveDetails: size = self.sqlite_path.stat().st_size - return ArchiveDetails(size=size) + all_values = self.value_ids + num_values = len(all_values) + + return ArchiveDetails( + root={ + "size": size, + "no_values": num_values, + "value_ids": sorted((str(x) for x in all_values)), + } + ) class SqliteDataStore(SqliteDataArchive[SqliteDataStoreConfig], BaseDataStore): diff --git a/src/kiara/registries/jobs/job_store/filesystem_store.py b/src/kiara/registries/jobs/job_store/filesystem_store.py index 1afc64028..5738de478 100644 --- a/src/kiara/registries/jobs/job_store/filesystem_store.py +++ b/src/kiara/registries/jobs/job_store/filesystem_store.py @@ -49,7 +49,7 @@ def get_archive_details(self) -> ArchiveDetails: size = sum( f.stat().st_size for f in self.job_store_path.glob("**/*") if f.is_file() ) - return ArchiveDetails(size=size) + return ArchiveDetails(root={"size": size}) def _retrieve_archive_metadata(self) -> Mapping[str, Any]: