From 11f505d5d4e96ec08ca3270a545f8e9b1db6e902 Mon Sep 17 00:00:00 2001 From: Markus Binsteiner Date: Mon, 19 Feb 2024 23:00:10 +0100 Subject: [PATCH] feat: add 'import_archive' API endpoint --- src/kiara/context/__init__.py | 2 +- src/kiara/defaults.py | 3 + src/kiara/interfaces/cli/archive/commands.py | 22 +- src/kiara/interfaces/cli/data/commands.py | 3 +- src/kiara/interfaces/python_api/__init__.py | 215 +++++++++++------- .../interfaces/python_api/models/archive.py | 16 +- src/kiara/interfaces/python_api/value.py | 34 ++- src/kiara/models/archives.py | 18 +- src/kiara/models/values/matchers.py | 4 + src/kiara/registries/__init__.py | 10 +- src/kiara/registries/aliases/__init__.py | 4 +- src/kiara/registries/aliases/archives.py | 2 +- src/kiara/registries/data/__init__.py | 17 +- .../data/data_store/filesystem_store.py | 2 +- src/kiara/registries/jobs/__init__.py | 2 +- .../jobs/job_store/filesystem_store.py | 2 +- src/kiara/registries/workflows/__init__.py | 2 +- src/kiara/registries/workflows/archives.py | 2 +- src/kiara/utils/stores.py | 8 +- 19 files changed, 238 insertions(+), 130 deletions(-) diff --git a/src/kiara/context/__init__.py b/src/kiara/context/__init__.py index c0a6e14af..40f70000b 100644 --- a/src/kiara/context/__init__.py +++ b/src/kiara/context/__init__.py @@ -336,7 +336,7 @@ def register_external_archive( for archive_type, _archive_inst in archive_instances.items(): log_message( "register.external.archive", - archive=_archive_inst.archive_alias, + archive=_archive_inst.archive_name, allow_write_access=allow_write_access, ) diff --git a/src/kiara/defaults.py b/src/kiara/defaults.py index 5e20e1783..c1940363a 100644 --- a/src/kiara/defaults.py +++ b/src/kiara/defaults.py @@ -295,3 +295,6 @@ class CHUNK_COMPRESSION_TYPE(Enum): DEFAULT_CHUNK_COMPRESSION = CHUNK_COMPRESSION_TYPE.ZSTD + +ARCHIVE_NAME_MARKER = "archive_name" +DATA_ARCHIVE_DEFAULT_VALUE_MARKER = "default_value" diff --git a/src/kiara/interfaces/cli/archive/commands.py b/src/kiara/interfaces/cli/archive/commands.py index 547b5d5fb..a46859318 100644 --- a/src/kiara/interfaces/cli/archive/commands.py +++ b/src/kiara/interfaces/cli/archive/commands.py @@ -3,7 +3,6 @@ # Copyright (c) 2021, Markus Binsteiner # # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) - import rich_click as click from kiara.utils.cli import ( @@ -35,19 +34,30 @@ def explain_archive( kiara_api: KiaraAPI = ctx.obj.kiara_api - info = kiara_api.retrieve_kiarchive_info(archive) + info = kiara_api.retrieve_archive_info(archive) terminal_print_model(info, format=format, in_panel=f"Archive info: {archive}") @archive.command("import") @click.argument("archive", nargs=1, required=True) +# @click.option( +# "--all-values", +# "-a", +# is_flag=True, +# default=False, +# help="Import all values from the archive, even if they don't have an alias associated with them.", +# ) @click.pass_context @handle_exception() -def import_archive(ctx, archive: str): +def import_archive(ctx, archive: str, all_values: bool = False): """Import an archive file.""" - # kiara_api: KiaraAPI = ctx.obj.kiara_api + from kiara.interfaces.python_api import KiaraAPI + + kiara_api: KiaraAPI = ctx.obj.kiara_api + + result = kiara_api.import_archive(archive) - raise NotImplementedError() - # kiara_api.import_archive(archive) + render_config = {"add_field_column": False} + terminal_print_model(result, **render_config) diff --git a/src/kiara/interfaces/cli/data/commands.py b/src/kiara/interfaces/cli/data/commands.py index c0de78d10..1ad4dc98c 100644 --- a/src/kiara/interfaces/cli/data/commands.py +++ b/src/kiara/interfaces/cli/data/commands.py @@ -15,6 +15,7 @@ import rich_click as click import structlog +from kiara.defaults import DATA_ARCHIVE_DEFAULT_VALUE_MARKER from kiara.exceptions import InvalidCommandLineInvocation from kiara.utils import log_exception, log_message from kiara.utils.cli import output_format_option, terminal_print, terminal_print_model @@ -647,7 +648,7 @@ def export_data_archive( if not no_default_value: try: data_store.set_archive_metadata_value( - "default_value", str(values[0][0].value_id) + DATA_ARCHIVE_DEFAULT_VALUE_MARKER, str(values[0][0].value_id) ) except Exception as e: data_store.delete_archive(archive_id=data_store.archive_id) diff --git a/src/kiara/interfaces/python_api/__init__.py b/src/kiara/interfaces/python_api/__init__.py index b158241a4..9029275b8 100644 --- a/src/kiara/interfaces/python_api/__init__.py +++ b/src/kiara/interfaces/python_api/__init__.py @@ -16,6 +16,7 @@ Dict, Iterable, List, + Literal, Mapping, MutableMapping, Set, @@ -29,6 +30,7 @@ from kiara.defaults import ( CHUNK_COMPRESSION_TYPE, + DATA_ARCHIVE_DEFAULT_VALUE_MARKER, OFFICIAL_KIARA_PLUGINS, VALID_VALUE_QUERY_CATEGORIES, VALUE_ATTR_DELIMITER, @@ -90,14 +92,14 @@ if TYPE_CHECKING: from kiara.context import Kiara, KiaraConfig, KiaraRuntimeConfig - from kiara.interfaces.python_api.models.archive import Kiarchive + from kiara.interfaces.python_api.models.archive import KiArchive from kiara.interfaces.python_api.models.doc import ( OperationsMap, PipelinesMap, WorkflowsMap, ) from kiara.interfaces.python_api.workflow import Workflow - from kiara.models.archives import KiarchiveInfo + from kiara.models.archives import KiArchiveInfo from kiara.models.module.pipeline import PipelineConfig, PipelineStructure from kiara.models.module.pipeline.pipeline import PipelineGroupInfo, PipelineInfo @@ -1697,6 +1699,7 @@ def store_value( store: Union[str, None] = None, data_store: Union[str, None] = None, alias_store: Union[str, None] = None, + set_as_store_default: bool = False, ) -> StoreValueResult: """ Store the specified value in a value store. @@ -1715,6 +1718,7 @@ def store_value( store: in case data and alias store names are the same, you can use this, if you specify one or both of the others, this will be overwritten data_store: the registered name (or archive id as string) of the store to write the data alias_store: the registered name (or archive id as string) of the store to persist the alias(es)/value_id mapping + set_as_store_default: whether to set the specified store as the default store for the value """ if isinstance(alias, str): alias = [alias] @@ -1738,6 +1742,12 @@ def store_value( allow_overwrite=allow_overwrite, alias_store=alias_store, ) + + if set_as_store_default: + store_instance = self.context.data_registry.get_archive(data_store) + store_instance.set_archive_metadata_value( + DATA_ARCHIVE_DEFAULT_VALUE_MARKER, str(value_obj.value_id) + ) result = StoreValueResult( value=value_obj, aliases=sorted(alias) if alias else [], @@ -1776,13 +1786,14 @@ def store_values( loop over the values on the frontend side, and call the 'store_value' method for each value. If you provide a non-mapping interable as 'values', the 'alias_map' argument must either be 'False', or a - map with a stringified uuid refering to the value in question as key, and a list of aliases as value. + map with a stringified uuid referring to the value in question as key, and a list of aliases as value. - If you use a mapping iterable as 'values': + If you use a mapping type as 'values': If alias_map is 'False', no aliases will be registered. If 'True', the key in the 'values' argument will be used. If the value is a string, all keys from the 'values' map will be used as alias, prefixed with the value of 'alias_map' + '.'. - Alternatively, if a map is provided, the key in the 'values' argument will be used to look up the alias(es) in the - 'alias_map' argument. + Alternatively, if another mapping is provided for 'alias_map', the key in the 'values' argument will be used to look up the alias(es) in the 'alias_map' argument, by looking up that key. + + Sorry, this is a bit convoluted, but it's the only way I could think of to make this work for all the requirements I had. In most keases, you'll only have to use 'True' or 'False' here, hopefully. This method does not raise an error if the storing of the value fails, so you have to investigate the 'StoreValuesResult' instance that is returned to see if the storing was successful. @@ -1867,52 +1878,12 @@ def store_values( # ------------------------------------------------------------------------------------------------------------------ # archive-related methods - # def create_kiarchive( - # self, - # kiarchive_uri: Union[str, Path], - # archive_alias: Union[str, None] = None, - # compression: CHUNK_COMPRESSION_TYPE = CHUNK_COMPRESSION_TYPE.ZSTD, - # allow_existing: bool = False, - # allow_write_access: bool = True, - # ) -> str: - # """Create a new kiarchive in a file at the specified path, containing a data & alias store. - # - # # NOTE: this is a preliminary endpoint, and might be changed in the future. If you have a use-case for this, please let me know. - # - # Arguments: - # kiarchive_uri: the uri of the archive (file path) - # archive_alias: the alias to use for the archive - # compression: the compression to use for the archive - # allow_existing: whether to allow a kiarchive to already exist at the location, and load that instead of creating a new one - # register_archive: whether to register the archive with the context - # register_alias: the alias to use for the archive in the context - # allow_write_access: whether to allow write access to the archive - # - # Returns: - # the name/alias that the archive was registered with - # - # """ - # from kiara.interfaces.python_api.models.archive import Kiarchive - # - # kiarchive = Kiarchive.create_kiarchive( - # kiara=self.context, - # kiarchive_uri=kiarchive_uri, - # allow_existing=allow_existing, - # archive_alias=archive_alias, - # compression=compression, - # allow_write_access=allow_write_access, - # ) - # - # result = self.register_kiarchive( - # kiarchive=kiarchive, allow_write_access=allow_write_access - # ) - # return result - - def register_kiarchive( + def register_archive( self, - kiarchive: Union[str, Path, "Kiarchive"], + archive: Union[str, Path, "KiArchive"], allow_write_access: bool = False, registered_name: Union[str, None] = None, + create_if_not_exists: bool = True, ) -> str: """Register a kiarchive with the current context. @@ -1924,31 +1895,40 @@ def register_kiarchive( # NOTE: this is a preliminary endpoint, and might be changed in the future. If you have a use-case for this, please let me know. Arguments: - kiarchive: the uri of the archive (file path), or a [Kiarchive][kiara.interfaces.python_api.models.archive.Kiarchive] instance + archive: the uri of the archive (file path), or a [Kiarchive][kiara.interfaces.python_api.models.archive.Kiarchive] instance allow_write_access: whether to allow write access to the archive + registered_name: the name/alias that the archive is registered in the context, and which can be used in the 'store_value(s)' endpoint, if not provided, it will be auto-determined from the file name + create_if_not_exists: if the file does not exist, create it. If this is 'False', an exception will be raised if the file does not exist. Returns: - the name/alias that the archive was registered with + the name/alias that the archive is registered in the context, and which can be used in the 'store_value(s)' endpoint """ - from kiara.interfaces.python_api.models.archive import Kiarchive + from kiara.interfaces.python_api.models.archive import KiArchive - if isinstance(kiarchive, str): - kiarchive = Path(kiarchive) + if isinstance(archive, str): + archive = Path(archive) - if isinstance(kiarchive, Path): + if not archive.name.endswith(".kiarchive"): + archive = archive.parent / f"{archive.name}.kiarchive" - if kiarchive.exists(): - kiarchive = Kiarchive.load_kiarchive( - kiara=self.context, path=kiarchive, archive_name=registered_name + if isinstance(archive, Path): + + if archive.exists(): + archive = KiArchive.load_kiarchive( + kiara=self.context, path=archive, archive_name=registered_name ) else: - kiarchive_alias = kiarchive.name + if not create_if_not_exists: + raise KiaraException( + f"Archive file '{archive.as_posix()}' does not exist." + ) + kiarchive_alias = archive.name if kiarchive_alias.endswith(".kiarchive"): kiarchive_alias = kiarchive_alias[:-10] - kiarchive = Kiarchive.create_kiarchive( + archive = KiArchive.create_kiarchive( kiara=self.context, - kiarchive_uri=kiarchive.as_posix(), + kiarchive_uri=archive.as_posix(), allow_existing=False, archive_name=kiarchive_alias, compression=CHUNK_COMPRESSION_TYPE.ZSTD, @@ -1956,44 +1936,123 @@ def register_kiarchive( ) data_alias = self.context.register_external_archive( - kiarchive.data_archive, + archive.data_archive, allow_write_access=allow_write_access, ) alias_alias = self.context.register_external_archive( - kiarchive.alias_archive, allow_write_access=allow_write_access + archive.alias_archive, allow_write_access=allow_write_access ) assert data_alias["data"] == alias_alias["alias"] - assert kiarchive.archive_name == data_alias["data"] + assert archive.archive_name == data_alias["data"] + + return archive.archive_name - return kiarchive.archive_name + def set_archive_metadata_value( + self, + archive: Union[str, uuid.UUID], + key: str, + value: Any, + archive_type: Literal["data", "alias"] = "data", + ) -> None: + + if archive_type == "data": + self.context.data_registry.get_archive(archive).set_archive_metadata_value( + key, value + ) + elif archive_type == "alias": + self.context.alias_registry.get_archive(archive).set_archive_metadata_value( + key, value + ) + else: + raise KiaraException( + f"Invalid archive type: {archive_type}. Valid types are: 'data', 'alias'." + ) - def retrieve_kiarchive_info( - self, kiarchive: Union[str, "Kiarchive"] - ) -> "KiarchiveInfo": - """Retrieve information about a kiarchive at the specified location. + def retrieve_archive_info( + self, archive: Union[str, "KiArchive"] + ) -> "KiArchiveInfo": + """Retrieve information about an archive at the specified local path - The kiarchive can't be registered into the context, this might change in the future. + Currently, this only works with an external archive file, not with an archive that is registered into the context. + This will probably be added later on, let me know if there is demand, then I'll prioritize. # NOTE: this is a preliminary endpoint, and might be changed in the future. If you have a use-case for this, please let me know. Arguments: - kiarchive: the uri of the archive (file path) + archive: the uri of the archive (file path) Returns: - a [KiarchiveInfo][kiara.interfaces.python_api.models.archive.KiarchiveInfo] instance, containing details about the kiarchive + a [KiarchiveInfo][kiara.interfaces.python_api.models.archive.KiarchiveInfo] instance, containing details about the archive """ - from kiara.interfaces.python_api.models.archive import Kiarchive - from kiara.models.archives import KiarchiveInfo + from kiara.interfaces.python_api.models.archive import KiArchive + from kiara.models.archives import KiArchiveInfo - if not isinstance(kiarchive, Kiarchive): - kiarchive = Kiarchive.load_kiarchive(kiara=self.context, path=kiarchive) + if not isinstance(archive, KiArchive): + archive = KiArchive.load_kiarchive(kiara=self.context, path=archive) - kiarchive_info = KiarchiveInfo.create_from_instance( - kiara=self.context, instance=kiarchive + kiarchive_info = KiArchiveInfo.create_from_instance( + kiara=self.context, instance=archive ) return kiarchive_info + def import_archive( + self, + archive: Union[str, Path], + # only_aliases: bool = True, + registered_name: Union[str, None] = None, + ) -> StoreValuesResult: + """Import all data from the specified archive into the current context. + + The archive will be registered into the context, either ussing the provided registered_name, otherwise the name + will be auto-determined from the archive metadata. + + Currently, this only works with an external archive file, not with an archive that is registered into the context. + This will be added later on. + + This method does not raise an error if the storing of the value fails, so you have to investigate the + 'StoreValuesResult' instance that is returned to see if the storing was successful + + Arguments: + archive: the uri of the archive (file path) + only_aliases: whether to only import the aliases, or all values, even if they don't have an alias + registered_name: the name/alias that the archive is registered in the context + + """ + + archive_ref = self.register_archive( + archive, registered_name=registered_name, allow_write_access=False + ) + + only_aliases = False + if not only_aliases: + values = self.list_values( + in_data_archives=[archive_ref], allow_internal=True, has_alias=False + ).values() + aliases = self.list_aliases(in_data_archives=[archive_ref]) + alias_map: Union[bool, Dict[str, List[str]]] = {} + for alias, value in aliases.items(): + # TODO: maybe add a matcher arg to the list_aliases endpoint + if not alias.startswith(f"{archive_ref}#"): + continue + alias_map.setdefault(str(value.value_id), []).append( + alias[len(archive_ref) + 1 :] + ) + else: + _values = self.list_aliases( + in_data_archives=[archive_ref], allow_internal=True + ) + values = {} + for alias, value in _values.items(): + # TODO: maybe add a matcher arg to the list_aliases endpoint + if not alias.startswith(f"{archive_ref}#"): + continue + values[alias[len(archive_ref) + 1 :]] = value + alias_map = True + + result = self.store_values(values, alias_map=alias_map) + return result + # ------------------------------------------------------------------------------------------------------------------ # operation-related methods diff --git a/src/kiara/interfaces/python_api/models/archive.py b/src/kiara/interfaces/python_api/models/archive.py index 71f6c01e4..d33d2357d 100644 --- a/src/kiara/interfaces/python_api/models/archive.py +++ b/src/kiara/interfaces/python_api/models/archive.py @@ -13,7 +13,7 @@ from kiara.registries.data import DataArchive -class Kiarchive(KiaraModel): +class KiArchive(KiaraModel): @classmethod def load_kiarchive( cls, @@ -21,7 +21,7 @@ def load_kiarchive( path: Union[str, Path], allow_write_access: bool = False, archive_name: Union[str, None] = None, - ) -> "Kiarchive": + ) -> "KiArchive": if isinstance(path, Path): path = path.as_posix() @@ -61,23 +61,23 @@ def load_kiarchive( raise Exception( f"Data and alias archives in file '{path}' have different IDs." ) - if data_archive.archive_alias != alias_archive.archive_alias: + if data_archive.archive_name != alias_archive.archive_name: raise Exception( f"Data and alias archives in file '{path}' have different aliases." ) archive_id = data_archive.archive_id - archive_alias = data_archive.archive_alias + archive_alias = data_archive.archive_name elif alias_archive: # we can assume data archive is None here archive_id = alias_archive.archive_id - archive_alias = alias_archive.archive_alias + archive_alias = alias_archive.archive_name else: raise Exception( "This should never happen, but we need to handle it anyway. Bug in code." ) - kiarchive = Kiarchive( + kiarchive = KiArchive( archive_id=archive_id, archive_name=archive_alias, data_archive_config=data_archive_config, @@ -102,7 +102,7 @@ def create_kiarchive( compression: CHUNK_COMPRESSION_TYPE = CHUNK_COMPRESSION_TYPE.ZSTD, allow_write_access: bool = True, allow_existing: bool = False, - ) -> "Kiarchive": + ) -> "KiArchive": if isinstance(kiarchive_uri, str): kiarchive_uri = Path(kiarchive_uri) @@ -145,7 +145,7 @@ def create_kiarchive( kiarchive_id = data_store.archive_id assert alias_store.archive_id == kiarchive_id - kiarchive = Kiarchive( + kiarchive = KiArchive( archive_id=kiarchive_id, archive_name=archive_name, archive_base_path=archive_base_path, diff --git a/src/kiara/interfaces/python_api/value.py b/src/kiara/interfaces/python_api/value.py index 75a078554..2d4161017 100644 --- a/src/kiara/interfaces/python_api/value.py +++ b/src/kiara/interfaces/python_api/value.py @@ -70,22 +70,42 @@ class StoreValuesResult(RootModel): def create_renderable(self, **config: Any) -> RenderableType: + add_field_column = config.get("add_field_column", True) + + errors = {} + for field_name, value_result in self.root.items(): + if value_result.error: + errors[field_name] = value_result.error + table = Table(show_header=True, show_lines=False, box=box.SIMPLE) - table.add_column("field", style="b") - table.add_column("data type", style="i") + if add_field_column: + table.add_column("field", style="b") table.add_column("stored id", style="i") + table.add_column("data type", style="i") table.add_column("alias(es)") + if errors: + table.add_column("error", style="red") for field_name, value_result in self.root.items(): - row = [ - field_name, - str(value_result.value.value_schema.type), - str(value_result.value.value_id), - ] + if add_field_column: + row = [ + field_name, + str(value_result.value.value_id), + str(value_result.value.value_schema.type), + ] + else: + row = [ + str(value_result.value.value_id), + str(value_result.value.value_schema.type), + ] + if value_result.aliases: row.append(", ".join(value_result.aliases)) else: row.append("") + + if errors.get(field_name) is not None: + row.append(errors[field_name]) table.add_row(*row) return table diff --git a/src/kiara/models/archives.py b/src/kiara/models/archives.py index 2793c7ff6..4787ab076 100644 --- a/src/kiara/models/archives.py +++ b/src/kiara/models/archives.py @@ -37,7 +37,7 @@ if TYPE_CHECKING: from kiara.context import Kiara - from kiara.interfaces.python_api import Kiarchive + from kiara.interfaces.python_api import KiArchive class ArchiveTypeInfo(TypeInfo): @@ -169,7 +169,7 @@ def create_from_archive( # archive_aliases = list(archive_aliases) return ArchiveInfo( archive_type_info=archive_type_info, - archive_alias=archive.archive_alias, + archive_alias=archive.archive_name, archive_id=archive.archive_id, type_name=str(archive.archive_id), documentation=doc, @@ -322,20 +322,20 @@ def create_renderable(self, **config: Any) -> RenderableType: return table -class KiarchiveInfo(ItemInfo): +class KiArchiveInfo(ItemInfo): @classmethod - def base_instance_class(cls) -> Type["Kiarchive"]: - from kiara.interfaces.python_api import Kiarchive + def base_instance_class(cls) -> Type["KiArchive"]: + from kiara.interfaces.python_api import KiArchive - return Kiarchive + return KiArchive @classmethod - def create_from_instance(cls, kiara: "Kiara", instance: "Kiarchive", **kwargs): + def create_from_instance(cls, kiara: "Kiara", instance: "KiArchive", **kwargs): return cls.create_from_kiarchive(kiarchive=instance, **kwargs) @classmethod - def create_from_kiarchive(cls, kiarchive: "Kiarchive"): + def create_from_kiarchive(cls, kiarchive: "KiArchive"): data_archive = kiarchive.data_archive alias_archive = kiarchive.alias_archive @@ -366,7 +366,7 @@ def create_from_kiarchive(cls, kiarchive: "Kiarchive"): if documentation is None or authors is None or context is None: raise ValueError("No documentation, authors or context found.") - return KiarchiveInfo( + return KiArchiveInfo( type_name=kiarchive.archive_file_name, data_archive_info=data_archive_info, alias_archive_info=alias_archive_info, diff --git a/src/kiara/models/values/matchers.py b/src/kiara/models/values/matchers.py index 9f8d1ace8..b3933e0cb 100644 --- a/src/kiara/models/values/matchers.py +++ b/src/kiara/models/values/matchers.py @@ -35,6 +35,10 @@ def create_matcher(self, **match_options: Any): description="Values must have an alias that matches one of the provided matchers. Assumes 'has_alias' is set to 'True'.", default=None, ) + in_data_archives: Union[None, List[str]] = Field( + description="A list of registered names of archives the value must be in. If 'None', all archives will be used.", + default=None, + ) @field_validator("alias_matchers") @classmethod diff --git a/src/kiara/registries/__init__.py b/src/kiara/registries/__init__.py index ce5180101..447c1e138 100644 --- a/src/kiara/registries/__init__.py +++ b/src/kiara/registries/__init__.py @@ -24,7 +24,11 @@ import structlog from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator -from kiara.defaults import CHUNK_COMPRESSION_TYPE, DEFAULT_CHUNK_COMPRESSION +from kiara.defaults import ( + ARCHIVE_NAME_MARKER, + CHUNK_COMPRESSION_TYPE, + DEFAULT_CHUNK_COMPRESSION, +) from kiara.utils import log_message try: @@ -231,11 +235,11 @@ def _set_archive_metadata_value(self, key: str, value: Any): ) @property - def archive_alias(self) -> str: + def archive_name(self) -> str: if self._archive_instance_name: return self._archive_instance_name - alias = self.get_archive_metadata("archive_alias") + alias = self.get_archive_metadata(ARCHIVE_NAME_MARKER) if not alias: alias = str(self.archive_id) self._archive_instance_name = alias diff --git a/src/kiara/registries/aliases/__init__.py b/src/kiara/registries/aliases/__init__.py index f7b8aca3e..61deccd61 100644 --- a/src/kiara/registries/aliases/__init__.py +++ b/src/kiara/registries/aliases/__init__.py @@ -133,13 +133,13 @@ def register_archive( mount_point: Union[str, None] = None, ) -> str: - alias = archive.archive_alias + alias = archive.archive_name if not alias: raise Exception("Invalid alias archive alias: can't be empty.") if not mount_point: - mount_point = archive.archive_alias + mount_point = archive.archive_name if "#" in mount_point: raise Exception( diff --git a/src/kiara/registries/aliases/archives.py b/src/kiara/registries/aliases/archives.py index 126548ef2..6bc010689 100644 --- a/src/kiara/registries/aliases/archives.py +++ b/src/kiara/registries/aliases/archives.py @@ -54,7 +54,7 @@ def _retrieve_archive_metadata(self) -> Mapping[str, Any]: _archive_metadata["archive_id"] = str(_archive_id) except Exception: raise Exception( - f"Could not retrieve archive id for alias archive '{self.archive_alias}'." + f"Could not retrieve archive id for alias archive '{self.archive_name}'." ) return _archive_metadata diff --git a/src/kiara/registries/data/__init__.py b/src/kiara/registries/data/__init__.py index 6fc0da2c3..c4a72aa13 100644 --- a/src/kiara/registries/data/__init__.py +++ b/src/kiara/registries/data/__init__.py @@ -30,6 +30,7 @@ from kiara.data_types import DataType from kiara.data_types.included_core_types import NoneType from kiara.defaults import ( + DATA_ARCHIVE_DEFAULT_VALUE_MARKER, INVALID_HASH_MARKER, NO_SERIALIZATION_MARKER, NONE_STORE_ID, @@ -159,7 +160,7 @@ def resolve_alias(self, alias: str) -> uuid.UUID: if not path_in_archive: default_value = data_archive.get_archive_metadata( - "default_value" + DATA_ARCHIVE_DEFAULT_VALUE_MARKER ) _value_id = uuid.UUID(default_value) else: @@ -309,7 +310,7 @@ def register_data_archive( set_as_default_store: Union[bool, None] = None, ) -> str: - alias = archive.archive_alias + alias = archive.archive_name if not alias: raise Exception("Invalid data archive alias: can't be empty.") @@ -502,12 +503,12 @@ def store_value( if not store.is_writeable(): if data_store: raise Exception( - f"Can't store value into store '{data_store}': not writable." + f"Can't write value into store '{data_store}': not writable." ) else: - raise Exception("Can't store value into store: not writable.") + raise Exception("Can't write value into store: not writable.") - _data_store = store.archive_alias + _data_store = store.archive_name # make sure all property values are available if _value.pedigree != ORPHAN: for value_id in _value.pedigree.inputs.values(): @@ -552,6 +553,10 @@ def find_values(self, matcher: ValueMatcher) -> Dict[uuid.UUID, Value]: matches: Dict[uuid.UUID, Value] = {} for store_id, store in self.data_archives.items(): + + if matcher.in_data_archives and store_id not in matcher.in_data_archives: + continue + try: _matches = store.find_values(matcher=matcher) for value in _matches: @@ -571,7 +576,7 @@ def find_values(self, matcher: ValueMatcher) -> Dict[uuid.UUID, Value]: log_message( "store.feature.missing", feature="find_value", - reasong="find_values not implemented for store: {store_id}", + reasong=f"find_values not implemented for store: {store_id}", solution="return all values", ) all_value_ids = store.value_ids diff --git a/src/kiara/registries/data/data_store/filesystem_store.py b/src/kiara/registries/data/data_store/filesystem_store.py index b35b30dd1..f7c0fd28a 100644 --- a/src/kiara/registries/data/data_store/filesystem_store.py +++ b/src/kiara/registries/data/data_store/filesystem_store.py @@ -101,7 +101,7 @@ def _retrieve_archive_metadata(self) -> Mapping[str, Any]: _archive_metadata["archive_id"] = str(_archive_id) except Exception: raise Exception( - f"Could not retrieve archive id for alias archive '{self.archive_alias}'." + f"Could not retrieve archive id for alias archive '{self.archive_name}'." ) return _archive_metadata diff --git a/src/kiara/registries/jobs/__init__.py b/src/kiara/registries/jobs/__init__.py index 21263dd7c..32722b518 100644 --- a/src/kiara/registries/jobs/__init__.py +++ b/src/kiara/registries/jobs/__init__.py @@ -251,7 +251,7 @@ def suppoerted_event_types(self) -> Iterable[Type[KiaraEvent]]: def register_job_archive(self, archive: JobArchive) -> str: - alias = archive.archive_alias + alias = archive.archive_name if not alias: raise Exception("Invalid job archive alias: can't be empty.") diff --git a/src/kiara/registries/jobs/job_store/filesystem_store.py b/src/kiara/registries/jobs/job_store/filesystem_store.py index 29081515d..048d77f4b 100644 --- a/src/kiara/registries/jobs/job_store/filesystem_store.py +++ b/src/kiara/registries/jobs/job_store/filesystem_store.py @@ -67,7 +67,7 @@ def _retrieve_archive_metadata(self) -> Mapping[str, Any]: _archive_metadata["archive_id"] = str(_archive_id) except Exception: raise Exception( - f"Could not retrieve archive id for alias archive '{self.archive_alias}'." + f"Could not retrieve archive id for alias archive '{self.archive_name}'." ) return _archive_metadata diff --git a/src/kiara/registries/workflows/__init__.py b/src/kiara/registries/workflows/__init__.py index 1d154e5bf..98c325337 100644 --- a/src/kiara/registries/workflows/__init__.py +++ b/src/kiara/registries/workflows/__init__.py @@ -158,7 +158,7 @@ def register_archive( archive: WorkflowArchive, set_as_default_store: Union[bool, None] = None, ): - alias = archive.archive_alias + alias = archive.archive_name if not alias: raise Exception("Invalid workflows archive alias: can't be empty.") diff --git a/src/kiara/registries/workflows/archives.py b/src/kiara/registries/workflows/archives.py index e8cc02646..b65669386 100644 --- a/src/kiara/registries/workflows/archives.py +++ b/src/kiara/registries/workflows/archives.py @@ -48,7 +48,7 @@ def _retrieve_archive_metadata(self) -> Mapping[str, Any]: _archive_metadata["archive_id"] = str(_archive_id) except Exception: raise Exception( - f"Could not retrieve archive id for alias archive '{self.archive_alias}'." + f"Could not retrieve archive id for alias archive '{self.archive_name}'." ) return _archive_metadata diff --git a/src/kiara/utils/stores.py b/src/kiara/utils/stores.py index b623f2582..0a79628b9 100644 --- a/src/kiara/utils/stores.py +++ b/src/kiara/utils/stores.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Mapping, Type, Union +from kiara.defaults import ARCHIVE_NAME_MARKER + if TYPE_CHECKING: from kiara.registries import KiaraArchive @@ -32,7 +34,7 @@ def create_new_archive( archive_instance = archive_cls(archive_name=archive_name, archive_config=config, force_read_only=force_read_only) # type: ignore if not force_read_only: - archive_instance.set_archive_metadata_value("archive_alias", archive_name) + archive_instance.set_archive_metadata_value(ARCHIVE_NAME_MARKER, archive_name) return archive_instance @@ -61,9 +63,9 @@ def check_external_archive( raise Exception( "Multiple archives of the same type are not supported." ) - if archive_name and _archive.archive_alias != archive_name: + if archive_name and _archive.archive_name != archive_name: raise Exception( - f"Archive alias '{_archive.archive_alias}' does not match expected alias '{archive_name}'" + f"Archive alias '{_archive.archive_name}' does not match expected alias '{archive_name}'" ) archive_instances[archive_type] = _archive continue