feat: sqlite-based job store

.gitignore

@@ -72,3 +72,5 @@ environment.yaml
 .pixi
 dev.py
 store.sqlite
+**.kiarchive
+**.kontext

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Version 0.5.9 (upcoming)
 
+- archive export & import feature
+- expanded input options for 'store_values' API endpoint
+
+
 ## Version 0.5.8
 
 - add 'mock' module type

pyproject.toml

@@ -125,6 +125,7 @@ sqlite_workflow_store = "kiara.registries.workflows.sqlite_store:SqliteWorkflowS
 run = "kiara.interfaces.cli.run:run"
 info = "kiara.interfaces.cli.info.commands:info"
 context = "kiara.interfaces.cli.context.commands:context"
+archive = "kiara.interfaces.cli.archive.commands:context"
 data = "kiara.interfaces.cli.data.commands:data"
 module = "kiara.interfaces.cli.module.commands:module"
 operation = "kiara.interfaces.cli.operation.commands:operation"

src/kiara/context/config.py

@@ -223,6 +223,82 @@ def archives(self) -> List["KiaraArchive"]:
 
 
 class KiaraContextConfig(BaseModel):
+    @classmethod
+    def create_from_sqlite_db(cls, db_path: Path) -> "KiaraContextConfig":
+
+        import sqlite3
+
+        if not db_path.exists():
+            context_id = str(uuid.uuid4())
+            conn = sqlite3.connect(db_path)
+            c = conn.cursor()
+            c.execute(
+                """CREATE TABLE context_metadata
+                         (key text PRIMARY KEY , value text NOT NULL)"""
+            )
+            c.execute(
+                "INSERT INTO context_metadata VALUES ('context_id', ?)", (context_id,)
+            )
+            c.execute(
+                """CREATE TABLE archive_metadata
+                         (key text PRIMARY KEY , value text NOT NULL)"""
+            )
+            c.execute(
+                "INSERT INTO archive_metadata VALUES ('archive_id', ?)", (context_id,)
+            )
+
+            conn.commit()
+            conn.close()
+        else:
+            try:
+
+                with sqlite3.connect(db_path) as conn:
+                    context_id = conn.execute(
+                        "SELECT value FROM context_metadata WHERE key = 'context_id'"
+                    ).fetchone()[0]
+            except Exception as e:
+                raise KiaraException(
+                    f"Can't read context from sqlite db '{db_path}': {e}"
+                )
+
+        base_path = os.path.abspath(kiara_app_dirs.user_data_dir)
+        stores_base_path = os.path.join(base_path, "stores")
+        workflow_base_path = os.path.join(
+            stores_base_path, "filesystem_stores", "workflows"
+        )
+        workflow_store_path = os.path.join(workflow_base_path, context_id)
+
+        data_store_config = KiaraArchiveConfig(
+            archive_type="sqlite_data_store",
+            config={"sqlite_db_path": db_path.as_posix()},
+        )
+        alias_store_config = KiaraArchiveConfig(
+            archive_type="sqlite_alias_store",
+            config={"sqlite_db_path": db_path.as_posix()},
+        )
+        job_store_config = KiaraArchiveConfig(
+            archive_type="sqlite_job_store",
+            config={"sqlite_db_path": db_path.as_posix()},
+        )
+        workflow_store_config = KiaraArchiveConfig(
+            archive_type="filesystem_workflow_store",
+            config={"archive_path": workflow_store_path},
+        )
+
+        archives = {
+            DEFAULT_DATA_STORE_MARKER: data_store_config,
+            DEFAULT_ALIAS_STORE_MARKER: alias_store_config,
+            DEFAULT_JOB_STORE_MARKER: job_store_config,
+            DEFAULT_WORKFLOW_STORE_MARKER: workflow_store_config,
+        }
+
+        context_config = cls(
+            context_id=context_id,
+            archives=archives,
+        )
+
+        return context_config
+
     model_config = ConfigDict(extra="forbid")
 
     context_id: str = Field(description="A globally unique id for this kiara context.")
@@ -515,7 +591,7 @@ def _validate_context(self, context_config: KiaraContextConfig) -> bool:
         def create_default_sqlite_archive_config() -> Dict[str, Any]:
 
             store_id = str(uuid.uuid4())
-            file_name = f"{store_id}.sqlite"
+            file_name = f"{store_id}.karchive"
             archive_path = Path(
                 os.path.abspath(os.path.join(sqlite_base_path, file_name))
             )
@@ -638,39 +714,51 @@ def create_context_config(
 
         if not context_alias:
             context_alias = DEFAULT_CONTEXT_NAME
+
         if context_alias in self.available_context_names:
             raise Exception(
                 f"Can't create kiara context '{context_alias}': context with that alias already registered."
             )
 
-        if os.path.sep in context_alias:
-            raise Exception(
-                f"Can't create context with alias '{context_alias}': no special characters allowed."
+        if context_alias.endswith(".kontext"):
+            context_db_file = Path(context_alias)
+            context_config: KiaraContextConfig = (
+                KiaraContextConfig.create_from_sqlite_db(db_path=context_db_file)
             )
+            self._validate_context(context_config=context_config)
+            context_config._context_config_path = context_db_file
+        else:
 
-        context_file = (
-            Path(os.path.join(self.context_search_paths[0])) / f"{context_alias}.yaml"
-        )
+            if os.path.sep in context_alias:
+                raise Exception(
+                    f"Can't create context with alias '{context_alias}': no special characters allowed."
+                )
 
-        archives: Dict[str, KiaraArchiveConfig] = {}
-        # create_default_archives(kiara_config=self)
-        context_id = ID_REGISTRY.generate(
-            obj_type=KiaraContextConfig, comment=f"new kiara context '{context_alias}'"
-        )
+            context_file = (
+                Path(os.path.join(self.context_search_paths[0]))
+                / f"{context_alias}.yaml"
+            )
 
-        context_config = KiaraContextConfig(
-            context_id=str(context_id), archives=archives, extra_pipelines=[]
-        )
+            archives: Dict[str, KiaraArchiveConfig] = {}
+            # create_default_archives(kiara_config=self)
+            context_id = ID_REGISTRY.generate(
+                obj_type=KiaraContextConfig,
+                comment=f"new kiara context '{context_alias}'",
+            )
 
-        self._validate_context(context_config=context_config)
+            context_config = KiaraContextConfig(
+                context_id=str(context_id), archives=archives, extra_pipelines=[]
+            )
+
+            self._validate_context(context_config=context_config)
 
-        context_file.parent.mkdir(parents=True, exist_ok=True)
-        with open(context_file, "wt") as f:
-            yaml.dump(context_config.model_dump(), f)
+            context_file.parent.mkdir(parents=True, exist_ok=True)
+            with open(context_file, "wt") as f:
+                yaml.dump(context_config.model_dump(), f)
 
-        context_config._context_config_path = context_file
+            context_config._context_config_path = context_file
+            self._available_context_files[context_alias] = context_file
 
-        self._available_context_files[context_alias] = context_file
         self._context_data[context_alias] = context_config
 
         return context_config
@@ -687,13 +775,20 @@ def create_context(
             with contextlib.suppress(Exception):
                 context = uuid.UUID(context)  # type: ignore
 
-        if isinstance(context, str) and os.path.exists(context):
+        if isinstance(context, str) and (
+            os.path.exists(context) or context.endswith(".kontext")
+        ):
             context = Path(context)
 
         if isinstance(context, Path):
-            with context.open("rt") as f:
-                data = yaml.load(f)
-            context_config = KiaraContextConfig(**data)
+            if context.name.endswith(".kontext"):
+                context_config = KiaraContextConfig.create_from_sqlite_db(
+                    db_path=context
+                )
+            else:
+                with context.open("rt") as f:
+                    data = yaml.load(f)
+                context_config = KiaraContextConfig(**data)
         elif isinstance(context, str):
             context_config = self.get_context_config(context_name=context)
         elif isinstance(context, uuid.UUID):

src/kiara/context/runtime_config.py

@@ -13,6 +13,11 @@ class JobCacheStrategy(Enum):
 
 
 class KiaraRuntimeConfig(BaseSettings):
+    """The runtime configuration for a *kiara* backend.
+
+    The most important option here is the 'job_cache' setting, which determines how the runtime will match a new job against the records of past ones, in order to find a matching one and not have to re-run the possibly expensive job again. By default, no matching is done, other options are matching based on exact input value ids, or (more expensive) matching based on the input data hashes.
+    """
+
     model_config = SettingsConfigDict(
         extra="forbid", validate_assignment=True, env_prefix="kiara_runtime_"
     )

src/kiara/interfaces/__init__.py

@@ -230,6 +230,7 @@ def __init__(
         ensure_plugins: Union[str, Iterable[str], None] = None,
         exit_process: bool = True,
     ):
+
         if not context:
             context = os.environ.get("KIARA_CONTEXT", None)
 

src/kiara/interfaces/cli/archive/commands.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+
+#  Copyright (c) 2021, Markus Binsteiner
+#
+#  Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/)
+from typing import TYPE_CHECKING
+
+import rich_click as click
+
+from kiara.utils.cli import (
+    OutputFormat,
+    output_format_option,
+    terminal_print_model,
+)
+
+if TYPE_CHECKING:
+    pass
+
+
+@click.group("archive")
+@click.pass_context
+def context(ctx):
+    """Kiara archive related sub-commands."""
+
+
+@context.command("explain")
+@click.argument("archive", nargs=1, required=True)
+@output_format_option()
+@click.pass_context
+def explain_archive(
+    ctx,
+    format: str,
+    archive: str,
+):
+    """Print details of an archive file."""
+
+    from kiara.api import KiaraAPI
+
+    kiara_api: KiaraAPI = ctx.obj.kiara_api
+
+    infos = kiara_api.get_archive_info(archive)
+
+    if not format or format == OutputFormat.TERMINAL:
+        for info in infos:
+            types = ", ".join(info.archive_type_info.supported_item_types)
+            terminal_print_model(info, in_panel=f"Archive type(s): {types}")
+    else:
+        terminal_print_model(*infos, format=format)

src/kiara/interfaces/cli/context/commands.py

@@ -7,6 +7,7 @@
 from typing import TYPE_CHECKING, Tuple, Union
 
 import rich_click as click
+from rich import box
 from rich.panel import Panel
 
 from kiara.interfaces import KiaraAPIWrap, get_console
@@ -42,10 +43,17 @@ def list_contexts(ctx) -> None:
 @context.command("explain")
 @click.argument("context_name", nargs=-1, required=False)
 @click.option("--value-ids", "-i", help="Show value ids.", is_flag=True, default=False)
+@click.option(
+    "--show-config", "-c", help="Also show kiara config.", is_flag=True, default=False
+)
 @output_format_option()
 @click.pass_context
 def explain_context(
-    ctx, format: str, value_ids: bool, context_name: Union[Tuple[str], None] = None
+    ctx,
+    format: str,
+    value_ids: bool,
+    context_name: Union[Tuple[str], None] = None,
+    show_config: bool = False,
 ):
     """Print details for one or several contexts."""
     kiara_config: KiaraConfig = ctx.obj.kiara_config
@@ -58,15 +66,36 @@ def explain_context(
 
     from kiara.models.context import ContextInfo
 
+    render_config = {
+        "show_lines": False,
+        "show_header": False,
+        "show_description": False,
+    }
+
+    if show_config:
+        from rich.table import Table
+
+        config = kiara_config.create_renderable(**render_config)
+        table = Table(show_header=False, show_lines=False, box=box.SIMPLE)
+        table.add_column("key", style="i")
+        table.add_column("value")
+        if kiara_config._config_path:
+            table.add_row("config file", f"  {kiara_config._config_path}")
+        table.add_row("config", config)
+        terminal_print(table, in_panel="Kiara config")
+
     if len(contexts) == 1:
 
         kcc = kiara_config.get_context_config(contexts[0])
-
         cs = ContextInfo.create_from_context_config(
             kcc, context_name=contexts[0], runtime_config=kiara_config.runtime_config
         )
         terminal_print_model(
-            cs, format=format, full_details=True, show_value_ids=value_ids
+            cs,
+            format=format,
+            full_details=True,
+            show_value_ids=value_ids,
+            in_panel=f"Context '{contexts[0]}'",
         )
 
     else: