diff --git a/docs/sections/user_guide/cli/tools/file.rst b/docs/sections/user_guide/cli/tools/file.rst index 4b292f57d..a59382dd5 100644 --- a/docs/sections/user_guide/cli/tools/file.rst +++ b/docs/sections/user_guide/cli/tools/file.rst @@ -107,7 +107,7 @@ Here, ``foo`` and ``bar`` are copies of their respective source files. ``link`` -------- -The ``link`` action stages files in a target directory by linking files. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`file block `. +The ``link`` action stages files in a target directory by linking files, directories, or other symbolic links. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`file block `. .. code-block:: text diff --git a/recipe/meta.json b/recipe/meta.json index c8c08bfc9..31d04c4c5 100644 --- a/recipe/meta.json +++ b/recipe/meta.json @@ -4,7 +4,7 @@ "name": "uwtools", "packages": { "dev": [ - "black =24.3.*", + "black =24.4.*", "coverage =7.4.*", "docformatter =1.7.*", "f90nml =1.4.*", diff --git a/recipe/meta.yaml b/recipe/meta.yaml index f3344b4c3..cdf4234e0 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -22,7 +22,7 @@ requirements: - pyyaml 6.0.* test: requires: - - black 24.3.* + - black 24.4.* - coverage 7.4.* - docformatter 1.7.* - isort 5.13.* diff --git a/src/uwtools/api/chgres_cube.py b/src/uwtools/api/chgres_cube.py index 2ef18398d..2609a8e41 100644 --- a/src/uwtools/api/chgres_cube.py +++ b/src/uwtools/api/chgres_cube.py @@ -4,21 +4,23 @@ import datetime as dt from pathlib import Path -from typing import Dict, Optional +from typing import Dict, Optional, Union import iotaa as _iotaa import uwtools.drivers.support as _support from uwtools.drivers.chgres_cube import ChgresCube as _ChgresCube +from uwtools.utils.api import ensure_data_source as _ensure_data_source def execute( task: str, cycle: dt.datetime, - config: Optional[Path] = None, + config: Optional[Union[Path, str]] = None, batch: bool = False, dry_run: bool = False, - graph_file: Optional[Path] = None, + graph_file: Optional[Union[Path, str]] = None, + stdin_ok: bool = False, ) -> bool: """ Execute a ``chgres_cube`` task. @@ -32,9 +34,12 @@ def execute( :param batch: Submit run to the batch system? :param dry_run: Do not run the executable, just report what would have been done. :param graph_file: Write Graphviz DOT output here. + :param stdin_ok: OK to read from stdin? :return: ``True`` if task completes without raising an exception. """ - obj = _ChgresCube(config=config, cycle=cycle, batch=batch, dry_run=dry_run) + obj = _ChgresCube( + cycle=cycle, config=_ensure_data_source(config, stdin_ok), batch=batch, dry_run=dry_run + ) getattr(obj, task)() if graph_file: with open(graph_file, "w", encoding="utf-8") as f: diff --git a/src/uwtools/api/config.py b/src/uwtools/api/config.py index be17fc50d..5a668f3ee 100644 --- a/src/uwtools/api/config.py +++ b/src/uwtools/api/config.py @@ -15,14 +15,16 @@ from uwtools.config.tools import compare_configs as _compare from uwtools.config.tools import realize_config as _realize from uwtools.config.validator import validate_yaml as _validate_yaml +from uwtools.utils.api import ensure_data_source as _ensure_data_source +from uwtools.utils.api import str2path as _str2path from uwtools.utils.file import FORMAT as _FORMAT # Public def compare( - config_1_path: Path, - config_2_path: Path, + config_1_path: Union[Path, str], + config_2_path: Union[Path, str], config_1_format: Optional[str] = None, config_2_format: Optional[str] = None, ) -> bool: @@ -30,88 +32,112 @@ def compare( NB: This docstring is dynamically replaced: See compare.__doc__ definition below. """ return _compare( - config_1_path=config_1_path, - config_2_path=config_2_path, + config_1_path=Path(config_1_path), + config_2_path=Path(config_2_path), config_1_format=config_1_format, config_2_format=config_2_format, ) -def get_fieldtable_config(config: Union[dict, Optional[Path]] = None) -> _FieldTableConfig: +def get_fieldtable_config( + config: Union[dict, Optional[Union[Path, str]]] = None, stdin_ok=False +) -> _FieldTableConfig: """ Get a ``FieldTableConfig`` object. :param config: FieldTable file to load (``None`` or unspecified => read ``stdin``), or initial ``dict`` + :param stdin_ok: OK to read from stdin? :return: An initialized ``FieldTableConfig`` object """ - return _FieldTableConfig(config=config) + return _FieldTableConfig(config=_ensure_data_source(config, stdin_ok)) -def get_ini_config(config: Union[dict, Optional[Path]] = None) -> _INIConfig: +def get_ini_config( + config: Union[dict, Optional[Union[Path, str]]] = None, + stdin_ok: bool = False, +) -> _INIConfig: """ Get an ``INIConfig`` object. :param config: INI file to load (``None`` or unspecified => read ``stdin``), or initial ``dict`` + :param stdin_ok: OK to read from stdin? :return: An initialized ``INIConfig`` object """ - return _INIConfig(config=config) + return _INIConfig(config=_ensure_data_source(config, stdin_ok)) -def get_nml_config(config: Union[dict, Optional[Path]] = None) -> _NMLConfig: +def get_nml_config( + config: Union[dict, Optional[Union[Path, str]]] = None, + stdin_ok: bool = False, +) -> _NMLConfig: """ Get an ``NMLConfig`` object. :param config: Fortran namelist file to load (``None`` or unspecified => read ``stdin``), or initial ``dict`` + :param stdin_ok: OK to read from stdin? :return: An initialized ``NMLConfig`` object """ - return _NMLConfig(config=config) + return _NMLConfig(config=_ensure_data_source(config, stdin_ok)) -def get_sh_config(config: Union[dict, Optional[Path]] = None) -> _SHConfig: +def get_sh_config( + config: Union[dict, Optional[Union[Path, str]]] = None, + stdin_ok: bool = False, +) -> _SHConfig: """ Get an ``SHConfig`` object. :param config: File of shell 'key=value' pairs to load (``None`` or unspecified => read ``stdin``), or initial ``dict`` + :param stdin_ok: OK to read from stdin? :return: An initialized ``SHConfig`` object """ - return _SHConfig(config=config) + return _SHConfig(config=_ensure_data_source(config, stdin_ok)) -def get_yaml_config(config: Union[dict, Optional[Path]] = None) -> _YAMLConfig: +def get_yaml_config( + config: Union[dict, Optional[Union[Path, str]]] = None, + stdin_ok: bool = False, +) -> _YAMLConfig: """ Get a ``YAMLConfig`` object. :param config: YAML file to load (``None`` or unspecified => read ``stdin``), or initial ``dict`` + :param stdin_ok: OK to read from stdin? :return: An initialized ``YAMLConfig`` object """ - return _YAMLConfig(config=config) + return _YAMLConfig(config=_ensure_data_source(config, stdin_ok)) def realize( - input_config: Union[dict, _Config, Optional[Path]] = None, + input_config: Optional[Union[dict, _Config, Path, str]] = None, input_format: Optional[str] = None, output_block: Optional[List[Union[str, int]]] = None, - output_file: Optional[Path] = None, + output_file: Optional[Union[Path, str]] = None, output_format: Optional[str] = None, - supplemental_configs: Optional[List[Union[dict, _Config, Path]]] = None, + supplemental_configs: Optional[List[Union[dict, _Config, Path, str]]] = None, values_needed: bool = False, total: bool = False, dry_run: bool = False, + stdin_ok: bool = False, ) -> None: """ NB: This docstring is dynamically replaced: See realize.__doc__ definition below. """ + input_config = ( + _YAMLConfig(config=input_config) if isinstance(input_config, dict) else input_config + ) + scs = [_str2path(x) for x in supplemental_configs] if supplemental_configs else None _realize( - input_config=_ensure_config_arg_type(input_config), + input_config=_ensure_data_source(input_config, stdin_ok), input_format=input_format, output_block=output_block, - output_file=output_file, + output_file=_str2path(output_file), output_format=output_format, - supplemental_configs=supplemental_configs, + supplemental_configs=scs, values_needed=values_needed, total=total, dry_run=dry_run, @@ -119,11 +145,12 @@ def realize( def realize_to_dict( # pylint: disable=unused-argument - input_config: Union[dict, _Config, Optional[Path]] = None, + input_config: Optional[Union[dict, _Config, Path, str]] = None, input_format: Optional[str] = None, - supplemental_configs: Optional[List[Union[dict, _Config, Path]]] = None, + supplemental_configs: Optional[List[Union[dict, _Config, Path, str]]] = None, values_needed: bool = False, dry_run: bool = False, + stdin_ok: bool = False, ) -> dict: """ Realize a config to a ``dict``, based on an input config and optional supplemental configs. @@ -134,7 +161,9 @@ def realize_to_dict( # pylint: disable=unused-argument def validate( - schema_file: Path, config: Optional[Union[dict, _YAMLConfig, Optional[Path]]] = None + schema_file: Union[Path, str], + config: Optional[Union[dict, _YAMLConfig, Path, str]] = None, + stdin_ok: bool = False, ) -> bool: """ Check whether the specified config conforms to the specified JSON Schema spec. @@ -144,25 +173,12 @@ def validate( :param schema_file: The JSON Schema file to use for validation :param config: The config to validate + :param stdin_ok: OK to read from stdin? :return: ``True`` if the YAML file conforms to the schema, ``False`` otherwise """ - return _validate_yaml(schema_file=schema_file, config=config) - - -# Private - - -def _ensure_config_arg_type( - config: Union[dict, _Config, Optional[Path]] -) -> Union[_Config, Optional[Path]]: - """ - Encapsulate a ``dict`` in a ``Config``; return a ``Config`` or path argument as-is. - - :param config: A config as a ``dict``, ``Config``, or ``Path`` - """ - if isinstance(config, dict): - return _YAMLConfig(config=config) - return config + return _validate_yaml( + schema_file=_ensure_data_source(schema_file, stdin_ok), config=_str2path(config) + ) # Import-time code @@ -219,6 +235,7 @@ def _ensure_config_arg_type( :param values_needed: Report complete, missing, and template values :param total: Require rendering of all Jinja2 variables/expressions :param dry_run: Log output instead of writing to output +:param stdin_ok: OK to read from stdin? :raises: UWConfigRealizeError if ``total`` is ``True`` and any Jinja2 variable/expression was not rendered """.format( extensions=", ".join(_FORMAT.extensions()) diff --git a/src/uwtools/api/file.py b/src/uwtools/api/file.py index 0ed26f2e4..8d09886da 100644 --- a/src/uwtools/api/file.py +++ b/src/uwtools/api/file.py @@ -7,13 +7,15 @@ from uwtools.file import FileCopier as _FileCopier from uwtools.file import FileLinker as _FileLinker +from uwtools.utils.api import ensure_data_source as _ensure_data_source def copy( - target_dir: Path, - config: Optional[Union[dict, Path]] = None, + target_dir: Union[Path, str], + config: Optional[Union[dict, Path, str]] = None, keys: Optional[List[str]] = None, dry_run: bool = False, + stdin_ok: bool = False, ) -> bool: """ Copy files. @@ -22,17 +24,24 @@ def copy( :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``). :param keys: YAML keys leading to file dst/src block :param dry_run: Do not copy files + :param stdin_ok: OK to read from stdin? :return: ``True`` if no exception is raised """ - _FileCopier(target_dir=target_dir, config=config, keys=keys, dry_run=dry_run).go() + _FileCopier( + target_dir=Path(target_dir), + config=_ensure_data_source(config, stdin_ok), + keys=keys, + dry_run=dry_run, + ).go() return True def link( - target_dir: Path, - config: Optional[Union[dict, Path]] = None, + target_dir: Union[Path, str], + config: Optional[Union[dict, Path, str]] = None, keys: Optional[List[str]] = None, dry_run: bool = False, + stdin_ok: bool = False, ) -> bool: """ Link files. @@ -41,7 +50,13 @@ def link( :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``). :param keys: YAML keys leading to file dst/src block :param dry_run: Do not link files + :param stdin_ok: OK to read from stdin? :return: ``True`` if no exception is raised """ - _FileLinker(target_dir=target_dir, config=config, keys=keys, dry_run=dry_run).go() + _FileLinker( + target_dir=Path(target_dir), + config=_ensure_data_source(config, stdin_ok), + keys=keys, + dry_run=dry_run, + ).go() return True diff --git a/src/uwtools/api/fv3.py b/src/uwtools/api/fv3.py index 458c9538e..27ca3875a 100644 --- a/src/uwtools/api/fv3.py +++ b/src/uwtools/api/fv3.py @@ -4,19 +4,21 @@ import datetime as dt from pathlib import Path -from typing import Dict, Optional +from typing import Dict, Optional, Union import uwtools.drivers.support as _support from uwtools.drivers.fv3 import FV3 as _FV3 +from uwtools.utils.api import ensure_data_source as _ensure_data_source def execute( task: str, cycle: dt.datetime, - config: Optional[Path] = None, + config: Optional[Union[Path, str]] = None, batch: bool = False, dry_run: bool = False, graph_file: Optional[Path] = None, + stdin_ok: bool = False, ) -> bool: """ Execute an FV3 task. @@ -30,9 +32,12 @@ def execute( :param batch: Submit run to the batch system? :param dry_run: Do not run forecast, just report what would have been done. :param graph_file: Write Graphviz DOT output here. + :param stdin_ok: OK to read from stdin? :return: ``True`` if task completes without raising an exception. """ - obj = _FV3(config=config, cycle=cycle, batch=batch, dry_run=dry_run) + obj = _FV3( + cycle=cycle, config=_ensure_data_source(config, stdin_ok), batch=batch, dry_run=dry_run + ) getattr(obj, task)() if graph_file: with open(graph_file, "w", encoding="utf-8") as f: diff --git a/src/uwtools/api/jedi.py b/src/uwtools/api/jedi.py index ef1367175..e1106c9fa 100644 --- a/src/uwtools/api/jedi.py +++ b/src/uwtools/api/jedi.py @@ -4,35 +4,40 @@ import datetime as dt from pathlib import Path -from typing import Dict, Optional +from typing import Dict, Optional, Union import uwtools.drivers.support as _support from uwtools.drivers.jedi import JEDI as _JEDI +from uwtools.utils.api import ensure_data_source as _ensure_data_source def execute( task: str, - config: Path, cycle: dt.datetime, + config: Optional[Union[Path, str]] = None, batch: bool = False, dry_run: bool = False, graph_file: Optional[Path] = None, + stdin_ok: bool = False, ) -> bool: """ Execute a JEDI task. If ``batch`` is specified, a runscript will be written and submitted to the batch system. Otherwise, the executable will be run directly on the current system. - :param task: The task to execute + + :param task: The task to execute. :param cycle: The cycle. - :param config: Path to YAML config file - :param cycle: The cycle to run - :param batch: Submit run to the batch system - :param dry_run: Do not run the executable, just report what would have been done - :param graph_file: Write Graphviz DOT output here - :return: True if task completes without raising an exception + :param config: Path to config file (read stdin if missing or None). + :param batch: Submit run to the batch system? + :param dry_run: Do not run the executable, just report what would have been done. + :param graph_file: Write Graphviz DOT output here. + :param stdin_ok: OK to read from stdin? + :return: ``True`` if task completes without raising an exception. """ - obj = _JEDI(config=config, cycle=cycle, batch=batch, dry_run=dry_run) + obj = _JEDI( + cycle=cycle, config=_ensure_data_source(config, stdin_ok), batch=batch, dry_run=dry_run + ) getattr(obj, task)() if graph_file: with open(graph_file, "w", encoding="utf-8") as f: diff --git a/src/uwtools/api/mpas.py b/src/uwtools/api/mpas.py index dfcee9fc8..ff0354a15 100644 --- a/src/uwtools/api/mpas.py +++ b/src/uwtools/api/mpas.py @@ -4,19 +4,21 @@ import datetime as dt from pathlib import Path -from typing import Dict, Optional +from typing import Dict, Optional, Union import uwtools.drivers.support as _support from uwtools.drivers.mpas import MPAS as _MPAS +from uwtools.utils.api import ensure_data_source as _ensure_data_source def execute( task: str, cycle: dt.datetime, - config: Optional[Path] = None, + config: Optional[Union[Path, str]] = None, batch: bool = False, dry_run: bool = False, graph_file: Optional[Path] = None, + stdin_ok: bool = False, ) -> bool: """ Execute an ``mpas`` task. @@ -30,9 +32,12 @@ def execute( :param batch: Submit run to the batch system. :param dry_run: Do not run the executable, just report what would have been done. :param graph_file: Write Graphviz DOT output here. + :param stdin_ok: OK to read from stdin? :return: ``True`` if task completes without raising an exception. """ - obj = _MPAS(config=config, cycle=cycle, batch=batch, dry_run=dry_run) + obj = _MPAS( + cycle=cycle, config=_ensure_data_source(config, stdin_ok), batch=batch, dry_run=dry_run + ) getattr(obj, task)() if graph_file: with open(graph_file, "w", encoding="utf-8") as f: diff --git a/src/uwtools/api/mpas_init.py b/src/uwtools/api/mpas_init.py index a698e5705..9df0a25f0 100644 --- a/src/uwtools/api/mpas_init.py +++ b/src/uwtools/api/mpas_init.py @@ -4,19 +4,21 @@ import datetime as dt from pathlib import Path -from typing import Dict, Optional +from typing import Dict, Optional, Union import uwtools.drivers.support as _support from uwtools.drivers.mpas_init import MPASInit as _MPASInit +from uwtools.utils.api import ensure_data_source as _ensure_data_source def execute( task: str, cycle: dt.datetime, - config: Optional[Path] = None, + config: Optional[Union[Path, str]] = None, batch: bool = False, dry_run: bool = False, graph_file: Optional[Path] = None, + stdin_ok: bool = False, ) -> bool: """ Execute an MPAS ``init-atmosphere`` task. @@ -30,9 +32,12 @@ def execute( :param batch: Submit run to the batch system? :param dry_run: Do not run the executable, just report what would have been done. :param graph_file: Write Graphviz DOT output here. + :param stdin_ok: OK to read from stdin? :return: ``True`` if task completes without raising an exception. """ - obj = _MPASInit(config=config, cycle=cycle, batch=batch, dry_run=dry_run) + obj = _MPASInit( + cycle=cycle, config=_ensure_data_source(config, stdin_ok), batch=batch, dry_run=dry_run + ) getattr(obj, task)() if graph_file: with open(graph_file, "w", encoding="utf-8") as f: diff --git a/src/uwtools/api/rocoto.py b/src/uwtools/api/rocoto.py index 702a65181..eb984efd9 100644 --- a/src/uwtools/api/rocoto.py +++ b/src/uwtools/api/rocoto.py @@ -8,9 +8,15 @@ from uwtools.config.formats.yaml import YAMLConfig as _YAMLConfig from uwtools.rocoto import realize_rocoto_xml as _realize from uwtools.rocoto import validate_rocoto_xml_file as _validate +from uwtools.utils.api import ensure_data_source as _ensure_data_source +from uwtools.utils.api import str2path as _str2path -def realize(config: Union[_YAMLConfig, Optional[Path]], output_file: Optional[Path] = None) -> bool: +def realize( + config: Optional[Union[_YAMLConfig, Path, str]], + output_file: Optional[Union[Path, str]] = None, + stdin_ok: bool = False, +) -> bool: """ Realize the Rocoto workflow defined in the given YAML as XML. @@ -22,17 +28,22 @@ def realize(config: Union[_YAMLConfig, Optional[Path]], output_file: Optional[Pa ``YAMLConfig`` object :param output_file: Path to write rendered XML file (``None`` or unspecified => write to ``stdout``) + :param stdin_ok: OK to read from stdin? :return: ``True`` """ - _realize(config=config, output_file=output_file) + _realize(config=_ensure_data_source(config, stdin_ok), output_file=_str2path(output_file)) return True -def validate(xml_file: Optional[Path] = None) -> bool: +def validate( + xml_file: Optional[Union[Path, str]] = None, + stdin_ok: bool = False, +) -> bool: """ Validate purported Rocoto XML file against its schema. :param xml_file: Path to XML file (``None`` or unspecified => read ``stdin``) + :param stdin_ok: OK to read from stdin? :return: ``True`` if the XML conforms to the schema, ``False`` otherwise """ - return _validate(xml_file=xml_file) + return _validate(xml_file=_ensure_data_source(xml_file, stdin_ok)) diff --git a/src/uwtools/api/sfc_climo_gen.py b/src/uwtools/api/sfc_climo_gen.py index bc1664574..5cc408949 100644 --- a/src/uwtools/api/sfc_climo_gen.py +++ b/src/uwtools/api/sfc_climo_gen.py @@ -3,18 +3,20 @@ """ from pathlib import Path -from typing import Dict, Optional +from typing import Dict, Optional, Union import uwtools.drivers.support as _support from uwtools.drivers.sfc_climo_gen import SfcClimoGen as _SfcClimoGen +from uwtools.utils.api import ensure_data_source as _ensure_data_source def execute( task: str, - config: Optional[Path] = None, + config: Optional[Union[Path, str]] = None, batch: bool = False, dry_run: bool = False, graph_file: Optional[Path] = None, + stdin_ok: bool = False, ) -> bool: """ Execute an ``sfc_climo_gen`` task. @@ -27,9 +29,10 @@ def execute( :param batch: Submit run to the batch system? :param dry_run: Do not run forecast, just report what would have been done. :param graph_file: Write Graphviz DOT output here. + :param stdin_ok: OK to read from stdin? :return: ``True`` if task completes without raising an exception. """ - obj = _SfcClimoGen(config=config, batch=batch, dry_run=dry_run) + obj = _SfcClimoGen(config=_ensure_data_source(config, stdin_ok), batch=batch, dry_run=dry_run) getattr(obj, task)() if graph_file: with open(graph_file, "w", encoding="utf-8") as f: diff --git a/src/uwtools/api/template.py b/src/uwtools/api/template.py index 768803bbf..2c2bc9e0a 100644 --- a/src/uwtools/api/template.py +++ b/src/uwtools/api/template.py @@ -9,18 +9,21 @@ from uwtools.config.atparse_to_jinja2 import convert as _convert_atparse_to_jinja2 from uwtools.config.jinja2 import render as _render from uwtools.exceptions import UWTemplateRenderError +from uwtools.utils.api import ensure_data_source as _ensure_data_source +from uwtools.utils.api import str2path as _str2path def render( - values_src: Optional[Union[dict, Path]] = None, + values_src: Optional[Union[dict, Path, str]] = None, values_format: Optional[str] = None, - input_file: Optional[Path] = None, - output_file: Optional[Path] = None, + input_file: Optional[Union[Path, str]] = None, + output_file: Optional[Union[Path, str]] = None, overrides: Optional[Dict[str, str]] = None, env: bool = False, searchpath: Optional[List[str]] = None, values_needed: bool = False, dry_run: bool = False, + stdin_ok: bool = False, ) -> str: """ Render a Jinja2 template to a file, based on specified values. @@ -42,14 +45,15 @@ def render( :param searchpath: Paths to search for extra templates :param values_needed: Just report variables needed to render the template? :param dry_run: Run in dry-run mode? + :param stdin_ok: OK to read from stdin? :return: The rendered template string :raises: UWTemplateRenderError if template could not be rendered """ result = _render( - values_src=values_src, + values_src=_str2path(values_src), values_format=values_format, - input_file=input_file, - output_file=output_file, + input_file=_ensure_data_source(input_file, stdin_ok), + output_file=_str2path(output_file), overrides=overrides, env=env, searchpath=searchpath, @@ -62,9 +66,9 @@ def render( def render_to_str( # pylint: disable=unused-argument - values_src: Optional[Union[dict, Path]] = None, + values_src: Optional[Union[dict, Path, str]] = None, values_format: Optional[str] = None, - input_file: Optional[Path] = None, + input_file: Optional[Union[Path, str]] = None, overrides: Optional[Dict[str, str]] = None, env: bool = False, searchpath: Optional[List[str]] = None, @@ -80,9 +84,10 @@ def render_to_str( # pylint: disable=unused-argument def translate( - input_file: Optional[Path] = None, - output_file: Optional[Path] = None, + input_file: Optional[Union[Path, str]] = None, + output_file: Optional[Union[Path, str]] = None, dry_run: bool = False, + stdin_ok: bool = False, ) -> bool: """ Translate an atparse template to a Jinja2 template. @@ -91,10 +96,16 @@ def translate( ``stdin`` is read. If no output file is specified, ``stdout`` is written to. In ``dry_run`` mode, output is written to ``stderr``. - :param input_file: Path to the template containing atparse syntax + :param input_file: Path to the template containing atparse syntax (``None`` or unspecified => + read ``stdin``) :param output_file: Path to the file to write the converted template to :param dry_run: Run in dry-run mode? + :param stdin_ok: OK to read from stdin? :return: ``True`` """ - _convert_atparse_to_jinja2(input_file=input_file, output_file=output_file, dry_run=dry_run) + _convert_atparse_to_jinja2( + input_file=_ensure_data_source(input_file, stdin_ok), + output_file=_str2path(output_file), + dry_run=dry_run, + ) return True diff --git a/src/uwtools/api/ungrib.py b/src/uwtools/api/ungrib.py index edf8cc9e0..533fe4a07 100644 --- a/src/uwtools/api/ungrib.py +++ b/src/uwtools/api/ungrib.py @@ -4,19 +4,21 @@ import datetime as dt from pathlib import Path -from typing import Dict, Optional +from typing import Dict, Optional, Union import uwtools.drivers.support as _support from uwtools.drivers.ungrib import Ungrib as _Ungrib +from uwtools.utils.api import ensure_data_source as _ensure_data_source def execute( task: str, cycle: dt.datetime, - config: Optional[Path] = None, + config: Optional[Union[Path, str]] = None, batch: bool = False, dry_run: bool = False, graph_file: Optional[Path] = None, + stdin_ok: bool = False, ) -> bool: """ Execute an ``ungrib`` task. @@ -30,9 +32,12 @@ def execute( :param batch: Submit run to the batch system? :param dry_run: Do not run the executable, just report what would have been done. :param graph_file: Write Graphviz DOT output here. + :param stdin_ok: OK to read from stdin? :return: ``True`` if task completes without raising an exception. """ - obj = _Ungrib(config=config, cycle=cycle, batch=batch, dry_run=dry_run) + obj = _Ungrib( + cycle=cycle, config=_ensure_data_source(config, stdin_ok), batch=batch, dry_run=dry_run + ) getattr(obj, task)() if graph_file: with open(graph_file, "w", encoding="utf-8") as f: diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index 77fd00d30..83fc0e28d 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -131,6 +131,7 @@ def _dispatch_chgres_cube(args: Args) -> bool: batch=args[STR.batch], dry_run=args[STR.dryrun], graph_file=args[STR.graphfile], + stdin_ok=True, ) @@ -266,6 +267,7 @@ def _dispatch_config_realize(args: Args) -> bool: values_needed=args[STR.valsneeded], total=args[STR.total], dry_run=args[STR.dryrun], + stdin_ok=True, ) except UWConfigRealizeError: log.error( @@ -281,7 +283,11 @@ def _dispatch_config_validate(args: Args) -> bool: :param args: Parsed command-line args. """ - return uwtools.api.config.validate(schema_file=args[STR.schemafile], config=args[STR.infile]) + return uwtools.api.config.validate( + schema_file=args[STR.schemafile], + config=args[STR.infile], + stdin_ok=True, + ) # Mode file @@ -362,6 +368,7 @@ def _dispatch_file_copy(args: Args) -> bool: config=args[STR.cfgfile], keys=args[STR.keys], dry_run=args[STR.dryrun], + stdin_ok=True, ) @@ -376,6 +383,7 @@ def _dispatch_file_link(args: Args) -> bool: config=args[STR.cfgfile], keys=args[STR.keys], dry_run=args[STR.dryrun], + stdin_ok=True, ) @@ -430,6 +438,7 @@ def _dispatch_fv3(args: Args) -> bool: batch=args[STR.batch], dry_run=args[STR.dryrun], graph_file=args[STR.graphfile], + stdin_ok=True, ) @@ -483,6 +492,7 @@ def _dispatch_jedi(args: Args) -> bool: batch=args[STR.batch], dry_run=args[STR.dryrun], graph_file=args[STR.graphfile], + stdin_ok=True, ) @@ -538,6 +548,7 @@ def _dispatch_mpas(args: Args) -> bool: batch=args[STR.batch], dry_run=args[STR.dryrun], graph_file=args[STR.graphfile], + stdin_ok=True, ) @@ -592,6 +603,7 @@ def _dispatch_mpas_init(args: Args) -> bool: batch=args[STR.batch], dry_run=args[STR.dryrun], graph_file=args[STR.graphfile], + stdin_ok=True, ) @@ -659,7 +671,11 @@ def _dispatch_rocoto_realize(args: Args) -> bool: :param args: Parsed command-line args. """ - return uwtools.api.rocoto.realize(config=args[STR.infile], output_file=args[STR.outfile]) + return uwtools.api.rocoto.realize( + config=args[STR.infile], + output_file=args[STR.outfile], + stdin_ok=True, + ) def _dispatch_rocoto_validate(args: Args) -> bool: @@ -668,7 +684,7 @@ def _dispatch_rocoto_validate(args: Args) -> bool: :param args: Parsed command-line args. """ - return uwtools.api.rocoto.validate(xml_file=args[STR.infile]) + return uwtools.api.rocoto.validate(xml_file=args[STR.infile], stdin_ok=True) # Mode sfc_climo_gen @@ -721,6 +737,7 @@ def _dispatch_sfc_climo_gen(args: Args) -> bool: batch=args[STR.batch], dry_run=args[STR.dryrun], graph_file=args[STR.graphfile], + stdin_ok=True, ) @@ -807,6 +824,7 @@ def _dispatch_template_render(args: Args) -> bool: searchpath=args[STR.searchpath], values_needed=args[STR.valsneeded], dry_run=args[STR.dryrun], + stdin_ok=True, ) except UWTemplateRenderError: if args[STR.valsneeded]: @@ -826,6 +844,7 @@ def _dispatch_template_translate(args: Args) -> bool: input_file=args[STR.infile], output_file=args[STR.outfile], dry_run=args[STR.dryrun], + stdin_ok=True, ) @@ -880,6 +899,7 @@ def _dispatch_ungrib(args: Args) -> bool: batch=args[STR.batch], dry_run=args[STR.dryrun], graph_file=args[STR.graphfile], + stdin_ok=True, ) diff --git a/src/uwtools/config/formats/yaml.py b/src/uwtools/config/formats/yaml.py index c497bfd10..0032830fa 100644 --- a/src/uwtools/config/formats/yaml.py +++ b/src/uwtools/config/formats/yaml.py @@ -12,6 +12,7 @@ add_yaml_representers, log_and_error, ) +from uwtools.exceptions import UWConfigError from uwtools.strings import FORMAT from uwtools.utils.file import readable, writable @@ -68,8 +69,12 @@ def _load(self, config_file: Optional[Path]) -> dict: with readable(config_file) as f: try: config = yaml.load(f.read(), Loader=loader) - assert isinstance(config, dict) - return config + if isinstance(config, dict): + return config + raise UWConfigError( + "Parsed a %s value from %s, expected a dict" + % (type(config).__name__, config_file or "stdin") + ) except yaml.constructor.ConstructorError as e: if e.problem: if "unhashable" in e.problem: diff --git a/src/uwtools/drivers/jedi.py b/src/uwtools/drivers/jedi.py index 75ed0ea7a..1d5abdb30 100644 --- a/src/uwtools/drivers/jedi.py +++ b/src/uwtools/drivers/jedi.py @@ -5,6 +5,7 @@ import logging from datetime import datetime from pathlib import Path +from typing import Optional from iotaa import asset, dryrun, refs, run, task, tasks @@ -19,12 +20,18 @@ class JEDI(Driver): A driver for the JEDI component. """ - def __init__(self, config: Path, cycle: datetime, dry_run: bool = False, batch: bool = False): + def __init__( + self, + cycle: datetime, + config: Optional[Path] = None, + dry_run: bool = False, + batch: bool = False, + ): """ The driver. - :param config: Path to config file. :param cycle: The forecast cycle. + :param config: Path to config file. :param dry_run: Run in dry-run mode? :param batch: Run component via the batch system? """ diff --git a/src/uwtools/file.py b/src/uwtools/file.py index 3e5dfe3fc..f749af6b6 100644 --- a/src/uwtools/file.py +++ b/src/uwtools/file.py @@ -58,6 +58,8 @@ def _file_map(self) -> dict: raise UWConfigError("Config navigation %s failed" % " -> ".join(nav)) log.debug("Following config key '%s'", key) cfg = cfg[key] + if not isinstance(cfg, dict): + raise UWConfigError("No file map found at %s" % " -> ".join(self._keys)) return cfg def _validate(self) -> bool: diff --git a/src/uwtools/tests/api/test_chgres_cube.py b/src/uwtools/tests/api/test_chgres_cube.py index 93ebeded4..1e529ad8e 100644 --- a/src/uwtools/tests/api/test_chgres_cube.py +++ b/src/uwtools/tests/api/test_chgres_cube.py @@ -1,26 +1,29 @@ -# pylint: disable=missing-function-docstring,protected-access +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name import datetime as dt +from pathlib import Path from unittest.mock import patch import iotaa +from pytest import fixture from uwtools.api import chgres_cube -def test_execute(tmp_path): - dot = tmp_path / "graph.dot" - args: dict = { +@fixture +def kwargs(): + return { "batch": False, "config": "config.yaml", "cycle": dt.datetime.now(), "dry_run": True, - "graph_file": dot, } + + +def test_execute(kwargs, tmp_path): with patch.object(chgres_cube, "_ChgresCube") as ChgresCube: - assert chgres_cube.execute(**args, task="foo") is True - del args["graph_file"] - ChgresCube.assert_called_once_with(**args) + assert chgres_cube.execute(**kwargs, task="foo", graph_file=tmp_path / "graph.dot") is True + ChgresCube.assert_called_once_with(**{**kwargs, "config": Path(kwargs["config"])}) ChgresCube().foo.assert_called_once_with() diff --git a/src/uwtools/tests/api/test_config.py b/src/uwtools/tests/api/test_config.py index 560a2d0d5..67859e2b8 100644 --- a/src/uwtools/tests/api/test_config.py +++ b/src/uwtools/tests/api/test_config.py @@ -20,7 +20,13 @@ def test_compare(): } with patch.object(config, "_compare") as _compare: config.compare(**kwargs) - _compare.assert_called_once_with(**kwargs) + _compare.assert_called_once_with( + **{ + **kwargs, + "config_1_path": Path(kwargs["config_1_path"]), + "config_2_path": Path(kwargs["config_2_path"]), + } + ) @pytest.mark.parametrize( @@ -54,7 +60,14 @@ def test_realize(): } with patch.object(config, "_realize") as _realize: config.realize(**kwargs) - _realize.assert_called_once_with(**kwargs) + _realize.assert_called_once_with( + **{ + **kwargs, + "input_config": Path(kwargs["input_config"]), + "output_file": Path(kwargs["output_file"]), + "supplemental_configs": [Path(x) for x in kwargs["supplemental_configs"]], + } + ) def test_realize_to_dict(): @@ -64,6 +77,7 @@ def test_realize_to_dict(): "supplemental_configs": ["path3"], "values_needed": True, "dry_run": True, + "stdin_ok": False, } with patch.object(config, "_realize") as _realize: config.realize_to_dict(**kwargs) @@ -78,7 +92,7 @@ def test_validate(cfg): with patch.object(config, "_validate_yaml", return_value=True) as _validate_yaml: assert config.validate(**kwargs) _validate_yaml.assert_called_once_with( - schema_file=kwargs["schema_file"], config=kwargs["config"] + schema_file=Path(kwargs["schema_file"]), config=kwargs["config"] ) @@ -89,23 +103,4 @@ def test_validate_config_file(tmp_path): kwargs: dict = {"schema_file": "schema-file", "config": cfg} with patch.object(config, "_validate_yaml", return_value=True) as _validate_yaml: assert config.validate(**kwargs) - _validate_yaml.assert_called_once_with(schema_file=kwargs["schema_file"], config=cfg) - - -def test__ensure_config_arg_type_config_obj(): - config_obj = YAMLConfig(config={}) - assert config._ensure_config_arg_type(config=config_obj) is config_obj - - -def test__ensure_config_arg_type_dict(): - config_dict = {"foo": 88} - config_obj = config._ensure_config_arg_type(config=config_dict) - assert isinstance(config_obj, YAMLConfig) - assert config_obj.data == config_dict - - -def test__ensure_config_arg_type_path(): - config_path = Path("/path/to/config.yaml") - config_obj = config._ensure_config_arg_type(config=config_path) - assert isinstance(config_obj, Path) - assert config_obj is config_path + _validate_yaml.assert_called_once_with(schema_file=Path(kwargs["schema_file"]), config=cfg) diff --git a/src/uwtools/tests/api/test_file.py b/src/uwtools/tests/api/test_file.py index f77efc0c8..f9e398b99 100644 --- a/src/uwtools/tests/api/test_file.py +++ b/src/uwtools/tests/api/test_file.py @@ -1,5 +1,6 @@ # pylint: disable=missing-function-docstring,redefined-outer-name +from pathlib import Path from unittest.mock import patch from pytest import fixture @@ -8,7 +9,7 @@ @fixture -def args(): +def kwargs(): return { "target_dir": "/target/dir", "config": "/config/file", @@ -17,15 +18,19 @@ def args(): } -def test_copy(args): +def test_copy(kwargs): with patch.object(file, "_FileCopier") as FileCopier: - file.copy(**args) - FileCopier.assert_called_once_with(**args) + file.copy(**kwargs) + FileCopier.assert_called_once_with( + **{**kwargs, "target_dir": Path(kwargs["target_dir"]), "config": Path(kwargs["config"])} + ) FileCopier().go.assert_called_once_with() -def test_link(args): +def test_link(kwargs): with patch.object(file, "_FileLinker") as FileLinker: - file.link(**args) - FileLinker.assert_called_once_with(**args) + file.link(**kwargs) + FileLinker.assert_called_once_with( + **{**kwargs, "target_dir": Path(kwargs["target_dir"]), "config": Path(kwargs["config"])} + ) FileLinker().go.assert_called_once_with() diff --git a/src/uwtools/tests/api/test_fv3.py b/src/uwtools/tests/api/test_fv3.py index 7dcc33e61..c9fe47cdb 100644 --- a/src/uwtools/tests/api/test_fv3.py +++ b/src/uwtools/tests/api/test_fv3.py @@ -1,24 +1,28 @@ -# pylint: disable=missing-function-docstring,protected-access +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name import datetime as dt +from pathlib import Path from unittest.mock import patch +from pytest import fixture + from uwtools.api import fv3 -def test_execute(tmp_path): - dot = tmp_path / "graph.dot" - args: dict = { +@fixture +def kwargs(): + return { "batch": False, "config": "config.yaml", "cycle": dt.datetime.now(), "dry_run": True, - "graph_file": dot, } + + +def test_execute(kwargs, tmp_path): with patch.object(fv3, "_FV3") as FV3: - assert fv3.execute(**args, task="foo") is True - del args["graph_file"] - FV3.assert_called_once_with(**args) + assert fv3.execute(**kwargs, task="foo", graph_file=tmp_path / "graph.dot") is True + FV3.assert_called_once_with(**{**kwargs, "config": Path(kwargs["config"])}) FV3().foo.assert_called_once_with() diff --git a/src/uwtools/tests/api/test_jedi.py b/src/uwtools/tests/api/test_jedi.py index 398448cc4..8fce7ade8 100644 --- a/src/uwtools/tests/api/test_jedi.py +++ b/src/uwtools/tests/api/test_jedi.py @@ -1,6 +1,7 @@ # pylint: disable=missing-function-docstring,protected-access import datetime as dt +from pathlib import Path from unittest.mock import patch from uwtools.api import jedi @@ -9,8 +10,8 @@ def test_execute(tmp_path): dot = tmp_path / "graph.dot" args: dict = { - "config": "config.yaml", "cycle": dt.datetime.now(), + "config": Path("config.yaml"), "batch": False, "dry_run": True, "graph_file": dot, diff --git a/src/uwtools/tests/api/test_mpas.py b/src/uwtools/tests/api/test_mpas.py index a406edc2b..96971978a 100644 --- a/src/uwtools/tests/api/test_mpas.py +++ b/src/uwtools/tests/api/test_mpas.py @@ -1,25 +1,29 @@ -# pylint: disable=missing-function-docstring,protected-access +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name import datetime as dt +from pathlib import Path from unittest.mock import patch +from pytest import fixture + from uwtools.api import mpas -def test_execute(tmp_path): +@fixture +def kwargs(): cycle = dt.datetime.now() - dot = tmp_path / "graph.dot" - args: dict = { + return { "batch": False, "config": "config.yaml", "cycle": cycle, "dry_run": True, - "graph_file": dot, } + + +def test_execute(kwargs, tmp_path): with patch.object(mpas, "_MPAS") as MPAS: - assert mpas.execute(**args, task="foo") is True - del args["graph_file"] - MPAS.assert_called_once_with(**args) + assert mpas.execute(**kwargs, task="foo", graph_file=tmp_path / "graph.dot") is True + MPAS.assert_called_once_with(**{**kwargs, "config": Path(kwargs["config"])}) MPAS().foo.assert_called_once_with() diff --git a/src/uwtools/tests/api/test_mpas_init.py b/src/uwtools/tests/api/test_mpas_init.py index 36568807d..9d855324a 100644 --- a/src/uwtools/tests/api/test_mpas_init.py +++ b/src/uwtools/tests/api/test_mpas_init.py @@ -1,25 +1,29 @@ -# pylint: disable=missing-function-docstring,protected-access +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name import datetime as dt +from pathlib import Path from unittest.mock import patch +from pytest import fixture + from uwtools.api import mpas_init -def test_execute(tmp_path): +@fixture +def kwargs(): cycle = dt.datetime.now() - dot = tmp_path / "graph.dot" - args: dict = { + return { "batch": False, "config": "config.yaml", "cycle": cycle, "dry_run": True, - "graph_file": dot, } + + +def test_execute(kwargs, tmp_path): with patch.object(mpas_init, "_MPASInit") as MPASInit: - assert mpas_init.execute(**args, task="foo") is True - del args["graph_file"] - MPASInit.assert_called_once_with(**args) + assert mpas_init.execute(**kwargs, task="foo", graph_file=tmp_path / "graph.dot") is True + MPASInit.assert_called_once_with(**{**kwargs, "config": Path(kwargs["config"])}) MPASInit().foo.assert_called_once_with() diff --git a/src/uwtools/tests/api/test_sfc_climo_gen.py b/src/uwtools/tests/api/test_sfc_climo_gen.py index e8f407f04..ccccee6d5 100644 --- a/src/uwtools/tests/api/test_sfc_climo_gen.py +++ b/src/uwtools/tests/api/test_sfc_climo_gen.py @@ -1,22 +1,28 @@ -# pylint: disable=missing-function-docstring,protected-access +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name +from pathlib import Path from unittest.mock import patch +from pytest import fixture + from uwtools.api import sfc_climo_gen -def test_execute(tmp_path): - dot = tmp_path / "graph.dot" - args: dict = { +@fixture +def kwargs(): + return { "batch": False, "config": "config.yaml", "dry_run": True, - "graph_file": dot, } + + +def test_execute(kwargs, tmp_path): with patch.object(sfc_climo_gen, "_SfcClimoGen") as SfcClimoGen: - assert sfc_climo_gen.execute(**args, task="foo") is True - del args["graph_file"] - SfcClimoGen.assert_called_once_with(**args) + assert ( + sfc_climo_gen.execute(**kwargs, task="foo", graph_file=tmp_path / "graph.dot") is True + ) + SfcClimoGen.assert_called_once_with(**{**kwargs, "config": Path(kwargs["config"])}) SfcClimoGen().foo.assert_called_once_with() diff --git a/src/uwtools/tests/api/test_template.py b/src/uwtools/tests/api/test_template.py index b54f9017c..a46e4cc97 100644 --- a/src/uwtools/tests/api/test_template.py +++ b/src/uwtools/tests/api/test_template.py @@ -28,7 +28,14 @@ def kwargs(): def test_render(kwargs): with patch.object(template, "_render") as _render: template.render(**kwargs) - _render.assert_called_once_with(**kwargs) + _render.assert_called_once_with( + **{ + **kwargs, + "input_file": Path(kwargs["input_file"]), + "output_file": Path(kwargs["output_file"]), + "values_src": Path(kwargs["values_src"]), + } + ) def test_render_fail(kwargs): @@ -53,7 +60,7 @@ def test_translate(): with patch.object(template, "_convert_atparse_to_jinja2") as _catj: assert template.translate(**kwargs) _catj.assert_called_once_with( - input_file=kwargs["input_file"], - output_file=kwargs["output_file"], + input_file=Path(kwargs["input_file"]), + output_file=Path(kwargs["output_file"]), dry_run=kwargs["dry_run"], ) diff --git a/src/uwtools/tests/api/test_ungrib.py b/src/uwtools/tests/api/test_ungrib.py index 24db3327d..440bded6c 100644 --- a/src/uwtools/tests/api/test_ungrib.py +++ b/src/uwtools/tests/api/test_ungrib.py @@ -1,25 +1,29 @@ -# pylint: disable=missing-function-docstring,protected-access +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name import datetime as dt +from pathlib import Path from unittest.mock import patch +from pytest import fixture + from uwtools.api import ungrib -def test_execute(tmp_path): +@fixture +def kwargs(): cycle = dt.datetime.now() - dot = tmp_path / "graph.dot" - args: dict = { + return { "batch": False, "config": "config.yaml", "cycle": cycle, "dry_run": True, - "graph_file": dot, } + + +def test_execute(kwargs, tmp_path): with patch.object(ungrib, "_Ungrib") as Ungrib: - assert ungrib.execute(**args, task="foo") is True - del args["graph_file"] - Ungrib.assert_called_once_with(**args) + assert ungrib.execute(**kwargs, task="foo", graph_file=tmp_path / "graph.dot") is True + Ungrib.assert_called_once_with(**{**kwargs, "config": Path(kwargs["config"])}) Ungrib().foo.assert_called_once_with() diff --git a/src/uwtools/tests/config/formats/test_yaml.py b/src/uwtools/tests/config/formats/test_yaml.py index 880e29c80..e9f2ff835 100644 --- a/src/uwtools/tests/config/formats/test_yaml.py +++ b/src/uwtools/tests/config/formats/test_yaml.py @@ -8,6 +8,7 @@ import logging import sys from io import StringIO +from textwrap import dedent from unittest.mock import patch import yaml @@ -106,20 +107,36 @@ def test_simple(tmp_path): def test_constructor_error_no_quotes(tmp_path): # Test that Jinja2 template without quotes raises UWConfigError. - tmpfile = tmp_path / "test.yaml" with tmpfile.open("w", encoding="utf-8") as f: - f.write( - """ -foo: {{ bar }} -bar: 2 -""" - ) + s = """ + foo: {{ bar }} + bar: 2 + """ + f.write(dedent(s).strip()) with raises(exceptions.UWConfigError) as e: YAMLConfig(tmpfile) assert "value is enclosed in quotes" in str(e.value) +def test_constructor_error_not_dict_from_file(tmp_path): + # Test that a useful exception is raised if the YAML file input is a non-dict value. + tmpfile = tmp_path / "test.yaml" + with tmpfile.open("w", encoding="utf-8") as f: + f.write("hello") + with raises(exceptions.UWConfigError) as e: + YAMLConfig(tmpfile) + assert f"Parsed a str value from {tmpfile}, expected a dict" in str(e.value) + + +def test_constructor_error_not_dict_from_stdin(): + # Test that a useful exception is raised if the YAML stdin input is a non-dict value. + with patch.object(sys, "stdin", new=StringIO("a string")): + with raises(exceptions.UWConfigError) as e: + YAMLConfig() + assert "Parsed a str value from stdin, expected a dict" in str(e.value) + + def test_constructor_error_unregistered_constructor(tmp_path): # Test that unregistered constructor raises UWConfigError. diff --git a/src/uwtools/tests/test_cli.py b/src/uwtools/tests/test_cli.py index cb6121b4a..bf6217d4f 100644 --- a/src/uwtools/tests/test_cli.py +++ b/src/uwtools/tests/test_cli.py @@ -6,6 +6,7 @@ import sys from argparse import ArgumentParser as Parser from argparse import _SubParsersAction +from pathlib import Path from typing import List from unittest.mock import patch @@ -54,6 +55,7 @@ def args_dispatch_file(): "config_file": "/config/file", "keys": ["a", "b"], "dry_run": False, + "stdin_ok": True, } @@ -337,11 +339,18 @@ def test__dispatch_chgres_cube(): "cycle": cycle, "dry_run": False, "graph_file": None, + "stdin_ok": True, } with patch.object(uwtools.api.chgres_cube, "execute") as execute: cli._dispatch_chgres_cube({**args, "action": "foo"}) execute.assert_called_once_with( - batch=True, config="config.yaml", cycle=cycle, dry_run=False, graph_file=None, task="foo" + batch=True, + config="config.yaml", + cycle=cycle, + dry_run=False, + graph_file=None, + task="foo", + stdin_ok=True, ) @@ -397,6 +406,7 @@ def test__dispatch_config_realize(): values_needed=7, total=8, dry_run=9, + stdin_ok=True, ) @@ -445,15 +455,21 @@ def test__dispatch_config_realize_no_optional(): values_needed=False, total=False, dry_run=False, + stdin_ok=True, ) def test__dispatch_config_validate_config_obj(): - config = uwtools.api.config._YAMLConfig(config={}) - _dispatch_config_validate_args = {STR.schemafile: 1, STR.infile: config} + _dispatch_config_validate_args = { + STR.schemafile: Path("/path/to/a.jsonschema"), + STR.infile: Path("/path/to/config.yaml"), + } with patch.object(uwtools.api.config, "_validate_yaml") as _validate_yaml: cli._dispatch_config_validate(_dispatch_config_validate_args) - _validate_yaml_args = {STR.schemafile: 1, STR.config: config} + _validate_yaml_args = { + STR.schemafile: _dispatch_config_validate_args[STR.schemafile], + STR.config: _dispatch_config_validate_args[STR.infile], + } _validate_yaml.assert_called_once_with(**_validate_yaml_args) @@ -468,26 +484,28 @@ def test__dispatch_file(action, funcname): def test__dispatch_file_copy(args_dispatch_file): - a = args_dispatch_file + args = args_dispatch_file with patch.object(cli.uwtools.api.file, "copy") as copy: - cli._dispatch_file_copy(a) + cli._dispatch_file_copy(args) copy.assert_called_once_with( - target_dir=a["target_dir"], - config=a["config_file"], - keys=a["keys"], - dry_run=a["dry_run"], + target_dir=args["target_dir"], + config=args["config_file"], + keys=args["keys"], + dry_run=args["dry_run"], + stdin_ok=args["stdin_ok"], ) def test__dispatch_file_link(args_dispatch_file): - a = args_dispatch_file + args = args_dispatch_file with patch.object(cli.uwtools.api.file, "link") as link: - cli._dispatch_file_link(a) + cli._dispatch_file_link(args) link.assert_called_once_with( - target_dir=a["target_dir"], - config=a["config_file"], - keys=a["keys"], - dry_run=a["dry_run"], + target_dir=args["target_dir"], + config=args["config_file"], + keys=args["keys"], + dry_run=args["dry_run"], + stdin_ok=args["stdin_ok"], ) @@ -499,11 +517,18 @@ def test__dispatch_fv3(): "cycle": cycle, "dry_run": False, "graph_file": None, + "stdin_ok": True, } with patch.object(uwtools.api.fv3, "execute") as execute: cli._dispatch_fv3({**args, "action": "foo"}) execute.assert_called_once_with( - batch=True, config="config.yaml", cycle=cycle, dry_run=False, graph_file=None, task="foo" + batch=True, + config="config.yaml", + cycle=cycle, + dry_run=False, + graph_file=None, + task="foo", + stdin_ok=True, ) @@ -519,7 +544,13 @@ def test__dispatch_jedi(): with patch.object(uwtools.api.jedi, "execute") as execute: cli._dispatch_jedi({**args, "action": "foo"}) execute.assert_called_once_with( - task="foo", config="config.yaml", cycle=cycle, batch=True, dry_run=False, graph_file=None + task="foo", + config="config.yaml", + cycle=cycle, + batch=True, + dry_run=False, + graph_file=None, + stdin_ok=True, ) @@ -531,11 +562,18 @@ def test__dispatch_mpas(): "cycle": cycle, "dry_run": False, "graph_file": None, + "stdin_ok": True, } with patch.object(uwtools.api.mpas, "execute") as execute: cli._dispatch_mpas({**args, "action": "foo"}) execute.assert_called_once_with( - batch=True, config="config.yaml", cycle=cycle, dry_run=False, graph_file=None, task="foo" + batch=True, + config="config.yaml", + cycle=cycle, + dry_run=False, + graph_file=None, + task="foo", + stdin_ok=True, ) @@ -547,11 +585,18 @@ def test__dispatch_mpas_init(): "cycle": cycle, "dry_run": False, "graph_file": None, + "stdin_ok": True, } with patch.object(uwtools.api.mpas_init, "execute") as execute: cli._dispatch_mpas_init({**args, "action": "foo"}) execute.assert_called_once_with( - batch=True, config="config.yaml", cycle=cycle, dry_run=False, graph_file=None, task="foo" + batch=True, + config="config.yaml", + cycle=cycle, + dry_run=False, + graph_file=None, + task="foo", + stdin_ok=True, ) @@ -610,11 +655,12 @@ def test__dispatch_sfc_climo_gen(): "config_file": "config.yaml", "dry_run": False, "graph_file": None, + "stdin_ok": True, } with patch.object(uwtools.api.sfc_climo_gen, "execute") as execute: cli._dispatch_sfc_climo_gen({**args, "action": "foo"}) execute.assert_called_once_with( - batch=True, config="config.yaml", dry_run=False, graph_file=None, task="foo" + batch=True, config="config.yaml", dry_run=False, graph_file=None, task="foo", stdin_ok=True ) @@ -671,6 +717,7 @@ def test__dispatch_template_render_no_optional(): searchpath=None, values_needed=False, dry_run=False, + stdin_ok=True, ) @@ -698,6 +745,7 @@ def test__dispatch_template_render_yaml(): searchpath=6, values_needed=7, dry_run=8, + stdin_ok=True, ) @@ -737,11 +785,18 @@ def test__dispatch_ungrib(): "cycle": cycle, "dry_run": False, "graph_file": None, + "stdin_ok": True, } with patch.object(uwtools.api.ungrib, "execute") as execute: cli._dispatch_ungrib({**args, "action": "foo"}) execute.assert_called_once_with( - task="foo", batch=True, config="config.yaml", cycle=cycle, dry_run=False, graph_file=None + task="foo", + batch=True, + config="config.yaml", + cycle=cycle, + dry_run=False, + graph_file=None, + stdin_ok=True, ) diff --git a/src/uwtools/tests/test_file.py b/src/uwtools/tests/test_file.py index 59d188df9..f9753b61e 100644 --- a/src/uwtools/tests/test_file.py +++ b/src/uwtools/tests/test_file.py @@ -36,7 +36,7 @@ def test_FileStager(assets, source): @pytest.mark.parametrize("source", ("dict", "file")) def test_FileStager_bad_key(assets, source): - dstdir, cfgfile, cfgdict = assets + dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile with raises(UWConfigError) as e: file.FileStager(target_dir=dstdir, config=config, keys=["a", "x"]) @@ -65,3 +65,12 @@ def test_FileLinker_config_file(assets, source): stager.go() assert (dstdir / "foo").is_symlink() assert (dstdir / "subdir" / "bar").is_symlink() + + +@pytest.mark.parametrize("val", [None, True, False, "str", 88, 3.14, [], tuple()]) +def test_FileStager_empty_val(assets, val): + dstdir, cfgdict, _ = assets + cfgdict["a"]["b"] = val + with raises(UWConfigError) as e: + file.FileStager(target_dir=dstdir, config=cfgdict, keys=["a", "b"]) + assert str(e.value) == "No file map found at a -> b" diff --git a/src/uwtools/tests/utils/test_api.py b/src/uwtools/tests/utils/test_api.py new file mode 100644 index 000000000..5f0ab6e58 --- /dev/null +++ b/src/uwtools/tests/utils/test_api.py @@ -0,0 +1,43 @@ +# pylint: disable=missing-function-docstring + +from pathlib import Path + +import pytest +from pytest import raises + +from uwtools.exceptions import UWError +from uwtools.utils import api + + +@pytest.mark.parametrize("val", [Path("/some/path"), {"foo": 88}]) +def test_ensure_data_source_passthrough(val): + assert api.ensure_data_source(data_source=val, stdin_ok=False) == val + + +def test_ensure_data_source_stdin_not_ok(): + with raises(UWError) as e: + api.ensure_data_source(data_source=None, stdin_ok=False) + assert str(e.value) == "Set stdin_ok=True to permit read from stdin" + + +def test_ensure_data_source_stdin_ok(): + assert api.ensure_data_source(data_source=None, stdin_ok=True) is None + + +def test_ensure_data_source_str_to_path(): + val = "/some/path" + result = api.ensure_data_source(data_source=val, stdin_ok=False) + assert isinstance(result, Path) + assert result == Path(val) + + +@pytest.mark.parametrize("val", [Path("/some/path"), {"foo": 88}]) +def test_str2path_passthrough(val): + assert api.str2path(val) == val + + +def test_str2path_convert(): + val = "/some/path" + result = api.str2path(val) + assert isinstance(result, Path) + assert result == Path(val) diff --git a/src/uwtools/tests/utils/test_tasks.py b/src/uwtools/tests/utils/test_tasks.py index 4f888edcc..f9c552a78 100644 --- a/src/uwtools/tests/utils/test_tasks.py +++ b/src/uwtools/tests/utils/test_tasks.py @@ -1,25 +1,42 @@ # pylint: disable=missing-function-docstring -from iotaa import refs +import os from uwtools.utils import tasks +def test_tasks_existing_missing(tmp_path): + path = tmp_path / "x" + assert not tasks.existing(path=path).ready() # type: ignore # pylint: disable=no-member + + +def test_tasks_existing_present_directory(tmp_path): + path = tmp_path / "directory" + path.mkdir() + assert tasks.existing(path=path).ready() # type: ignore # pylint: disable=no-member + + +def test_tasks_existing_present_file(tmp_path): + path = tmp_path / "file" + path.touch() + assert tasks.existing(path=path).ready() # type: ignore # pylint: disable=no-member + + +def test_tasks_existing_present_symlink(tmp_path): + path = tmp_path / "symlink" + path.symlink_to(os.devnull) + assert tasks.existing(path=path).ready() # type: ignore # pylint: disable=no-member + + def test_tasks_file_missing(tmp_path): path = tmp_path / "file" - assert not path.is_file() - asset_id = refs(tasks.file(path=path)) - assert asset_id == path - assert not asset_id.is_file() + assert not tasks.file(path=path).ready() # type: ignore # pylint: disable=no-member def test_tasks_file_present(tmp_path): path = tmp_path / "file" path.touch() - assert path.is_file() - asset_id = refs(tasks.file(path=path)) - assert asset_id == path - assert asset_id.is_file() + assert tasks.file(path=path).ready() # type: ignore # pylint: disable=no-member def test_tasks_filecopy(tmp_path): diff --git a/src/uwtools/utils/api.py b/src/uwtools/utils/api.py new file mode 100644 index 000000000..7b0889366 --- /dev/null +++ b/src/uwtools/utils/api.py @@ -0,0 +1,34 @@ +""" +Support for API modules. +""" + +from pathlib import Path +from typing import Any, Optional, Union + +from uwtools.config.formats.base import Config +from uwtools.exceptions import UWError + + +def ensure_data_source( + data_source: Optional[Union[dict, Config, Path, str]], stdin_ok: bool +) -> Any: + """ + If stdin read is disabled, ensure that a data source was provided. Convert str -> Path. + + :param data_source: Data source as provided to API. + :param stdin_ok: OK to read from stdin? + :return: Data source, with a str converted to Path. + :raises: UWError if no data source was provided and stdin read is disabled. + """ + if data_source is None and not stdin_ok: + raise UWError("Set stdin_ok=True to permit read from stdin") + return str2path(data_source) + + +def str2path(val: Any) -> Any: + """ + Return str value as Path, other types unmodified. + + :param val: Any value. + """ + return Path(val) if isinstance(val, str) else val diff --git a/src/uwtools/utils/tasks.py b/src/uwtools/utils/tasks.py index 4c91ee2a8..9f716933d 100644 --- a/src/uwtools/utils/tasks.py +++ b/src/uwtools/utils/tasks.py @@ -9,6 +9,17 @@ from iotaa import asset, external, task +@external +def existing(path: Path): + """ + An existing filesystem item (file, directory, or symlink). + + :param path: Path to the item. + """ + yield "Filesystem item %s" % path + yield asset(path, path.exists) + + @external def file(path: Path): """ @@ -45,6 +56,6 @@ def symlink(target: Path, linkname: Path): """ yield "Link %s -> %s" % (linkname, target) yield asset(linkname, linkname.exists) - yield file(target) + yield existing(target) linkname.parent.mkdir(parents=True, exist_ok=True) os.symlink(src=target, dst=linkname)