Skip to content

Commit

Permalink
default kwargs for all modules
Browse files Browse the repository at this point in the history
  • Loading branch information
@bnb32
bnb32 committed Jul 8, 2024
1 parent 71d20c0 commit 9db5f7e
Show file tree
Hide file tree
Showing 2 changed files with 58 additions and 75 deletions.
36 changes: 10 additions & 26 deletions nsrdb/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,32 +286,16 @@ def create_configs(
init_logger('nsrdb.create_configs', log_level='DEBUG')

ctx.ensure_object(dict)
if run_type == 'full':
CreateConfigs.full(config)
elif run_type == 'main':
if all_domains:
CreateConfigs.main_all(config)
else:
CreateConfigs.main(config)
elif run_type == 'aggregate':
if collect:
CreateConfigs.collect_aggregate(config)
else:
CreateConfigs.aggregate(config)
elif run_type == 'blend':
if collect:
CreateConfigs.collect_blend(config)
else:
CreateConfigs.blend(config)
elif run_type == 'post':
CreateConfigs.post(config)
else:
msg = (
f'Received unknown "run_type" {run_type}. Accepted values are '
'"main", "post", "aggregate", and "blend"'
)
logger.error(msg)
raise ValueError(msg)
func_name = f'collect_{run_type}' if collect else run_type
func_name = 'main_all' if run_type == 'main' and all_domains else func_name
valid_types = ['full', 'main', 'aggregate', 'blend', 'post']
msg = (
f'Received unknown "run_type" {run_type}. Accepted values are '
f'{valid_types}'
)
assert hasattr(CreateConfigs, func_name), msg
config_func = getattr(CreateConfigs, func_name)
config_func(config)


@main.command()
Expand Down
97 changes: 48 additions & 49 deletions nsrdb/create_configs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,47 @@

DEFAULT_META_DIR = '/projects/pxs/reference_grids/'

DEFAULT_KWARGS = {

BASE_KWARGS = {
'basename': 'nsrdb',
'out_dir': './',
'execution_control': DEFAULT_EXEC_CONFIG,
'meta_dir': DEFAULT_META_DIR,
}

MAIN_KWARGS = {
**BASE_KWARGS,
'freq': '30min',
'spatial': '4km',
'satellite': 'east',
'extent': 'full',
'out_dir': './',
'execution_control': DEFAULT_EXEC_CONFIG,
'meta_dir': DEFAULT_META_DIR,
}

BLEND_KWARGS = {
**BASE_KWARGS,
'file_tag': 'all',
'extent': 'full',
'main_dir': '../',
}

AGG_KWARGS = {
**BASE_KWARGS,
'full_spatial': '2km',
'conus_spatial': '2km',
'final_spatial': '4km',
'data_dir': './',
'full_freq': '10min',
'conus_freq': '5min',
'final_freq': '30min',
'n_chunks': 32,
'source_priority': ['conus', 'full_disk'],
}

COLLECT_AGG_KWARGS = {
**BASE_KWARGS,
'final_spatial': '4km',
'final_freq': '30min',
'max_workers': 10,
}


Expand All @@ -65,7 +97,7 @@ def main(cls, kwargs):
Dictionary of parameters including year, basename, satellite,
extent, freq, spatial, meta_file, doy_range
"""
config = copy.deepcopy(DEFAULT_KWARGS)
config = copy.deepcopy(MAIN_KWARGS)
config.update(kwargs)
config['out_dir'] = os.path.abspath(config['out_dir'])
os.makedirs(config['out_dir'], exist_ok=True)
Expand Down Expand Up @@ -209,7 +241,7 @@ def _update_run_templates(cls, config):
def _get_run_name(cls, config, run_type='main'):
"""Get name of run for given main run input."""
config.update(
{k: v for k, v in DEFAULT_KWARGS.items() if k not in config}
{k: v for k, v in MAIN_KWARGS.items() if k not in config}
)
pattern_dict = {
'main': cls.MAIN_RUN_NAME,
Expand Down Expand Up @@ -344,20 +376,7 @@ def _aggregate(cls, kwargs):
Dictionary with keys specifying the case for which to aggregate
files
"""
default_kwargs = {
'full_spatial': '2km',
'conus_spatial': '2km',
'final_spatial': '4km',
'data_dir': './',
'full_freq': '10min',
'conus_freq': '5min',
'final_freq': '30min',
'n_chunks': 32,
'source_priority': ['conus', 'full_disk'],
'meta_dir': DEFAULT_META_DIR,
}

config = copy.deepcopy(default_kwargs)
config = copy.deepcopy(AGG_KWARGS)
config.update(kwargs)

if config['year'] == 2018:
Expand Down Expand Up @@ -409,16 +428,7 @@ def _blend(cls, kwargs):
Dictionary with keys specifying the case for which to blend data
files
"""
default_kwargs = {
'file_tag': 'all',
'spatial': '4km',
'extent': 'full',
'main_dir': '../',
'chunk_size': 100000,
'freq': '30min',
'meta_dir': DEFAULT_META_DIR,
}
config = copy.deepcopy(default_kwargs)
config = copy.deepcopy(BLEND_KWARGS)
config.update(kwargs)

if config['year'] > 2017:
Expand All @@ -442,13 +452,15 @@ def _blend(cls, kwargs):
meta_file += '.csv'
config['meta_file'] = os.path.join(config['meta_dir'], meta_file)

config['satellite'] = 'east'
config['east_dir'] = os.path.join(
config['main_dir'], cls._get_run_name(config), 'final'
config['main_dir'],
cls._get_run_name({'satellite': 'east', **config}),
'final',
)
config['satellite'] = 'west'
config['west_dir'] = os.path.join(
config['main_dir'], cls._get_run_name(config), 'final'
config['main_dir'],
cls._get_run_name({'satellite': 'west', **config}),
'final',
)
config['run_name'] = config.get(
'run_name', cls._get_run_name(config, run_type='blend')
Expand Down Expand Up @@ -518,14 +530,7 @@ def _collect_blend(cls, kwargs):
Dictionary with keys specifying the case for blend collection
"""

default_kwargs = {
'spatial': '4km',
'freq': '30min',
'extent': 'full',
'meta_dir': DEFAULT_META_DIR,
}

config = copy.deepcopy(default_kwargs)
config = copy.deepcopy(BASE_KWARGS)
config.update(kwargs)

config['meta'] = os.path.join(
Expand Down Expand Up @@ -582,13 +587,7 @@ def _collect_aggregate(cls, kwargs):
kwargs : dict
Dictionary with keys specifying the case for aggregation collection
"""
default_kwargs = {
'final_spatial': '4km',
'final_freq': '30min',
'meta_dir': DEFAULT_META_DIR,
}

config = copy.deepcopy(default_kwargs)
config = copy.deepcopy(COLLECT_AGG_KWARGS)
config.update(kwargs)

meta_file = f'nsrdb_meta_{config["final_spatial"]}.csv'
Expand Down

0 comments on commit 9db5f7e

Please sign in to comment.