Merge pull request #78 from NREL/bnb/post_proc_fix

fix config creation for post proc blend collection
NREL · Jan 7, 2025 · 6e255e0 · 6e255e0
2 parents 1828909 + 386bcc6
commit 6e255e0
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 15 deletions.
diff --git a/nsrdb/blend/blend.py b/nsrdb/blend/blend.py
@@ -140,17 +140,6 @@ def _parse_blended_meta(self):
         self._meta_west = self._meta_west[west_mask]
         self._meta_east = self._meta_east[east_mask]
 
-        """
-        if len(self._meta_east) < 10:
-            e = 'Eastern meta got totally eliminated by seam mask!'
-            logger.error(e)
-            raise RuntimeError(e)
-        if len(self._meta_west) < 10:
-            e = 'Western meta got totally eliminated by seam mask!'
-            logger.error(e)
-            raise RuntimeError(e)
-        """
-
         west_gid_full = self._meta_west[self._map_col].values.tolist()
         east_gid_full = self._meta_east[self._map_col].values.tolist()
         gid_full_all = list(set(west_gid_full + east_gid_full))

diff --git a/nsrdb/config/create_configs.py b/nsrdb/config/create_configs.py
@@ -385,6 +385,7 @@ def post(cls, kwargs):
         pipeline_config = {'pipeline': []}
         out_dir = os.path.abspath(kwargs.get('out_dir', './'))
         post_proc_dir = os.path.join(out_dir, 'post_proc')
+        kwargs['post_proc_dir'] = kwargs.get('post_proc_dir', post_proc_dir)
         kwargs['main_dir'] = out_dir
 
         if kwargs['year'] > 2018:
@@ -411,6 +412,10 @@ def post(cls, kwargs):
                 config['out_dir'] = os.path.join(
                     post_proc_dir, config['run_name']
                 )
+            if mod_name == 'collect-blend':
+                config['collect_dir'] = os.path.join(
+                    post_proc_dir, config['run_name'].replace('_collect', '')
+                )
             cls._write_config(
                 config,
                 os.path.join(post_proc_dir, fname),
@@ -611,16 +616,19 @@ def _collect_blend(cls, kwargs):
 
         config = cls.init_kwargs(kwargs, COLLECT_BLEND_KWARGS)
         config['meta_final'] = cls._get_meta(config, run_type='collect-blend')
-        config['collect_dir'] = cls._get_run_name(config, run_type='blend')
-        config['collect_tag'] = config['collect_dir'].replace('_blend', '')
+        config['collect_dir'] = os.path.join(
+            config['out_dir'],
+            'post_proc',
+            cls._get_run_name(config, run_type='blend'),
+        )
         config['fout'] = os.path.join(
             f'{config["out_dir"]}',
             f'{config["basename"]}_{config["year"]}.h5',
         )
-
         config['run_name'] = cls._get_run_name(
             config, run_type='collect-blend'
         )
+        config['collect_tag'] = f'{config["basename"]}_{config["extent"]}'
         return config
 
     @classmethod

diff --git a/nsrdb/file_handlers/collection.py b/nsrdb/file_handlers/collection.py
@@ -356,6 +356,12 @@ def _get_collection_attrs(
             Output (collected) dataset shape
         """
 
+        logger.info(
+            'Getting collection attrs from collect_dir=%s, flist=%s',
+            collect_dir,
+            flist,
+        )
+
         if sort:
             flist = sorted(flist, key=sort_key)
 

diff --git a/nsrdb/gap_fill/mlclouds_fill.py b/nsrdb/gap_fill/mlclouds_fill.py
@@ -74,7 +74,7 @@ def __init__(
             'Initializing MLCloudsFill with model: {}'.format(model_path)
         )
         logger.info(
-            'MLCloudsFill fill filling all cloud properties: {}'.format(
+            'MLCloudsFill filling all cloud properties: {}'.format(
                 self._fill_all
             )
         )