[WIP]: Monitor status table

CHANGES.rst

@@ -1,5 +1,21 @@
 ## What's Changed
 
+1.2.5 (2024-03-19)
+==================
+
+Web Application
+~~~~~~~~~~~~~~~
+- Fix Bokeh `file_html` Call by @mfixstsci
+- Update Bad Pix Exclude Line by @mfixstsci
+- Interactive preview image - updates for Bokeh 3 by @bhilbert4
+
+Project & API Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- Allow creation of pngs from 3D and 4D arrays by @bhilbert4
+- Add max length to charfield by @BradleySappington
+- Header fix by @BradleySappington
+
+
 1.2.4 (2024-03-11)
 ==================
 

jwql/instrument_monitors/nircam_monitors/claw_monitor.py

@@ -192,6 +192,7 @@ def make_background_plots(self, plot_type='bkg'):
                     df = df[df['stddev'] != 0]  # older data has no accurate stddev measures
                     plot_data = df['stddev'].values
                 if plot_type == 'model':
+                    df = df[np.isfinite(df['total_bkg'])]  # the claw monitor did not track model measurements at first
                     plot_data = df['median'].values / df['total_bkg'].values
                 plot_expstarts = df['expstart_mjd'].values
 
@@ -300,7 +301,11 @@ def process(self):
 
                 # Get predicted background level using JWST background tool
                 ra, dec = hdu[1].header['RA_V1'], hdu[1].header['DEC_V1']
-                wv = self.filter_wave[self.fltr.upper()]
+                if ('N' in self.pupil.upper()) | ('M' in self.pupil.upper()):
+                    fltr_wv = self.pupil.upper()
+                else:
+                    fltr_wv = self.fltr.upper()
+                wv = self.filter_wave[fltr_wv]
                 date = hdu[0].header['DATE-BEG']
                 doy = int(Time(date).yday.split(':')[1])
                 try:
@@ -332,7 +337,7 @@ def process(self):
                                  'skyflat_filename': os.path.basename(self.outfile),
                                  'doy': float(doy),
                                  'total_bkg': float(total_bkg),
-                                 'entry_date': datetime.datetime.now()
+                                 'entry_date': datetime.datetime.now(datetime.timezone.utc)
                                  }
                 entry = self.stats_table(**claw_db_entry)
                 entry.save()
@@ -423,11 +428,13 @@ def run(self):
         mast_table = self.query_mast()
         logging.info('{} files found between {} and {}.'.format(len(mast_table), self.query_start_mjd, self.query_end_mjd))
 
-        # Define pivot wavelengths
-        self.filter_wave = {'F070W': 0.704, 'F090W': 0.902, 'F115W': 1.154, 'F150W': 1.501, 'F150W2': 1.659,
-                            'F200W': 1.989, 'F212N': 2.121, 'F250M': 2.503, 'F277W': 2.762, 'F300M': 2.989,
-                            'F322W2': 3.232, 'F356W': 3.568, 'F410M': 4.082, 'F430M': 4.281, 'F444W': 4.408,
-                            'F480M': 4.874}
+        # Define pivot wavelengths - last downloaded March 8 2024 from:
+        # https://jwst-docs.stsci.edu/jwst-near-infrared-camera/nircam-instrumentation/nircam-filters
+        self.filter_wave = {'F070W': 0.704, 'F090W': 0.901, 'F115W': 1.154, 'F140M': 1.404, 'F150W': 1.501, 'F162M': 1.626, 'F164N': 1.644,
+                            'F150W2': 1.671, 'F182M': 1.845, 'F187N': 1.874, 'F200W': 1.99, 'F210M': 2.093, 'F212N': 2.12, 'F250M': 2.503,
+                            'F277W': 2.786, 'F300M': 2.996, 'F322W2': 3.247, 'F323N': 3.237, 'F335M': 3.365, 'F356W': 3.563, 'F360M': 3.621,
+                            'F405N': 4.055, 'F410M': 4.092, 'F430M': 4.28, 'F444W': 4.421, 'F460M': 4.624, 'F466N': 4.654, 'F470N': 4.707,
+                            'F480M': 4.834}
 
         # Create observation-level median stacks for each filter/pupil combo, in pixel-space
         combos = np.array(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in mast_table])
@@ -469,7 +476,7 @@ def run(self):
                      'start_time_mjd': self.query_start_mjd,
                      'end_time_mjd': self.query_end_mjd,
                      'run_monitor': monitor_run,
-                     'entry_date': datetime.datetime.now()}
+                     'entry_date': datetime.datetime.now(datetime.timezone.utc)}
         entry = self.query_table(**new_entry)
         entry.save()
 

jwql/pull_jwql_branch.sh

@@ -62,6 +62,7 @@ echo "Reset: $reset";
 echo "Notify: $notify $recipient";
 
 # 1. Pull updated code from GitHub deployment branch (keep second checkout in case its already defined for some weird reason)
+git fetch origin
 git checkout -b $branch_name --track origin/$branch_name
 git checkout $branch_name
 git fetch origin $branch_name

jwql/tests/test_data_containers.py

@@ -31,7 +31,7 @@
 import pandas as pd
 import pytest
 
-from jwql.utils.constants import ON_GITHUB_ACTIONS
+from jwql.utils.constants import ON_GITHUB_ACTIONS, DEFAULT_MODEL_CHARFIELD
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings")
 
@@ -45,7 +45,7 @@
     from jwql.utils.utils import get_config  # noqa: E402 (module level import not at top of file)
     from jwql.website.apps.jwql.models import RootFileInfo
 
-    
+
 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.')
 def test_build_table():
     tab = data_containers.build_table('filesystem_general')
@@ -199,7 +199,6 @@ def test_get_all_proposals():
                            (['uncal', 'rate', 'o001_crf', 'o006_crfints', 'bad'], {'bad'})),
                           (False, ['rate', 'uncal', 'bad', 'o006_crfints', 'o001_crf'],
                            ['uncal', 'rate', 'o001_crf', 'o006_crfints', 'bad'])])
-
 def test_get_available_suffixes(untracked, input_suffixes, expected):
     result = data_containers.get_available_suffixes(
         input_suffixes, return_untracked=untracked)
@@ -339,6 +338,7 @@ def test_get_anomaly_form_post_group(mocker):
     assert update_mock.call_count == 2
 """
 
+
 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.')
 def test_get_dashboard_components():
     request = MockPostRequest()
@@ -607,42 +607,42 @@ def test_mast_query_by_rootname():
     instrument = 'NIRCam'
     rootname1 = 'jw02767002001_02103_00005_nrcb4'
     dict_stuff = data_containers.mast_query_by_rootname(instrument, rootname1)
-    defaults = dict(filter=dict_stuff.get('filter', ''),
-                    detector=dict_stuff.get('detector', ''),
-                    exp_type=dict_stuff.get('exp_type', ''),
-                    read_pat=dict_stuff.get('readpatt', ''),
-                    grating=dict_stuff.get('grating', ''),
+    defaults = dict(filter=dict_stuff.get('filter', DEFAULT_MODEL_CHARFIELD),
+                    detector=dict_stuff.get('detector', DEFAULT_MODEL_CHARFIELD),
+                    exp_type=dict_stuff.get('exp_type', DEFAULT_MODEL_CHARFIELD),
+                    read_pat=dict_stuff.get('readpatt', DEFAULT_MODEL_CHARFIELD),
+                    grating=dict_stuff.get('grating', DEFAULT_MODEL_CHARFIELD),
                     patt_num=dict_stuff.get('patt_num', 0),
-                    aperture=dict_stuff.get('apername', ''),
-                    subarray=dict_stuff.get('subarray', ''),
-                    pupil=dict_stuff.get('pupil', ''))
+                    aperture=dict_stuff.get('apername', DEFAULT_MODEL_CHARFIELD),
+                    subarray=dict_stuff.get('subarray', DEFAULT_MODEL_CHARFIELD),
+                    pupil=dict_stuff.get('pupil', DEFAULT_MODEL_CHARFIELD))
     assert isinstance(defaults, dict)
 
     rootname2 = 'jw02084001001_04103_00001-seg003_nrca3'
     dict_stuff = data_containers.mast_query_by_rootname(instrument, rootname2)
-    defaults = dict(filter=dict_stuff.get('filter', ''),
-                    detector=dict_stuff.get('detector', ''),
-                    exp_type=dict_stuff.get('exp_type', ''),
-                    read_pat=dict_stuff.get('readpatt', ''),
-                    grating=dict_stuff.get('grating', ''),
+    defaults = dict(filter=dict_stuff.get('filter', DEFAULT_MODEL_CHARFIELD),
+                    detector=dict_stuff.get('detector', DEFAULT_MODEL_CHARFIELD),
+                    exp_type=dict_stuff.get('exp_type', DEFAULT_MODEL_CHARFIELD),
+                    read_pat=dict_stuff.get('readpatt', DEFAULT_MODEL_CHARFIELD),
+                    grating=dict_stuff.get('grating', DEFAULT_MODEL_CHARFIELD),
                     patt_num=dict_stuff.get('patt_num', 0),
-                    aperture=dict_stuff.get('apername', ''),
-                    subarray=dict_stuff.get('subarray', ''),
-                    pupil=dict_stuff.get('pupil', ''))
+                    aperture=dict_stuff.get('apername', DEFAULT_MODEL_CHARFIELD),
+                    subarray=dict_stuff.get('subarray', DEFAULT_MODEL_CHARFIELD),
+                    pupil=dict_stuff.get('pupil', DEFAULT_MODEL_CHARFIELD))
     assert isinstance(defaults, dict)
 
     instrument2 = 'FGS'
     rootname3 = 'jw01029003001_06201_00001_guider2'
     dict_stuff = data_containers.mast_query_by_rootname(instrument2, rootname3)
-    defaults = dict(filter=dict_stuff.get('filter', ''),
-                    detector=dict_stuff.get('detector', ''),
-                    exp_type=dict_stuff.get('exp_type', ''),
-                    read_pat=dict_stuff.get('readpatt', ''),
-                    grating=dict_stuff.get('grating', ''),
+    defaults = dict(filter=dict_stuff.get('filter', DEFAULT_MODEL_CHARFIELD),
+                    detector=dict_stuff.get('detector', DEFAULT_MODEL_CHARFIELD),
+                    exp_type=dict_stuff.get('exp_type', DEFAULT_MODEL_CHARFIELD),
+                    read_pat=dict_stuff.get('readpatt', DEFAULT_MODEL_CHARFIELD),
+                    grating=dict_stuff.get('grating', DEFAULT_MODEL_CHARFIELD),
                     patt_num=dict_stuff.get('patt_num', 0),
-                    aperture=dict_stuff.get('apername', ''),
-                    subarray=dict_stuff.get('subarray', ''),
-                    pupil=dict_stuff.get('pupil', ''))
+                    aperture=dict_stuff.get('apername', DEFAULT_MODEL_CHARFIELD),
+                    subarray=dict_stuff.get('subarray', DEFAULT_MODEL_CHARFIELD),
+                    pupil=dict_stuff.get('pupil', DEFAULT_MODEL_CHARFIELD))
     assert isinstance(defaults, dict)
 
 

jwql/utils/constants.py

@@ -364,6 +364,9 @@
     "wfscmb",
 ]
 
+# Default Model Values
+DEFAULT_MODEL_CHARFIELD = "empty"
+
 # Filename Component Lengths
 FILE_AC_CAR_ID_LEN = 4
 FILE_AC_O_ID_LEN = 3
@@ -669,6 +672,16 @@
     },
 }
 
+# Names of monitor Django models where information about the
+# hitory of monitor runs is stored
+MONITOR_HISTORY_MODELS = {'fgs': [('Bad Pixel', 'FGSBadPixelQueryHistory'),
+                                  ''],
+                          'nircam': [('Bad Pixel': 'NIRCamBadPixelQueryHistory'),
+                                     ('Bias', 'NIRCamBiasQueryHistory'),
+                                     ('Claw', 'NIRCamClawQueryHistory'),
+                                     ('Dark Current', 'NIRCamDarkQueryHistory'),
+                                     ('Readnoise', 'NIRCamReadnoiseQueryHistory')]}
+
 # Names of all of the monitor database tables
 MONITOR_TABLE_NAMES = [
     "fgs_bad_pixel_query_history", "fgs_bad_pixel_stats",

jwql/utils/interactive_preview_image.py

@@ -554,7 +554,7 @@ def add_interactive_controls(self, images, color_bars):
         # JS callbacks for client side controls
 
         # set alternate image visibility when scale selection changes
-        scale_group.js_on_click(CustomJS(args={'i1': images[0], 'c1': color_bars[0],
+        scale_group.js_on_change('active', CustomJS(args={'i1': images[0], 'c1': color_bars[0],
                                                'i2': images[1], 'c2': color_bars[1]},
                                          code="""
             if (i1.visible == true) {
@@ -594,10 +594,10 @@ def add_interactive_controls(self, images, color_bars):
             limit_high.js_link('value', color_bars[i].color_mapper, 'high')
 
         # reset boxes to preset range on button click
-        reset.js_on_click(limit_reset)
+        reset.js_on_event('button_click', limit_reset)
 
         # also reset when swapping limit style
-        scale_group.js_on_click(limit_reset)
+        scale_group.js_on_change('active', limit_reset)
 
         # return widgets
         spacer = Spacer(height=20)

jwql/website/apps/jwql/archive_database_update.py

@@ -26,7 +26,7 @@
         Use the '--fill_empty' argument to provide a model and field.  Updates ALL fields for any model with empty/null/0 specified field
         $ python archive_database_update.py --fill_empty rootfileinfo expstart
         WARNING: Not all fields will be populated by all model objects. This will result in updates that may not be necessary.
-                 While this will not disturb the data, it has the potential to increase run time.  
+                 While this will not disturb the data, it has the potential to increase run time.
                  Select the field that is most pertient to the models you need updated minimize run time
 
         Use the 'update' argument to update every rootfileinfo data model with the most complete information from MAST
@@ -49,6 +49,7 @@
 
 from django.apps import apps
 from jwql.utils.protect_module import lock_module
+from jwql.utils.constants import DEFAULT_MODEL_CHARFIELD
 
 # These lines are needed in order to use the Django models in a standalone
 # script (as opposed to code run as a result of a webpage request). If these
@@ -160,6 +161,16 @@ def get_updates(update_database):
         create_archived_proposals_context(inst)
 
 
+@log_info
+@log_fail
+def cleanup_past_runs():
+    logging.info("Starting cleanup_past_runs")
+    rootfileinfo_field_set = ["filter", "detector", "exp_type", "read_patt", "grating", "read_patt_num", "aperture", "subarray", "pupil", "expstart"]
+    # Consume iterator created in map with list in order to make it run
+    list(map(lambda x: fill_empty_model("rootfileinfo", x), rootfileinfo_field_set))
+    logging.info("Finished cleanup_past_runs")
+
+
 def get_all_possible_filenames_for_proposal(instrument, proposal_num):
     """Wrapper around a MAST query for filenames from a given instrument/proposal
 
@@ -332,15 +343,15 @@ def update_database_table(update, instrument, prop, obs, thumbnail, obsfiles, ty
                 # Updating defaults only on update or creation to prevent call to mast_query_by_rootname on every file name.
                 defaults_dict = mast_query_by_rootname(instrument, file)
 
-                defaults = dict(filter=defaults_dict.get('filter', ''),
-                                detector=defaults_dict.get('detector', ''),
-                                exp_type=defaults_dict.get('exp_type', ''),
-                                read_patt=defaults_dict.get('readpatt', ''),
-                                grating=defaults_dict.get('grating', ''),
-                                read_patt_num=defaults_dict.get('patt_num', 0),
-                                aperture=defaults_dict.get('apername', ''),
-                                subarray=defaults_dict.get('subarray', ''),
-                                pupil=defaults_dict.get('pupil', ''),
+                defaults = dict(filter=defaults_dict.get('filter', DEFAULT_MODEL_CHARFIELD),
+                                detector=defaults_dict.get('detector', DEFAULT_MODEL_CHARFIELD),
+                                exp_type=defaults_dict.get('exp_type', DEFAULT_MODEL_CHARFIELD),
+                                read_patt=defaults_dict.get('readpatt', DEFAULT_MODEL_CHARFIELD),
+                                grating=defaults_dict.get('grating', DEFAULT_MODEL_CHARFIELD),
+                                read_patt_num=defaults_dict.get('patt_num', 1),
+                                aperture=defaults_dict.get('apername', DEFAULT_MODEL_CHARFIELD),
+                                subarray=defaults_dict.get('subarray', DEFAULT_MODEL_CHARFIELD),
+                                pupil=defaults_dict.get('pupil', DEFAULT_MODEL_CHARFIELD),
                                 expstart=defaults_dict.get('expstart', 0.0))
 
                 for key, value in defaults.items():
@@ -369,10 +380,14 @@ def fill_empty_model(model_name, model_field):
 
     '''
 
+    is_proposal = (model_name == "proposal")
+    is_rootfileinfo = (model_name == "rootfileinfo")
+    rootfile_info_fields_default_ok = ["filter", "grating", "pupil"]
+
     model_field_null = model_field + "__isnull"
     model_field_empty = model_field + "__exact"
 
-    model = apps.get_model('jwql', model_name)
+    model = apps.get_model("jwql", model_name)
     null_models = empty_models = zero_models = model.objects.none()
 
     # filter(field__isnull=True)
@@ -387,6 +402,13 @@ def fill_empty_model(model_name, model_field):
     except ValueError:
         pass
 
+    # filter(field__exact=DEFAULT_MODEL_CHARFIELD)
+    try:
+        if is_proposal or model_field not in rootfile_info_fields_default_ok:
+            empty_models = model.objects.filter(**{model_field_empty: DEFAULT_MODEL_CHARFIELD})
+    except ValueError:
+        pass
+
     # filter(field=0)
     try:
         zero_models = model.objects.filter(**{model_field: 0})
@@ -396,9 +418,9 @@ def fill_empty_model(model_name, model_field):
     model_set = null_models | empty_models | zero_models
     if model_set.exists():
         logging.info(f'{model_set.count()} models to be updated')
-        if model_name == 'proposal':
+        if is_proposal:
             fill_empty_proposals(model_set)
-        elif model_name == 'rootfileinfo':
+        elif is_rootfileinfo:
             fill_empty_rootfileinfo(model_set)
         else:
             logging.warning(f'Filling {model_name} model is not currently implemented')
@@ -458,18 +480,21 @@ def fill_empty_rootfileinfo(rootfileinfo_set):
     for rootfileinfo_mod in rootfileinfo_set:
         defaults_dict = mast_query_by_rootname(rootfileinfo_mod.instrument, rootfileinfo_mod.root_name)
 
-        defaults = dict(filter=defaults_dict.get('filter', ''),
-                        detector=defaults_dict.get('detector', ''),
-                        exp_type=defaults_dict.get('exp_type', ''),
-                        read_patt=defaults_dict.get('readpatt', ''),
-                        grating=defaults_dict.get('grating', ''),
-                        read_patt_num=defaults_dict.get('patt_num', 0),
-                        aperture=defaults_dict.get('apername', ''),
-                        subarray=defaults_dict.get('subarray', ''),
-                        pupil=defaults_dict.get('pupil', ''),
+        defaults = dict(filter=defaults_dict.get('filter', DEFAULT_MODEL_CHARFIELD),
+                        detector=defaults_dict.get('detector', DEFAULT_MODEL_CHARFIELD),
+                        exp_type=defaults_dict.get('exp_type', DEFAULT_MODEL_CHARFIELD),
+                        read_patt=defaults_dict.get('readpatt', DEFAULT_MODEL_CHARFIELD),
+                        grating=defaults_dict.get('grating', DEFAULT_MODEL_CHARFIELD),
+                        read_patt_num=defaults_dict.get('patt_num', 1),
+                        aperture=defaults_dict.get('apername', DEFAULT_MODEL_CHARFIELD),
+                        subarray=defaults_dict.get('subarray', DEFAULT_MODEL_CHARFIELD),
+                        pupil=defaults_dict.get('pupil', DEFAULT_MODEL_CHARFIELD),
                         expstart=defaults_dict.get('expstart', 0.0))
 
         for key, value in defaults.items():
+            # Final check to verify no None exists
+            if value is None:
+                value = DEFAULT_MODEL_CHARFIELD
             setattr(rootfileinfo_mod, key, value)
         try:
             rootfileinfo_mod.save()
@@ -496,6 +521,7 @@ def protected_code(update_database, fill_empty_list):
         fill_empty_model(fill_empty_list[0], fill_empty_list[1])
     else:
         get_updates(update_database)
+        cleanup_past_runs()
 
 
 if __name__ == '__main__':