Merge branch 'develop' into add-ML-wisp-finder

jwql/jwql_monitors/generate_preview_images.py

@@ -36,8 +36,11 @@
 import numpy as np
 
 from jwql.utils import permissions
-from jwql.utils.constants import IGNORED_SUFFIXES, JWST_INSTRUMENT_NAMES_SHORTHAND, NIRCAM_LONGWAVE_DETECTORS, \
-    NIRCAM_SHORTWAVE_DETECTORS, PREVIEW_IMAGE_LISTFILE, THUMBNAIL_LISTFILE
+from jwql.utils.constants import (IGNORED_SUFFIXES,
+                                  JWST_INSTRUMENT_NAMES_SHORTHAND,
+                                  NIRCAM_LONGWAVE_DETECTORS,
+                                  NIRCAM_SHORTWAVE_DETECTORS
+                                  )
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.protect_module import lock_module
 from jwql.utils.preview_image import PreviewImage
@@ -616,22 +619,6 @@ def generate_preview_images(overwrite, programs=None):
         full_preview_files.extend(r[0])
         full_thumbnail_files.extend(r[1])
 
-    # Filter the preview and thumbnail images by instrument and update the listfiles.
-    # We do this by looking for instrument abbreviations in the filenames. But will
-    # this work for level 3 files?? If an instrument abbreviation is not in the filename,
-    # then the preview/thubnail images won't be caught and added here.
-    for abbrev, inst_name in JWST_INSTRUMENT_NAMES_SHORTHAND.items():
-        inst_previews = [ele for ele in full_preview_files if re.search(abbrev, ele, re.IGNORECASE)]
-        inst_thumbs = [ele for ele in full_thumbnail_files if abbrev in ele]
-
-        # Read in the preview image listfile and the thumbnail image list file
-        # and add these new files to each
-        preview_image_listfile = os.path.join(SETTINGS['preview_image_filesystem'], f"{PREVIEW_IMAGE_LISTFILE}_{inst_name}.txt")
-        update_listfile(preview_image_listfile, inst_previews, 'preview')
-
-        thumbnail_image_listfile = os.path.join(SETTINGS['thumbnail_filesystem'], f"{THUMBNAIL_LISTFILE}_{inst_name}.txt")
-        update_listfile(thumbnail_image_listfile, inst_thumbs, 'thumbnail')
-
     # Complete logging:
     logging.info("Completed.")
 
@@ -842,43 +829,6 @@ def process_program(program, overwrite):
     return preview_image_files, thumbnail_files
 
 
-def update_listfile(filename, file_list, filetype):
-    """Add a list of files to a text file. Designed to add new files to the
-    file containing the list of all preview images and the file containing the
-    list of all thumbnail images.
-
-    Parameters
-    ----------
-    filename : str
-        Name, including path, of the file to be amended/created
-
-    file_list : list
-        List of filenames to be added to filename
-
-    filetype : str
-        Descriptor of the contents of the file being amended. Used only for
-        the logging statement
-    """
-    if len(file_list) > 0:
-        if not os.path.isfile(filename):
-            logging.warning(f"{filetype} image listfile not found!! Expected to be at {filename}. Creating a new file.")
-
-        with open(filename, 'a+') as fobj:
-            # Move read cursor to the start of file.
-            fobj.seek(0)
-
-            # If file is not empty then append '\n'
-            data = fobj.read(100)
-            if len(data) > 0:
-                fobj.write("\n")
-
-            # Append file_list at the end of file
-            for file_to_add in file_list:
-                fobj.write(f'{file_to_add}\n')
-
-        logging.info(f"{filetype} image listfile {filename} updated with new entries.")
-
-
 @lock_module
 def protected_code(overwrite, programs):
     """Protected code ensures only 1 instance of module will run at any given time

jwql/tests/test_bokeh_dashboard.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+
+"""Tests for the ``bokeh_dashboard`` module in the ``jwql`` web
+application.
+
+Authors
+-------
+
+    - Bryan Hilbert
+
+Use
+---
+
+    These tests can be run via the command line (omit the -s to
+    suppress verbose output to stdout):
+
+    ::
+
+        pytest -s test_bokeh_dashboard.py
+"""
+
+import os
+
+from django import setup
+import pandas as pd
+import pytest
+
+from jwql.utils.constants import DEFAULT_MODEL_CHARFIELD, ON_GITHUB_ACTIONS, ON_READTHEDOCS
+
+# Skip testing this module if on Github Actions
+if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS:
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings")
+    setup()
+    from jwql.website.apps.jwql import bokeh_dashboard  # noqa: E402 (module level import not at top of file)
+
+
+@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.')
+def test_build_table_latest_entry():
+    tab = bokeh_dashboard.build_table('FilesystemCharacteristics')
+    assert isinstance(tab, pd.DataFrame)
+    assert len(tab['date']) > 0

jwql/tests/test_data_containers.py

@@ -48,7 +48,7 @@
 
 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.')
 def test_build_table():
-    tab = data_containers.build_table('filesystem_general')
+    tab = data_containers.build_table('FilesystemGeneral')
     assert isinstance(tab, pd.DataFrame)
     assert len(tab['date']) > 0
 

jwql/utils/constants.py

@@ -795,10 +795,6 @@
 # Determine if the code is being run as part of a Readthedocs build
 ON_READTHEDOCS = os.environ.get('READTHEDOCS', False)
 
-# Base name for the file listing the preview images for a given instrument.
-# The complete name will have "_{instrument.lower}.txt" added to the end of this.
-PREVIEW_IMAGE_LISTFILE = "preview_image_inventory"
-
 # All possible proposal categories
 PROPOSAL_CATEGORIES = ["AR", "CAL", "COM", "DD", "ENG", "GO", "GTO", "NASA", "SURVEY"]
 
@@ -1008,10 +1004,6 @@ class QueryConfigKeys:
 # boolean accessed according to a viewed flag
 THUMBNAIL_FILTER_LOOK = ["New", "Viewed"]
 
-# Base name for the file listing the thumbnail images for a given instrument.
-# The complete name will have "_{instrument.lower}.txt" added to the end of this.
-THUMBNAIL_LISTFILE = "thumbnail_inventory"
-
 # Possible suffix types for time-series exposures
 TIME_SERIES_SUFFIX_TYPES = ["phot", "whtlt"]
 

jwql/website/apps/jwql/apps.py

@@ -0,0 +1,30 @@
+#! /usr/bin/env python
+
+"""
+apps.py is the standard and recommended way to configure application-specific settings
+in Django, including tasks like importing additional modules during initialization.
+
+Author
+------
+
+B. Hilbert
+"""
+
+from django.apps import AppConfig
+
+
+class JwqlAppConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'jwql'
+
+    def ready(self):
+        # Import models not defined in models.py here
+        # By importing these models here, they will be available
+        # to the build_table() function.
+        import jwql.website.apps.jwql.monitor_models.bad_pixel
+        import jwql.website.apps.jwql.monitor_models.bias
+        import jwql.website.apps.jwql.monitor_models.claw
+        import jwql.website.apps.jwql.monitor_models.common
+        import jwql.website.apps.jwql.monitor_models.dark_current
+        import jwql.website.apps.jwql.monitor_models.readnoise
+        import jwql.website.apps.jwql.monitor_models.ta