diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..c6e9e36a2 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ + +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "monthly" \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b047377a..54147a822 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "3.x" @@ -32,7 +32,7 @@ jobs: python-version: [3.9, "3.10"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: mamba-org/provision-with-micromamba@v15 with: @@ -43,7 +43,7 @@ jobs: - run: pip install -e .[test] - run: conda env export - + - uses: supercharge/redis-github-action@1.4.0 with: redis-version: 5.0 diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py index 546fc6d80..cc4b7fb68 100755 --- a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py @@ -984,9 +984,10 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun else: index += 1 - min_dark_time = min(dark_obstimes) - max_dark_time = max(dark_obstimes) - mid_dark_time = instrument_properties.mean_time(dark_obstimes) + if len(dark_slope_files) > 0: + min_dark_time = min(dark_obstimes) + max_dark_time = max(dark_obstimes) + mid_dark_time = instrument_properties.mean_time(dark_obstimes) # Check whether there are still enough files left to meet the threshold if illuminated_slope_files is None: diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 0531f0337..07191e49a 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -10,6 +10,7 @@ import shutil import sys import time +import traceback from jwst import datamodels from jwst.dq_init import DQInitStep @@ -51,18 +52,27 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co status_f.write("\t start_dir is {} ({})\n".format(start_dir, type(start_dir))) status_f.write("\t uncal_file is {} ({})\n".format(uncal_file, type(uncal_file))) status_f.write(f"\t outputs is {outputs}\n") + sys.stderr.write("Running run_pipe\n") + sys.stderr.write("\t input_file_basename is {} ({})\n".format(input_file_basename, type(input_file_basename))) + sys.stderr.write("\t start_dir is {} ({})\n".format(start_dir, type(start_dir))) + sys.stderr.write("\t uncal_file is {} ({})\n".format(uncal_file, type(uncal_file))) + sys.stderr.write(f"\t outputs is {outputs}\n") try: + sys.stderr.write("Copying file {} to working directory.\n".format(input_file)) copy_files([input_file], work_directory) + sys.stderr.write("Setting permissions on {}\n".format(uncal_file)) set_permissions(uncal_file) steps = get_pipeline_steps(instrument) + sys.stderr.write("Pipeline steps initialized to {}\n".format(steps)) # If the input file is a file other than uncal.fits, then we may only need to run a # subset of steps. Check the completed steps in the input file. Find the latest step # that has been completed, and skip that plus all prior steps if 'uncal' not in input_file: completed_steps = completed_pipeline_steps(input_file) + sys.stderr.write("Steps {} already completed.\n".format(completed_steps)) # Reverse the boolean value, so that now steps answers the question: "Do we need # to run this step?"" @@ -80,18 +90,21 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co for step in steps: if not steps[step]: + sys.stderr.write("Setting last_run to {}.\n".format(step)) last_run = deepcopy(step) for step in steps: if step == last_run: break if step != last_run: + sys.stderr.write("Setting {} to skip while looking for last_run.\n".format(step)) steps[step] = False # Set any steps the user specifically asks to skip for step, step_dict in step_args.items(): if 'skip' in step_dict: if step_dict['skip']: + sys.stderr.write("Setting step {} to skip by user request.\n".format(step)) steps[step] = False # Run each specified step @@ -161,7 +174,8 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co with open(status_file, "a+") as status_f: status_f.write("EXCEPTION\n") status_f.write("{}\n".format(e)) - status_f.write("FAILED") + status_f.write("FAILED\n") + status_f.write(traceback.format_exc()) sys.exit(1) with open(status_file, "a+") as status_f: @@ -217,6 +231,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T params['refpix'] = dict(odd_even_rows=False) # Default CR rejection threshold is too low + params['jump'] = {} params['jump']['rejection_threshold'] = 15 # Set up to save jump step output @@ -284,7 +299,8 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T with open(status_file, "a+") as status_f: status_f.write("EXCEPTION\n") status_f.write("{}\n".format(e)) - status_f.write("FAILED") + status_f.write("FAILED\n") + status_f.write(traceback.format_exc()) sys.exit(1) with open(status_file, "a+") as status_f: @@ -344,7 +360,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T outputs = args.outputs step_args = args.step_args - status_file = os.path.join(working_path, short_name+"_status.txt") + status_file = os.path.join(working_path, short_name + "_status.txt") with open(status_file, 'w') as out_file: out_file.write("Starting Process\n") out_file.write("\tpipeline is {} ({})\n".format(pipe_type, type(pipe_type))) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index be232935d..f076e2035 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -1,4 +1,4 @@ - #! /usr/bin/env python +#! /usr/bin/env python """This module contains code for the celery application, which is used for any demanding work which should be restricted in terms of how many iterations are run simultaneously, or @@ -204,7 +204,7 @@ def log_subprocess_output(pipe): If a subprocess STDOUT has been set to subprocess.PIPE, this function will log each line to the logging output. """ - for line in iter(pipe.readline, b''): # b'\n'-separated lines + for line in iter(pipe.readline, b''): # b'\n'-separated lines logging.info("\t{}".format(line.decode('UTF-8').strip())) @@ -224,6 +224,7 @@ def after_setup_celery_logger(logger, **kwargs): def collect_after_task(**kwargs): gc.collect() + def convert_step_args_to_string(args_dict): """Convert the nested dictionary containing pipeline step parameter keyword/value pairs to a string so that it can be passed via command line @@ -239,17 +240,17 @@ def convert_step_args_to_string(args_dict): args_str : str String representation of ``args_dict`` """ - args_str="'{" + args_str = "'{" for i, step in enumerate(args_dict): args_str += f'"{step}":' args_str += '{' for j, (param, val) in enumerate(args_dict[step].items()): args_str += f'"{param}":"{val}"' - if j < len(args_dict[step])-1: + if j < len(args_dict[step]) - 1: args_str += ', ' args_str += "}" - if i < len(args_dict)-1: + if i < len(args_dict) - 1: args_str += ',' args_str += "}'" return args_str @@ -331,7 +332,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, current_dir = os.path.dirname(__file__) cmd_name = os.path.join(current_dir, "run_pipeline.py") outputs = ",".join(ext_or_exts) - result_file = os.path.join(cal_dir, short_name+"_status.txt") + result_file = os.path.join(cal_dir, short_name + "_status.txt") if "all" in ext_or_exts: logging.info("All outputs requested") if instrument.lower() != 'miri': @@ -399,7 +400,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, set_permissions(os.path.join(output_dir, file)) logging.info("Removing local files.") - files_to_remove = glob(os.path.join(cal_dir, short_name+"*")) + files_to_remove = glob(os.path.join(cal_dir, short_name + "*")) for file_name in files_to_remove: logging.info("\tRemoving {}".format(file_name)) os.remove(file_name) @@ -408,7 +409,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, @celery_app.task(name='jwql.shared_tasks.shared_tasks.calwebb_detector1_save_jump') -def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save_fitopt=True): +def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save_fitopt=True, step_args={}): """Call ``calwebb_detector1`` on the provided file, running all steps up to the ``ramp_fit`` step, and save the result. Optionally run the ``ramp_fit`` step and save the resulting slope file as well. @@ -430,6 +431,13 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save If ``True``, the file of optional outputs from the ramp fitting step of the pipeline is saved. + step_args : dict + A dictionary containing custom arguments to supply to individual pipeline steps. + When a step is run, the dictionary will be checked for a key matching the step + name (as defined in jwql.utils.utils.get_pipeline_steps() for the provided + instrument). The value matching the step key should, itself, be a dictionary that + can be spliced in to step.call() via dereferencing (**dict) + Returns ------- jump_output : str @@ -465,11 +473,11 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save output_dir = os.path.join(config["transfer_dir"], "outgoing") cmd_name = os.path.join(os.path.dirname(__file__), "run_pipeline.py") - result_file = os.path.join(cal_dir, short_name+"_status.txt") + result_file = os.path.join(cal_dir, short_name + "_status.txt") cores = 'all' status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, - short_name, result_file, cores) + short_name, result_file, cores, step_args) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") @@ -484,7 +492,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save if core_fail: cores = "half" status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, - input_file, short_name, result_file, cores) + input_file, short_name, result_file, cores, step_args) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") managed = True @@ -498,7 +506,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save if core_fail: cores = "none" status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, - input_file, short_name, result_file, cores) + input_file, short_name, result_file, cores, step_args) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") managed = True @@ -524,7 +532,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save files["fitopt_output"] = os.path.join(output_dir, file) logging.info("Removing local files.") - files_to_remove = glob(os.path.join(cal_dir, short_name+"*")) + files_to_remove = glob(os.path.join(cal_dir, short_name + "*")) for file_name in files_to_remove: logging.info("\tRemoving {}".format(file_name)) os.remove(file_name) diff --git a/jwql/website/apps/jwql/monitor_pages/__init__.py b/jwql/website/apps/jwql/monitor_pages/__init__.py index 1405523c0..ed184d7ff 100644 --- a/jwql/website/apps/jwql/monitor_pages/__init__.py +++ b/jwql/website/apps/jwql/monitor_pages/__init__.py @@ -1,2 +1 @@ from .monitor_cosmic_rays_bokeh import CosmicRayMonitor -from .monitor_readnoise_bokeh import ReadnoiseMonitor diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py index 8c8be095e..552cac71f 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py @@ -12,7 +12,7 @@ :: - from jwql.website.apps.jwql import monitor_pages + . monitor_template = monitor_pages.ReadnoiseMonitor() monitor_template.input_parameters = ('NIRCam', 'NRCA1_FULL') """ @@ -20,34 +20,54 @@ from datetime import datetime, timedelta import os +from bokeh.embed import components +from bokeh.layouts import column, row +from bokeh.models import Panel, Tabs # bokeh <= 3.0 +from bokeh.models import ColumnDataSource, HoverTool +# from bokeh.models import TabPanel, Tabs # bokeh >= 3.0 +from bokeh.plotting import figure import numpy as np -from jwql.bokeh_templating import BokehTemplate from jwql.database.database_interface import session from jwql.database.database_interface import FGSReadnoiseStats, MIRIReadnoiseStats, NIRCamReadnoiseStats, NIRISSReadnoiseStats, NIRSpecReadnoiseStats -from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) - - -class ReadnoiseMonitor(BokehTemplate): - - # Combine the input parameters into a single property because we - # do not want to invoke the setter unless all are updated - @property - def input_parameters(self): - return (self._instrument, self._aperture) - - @input_parameters.setter - def input_parameters(self, info): - self._instrument, self._aperture = info - self.pre_init() - self.post_init() +from jwql.utils.constants import FULL_FRAME_APERTURES, JWST_INSTRUMENT_NAMES_MIXEDCASE + + +class ReadnoiseMonitorData(): + """Class to hold bias data to be plotted + + Parameters + ---------- + + instrument : str + Instrument name (e.g. nircam) + aperture : str + Aperture name (e.g. apername) + + Attributes + ---------- + + instrument : str + Instrument name (e.g. nircam) + aperture : str + Aperture name (e.g. apername) + query_results : list + Results from read noise statistics table based on + instrument, aperture and exposure start time + stats_table : sqlalchemy.orm.decl_api.DeclarativeMeta + Statistics table object to query based on instrument + and aperture + """ + + def __init__(self, instrument, aperture): + self.instrument = instrument + self.aperture = aperture + self.load_data() def identify_tables(self): """Determine which database tables to use for the given instrument""" - mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self._instrument.lower()] + mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument.lower()] self.stats_table = eval('{}ReadnoiseStats'.format(mixed_case_name)) def load_data(self): @@ -59,102 +79,137 @@ def load_data(self): # Query database for all data in readnoise stats with a matching aperture, # and sort the data by exposure start time. self.query_results = session.query(self.stats_table) \ - .filter(self.stats_table.aperture == self._aperture) \ + .filter(self.stats_table.aperture == self.aperture) \ .order_by(self.stats_table.expstart) \ .all() session.close() - def pre_init(self): - # Start with default values for instrument and aperture because - # BokehTemplate's __init__ method does not allow input arguments - try: - dummy_instrument = self._instrument - dummy_aperture = self._aperture - except AttributeError: - self._instrument = 'NIRCam' - self._aperture = '' +class ReadNoiseFigure(): + """Generate tabbed plot displayed in JWQL web application + """ + def __init__(self, instrument): + instrument_apertures = FULL_FRAME_APERTURES[instrument.upper()] - self._embed = True - self.format_string = None - self.interface_file = os.path.join(SCRIPT_DIR, 'yaml', 'monitor_readnoise_interface.yaml') + self.tabs = [] + for aperture in instrument_apertures: + readnoise_tab = ReadNoisePlotTab(instrument, aperture) + self.tabs.append(readnoise_tab.tab) - def post_init(self): + self.plot = Tabs(tabs=self.tabs) + self.tab_components = components(self.plot) - # Load the readnoise data - self.load_data() - # Update the mean readnoise figures - self.update_mean_readnoise_figures() +class ReadNoisePlotTab(): + """Class to make instrument/aperture panels + """ + def __init__(self, instrument, aperture): + self.instrument = instrument + self.aperture = aperture - # Update the readnoise difference image and histogram - self.update_readnoise_diff_plots() + self.db = ReadnoiseMonitorData(self.instrument, self.aperture) - def update_mean_readnoise_figures(self): - """Updates the mean readnoise bokeh plots""" + self.plot_readnoise_amplifers() + self.plot_readnoise_difference_image() + self.plot_readnoise_histogram() - # Get the dark exposures info - filenames = [os.path.basename(result.uncal_filename).replace('_uncal.fits', '') for result in self.query_results] - expstarts_iso = np.array([result.expstart for result in self.query_results]) - expstarts = np.array([datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%f') for date in expstarts_iso]) - nints = [result.nints for result in self.query_results] - ngroups = [result.ngroups for result in self.query_results] + self.tab = Panel(child=column(row(*self.amp_plots), + self.diff_image_plot, + self.readnoise_histogram), + title=self.aperture) - # Update the mean readnoise figures for all amps + def plot_readnoise_amplifers(self): + """Class to create readnoise scatter plots per amplifier. + """ + self.amp_plots = [] for amp in ['1', '2', '3', '4']: - readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.query_results]) - self.refs['mean_readnoise_source_amp{}'.format(amp)].data = {'time': expstarts, - 'time_iso': expstarts_iso, - 'mean_rn': readnoise_vals, - 'filename': filenames, - 'nints': nints, - 'ngroups': ngroups} - self.refs['mean_readnoise_figure_amp{}'.format(amp)].title.text = 'Amp {}'.format(amp) - self.refs['mean_readnoise_figure_amp{}'.format(amp)].hover.tooltips = [('file', '@filename'), - ('time', '@time_iso'), - ('nints', '@nints'), - ('ngroups', '@ngroups'), - ('readnoise', '@mean_rn')] - - # Update plot limits if data exists - if len(readnoise_vals) != 0: - self.refs['mean_readnoise_xr_amp{}'.format(amp)].start = expstarts.min() - timedelta(days=3) - self.refs['mean_readnoise_xr_amp{}'.format(amp)].end = expstarts.max() + timedelta(days=3) - min_val, max_val = min(x for x in readnoise_vals if x is not None), max(x for x in readnoise_vals if x is not None) - if min_val == max_val: - self.refs['mean_readnoise_yr_amp{}'.format(amp)].start = min_val - 1 - self.refs['mean_readnoise_yr_amp{}'.format(amp)].end = max_val + 1 - else: - offset = (max_val - min_val) * .1 - self.refs['mean_readnoise_yr_amp{}'.format(amp)].start = min_val - offset - self.refs['mean_readnoise_yr_amp{}'.format(amp)].end = max_val + offset - - def update_readnoise_diff_plots(self): - """Updates the readnoise difference image and histogram""" + + amp_plot = figure(title='Amp {}'.format(amp), width=280, height=280, x_axis_type='datetime') + amp_plot.xaxis[0].ticker.desired_num_ticks = 4 + + if self.db.query_results: + readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.db.query_results]) + else: + readnoise_vals = np.array(list()) + + filenames = [os.path.basename(result.uncal_filename).replace('_uncal.fits', '') for result in self.db.query_results] + expstarts_iso = np.array([result.expstart for result in self.db.query_results]) + expstarts = np.array([datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%f') for date in expstarts_iso]) + nints = [result.nints for result in self.db.query_results] + ngroups = [result.ngroups for result in self.db.query_results] + + source = ColumnDataSource(data=dict( + file=filenames, + expstarts=expstarts, + nints=nints, + ngroups=ngroups, + readnoise=readnoise_vals)) + + amp_plot.add_tools(HoverTool(tooltips=[("file", "@file"), + ("time", "@expstarts"), + ("nints", "@nints"), + ("ngroups", "@ngroups"), + ("readnoise", "@readnoise")])) + + amp_plot.circle(x='expstarts', y='readnoise', source=source) + + amp_plot.xaxis.axis_label = 'Date' + amp_plot.yaxis.axis_label = 'Mean Readnoise [DN]' + + self.amp_plots.append(amp_plot) + + def plot_readnoise_difference_image(self): + """Updates the readnoise difference image""" # Update the readnoise difference image and histogram, if data exists - if len(self.query_results) != 0: - # Get the most recent data; the entries were sorted by time when - # loading the database, so the last entry will always be the most recent. - diff_image_png = self.query_results[-1].readnoise_diff_image + + self.diff_image_plot = figure(title='Readnoise Difference (most recent dark - pipeline reffile)', + height=500, width=500, sizing_mode='scale_width') + + if len(self.db.query_results) != 0: + diff_image_png = self.db.query_results[-1].readnoise_diff_image diff_image_png = os.path.join('/static', '/'.join(diff_image_png.split('/')[-6:])) - diff_image_n = np.array(self.query_results[-1].diff_image_n) - diff_image_bin_centers = np.array(self.query_results[-1].diff_image_bin_centers) - - # Update the readnoise difference image and histogram - self.refs['readnoise_diff_image'].image_url(url=[diff_image_png], x=0, y=0, w=2048, h=2048, anchor="bottom_left") - self.refs['diff_hist_source'].data = {'n': diff_image_n, - 'bin_centers': diff_image_bin_centers} - self.refs['diff_hist_xr'].start = diff_image_bin_centers.min() - self.refs['diff_hist_xr'].end = diff_image_bin_centers.max() - self.refs['diff_hist_yr'].start = diff_image_n.min() - self.refs['diff_hist_yr'].end = diff_image_n.max() + diff_image_n.max() * 0.05 - - # Update the readnoise difference image style - self.refs['readnoise_diff_image'].xaxis.visible = False - self.refs['readnoise_diff_image'].yaxis.visible = False - self.refs['readnoise_diff_image'].xgrid.grid_line_color = None - self.refs['readnoise_diff_image'].ygrid.grid_line_color = None - self.refs['readnoise_diff_image'].title.text_font_size = '22px' - self.refs['readnoise_diff_image'].title.align = 'center' + self.diff_image_plot.image_url(url=[diff_image_png], x=0, y=0, w=2048, h=2048, anchor="bottom_left") + + self.diff_image_plot.xaxis.visible = False + self.diff_image_plot.yaxis.visible = False + self.diff_image_plot.xgrid.grid_line_color = None + self.diff_image_plot.ygrid.grid_line_color = None + self.diff_image_plot.title.text_font_size = '22px' + self.diff_image_plot.title.align = 'center' + + + def plot_readnoise_histogram(self): + """Updates the readnoise histogram""" + + if len(self.db.query_results) != 0: + diff_image_n = np.array(self.db.query_results[-1].diff_image_n) + diff_image_bin_centers = np.array(self.db.query_results[-1].diff_image_bin_centers) + else: + diff_image_n = np.array(list()) + diff_image_bin_centers = np.array(list()) + + hist_xr_start = diff_image_bin_centers.min() + hist_xr_end = diff_image_bin_centers.max() + hist_yr_start = diff_image_n.min() + hist_yr_end = diff_image_n.max() + diff_image_n.max() * 0.05 + + self.readnoise_histogram = figure(height=500, width=500, + x_range=(hist_xr_start, hist_xr_end), + y_range=(hist_yr_start, hist_yr_end), + sizing_mode='scale_width') + + source = ColumnDataSource(data=dict( + x=diff_image_bin_centers, + y=diff_image_n, + )) + + self.readnoise_histogram.add_tools(HoverTool(tooltips=[("Data (x, y)", "(@x, @y)"),])) + + self.readnoise_histogram.circle(x='x', y='y', source=source) + + self.readnoise_histogram.xaxis.axis_label = 'Readnoise Difference [DN]' + self.readnoise_histogram.yaxis.axis_label = 'Number of Pixels' + self.readnoise_histogram.xaxis.axis_label_text_font_size = "15pt" + self.readnoise_histogram.yaxis.axis_label_text_font_size = "15pt" diff --git a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_readnoise_interface.yaml b/jwql/website/apps/jwql/monitor_pages/yaml/monitor_readnoise_interface.yaml deleted file mode 100644 index 4d2fd8a78..000000000 --- a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_readnoise_interface.yaml +++ /dev/null @@ -1,189 +0,0 @@ -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Mean Readnoise vs Time Figures Amp1 -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &mean_readnoise_source_amp1 - ref: "mean_readnoise_source_amp1" - data: - time: [] - time_iso: [] - mean_rn: [] - filename: [] - nints: [] - ngroups: [] -- !Range1d: &mean_readnoise_xr_amp1 - ref: "mean_readnoise_xr_amp1" - start: 0 - end: 1 - bounds: 'auto' -- !Range1d: &mean_readnoise_yr_amp1 - ref: "mean_readnoise_yr_amp1" - start: 0 - end: 10 - bounds: 'auto' -- !Figure: &mean_readnoise_figure_amp1 - ref: "mean_readnoise_figure_amp1" - title: "Amp 1" - x_axis_label: "Date" - x_axis_type: "datetime" - y_axis_label: "Mean readnoise [DN]" - x_range: *mean_readnoise_xr_amp1 - y_range: *mean_readnoise_yr_amp1 - height: 800 - width: 800 - tools: "hover, wheel_zoom, pan, reset" - elements: - - {'kind': 'circle', 'x': 'time', 'y': 'mean_rn', 'size': 6, 'source': *mean_readnoise_source_amp1} -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Mean Readnoise vs Time Figures Amp2 -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &mean_readnoise_source_amp2 - ref: "mean_readnoise_source_amp2" - data: - time: [] - time_iso: [] - mean_rn: [] - filename: [] - nints: [] - ngroups: [] -- !Range1d: &mean_readnoise_xr_amp2 - ref: "mean_readnoise_xr_amp2" - start: 0 - end: 1 - bounds: 'auto' -- !Range1d: &mean_readnoise_yr_amp2 - ref: "mean_readnoise_yr_amp2" - start: 0 - end: 10 - bounds: 'auto' -- !Figure: &mean_readnoise_figure_amp2 - ref: "mean_readnoise_figure_amp2" - title: "Amp 2" - x_axis_label: "Date" - x_axis_type: "datetime" - y_axis_label: "Mean readnoise [DN]" - x_range: *mean_readnoise_xr_amp2 - y_range: *mean_readnoise_yr_amp2 - height: 800 - width: 800 - tools: "hover, wheel_zoom, pan, reset" - elements: - - {'kind': 'circle', 'x': 'time', 'y': 'mean_rn', 'size': 6, 'source': *mean_readnoise_source_amp2} -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Mean Readnoise vs Time Figures Amp3 -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &mean_readnoise_source_amp3 - ref: "mean_readnoise_source_amp3" - data: - time: [] - time_iso: [] - mean_rn: [] - filename: [] - nints: [] - ngroups: [] -- !Range1d: &mean_readnoise_xr_amp3 - ref: "mean_readnoise_xr_amp3" - start: 0 - end: 1 - bounds: 'auto' -- !Range1d: &mean_readnoise_yr_amp3 - ref: "mean_readnoise_yr_amp3" - start: 0 - end: 10 - bounds: 'auto' -- !Figure: &mean_readnoise_figure_amp3 - ref: "mean_readnoise_figure_amp3" - title: "Amp 3" - x_axis_label: "Date" - x_axis_type: "datetime" - y_axis_label: "Mean readnoise [DN]" - x_range: *mean_readnoise_xr_amp3 - y_range: *mean_readnoise_yr_amp3 - height: 800 - width: 800 - tools: "hover, wheel_zoom, pan, reset" - elements: - - {'kind': 'circle', 'x': 'time', 'y': 'mean_rn', 'size': 6, 'source': *mean_readnoise_source_amp3} -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Mean Readnoise vs Time Figures Amp4 -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &mean_readnoise_source_amp4 - ref: "mean_readnoise_source_amp4" - data: - time: [] - time_iso: [] - mean_rn: [] - filename: [] - nints: [] - ngroups: [] -- !Range1d: &mean_readnoise_xr_amp4 - ref: "mean_readnoise_xr_amp4" - start: 0 - end: 1 - bounds: 'auto' -- !Range1d: &mean_readnoise_yr_amp4 - ref: "mean_readnoise_yr_amp4" - start: 0 - end: 10 - bounds: 'auto' -- !Figure: &mean_readnoise_figure_amp4 - ref: "mean_readnoise_figure_amp4" - title: "Amp 4" - x_axis_label: "Date" - x_axis_type: "datetime" - y_axis_label: "Mean readnoise [DN]" - x_range: *mean_readnoise_xr_amp4 - y_range: *mean_readnoise_yr_amp4 - height: 800 - width: 800 - tools: "hover, wheel_zoom, pan, reset" - elements: - - {'kind': 'circle', 'x': 'time', 'y': 'mean_rn', 'size': 6, 'source': *mean_readnoise_source_amp4} -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Readnoise Difference Image -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &diff_source - ref: "diff_source" - data: - dh: [1] - dw: [1] - image: [[[0,0], [0, 0]]] -- !Figure: &readnoise_diff_image - ref: "readnoise_diff_image" - title: 'Readnoise Difference (most recent dark - pipeline reffile)' - elements: - - {"kind": "image", "image": "image", "x": 0, "y": 0, "dh": 'dh', "dw": 'dh', "source": *diff_source} - tools: "" -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Readnoise Difference Histogram -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &diff_hist_source - ref: "diff_hist_source" - data: - n: [] - bin_centers: [] -- !Range1d: &diff_hist_xr - ref: "diff_hist_xr" - start: 0 - end: 1 - bounds: 'auto' -- !Range1d: &diff_hist_yr - ref: "diff_hist_yr" - start: 0 - end: 10 - bounds: 'auto' -- !Figure: &readnoise_diff_hist - ref: "readnoise_diff_hist" - x_axis_label: "Readnoise Difference [DN]" - y_axis_label: "Number of Pixels" - x_range: *diff_hist_xr - y_range: *diff_hist_yr - height: 250 - width: 300 - tools: "hover, wheel_zoom, pan, reset" - elements: - - {'kind': 'circle', 'x': 'bin_centers', 'y': 'n', 'size': 4, 'source': *diff_hist_source} - -# Document structure -# - !Document: -# - !row: -# - *mean_readnoise_figure \ No newline at end of file diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index d8ff30f3f..a724f420c 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -41,12 +41,14 @@ from jwql.database.database_interface import session from jwql.database.database_interface import NIRCamClawStats from jwql.website.apps.jwql import bokeh_containers +from jwql.website.apps.jwql.monitor_pages.monitor_readnoise_bokeh import ReadNoiseFigure from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.utils import get_config, get_base_url from jwql.instrument_monitors.nirspec_monitors.ta_monitors import msata_monitor from jwql.instrument_monitors.nirspec_monitors.ta_monitors import wata_monitor from jwql.utils import monitor_utils + CONFIG = get_config() FILESYSTEM_DIR = os.path.join(CONFIG['jwql_dir'], 'filesystem') @@ -100,7 +102,7 @@ def bad_pixel_monitor(request, inst): context = { 'inst': inst, - } + } return render(request, template, context) @@ -283,7 +285,7 @@ def readnoise_monitor(request, inst): inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] # Get the html and JS needed to render the readnoise tab plots - tabs_components = bokeh_containers.readnoise_monitor_tabs(inst) + tabs_components = ReadNoiseFigure(inst).tab_components template = "readnoise_monitor.html"