From c3df1bbbc122c06e2910aa283a7c3fa08745559b Mon Sep 17 00:00:00 2001 From: Julie Prestopnik Date: Wed, 13 Nov 2024 13:09:36 -0700 Subject: [PATCH 01/10] Update version --- docs/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/version b/docs/version index d508d7ba..5c8498d0 100644 --- a/docs/version +++ b/docs/version @@ -1 +1 @@ -__version__="3.0.0-rc1" \ No newline at end of file +__version__="3.0.0-dev" From fdbe111052ef29010ca872303acdf7ca2c36e552 Mon Sep 17 00:00:00 2001 From: Julie Prestopnik Date: Wed, 13 Nov 2024 13:09:51 -0700 Subject: [PATCH 02/10] Update conf.py --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 66619627..a2fe713e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,7 +22,7 @@ copyright = '2024, NCAR' author = 'UCAR/NCAR, NOAA, CSU/CIRA, and CU/CIRES' author_list = 'Fisher, H., C. Kalb, D. Adriaansen, D. Fillmore, M. Win-Gildenmeister, T. Burek, M. Smith, and T. Jensen' -version = '3.0.0-rc1' +version = '3.0.0-dev' verinfo = version release = f'{version}' release_year = '2024' From d5e48f12b081aef2bc70eeb8c6a3bb6963dd3899 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 13 Nov 2024 22:07:35 -0700 Subject: [PATCH 03/10] added missing import (#412) --- metcalcpy/util/write_mpr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metcalcpy/util/write_mpr.py b/metcalcpy/util/write_mpr.py index c79e784c..e747de19 100644 --- a/metcalcpy/util/write_mpr.py +++ b/metcalcpy/util/write_mpr.py @@ -8,6 +8,7 @@ import os import numpy as np +from metcalcpy.util.safe_log import safe_log def write_mpr_file(data_fcst,data_obs,lats_in,lons_in,fcst_lead,fcst_valid,obs_lead,obs_valid,mod_name,desc,fcst_var,fcst_unit,fcst_lev,obs_var,obs_unit,obs_lev,maskname,obsslev,outdir,outfile_prefix, logger=None): From d4adc3754fa9f6fbd870fd726f4a8540c794c00b Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 15 Nov 2024 08:52:56 -0700 Subject: [PATCH 04/10] Per dtcenter/METplus#2336, call custom GHA to trigger METplus use case tests when changes are pushed to main_vX.Y and develop branches --- .github/workflows/trigger_metplus.yaml | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/.github/workflows/trigger_metplus.yaml b/.github/workflows/trigger_metplus.yaml index bd70bc1a..de2fb45e 100644 --- a/.github/workflows/trigger_metplus.yaml +++ b/.github/workflows/trigger_metplus.yaml @@ -4,6 +4,7 @@ on: push: branches: - develop + - 'main_v[0-9]+.[0-9]+' paths-ignore: - 'docs/**' - '.github/pull_request_template.md' @@ -16,24 +17,6 @@ jobs: name: Trigger METplus testing workflow runs-on: ubuntu-latest steps: - - name: Print GitHub values for reference - env: - GITHUB_CONTEXT: ${{ toJson(github) }} - run: echo "$GITHUB_CONTEXT" - - uses: actions/github-script@v7 + - uses: dtcenter/metplus-action-trigger-use-cases@v1 with: - github-token: ${{ secrets.METPLUS_BOT_TOKEN }} - script: | - await github.rest.actions.createWorkflowDispatch({ - owner: 'dtcenter', - repo: 'METplus', - workflow_id: 'testing.yml', - ref: 'develop', - inputs: { - repository: '${{ github.repository }}', - ref: '${{ github.ref }}', - actor: '${{ github.actor }}', - sha: '${{ github.sha }}', - pusher_email: '${{ github.event.pusher.email }}' - }, - }); + token: ${{ secrets.METPLUS_BOT_TOKEN }} From 313fdb89ab78fcfc048bffba574ba63f0c3f249e Mon Sep 17 00:00:00 2001 From: John Sharples <41682323+John-Sharples@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:07:42 +1100 Subject: [PATCH 05/10] 401: Add basic tests for untested modules (#415) * 401: Add basic tests for untested modules * 401: fix sonarQube issue * 401: more sonarqube fixes --- metcalcpy/agg_stat_bootstrap.py | 104 +++++++++--------- metcalcpy/pre_processing/directional_means.py | 8 +- test/logs/log_agg_eclv.txt | 96 ---------------- test/test_agg_stat_bootstrap.py | 15 +++ test/test_preprocessing_directional_means.py | 46 ++++++++ test/utils.py | 8 ++ 6 files changed, 123 insertions(+), 154 deletions(-) delete mode 100644 test/logs/log_agg_eclv.txt create mode 100644 test/test_agg_stat_bootstrap.py create mode 100644 test/test_preprocessing_directional_means.py create mode 100644 test/utils.py diff --git a/metcalcpy/agg_stat_bootstrap.py b/metcalcpy/agg_stat_bootstrap.py index cc6104ec..6e65e436 100644 --- a/metcalcpy/agg_stat_bootstrap.py +++ b/metcalcpy/agg_stat_bootstrap.py @@ -71,7 +71,7 @@ def __init__(self, in_params): """ self.logger = setup_logging(in_params) logger = self.logger - safe.logger(logger, "debug", "Initializing AggStatBootstrap with parameters.") + safe_log(logger, "debug", "Initializing AggStatBootstrap with parameters.") self.statistic = None self.derived_name_to_values = {} self.params = in_params @@ -94,14 +94,14 @@ def _init_out_frame(self, series_fields, series): pandas data frame """ logger = self.logger - safe.logger(logger, "debug", "Initializing output data frame.") + safe_log(logger, "debug", "Initializing output data frame.") result = pd.DataFrame() row_number = len(series) - safe.logger(logger, "debug", f"Number of rows to initialize: {row_number}") + safe_log(logger, "debug", f"Number of rows to initialize: {row_number}") # fill series variables and values for field_ind, field in enumerate(series_fields): result[field] = [row[field_ind] for row in series] - safe.logger(logger, "debug", f"Field '{field}' initialized with {len(result[field])} entries.") + safe_log(logger, "debug", f"Field '{field}' initialized with {len(result[field])} entries.") # fill the stats and CI values placeholders with None result['fcst_var'] = [None] * row_number result['stat_value'] = [None] * row_number @@ -109,38 +109,38 @@ def _init_out_frame(self, series_fields, series): result['stat_btcu'] = [None] * row_number result['nstats'] = [None] * row_number - safe.logger(logger, "debug", "Stats and confidence interval placeholders added.") - safe.logger(logger, "debug", f"DataFrame initialized with columns: {result.columns.tolist()}") + safe_log(logger, "debug", "Stats and confidence interval placeholders added.") + safe_log(logger, "debug", f"DataFrame initialized with columns: {result.columns.tolist()}") return result def _proceed_with_axis(self, axis="1"): logger = self.logger - safe.logger(logger, "info", f"Proceeding with axis: {axis}") + safe_log(logger, "info", f"Proceeding with axis: {axis}") if not self.input_data.empty: # identify all possible points values by adding series values, indy values # and statistics and then permute them - safe.logger(logger, "debug", "Input data is not empty. Proceeding with calculations.") + safe_log(logger, "debug", "Input data is not empty. Proceeding with calculations.") indy_vals = self.params['indy_vals'] series_val = self.params['series_val_' + axis] all_fields_values = series_val.copy() all_fields_values[self.params['indy_var']] = indy_vals all_fields_values['stat_name'] = self.params['list_stat_' + axis] all_points = list(itertools.product(*all_fields_values.values())) - safe.logger(logger, "debug", f"All points generated: {len(all_points)} points created for axis {axis}.") + safe_log(logger, "debug", f"All points generated: {len(all_points)} points created for axis {axis}.") fcst_var = None if len(self.params['fcst_var_val_' + axis]) > 0 and 'fcst_var' in self.input_data.columns: fcst_var = list(self.params['fcst_var_val_' + axis].keys())[0] - safe.logger(logger, "debug", f"Forecast variable identified: {fcst_var}") + safe_log(logger, "debug", f"Forecast variable identified: {fcst_var}") cases = [] out_frame = self._init_out_frame(all_fields_values.keys(), all_points) - safe.logger(logger, "debug", f"Output DataFrame initialized with {len(out_frame)} rows.") + safe_log(logger, "debug", f"Output DataFrame initialized with {len(out_frame)} rows.") point_to_distrib = {} # run the bootstrap flow for each independent variable value for indy_val in indy_vals: - safe.logger(logger, "debug", f"Processing independent value: {indy_val}") + safe_log(logger, "debug", f"Processing independent value: {indy_val}") # extract the records for the current indy value if is_string_integer(indy_val): filtered_by_indy_data = \ @@ -155,7 +155,7 @@ def _proceed_with_axis(self, axis="1"): all_fields_values = series_val.copy() all_points = list(itertools.product(*all_fields_values.values())) - safe.logger(logger, "debug", f"Number of points for independent value '{indy_val}': {len(all_points)}.") + safe_log(logger, "debug", f"Number of points for independent value '{indy_val}': {len(all_points)}.") for point in all_points: all_filters = [] @@ -182,7 +182,7 @@ def _proceed_with_axis(self, axis="1"): # use numpy to select the rows where any record evaluates to True mask = np.array(all_filters).all(axis=0) point_data = filtered_by_indy_data.loc[mask] - safe.logger(logger, "debug", f"Point data filtered for point {point}. Number of records: {len(point_data)}") + safe_log(logger, "debug", f"Point data filtered for point {point}. Number of records: {len(point_data)}") # build a list of cases to sample fcst_valid = point_data.loc[:, 'fcst_valid'].astype(str) @@ -193,7 +193,7 @@ def _proceed_with_axis(self, axis="1"): # calculate bootstrap for cases for stat_upper in self.params['list_stat_' + axis]: self.statistic = stat_upper.lower() - safe.logger(logger, "debug", f"Calculating bootstrap for statistic: {self.statistic}") + safe_log(logger, "debug", f"Calculating bootstrap for statistic: {self.statistic}") for point in all_points: all_filters = [] out_frame_filter = [] @@ -218,7 +218,7 @@ def _proceed_with_axis(self, axis="1"): mask_out_frame = np.array(out_frame_filter).all(axis=0) point_data = filtered_by_indy_data.loc[mask] bootstrap_results = self._get_bootstrapped_stats(point_data, cases) - safe.logger(logger, "debug", f"Bootstrap results calculated for point {point}: {bootstrap_results.value}") + safe_log(logger, "debug", f"Bootstrap results calculated for point {point}: {bootstrap_results.value}") # save bootstrap results point_to_distrib[point] = bootstrap_results n_stats = len(point_data) @@ -235,32 +235,32 @@ def _proceed_with_axis(self, axis="1"): out_frame.loc[index, 'stat_btcl'] = bootstrap_results.lower_bound out_frame.loc[index, 'stat_btcu'] = bootstrap_results.upper_bound out_frame.loc[index, 'nstats'] = n_stats - safe.logger(logger, "debug", f"Results saved to output DataFrame at index {index} for point {point}.") + safe_log(logger, "debug", f"Results saved to output DataFrame at index {index} for point {point}.") else: out_frame = pd.DataFrame() - safe.logger(logger, "warning", "Input data is empty. Returning an empty DataFrame.") + safe_log(logger, "warning", "Input data is empty. Returning an empty DataFrame.") - safe.logger(logger, "info", f"Completed processing for axis: {axis}") + safe_log(logger, "info", f"Completed processing for axis: {axis}") return out_frame def _get_bootstrapped_stats(self, series_data, cases): logger = self.logger - safe.logger(logger, "info", "Starting bootstrapping process.") + safe_log(logger, "info", "Starting bootstrapping process.") - safe.logger(logger, "debug", "Sorting series data.") + safe_log(logger, "debug", "Sorting series data.") self.series_data = sort_data(series_data) - safe.logger(logger, "debug", f"Data sorted. Number of rows: {len(self.series_data)}") + safe_log(logger, "debug", f"Data sorted. Number of rows: {len(self.series_data)}") if self.params['num_iterations'] == 1: - safe.logger(logger, "info", "Only one iteration specified. Skipping bootstrapping.") + safe_log(logger, "info", "Only one iteration specified. Skipping bootstrapping.") stat_val = self._calc_stats(cases)[0] - safe.logger(logger, "debug", f"Statistic calculated: {stat_val}") + safe_log(logger, "debug", f"Statistic calculated: {stat_val}") results = BootstrapResults(lower_bound=None, value=stat_val, upper_bound=None) - safe.logger(logger, "info", "Statistic calculated without bootstrapping.") + safe_log(logger, "info", "Statistic calculated without bootstrapping.") else: # need bootstrapping and CI calculation in addition to - safe.logger(logger, "info", "Performing bootstrapping and confidence interval calculation.") + safe_log(logger, "info", "Performing bootstrapping and confidence interval calculation.") try: results = bootstrap_and_value_mode( self.series_data, @@ -271,13 +271,13 @@ def _get_bootstrapped_stats(self, series_data, cases): ci_method=self.params['method'], logger=logger ) - safe.logger(logger, "debug", "Bootstrapping completed successfully.") + safe_log(logger, "debug", "Bootstrapping completed successfully.") except KeyError as err: - safe.logger(logger, "error", f"Error during bootstrapping: {err}") + safe_log(logger, "error", f"Error during bootstrapping: {err}") results = BootstrapResults(None, None, None) - safe.logger(logger, "info", "Returning empty BootstrapResults due to error.") + safe_log(logger, "info", "Returning empty BootstrapResults due to error.") print(err) - safe.logger(logger, "info", "Bootstrapping process completed.") + safe_log(logger, "info", "Bootstrapping process completed.") return results def _calc_stats(self, cases): @@ -294,23 +294,23 @@ def _calc_stats(self, cases): """ logger = self.logger func_name = f'calculate_{self.statistic}' - safe.logger(logger, "info", f"Starting statistic calculation using function: {func_name}") + safe_log(logger, "info", f"Starting statistic calculation using function: {func_name}") if cases is not None and cases.ndim == 2: # The single value case - safe.logger(logger, "debug", "Processing single-value case.") + safe_log(logger, "debug", "Processing single-value case.") # build a data frame with the sampled data data_cases = np.asarray(self.series_data['case']) flat_cases = cases.flatten() values = self.series_data[np.in1d(data_cases, flat_cases)].to_numpy() - safe.logger(logger, "debug", f"Number of values selected for single case: {len(values)}") + safe_log(logger, "debug", f"Number of values selected for single case: {len(values)}") # Calculate the statistic for each bootstrap iteration try: stat_value = globals()[func_name](values, self.column_names, logger=logger) stat_values.append([stat_value]) - safe.logger(logger, "info", f"Statistic calculated for bootstrap iteration: {stat_value}") + safe_log(logger, "info", f"Statistic calculated for bootstrap iteration: {stat_value}") except Exception as e: - safe.logger(logger, "error", f"Error calculating statistic for bootstrap iteration: {e}") + safe_log(logger, "error", f"Error calculating statistic for bootstrap iteration: {e}") raise elif cases is not None and cases.ndim == 3: @@ -319,17 +319,17 @@ def _calc_stats(self, cases): for row in cases: values_ind = self.series_data['case'].isin(row.flatten()) values = self.series_data[values_ind] - safe.logger(logger, "debug", f"Number of values selected for bootstrap iteration: {len(values)}") + safe_log(logger, "debug", f"Number of values selected for bootstrap iteration: {len(values)}") # Calculate the statistic for each bootstrap iteration try: stat_value = globals()[func_name](values, self.column_names, logger=logger) stat_values.append([stat_value]) - safe.logger(logger, "info", f"Statistic calculated for bootstrap iteration: {stat_value}") + safe_log(logger, "info", f"Statistic calculated for bootstrap iteration: {stat_value}") except Exception as e: - safe.logger(logger, "error", f"Error calculating statistic for bootstrap iteration: {e}") + safe_log(logger, "error", f"Error calculating statistic for bootstrap iteration: {e}") raise else: - safe.logger(logger, "error", "Invalid input for cases. Cannot calculate statistic.") + safe_log(logger, "error", "Invalid input for cases. Cannot calculate statistic.") raise KeyError("can't calculate statistic") return stat_values @@ -338,46 +338,46 @@ def calculate_values(self): Writes output data to the file """ logger = self.logger - safe.logger(logger, "info", "Starting calculation of values.") + safe_log(logger, "info", "Starting calculation of values.") if not self.input_data.empty: - safe.logger(logger, "debug", "Input data is not empty. Proceeding with calculations.") + safe_log(logger, "debug", "Input data is not empty. Proceeding with calculations.") if self.params['random_seed'] is not None and self.params['random_seed'] != 'None': - safe.logger(logger, "debug", f"Random seed set to: {self.params['random_seed']}") + safe_log(logger, "debug", f"Random seed set to: {self.params['random_seed']}") np.random.seed(self.params['random_seed']) # perform EE if needed is_event_equal = parse_bool(self.params['event_equal']) if is_event_equal: - safe.logger(logger, "info", "Event equalization required. Performing event equalization.") + safe_log(logger, "info", "Event equalization required. Performing event equalization.") self._perform_event_equalization() - safe.logger(logger, "debug", "Event equalization completed.") + safe_log(logger, "debug", "Event equalization completed.") # build the case information for each record - safe.logger(logger, "debug", "Building case information for each record.") + safe_log(logger, "debug", "Building case information for each record.") fcst_valid = self.input_data.loc[:, 'fcst_valid'].astype(str) indy_var = self.input_data.loc[:, self.params['indy_var']].astype(str) self.input_data['case'] = fcst_valid + '#' + indy_var - safe.logger(logger, "debug", "Case information added to the input data.") + safe_log(logger, "debug", "Case information added to the input data.") # get results for axis1 - safe.logger(logger, "info", "Calculating results for axis 1.") + safe_log(logger, "info", "Calculating results for axis 1.") out_frame = self._proceed_with_axis("1") if self.params['series_val_2']: - safe.logger(logger, "info", "Series values for axis 2 detected. Calculating results for axis 2.") + safe_log(logger, "info", "Series values for axis 2 detected. Calculating results for axis 2.") out_frame = pd.concat([out_frame, self._proceed_with_axis("2")]) - safe.logger(logger, "debug", "Results for axis 2 calculated and combined with axis 1.") + safe_log(logger, "debug", "Results for axis 2 calculated and combined with axis 1.") else: - safe.logger(logger, "warning", "Input data is empty. Returning an empty DataFrame.") + safe_log(logger, "warning", "Input data is empty. Returning an empty DataFrame.") out_frame = pd.DataFrame() header = True mode = 'w' - safe.logger(logger, "info", f"Exporting results to {self.params['agg_stat_output']}") + safe_log(logger, "info", f"Exporting results to {self.params['agg_stat_output']}") export_csv = out_frame.to_csv(self.params['agg_stat_output'], index=None, header=header, mode=mode, sep="\t", na_rep="NA") - safe.logger(logger, "info", "Results successfully exported to CSV.") + safe_log(logger, "info", "Results successfully exported to CSV.") def _perform_event_equalization(self): diff --git a/metcalcpy/pre_processing/directional_means.py b/metcalcpy/pre_processing/directional_means.py index 579e03e2..0701e70e 100644 --- a/metcalcpy/pre_processing/directional_means.py +++ b/metcalcpy/pre_processing/directional_means.py @@ -8,9 +8,7 @@ # ============================* - import numpy as np -import xarray as xr def zonal_mean(dat,dimvar='longitude'): @@ -52,10 +50,8 @@ def meridional_mean(dat, lat1, lat2, dimvar='latitude'): """ # Check inputs - if lat1 > lat2: - raise ValueError('lat1 is greater than lat2, but it must be less than lat2') - elif lat1 == lat2: - raise ValueError('lat1 is equal to lat2, but it must be less than lat2') + if lat1 >= lat2: + raise ValueError('lat1 is greater than or equal to lat2, but it must be less than lat2') wgts = np.cos(np.deg2rad(dat[dimvar].where((dat[dimvar] >= lat1) & (dat[dimvar] <= lat2),drop=True))) diff --git a/test/logs/log_agg_eclv.txt b/test/logs/log_agg_eclv.txt deleted file mode 100644 index 17bc673f..00000000 --- a/test/logs/log_agg_eclv.txt +++ /dev/null @@ -1,96 +0,0 @@ -2024-09-04 23:27:28 UTC - ishitas - INFO - User: ishitas has started the script with command: /home/ishitas/.local/lib/python3.10/site-packages/pytest/__main__.py -s --log-cli-level=INFO test_agg_eclv.py -2024-09-04 23:27:28 UTC - ishitas - INFO - Successfully loaded data from /d1/personal/ishitas/METcalcpy/test/data/agg_eclv_data.data -2024-09-04 23:27:28 UTC - ishitas - INFO - Random seed set to 1. -2024-09-04 23:27:28 UTC - ishitas - INFO - Generated all combinations for points to be processed: 2 combinations. -2024-09-04 23:27:28 UTC - ishitas - INFO - Output DataFrame initialized successfully with fields: ['model', 'fcst_lev', 'thresh_i', 'x_pnt_i', 'y_pnt_i', 'stat_btcl', 'stat_btcu', 'nstats']. -2024-09-04 23:27:28 UTC - ishitas - INFO - Output DataFrame initialized successfully with fields: ['model', 'fcst_lev', 'thresh_i', 'x_pnt_i', 'y_pnt_i', 'stat_btcl', 'stat_btcu', 'nstats']. -2024-09-04 23:27:28 UTC - ishitas - INFO - Statistics calculated successfully for single value case. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Bootstrapped statistics calculated for threshold 0. -2024-09-04 23:27:29 UTC - ishitas - INFO - Completed processing for point ('WRF', 'Z10') -2024-09-04 23:27:29 UTC - ishitas - INFO - Output DataFrame initialized successfully with fields: ['model', 'fcst_lev', 'thresh_i', 'x_pnt_i', 'y_pnt_i', 'stat_btcl', 'stat_btcu', 'nstats']. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for single value case. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics calculated successfully for all bootstrap samples. -2024-09-04 23:27:29 UTC - ishitas - INFO - Bootstrapped statistics calculated for threshold 0. -2024-09-04 23:27:29 UTC - ishitas - INFO - Completed processing for point ('WRF', 'P850-700') -2024-09-04 23:27:29 UTC - ishitas - INFO - All data processed successfully. Returning compiled DataFrame. -2024-09-04 23:27:29 UTC - ishitas - INFO - Statistics and confidence intervals calculation completed. -2024-09-04 23:27:29 UTC - ishitas - INFO - Data successfully written to /d1/personal/ishitas/METcalcpy/test/data/agg_eclv_data_output.data in mode w. diff --git a/test/test_agg_stat_bootstrap.py b/test/test_agg_stat_bootstrap.py new file mode 100644 index 00000000..db5d31c4 --- /dev/null +++ b/test/test_agg_stat_bootstrap.py @@ -0,0 +1,15 @@ +import metcalcpy.agg_stat_bootstrap as asb +from test.utils import AGG_STAT_AND_BOOT_DATA + + +DEFAULT_CONF = { + "log_filename": "tmp.log", + "log_level": "DEBUG", + "agg_stat_input": AGG_STAT_AND_BOOT_DATA, +} + + +def test_smoke(tmp_path): + """Basic test to check object instantiation""" + DEFAULT_CONF["log_dir"] = tmp_path + asb.AggStatBootstrap(DEFAULT_CONF) diff --git a/test/test_preprocessing_directional_means.py b/test/test_preprocessing_directional_means.py new file mode 100644 index 00000000..82a6294b --- /dev/null +++ b/test/test_preprocessing_directional_means.py @@ -0,0 +1,46 @@ +import pytest +import xarray as xr +import numpy as np +from metcalcpy.pre_processing import directional_means as dm + +TEST_DATA = xr.Dataset( + { + "variable": xr.DataArray( + [ + [1.1, 2.2, 3.3], + [2.2, 3.3, 4.4], + [4.4, 5.5, 6.6], + ], + coords={ + "latitude": [-10.25, -11.25, -12.25], + "longitude": [112.25, 113.25, 114.25], +}, + dims=["latitude", "longitude"], + ), + }, + attrs={}, +) + + +def test_zonal_mean(): + actual = dm.zonal_mean(TEST_DATA) + np.testing.assert_almost_equal(actual.variable.values, np.array([2.2, 3.3, 5.5])) + + actual = dm.zonal_mean(TEST_DATA, "latitude") + np.testing.assert_almost_equal(actual.variable.values, np.array([2.57, 3.67, 4.77]), 2) + + +def test_meridional_mean(): + actual = dm.meridional_mean(TEST_DATA, -12.25, -10.25) + np.testing.assert_almost_equal(actual.variable.values, np.array([2.562829, 3.662829, 4.762829])) + + actual = dm.meridional_mean(TEST_DATA, 112.25, 114.25, "longitude") + np.testing.assert_almost_equal(actual.variable.values, np.array([2.2297922, 3.3297922, 5.5297922])) + + +def test_meridional_mean_rasied(): + with pytest.raises(ValueError): + dm.meridional_mean(TEST_DATA, -10.25, -10.25) + + with pytest.raises(ValueError): + dm.meridional_mean(TEST_DATA, -10.25, -12.25) diff --git a/test/utils.py b/test/utils.py new file mode 100644 index 00000000..c52e0636 --- /dev/null +++ b/test/utils.py @@ -0,0 +1,8 @@ +from pathlib import Path + +def abs_path(rel_path): + """Turn a relative path into abs path""" + return str(Path(str(Path(__file__).parents[2])) / rel_path) + +# Datafile access +AGG_STAT_AND_BOOT_DATA = abs_path("METcalcpy/test/data/agg_stat_and_boot_data.data") \ No newline at end of file From e8b760350951b784967269bd20fcfa47d8793dc5 Mon Sep 17 00:00:00 2001 From: John Sharples <41682323+John-Sharples@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:09:07 +1100 Subject: [PATCH 06/10] 401: Migrate install to pyproject.toml (#414) * 401: Migrate install to pyproject.toml * remove modification to __init__.py * Remove setup.py references from docs --- .coveragerc | 8 ---- README.md | 4 +- docs/Users_Guide/installation.rst | 2 +- docs/version | 2 +- pyproject.toml | 62 +++++++++++++++++++++++++++++++ setup.py | 29 --------------- 6 files changed, 66 insertions(+), 41 deletions(-) delete mode 100644 .coveragerc create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 2acbc5ae..00000000 --- a/.coveragerc +++ /dev/null @@ -1,8 +0,0 @@ -[run] -source = metcalcpy -relative_files = True -omit = - config.py - config-3.py - metcalcpy/contributed/* - diff --git a/README.md b/README.md index 36d28fca..cea21286 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,10 @@ For information about the support provided for releases, see our [Release Suppor Instructions for installing the metcalcpy package locally --------------------------------------------------------- - activate your conda environment (i.e. 'conda activate your-conda-env-name') -- from within your active conda environment, cd to the METcalcpy/ directory, where you will see the setup.py script +- from within your active conda environment, cd to the METcalcpy/ directory, where you will see the file pyproject.toml - from this directory, run the following on the command line: pip install -e . - the -e option stands for editable, which is useful in that you can update your METcalcpy/metcalcpy source without reinstalling it -- the . indicates that you should search the current directory for the setup.py script +- the . indicates that you should search the current directory for the pyproject.toml file. - use metcalcpy package via import statement: - Examples: diff --git a/docs/Users_Guide/installation.rst b/docs/Users_Guide/installation.rst index 17bf80e3..6b96a17f 100644 --- a/docs/Users_Guide/installation.rst +++ b/docs/Users_Guide/installation.rst @@ -39,7 +39,7 @@ METcalcpy source code, e.g. `/User/someuser/METcalcpy`. From this directory, ru `pip install -e .` -This instructs pip to install the package based on instructios in the setup.py file located in the current directory +This instructs pip to install the package based on instructios in the pyproject.toml file located in the current directory (as indicated by the '.'). The `-e` directs pip to install the package in edit mode, so if one wishes to make changes to this source code, the changes are automatically applied without the need to re-install the package. diff --git a/docs/version b/docs/version index 5c8498d0..f0c3e423 100644 --- a/docs/version +++ b/docs/version @@ -1 +1 @@ -__version__="3.0.0-dev" +3.0.0-rc1-dev diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..82cf4242 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,62 @@ +[build-system] +requires = [ + "setuptools", + "wheel", + "setuptools-git-versioning>=2.0,<3", +] +build-backend = "setuptools.build_meta" + +[project] +name = "metcalcpy" +dynamic = ["version", "dependencies"] +description = "statistics and util package for METplus" +authors = [ + {name = "METplus"}, +] +requires-python = ">=3.10.4" +readme = "README.md" +license = {text = "MIT"} +classifiers = [ + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", +] + +[project.urls] +Homepage = "https://github.com/dtcenter/METcalcpy" + +[tool.setuptools.packages] +find = {include = ["metcalcpy*"]} + +[tool.setuptools.package-data] +metplus = [ + "docs/version" +] + +[tool.setuptools.exclude-package-data] +metplus = ["README", "__pycache__", "*~"] + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} + +[tool.setuptools-git-versioning] +enabled = true +version_file = "docs/version" + +[tool.pytest.ini_options] +testpaths = ["test"] + +[tool.coverage.run] +source = ["metcalcpy"] +omit = [ + "config.py", + "config-3.py", + "metcalcpy/contributed/*", + ] +relative_files = true + +[tool.coverage.report] +exclude_also = [ + "def __repr__", + "if __name__ == .__main__.:", + ] \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 2c06f5f8..00000000 --- a/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -import setuptools -from setuptools import setup, find_packages -from distutils.util import convert_path - -with open("README.md", "r") as fh: - long_description = fh.read() - -main_ns = {} -version_path = convert_path('docs/version') -with open(version_path) as version_file: - exec(version_file.read(), main_ns) - -setuptools.setup( - name="metcalcpy", - version=main_ns['__version__'], - author="METplus", - author_email="met-help@ucar.edu", - description="statistics and util package for METplus", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/dtcenter/METcalcpy", - packages=setuptools.find_packages(), - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - ], - python_requires='>=3.6', -) From bfa8b5363048def5ce445a52c5ba41ee95c5a8d5 Mon Sep 17 00:00:00 2001 From: bikegeek <3753118+bikegeek@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:18:59 -0700 Subject: [PATCH 07/10] Update version --- docs/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/version b/docs/version index f0c3e423..7232a97d 100644 --- a/docs/version +++ b/docs/version @@ -1 +1 @@ -3.0.0-rc1-dev +3.0.0-beta1-dev From d2a79923683ea37d5607f7e9df3f7d8d7e36794c Mon Sep 17 00:00:00 2001 From: bikegeek <3753118+bikegeek@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:19:40 -0700 Subject: [PATCH 08/10] Update docs/Users_Guide/installation.rst Co-authored-by: Julie Prestopnik --- docs/Users_Guide/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Users_Guide/installation.rst b/docs/Users_Guide/installation.rst index 6b96a17f..e60ce1c5 100644 --- a/docs/Users_Guide/installation.rst +++ b/docs/Users_Guide/installation.rst @@ -39,7 +39,7 @@ METcalcpy source code, e.g. `/User/someuser/METcalcpy`. From this directory, ru `pip install -e .` -This instructs pip to install the package based on instructios in the pyproject.toml file located in the current directory +This instructs pip to install the package based on instructions in the pyproject.toml file located in the current directory (as indicated by the '.'). The `-e` directs pip to install the package in edit mode, so if one wishes to make changes to this source code, the changes are automatically applied without the need to re-install the package. From 30610b234504374727f567ef5a1ea608ed2e1f91 Mon Sep 17 00:00:00 2001 From: John Sharples <41682323+John-Sharples@users.noreply.github.com> Date: Tue, 10 Dec 2024 03:15:43 +1100 Subject: [PATCH 09/10] 401 3d volrat tests (#417) * 401: initial commit * 401: improve 3d volrat test --- metcalcpy/util/mode_3d_volrat_statistics.py | 58 +++++----- metcalcpy/util/utils.py | 2 +- test/test_mode_3d_volrat_statistics.py | 113 ++++++++++++++++++++ 3 files changed, 144 insertions(+), 29 deletions(-) create mode 100644 test/test_mode_3d_volrat_statistics.py diff --git a/metcalcpy/util/mode_3d_volrat_statistics.py b/metcalcpy/util/mode_3d_volrat_statistics.py index 0ca9fb79..956cb13c 100644 --- a/metcalcpy/util/mode_3d_volrat_statistics.py +++ b/metcalcpy/util/mode_3d_volrat_statistics.py @@ -11,6 +11,8 @@ """ Program Name: mode_3d_volrat_statistics.py """ +import numpy as np + from metcalcpy.util.mode_arearat_statistics import * from metcalcpy.util.utils import column_data_by_name_value, THREE_D_DATA_FILTER from metcalcpy.util.safe_log import safe_log @@ -214,7 +216,7 @@ def calculate_3d_volrat_osm_osa(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio OSM/OSA calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -244,7 +246,7 @@ def calculate_3d_volrat_osu_osa(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio OSU/OSA calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -273,7 +275,7 @@ def calculate_3d_volrat_fsm_asm(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio FSM/ASM calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -303,7 +305,7 @@ def calculate_3d_volrat_osm_asm(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio OSM/ASM calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -333,7 +335,7 @@ def calculate_3d_volrat_osu_asu(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio OSU/ASU calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -363,7 +365,7 @@ def calculate_3d_volrat_fsa_aaa(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio FSA/AAA calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -393,7 +395,7 @@ def calculate_3d_volrat_osa_aaa(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio OSA/AAA calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -424,7 +426,7 @@ def calculate_3d_volrat_fsa_faa(input_data, columns_names, logger=None): try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio FSA/FAA calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -454,7 +456,7 @@ def calculate_3d_volrat_fca_faa(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio FCA/FAA calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -484,7 +486,7 @@ def calculate_3d_volrat_osa_oaa(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio OSA/OAA calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -514,7 +516,7 @@ def calculate_3d_volrat_oca_oaa(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio OCA/OAA calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -544,7 +546,7 @@ def calculate_3d_volrat_fca_aca(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio FCA/ACA calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -574,7 +576,7 @@ def calculate_3d_volrat_oca_aca(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio OCA/ACA calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -605,7 +607,7 @@ def calculate_3d_volrat_fsa_osa(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio FSA/OSA calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -636,7 +638,7 @@ def calculate_3d_volrat_osa_fsa(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio OSA/FSA calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -666,7 +668,7 @@ def calculate_3d_volrat_aca_asa(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio ACA/ASA calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -696,7 +698,7 @@ def calculate_3d_volrat_asa_aca(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio ASA/ACA calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -727,7 +729,7 @@ def calculate_3d_volrat_fca_fsa(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio FCA/FSA calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -758,7 +760,7 @@ def calculate_3d_volrat_fsa_fca(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio FSA/FCA calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -789,7 +791,7 @@ def calculate_3d_volrat_oca_osa(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio OCA/OSA calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -820,7 +822,7 @@ def calculate_3d_volrat_osa_oca(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Ratio OSA/OCA calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -850,7 +852,7 @@ def calculate_3d_objvhits(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Hits calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -880,7 +882,7 @@ def calculate_3d_objvmisses(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Renaming columns for 3D Volume Misses calculation.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", "Filtering data based on THREE_D_DATA_FILTER.") filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) @@ -909,7 +911,7 @@ def calculate_3d_objvfas(input_data, columns_names, logger=None): or None if some of the data values are missing or invalid """ try: - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) filtered_data = column_data_by_name_value(input_data, columns_names_new, THREE_D_DATA_FILTER) result = calculate_objafas(filtered_data, columns_names_new) except Exception as e: @@ -933,7 +935,7 @@ def calculate_3d_objvcsi(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Starting the renaming of columns.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", f"Renamed columns: {columns_names_new}") safe_log(logger, "debug", "Filtering data based on the new column names.") @@ -964,7 +966,7 @@ def calculate_3d_objvpody(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Starting the renaming of columns.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", f"Renamed columns: {columns_names_new}") safe_log(logger, "debug", "Filtering data based on the new column names.") @@ -995,7 +997,7 @@ def calculate_3d_objvfar(input_data, columns_names, logger=None): """ try: safe_log(logger, "debug", "Starting the renaming of columns.") - columns_names_new = rename_column(column_names, logger=logger) + columns_names_new = rename_column(columns_names, logger=logger) safe_log(logger, "debug", f"Renamed columns: {columns_names_new}") safe_log(logger, "debug", "Filtering data based on the new column names.") @@ -1027,4 +1029,4 @@ def rename_column(columns_names, logger=None): columns_names_new.insert(index, 'area') else: columns_names_new.insert(index, name) - return columns_names_new + return np.array(columns_names_new) diff --git a/metcalcpy/util/utils.py b/metcalcpy/util/utils.py index 9a0ff8ff..38f7c2f5 100644 --- a/metcalcpy/util/utils.py +++ b/metcalcpy/util/utils.py @@ -347,7 +347,7 @@ def column_data_by_name_value(input_data, columns, filters): try: # for each filter for key, value in filters.items(): - # get an index og the column + # get an index of the column index_array = np.where(columns == key)[0] if index_array.size == 0: return 0 diff --git a/test/test_mode_3d_volrat_statistics.py b/test/test_mode_3d_volrat_statistics.py new file mode 100644 index 00000000..a8a7ae03 --- /dev/null +++ b/test/test_mode_3d_volrat_statistics.py @@ -0,0 +1,113 @@ +import pytest +from unittest.mock import patch +import numpy as np +import metcalcpy.util.mode_3d_volrat_statistics as m3vs + +column_names = np.array( + ["object_type", "volume", "fcst_flag", "simple_flag", "matched_flag"] +) + + +data_simple = np.array( + [ + ["3d", 100, 1, 1, 1], + ["3d", 30, 0, 1, 1], + ] +) + + +data_complex = np.array( + [ + ["3d", 100, 1, 1, 1], + ["3d", 30, 0, 1, 1], + ["3d", 120, 1, 1, 0], + ["3d", 12, 0, 1, 1], + ["3d", 1, 1, 0, 1], + ["3d", 17, 0, 1, 1], + ["2d", 200, 1, 1, 1], + ["3d", 66, 0, 1, 0], + ] +) + + +@pytest.mark.parametrize( + "func_name,data_values,expected", + [ + ("volrat_fsa_asa", data_simple, 0.7692308), + ("volrat_osa_asa", data_simple, 0.2307692), + ("volrat_asm_asa", data_simple, 1.0), + ("volrat_asu_asa", data_simple, None), + ("volrat_fsm_fsa", data_simple, 1.0), + ("volrat_fsu_fsa", data_simple, None), + ("volrat_osm_osa", data_simple, 1.0), + ("volrat_osu_osa", data_simple, None), + ("volrat_fsm_asm", data_simple, None), + ("volrat_osm_asm", data_simple, 0.2307692), + ("volrat_osu_asu", data_simple, None), + ("volrat_fsa_aaa", data_simple, 0.7692308), + ("volrat_osa_aaa", data_simple, 0.2307692), + ("volrat_fsa_faa", data_simple, 1.0), + ("volrat_fca_faa", data_simple, None), + ("volrat_osa_oaa", data_simple, 1.0), + ("volrat_oca_oaa", data_simple, None), + ("volrat_fca_aca", data_simple, None), + ("volrat_oca_aca", data_simple, None), + ("volrat_fsa_osa", data_simple, 3.3333333), + ("volrat_osa_fsa", data_simple, 1.0), + ("volrat_aca_asa", data_simple, None), + ("volrat_asa_aca", data_simple, None), + ("volrat_fca_fsa", data_simple, None), + ("volrat_fsa_fca", data_simple, None), + ("volrat_oca_osa", data_simple, None), + ("volrat_osa_oca", data_simple, None), + ("objvhits", data_simple, 65.0), + ("objvmisses", data_simple, None), + ("objvfas", data_simple, None), + ("objvcsi", data_simple, None), + ("objvpody", data_simple, None), + ("objvfar", data_simple, None), + ("volrat_fsa_asa", data_complex, 0.6376812), + ("volrat_osa_asa", data_complex, 0.3623188), + ("volrat_asm_asa", data_complex, 0.4608696), + ("volrat_asu_asa", data_complex, 0.5391304), + ("volrat_fsm_fsa", data_complex, 0.4545455), + ("volrat_fsu_fsa", data_complex, 0.5454545), + ("volrat_osm_osa", data_complex, 0.472), + ("volrat_osu_osa", data_complex, 0.528), + ("volrat_fsm_asm", data_complex, 0.5454545), + ("volrat_osm_asm", data_complex, 0.3710692), + ("volrat_osu_asu", data_complex, 0.3548387), + ("volrat_fsa_aaa", data_complex, 0.6358382), + ("volrat_osa_aaa", data_complex, 0.3612717), + ("volrat_fsa_faa", data_complex, 0.9954751), + ("volrat_fca_faa", data_complex, 0.0045249), + ("volrat_osa_oaa", data_complex, 1.0), + ("volrat_oca_oaa", data_complex, None), + ("volrat_fca_aca", data_complex, 1.0), + ("volrat_oca_aca", data_complex, None), + ("volrat_fsa_osa", data_complex, 1.76), + ("volrat_osa_fsa", data_complex, 1.0), + ("volrat_aca_asa", data_complex, 0.0028986), + ("volrat_asa_aca", data_complex, 345.0), + ("volrat_fca_fsa", data_complex, 0.0045455), + ("volrat_fsa_fca", data_complex, 220.0), + ("volrat_oca_osa", data_complex, None), + ("volrat_osa_oca", data_complex, None), + ("objvhits", data_complex, 79.5), + ("objvmisses", data_complex, 66.0), + ("objvfas", data_complex, 120.0), + ("objvcsi", data_complex, 0.299435), + ("objvpody", data_complex, 0.5463918), + ("objvfar", data_complex, 0.2739726), + ], +) +def test_calculate_3d_volrat(func_name, data_values, expected): + func_str = f"calculate_3d_{func_name}" + func = getattr(m3vs, func_str) + actual = func(data_values, column_names) + assert actual == expected + + # Check None is returned on Exception + with patch.object(m3vs, "rename_column", side_effect=TypeError): + actual = func(data_values, column_names) + assert actual == None From 46ecf95f81cef8e87d0db032e685b1ec1ef6cd4a Mon Sep 17 00:00:00 2001 From: Julie Prestopnik Date: Thu, 12 Dec 2024 11:30:19 -0700 Subject: [PATCH 10/10] Adding In Memoriam --- docs/index.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index b27ff749..e1606462 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,6 +7,27 @@ Boulder, CO .. image:: _static/METplus_banner_photo_web.png +In Memoriam +----------- +This coordinated release is dedicated to three remarkable team members +we lost in 2024, whose contributions have left an indelible mark on our work. + +To `Tara Jensen `_, +for her vision and leadership in creating METplus as well as her +dedication, dogged determination, and mentorship that shaped its growth and +trajectory, leaving a legacy of innovation in the field of verification. + +To `Randy Bullock `_, +whose verification libraries formed the basis of MET and +whose mathematical brilliance, passion for maps, grid projections, and +graphics enriched and inspired new capabilities. + +To `Venita Hagerty `_, +for her pivotal expertise, support, and attention to +detail that ensured the success of METdataio and METexpress. + +Their contributions to METplus continue to guide and inspire us each day. + History -------