From 7278900aa6b6749bf5766e103e31773525d4f81c Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 10 Jul 2023 16:48:01 -0600 Subject: [PATCH 01/16] Standardize time_bounds long_name. --- src/main/histFileMod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/histFileMod.F90 b/src/main/histFileMod.F90 index 92ce3dfa95..3e0d4c7597 100644 --- a/src/main/histFileMod.F90 +++ b/src/main/histFileMod.F90 @@ -3301,7 +3301,7 @@ subroutine htape_timeconst(t, mode) dim2id(1) = hist_interval_dimid; dim2id(2) = time_dimid call ncd_defvar(nfid(t), 'time_bounds', ncd_double, 2, dim2id, varid, & - long_name = 'history time interval endpoints') + long_name = 'time interval endpoints') dim2id(1) = strlen_dimid; dim2id(2) = time_dimid call ncd_defvar(nfid(t), 'date_written', ncd_char, 2, dim2id, varid) From 0ae9dcfa6579f858b05cf0c68532560387bc7f39 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 10 Jul 2023 16:49:40 -0600 Subject: [PATCH 02/16] Add units to time_bounds. --- src/main/histFileMod.F90 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/histFileMod.F90 b/src/main/histFileMod.F90 index 3e0d4c7597..f2020d4d3f 100644 --- a/src/main/histFileMod.F90 +++ b/src/main/histFileMod.F90 @@ -3301,7 +3301,8 @@ subroutine htape_timeconst(t, mode) dim2id(1) = hist_interval_dimid; dim2id(2) = time_dimid call ncd_defvar(nfid(t), 'time_bounds', ncd_double, 2, dim2id, varid, & - long_name = 'time interval endpoints') + long_name = 'time interval endpoints', & + units = str) dim2id(1) = strlen_dimid; dim2id(2) = time_dimid call ncd_defvar(nfid(t), 'date_written', ncd_char, 2, dim2id, varid) From 21fae2979961652b58809fcdca4ed60e376237c6 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 10 Jul 2023 16:50:15 -0600 Subject: [PATCH 03/16] Add calendar to time_bounds. --- src/main/histFileMod.F90 | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/histFileMod.F90 b/src/main/histFileMod.F90 index f2020d4d3f..1163c6a77d 100644 --- a/src/main/histFileMod.F90 +++ b/src/main/histFileMod.F90 @@ -3303,6 +3303,7 @@ subroutine htape_timeconst(t, mode) call ncd_defvar(nfid(t), 'time_bounds', ncd_double, 2, dim2id, varid, & long_name = 'time interval endpoints', & units = str) + call ncd_putatt(nfid(t), varid, 'calendar', caldesc) dim2id(1) = strlen_dimid; dim2id(2) = time_dimid call ncd_defvar(nfid(t), 'date_written', ncd_char, 2, dim2id, varid) From 1eb46c4abe5cee47aae205b81385166f4c2a944c Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 10 Jul 2023 16:51:15 -0600 Subject: [PATCH 04/16] Renamed dimension hist_interval to nbnd. Also renamed related dimid variable. --- src/main/histFileMod.F90 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/histFileMod.F90 b/src/main/histFileMod.F90 index 1163c6a77d..3c28d0ed5b 100644 --- a/src/main/histFileMod.F90 +++ b/src/main/histFileMod.F90 @@ -324,7 +324,7 @@ end subroutine copy_entry_interface type(file_desc_t), target :: nfid(max_tapes) ! file ids type(file_desc_t), target :: ncid_hist(max_tapes) ! file ids for history restart files integer :: time_dimid ! time dimension id - integer :: hist_interval_dimid ! time bounds dimension id + integer :: nbnd_dimid ! time bounds dimension id integer :: strlen_dimid ! string dimension id ! ! Time Constant variable names and filename @@ -2501,7 +2501,7 @@ subroutine htape_create (t, histrest) end if if ( .not. lhistrest )then - call ncd_defdim(lnfid, 'hist_interval', 2, hist_interval_dimid) + call ncd_defdim(lnfid, 'nbnd', 2, nbnd_dimid) call ncd_defdim(lnfid, 'time', ncd_unlimited, time_dimid) if (masterproc)then write(iulog,*) trim(subname), & @@ -3299,7 +3299,7 @@ subroutine htape_timeconst(t, mode) call ncd_defvar(nfid(t) , 'nstep' , ncd_int, 1, dim1id , varid, & long_name = 'time step') - dim2id(1) = hist_interval_dimid; dim2id(2) = time_dimid + dim2id(1) = nbnd_dimid; dim2id(2) = time_dimid call ncd_defvar(nfid(t), 'time_bounds', ncd_double, 2, dim2id, varid, & long_name = 'time interval endpoints', & units = str) From bbb80802023c6553752dc8f549b9c10c38b82c05 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 11 Jul 2023 13:44:25 -0600 Subject: [PATCH 05/16] Use my RTM/MOSART forks' standardize-time-metadata branches. --- Externals.cfg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index cd6855c9dc..fc9a1e7cb2 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -15,15 +15,15 @@ required = True [rtm] local_path = components/rtm protocol = git -repo_url = https://github.com/ESCOMP/RTM -tag = rtm1_0_78 +repo_url = https://github.com/samsrabin/RTM +tag = d8d9030bd7cb978d0b5da1b2e11a09c81103e9eb required = True [mosart] local_path = components/mosart protocol = git -repo_url = https://github.com/ESCOMP/MOSART -tag = mosart1_0_48 +repo_url = https://github.com/samsrabin/MOSART +tag = adbdcb50addcba657de6846623ca84e29c1a8e8a required = True [mizuRoute] From f255b06114e30acaf60b28e5d81cddc3807f847f Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 11 Jul 2023 14:19:23 -0600 Subject: [PATCH 06/16] Add calendar attribute to mcdate, mcsec, mdcur, and mscur. Also update Externals.cfg to use RTM and MOSART versions that have this same change. --- Externals.cfg | 4 ++-- src/main/histFileMod.F90 | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index fc9a1e7cb2..e3ed9b4818 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -16,14 +16,14 @@ required = True local_path = components/rtm protocol = git repo_url = https://github.com/samsrabin/RTM -tag = d8d9030bd7cb978d0b5da1b2e11a09c81103e9eb +tag = 8ef4d4e1e0979a31edcf21176c31fdd0302f32bc required = True [mosart] local_path = components/mosart protocol = git repo_url = https://github.com/samsrabin/MOSART -tag = adbdcb50addcba657de6846623ca84e29c1a8e8a +tag = 87f09b6a7c508500574271f2bdb8a31bf1f148fc required = True [mizuRoute] diff --git a/src/main/histFileMod.F90 b/src/main/histFileMod.F90 index 3c28d0ed5b..512c0825a4 100644 --- a/src/main/histFileMod.F90 +++ b/src/main/histFileMod.F90 @@ -3264,6 +3264,7 @@ subroutine htape_timeconst(t, mode) dim1id(1) = time_dimid call ncd_defvar(nfid(t) , 'mcdate', ncd_int, 1, dim1id , varid, & long_name = 'current date (YYYYMMDD)') + call ncd_putatt(nfid(t), varid, 'calendar', caldesc) ! ! add global attribute time_period_freq ! @@ -3292,10 +3293,13 @@ subroutine htape_timeconst(t, mode) call ncd_defvar(nfid(t) , 'mcsec' , ncd_int, 1, dim1id , varid, & long_name = 'current seconds of current date', units='s') + call ncd_putatt(nfid(t), varid, 'calendar', caldesc) call ncd_defvar(nfid(t) , 'mdcur' , ncd_int, 1, dim1id , varid, & long_name = 'current day (from base day)') + call ncd_putatt(nfid(t), varid, 'calendar', caldesc) call ncd_defvar(nfid(t) , 'mscur' , ncd_int, 1, dim1id , varid, & long_name = 'current seconds of current day') + call ncd_putatt(nfid(t), varid, 'calendar', caldesc) call ncd_defvar(nfid(t) , 'nstep' , ncd_int, 1, dim1id , varid, & long_name = 'time step') From 1c3ae10584101e01f5fb982442060f6653c675d9 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Wed, 20 Nov 2024 15:59:39 -0700 Subject: [PATCH 07/16] Move line inside if-statement --- src/main/histFileMod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/histFileMod.F90 b/src/main/histFileMod.F90 index c3c17c6b64..8f51d03ec1 100644 --- a/src/main/histFileMod.F90 +++ b/src/main/histFileMod.F90 @@ -3443,8 +3443,8 @@ subroutine htape_timeconst(t, mode) call ncd_defvar(nfid(t), 'time_bounds', ncd_double, 2, dim2id, varid, & long_name = 'time interval endpoints', & units = str) + call ncd_putatt(nfid(t), varid, 'calendar', caldesc) end if - call ncd_putatt(nfid(t), varid, 'calendar', caldesc) dim2id(1) = strlen_dimid; dim2id(2) = time_dimid call ncd_defvar(nfid(t), 'date_written', ncd_char, 2, dim2id, varid) From 69bd98cb4ff5d99dcfbe104169c74b8b6a1f2855 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 22 Nov 2024 11:23:19 -0700 Subject: [PATCH 08/16] crop_calendars scripts: Handle center-of-period timesteps. --- .../crop_calendars/convert_axis_time2gs.py | 3 +- python/ctsm/crop_calendars/cropcal_module.py | 16 +++---- python/ctsm/crop_calendars/cropcal_utils.py | 42 +++++++++++++++++++ 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/python/ctsm/crop_calendars/convert_axis_time2gs.py b/python/ctsm/crop_calendars/convert_axis_time2gs.py index d48514370d..004dca5518 100644 --- a/python/ctsm/crop_calendars/convert_axis_time2gs.py +++ b/python/ctsm/crop_calendars/convert_axis_time2gs.py @@ -5,6 +5,7 @@ import sys import numpy as np import xarray as xr +from ctsm.crop_calendars.cropcal_utils import get_integer_years try: import pandas as pd @@ -85,7 +86,7 @@ def set_up_ds_with_gs_axis(ds_in): if not any(x in ["mxsowings", "mxharvests"] for x in ds_in[var].dims): data_vars[var] = ds_in[var] # Set up the new dataset - gs_years = [t.year - 1 for t in ds_in.time.values[:-1]] + gs_years = get_integer_years(ds_in)[:-1] coords = ds_in.coords coords["gs"] = gs_years ds_out = xr.Dataset(data_vars=data_vars, coords=coords, attrs=ds_in.attrs) diff --git a/python/ctsm/crop_calendars/cropcal_module.py b/python/ctsm/crop_calendars/cropcal_module.py index 3ea084e1d2..1993225a9c 100644 --- a/python/ctsm/crop_calendars/cropcal_module.py +++ b/python/ctsm/crop_calendars/cropcal_module.py @@ -20,23 +20,19 @@ def check_and_trim_years(year_1, year_n, ds_in): """ After importing a file, restrict it to years of interest. """ - ### In annual outputs, file with name Y is actually results from year Y-1. - ### Note that time values refer to when it was SAVED. So 1981-01-01 is for year 1980. - - def get_year_from_cftime(cftime_date): - # Subtract 1 because the date for annual files is when it was SAVED - return cftime_date.year - 1 # Check that all desired years are included - if get_year_from_cftime(ds_in.time.values[0]) > year_1: + year = utils.get_timestep_year(ds_in, ds_in.time.values[0]) + if year > year_1: raise RuntimeError( f"Requested year_1 is {year_1} but first year in outputs is " - + f"{get_year_from_cftime(ds_in.time.values[0])}" + + f"{year}" ) - if get_year_from_cftime(ds_in.time.values[-1]) < year_1: + year = utils.get_timestep_year(ds_in, ds_in.time.values[-1]) + if year < year_1: raise RuntimeError( f"Requested year_n is {year_n} but last year in outputs is " - + f"{get_year_from_cftime(ds_in.time.values[-1])}" + + f"{year}" ) # Remove years outside range of interest diff --git a/python/ctsm/crop_calendars/cropcal_utils.py b/python/ctsm/crop_calendars/cropcal_utils.py index 584046edee..176ce18e9d 100644 --- a/python/ctsm/crop_calendars/cropcal_utils.py +++ b/python/ctsm/crop_calendars/cropcal_utils.py @@ -430,3 +430,45 @@ def make_lon_increasing(xr_obj): raise RuntimeError("Unable to rearrange longitude axis so it's monotonically increasing") return xr_obj.roll(lon=shift, roll_coords=True) + + +def is_inst_file(dsa): + """ + Check whether Dataset or DataArray has time data from an "instantaneous file" + """ + return "at end of" in dsa["time"].attrs["long_name"] + + +def get_beg_inst_timestep_year(timestep): + """ + Get year associated with the BEGINNING of a timestep in an + instantaneous file + """ + year = timestep.year + + is_jan1 = timestep.dayofyr == 1 + is_midnight = timestep.hour == timestep.minute == timestep.second == 0 + if is_jan1 and is_midnight: + year -= 1 + + return year + + +def get_timestep_year(dsa, timestep): + """ + Get the year associated with a timestep, with different handling + depending on whether the file is instantaneous + """ + if is_inst_file(dsa): + year = get_beg_inst_timestep_year(timestep) + else: + year = timestep.year + return year + + +def get_integer_years(dsa): + """ + Convert time axis to numpy array of integer years + """ + out_array = [get_timestep_year(dsa, t) for t in dsa["time"].values] + return out_array From bd7ecaa38bac304c1b5795b5516fd1b77d755356 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 22 Nov 2024 17:09:32 -0700 Subject: [PATCH 09/16] Add SystemTests for RXCROPMATURITY with instantaneous h1. --- cime_config/SystemTests/rxcropmaturity.py | 12 +++++----- cime_config/SystemTests/rxcropmaturityinst.py | 6 +++++ .../SystemTests/rxcropmaturityskipgeninst.py | 6 +++++ cime_config/config_tests.xml | 20 +++++++++++++++++ cime_config/testdefs/testlist_clm.xml | 22 +++++++++++++++++++ 5 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 cime_config/SystemTests/rxcropmaturityinst.py create mode 100644 cime_config/SystemTests/rxcropmaturityskipgeninst.py diff --git a/cime_config/SystemTests/rxcropmaturity.py b/cime_config/SystemTests/rxcropmaturity.py index fb254c408f..a6569da7dd 100644 --- a/cime_config/SystemTests/rxcropmaturity.py +++ b/cime_config/SystemTests/rxcropmaturity.py @@ -106,7 +106,7 @@ def __init__(self, case): # Which conda environment should we use? self._get_conda_env() - def _run_phase(self, skip_gen=False): + def _run_phase(self, skip_gen=False, h1_inst=False): # Modeling this after the SSP test, we create a clone to be the case whose outputs we don't # want to be saved as baseline. @@ -129,7 +129,7 @@ def _run_phase(self, skip_gen=False): self._set_active_case(case_gddgen) # Set up stuff that applies to both tests - self._setup_all() + self._setup_all(h1_inst) # Add stuff specific to GDD-Generating run logger.info("RXCROPMATURITY log: modify user_nl files: generate GDDs") @@ -264,7 +264,7 @@ def _get_rx_dates(self): logger.error(error_message) raise RuntimeError(error_message) - def _setup_all(self): + def _setup_all(self, h1_inst): logger.info("RXCROPMATURITY log: _setup_all start") # Get some info @@ -274,7 +274,7 @@ def _setup_all(self): # Set sowing dates file (and other crop calendar settings) for all runs logger.info("RXCROPMATURITY log: modify user_nl files: all tests") - self._modify_user_nl_allruns() + self._modify_user_nl_allruns(h1_inst) logger.info("RXCROPMATURITY log: _setup_all done") # Make a surface dataset that has every crop in every gridcell @@ -399,7 +399,7 @@ def _run_check_rxboth_run(self, skip_gen): tool_path, ) - def _modify_user_nl_allruns(self): + def _modify_user_nl_allruns(self, h1_inst): nl_additions = [ "cropcals_rx = .true.", "cropcals_rx_adapt = .false.", @@ -417,6 +417,8 @@ def _modify_user_nl_allruns(self): "hist_type1d_pertape(2) = 'PFTS'", "hist_dov2xy(2) = .false.", ] + if h1_inst: + nl_additions.append("hist_avgflag_pertape(2) = 'I'") self._append_to_user_nl_clm(nl_additions) def _run_generate_gdds(self, case_gddgen): diff --git a/cime_config/SystemTests/rxcropmaturityinst.py b/cime_config/SystemTests/rxcropmaturityinst.py new file mode 100644 index 0000000000..bf8bf7750b --- /dev/null +++ b/cime_config/SystemTests/rxcropmaturityinst.py @@ -0,0 +1,6 @@ +from rxcropmaturity import RXCROPMATURITYSHARED + + +class RXCROPMATURITYINST(RXCROPMATURITYSHARED): + def run_phase(self): + self._run_phase(h1_inst=True) diff --git a/cime_config/SystemTests/rxcropmaturityskipgeninst.py b/cime_config/SystemTests/rxcropmaturityskipgeninst.py new file mode 100644 index 0000000000..4cab9bd7c0 --- /dev/null +++ b/cime_config/SystemTests/rxcropmaturityskipgeninst.py @@ -0,0 +1,6 @@ +from rxcropmaturity import RXCROPMATURITYSHARED + + +class RXCROPMATURITYSKIPGENINST(RXCROPMATURITYSHARED): + def run_phase(self): + self._run_phase(skip_gen=True, h1_inst=True) diff --git a/cime_config/config_tests.xml b/cime_config/config_tests.xml index 12859b9131..ee80087a08 100644 --- a/cime_config/config_tests.xml +++ b/cime_config/config_tests.xml @@ -145,6 +145,16 @@ This defines various CTSM-specific system tests $STOP_N + + As RXCROPMATURITY but ensure instantaneous h1. Can be removed once instantaneous and other variables are on separate files. + 1 + FALSE + FALSE + never + $STOP_OPTION + $STOP_N + + As RXCROPMATURITY but don't actually generate GDDs. Allows short testing with existing GDD inputs. 1 @@ -155,6 +165,16 @@ This defines various CTSM-specific system tests $STOP_N + + As RXCROPMATURITYSKIPGEN but ensure instantaneous h1. Can be removed once instantaneous and other variables are on separate files. + 1 + FALSE + FALSE + never + $STOP_OPTION + $STOP_N + +