From 2da40a3861d9e5052116e331dfaf3d9509d1372b Mon Sep 17 00:00:00 2001
From: James Brown <64858662+james-d-brown@users.noreply.github.com>
Date: Tue, 26 Nov 2024 14:41:53 +0000
Subject: [PATCH] Adjust the earliest valid time to account for the timescale
period when conducting an evaluation in memory, #369
---
.../wres/datamodel/time/TimeSeriesSlicer.java | 93 +++++++++++-------
.../datamodel/time/TimeSeriesSlicerTest.java | 28 +++++-
.../io/retrieving/RetrieverUtilities.java | 4 +-
.../database/TimeSeriesRetriever.java | 96 ++-----------------
.../EnsembleRetrieverFactoryInMemory.java | 8 +-
...eSingleValuedRetrieverFactoryInMemory.java | 8 +-
.../SingleValuedRetrieverFactoryInMemory.java | 8 +-
7 files changed, 107 insertions(+), 138 deletions(-)
diff --git a/wres-datamodel/src/wres/datamodel/time/TimeSeriesSlicer.java b/wres-datamodel/src/wres/datamodel/time/TimeSeriesSlicer.java
index 8ae01b4bc2..be759a9b63 100644
--- a/wres-datamodel/src/wres/datamodel/time/TimeSeriesSlicer.java
+++ b/wres-datamodel/src/wres/datamodel/time/TimeSeriesSlicer.java
@@ -1616,58 +1616,81 @@ public static TimeSeries snip( TimeSeries toSnip,
}
/**
- * Adjusts the earliest lead duration of the time window to account for the period associated with the desired time
- * scale in order to capture sufficient data for rescaling. If the time scale is instantaneous, no adjustment is
- * made.
+ * Subtracts any non-instantaneous desired timescale from the earliest lead duration and the earliest valid time to
+ * ensure that sufficient data is retrieved for upscaling.
*
- * @param timeWindow the time window to adjust, required
- * @param desiredTimeScale the desired time scale, lenient if null (returns the input time window)
+ * @param timeWindow the time window to adjust
+ * @param timeScale the timescale to use
* @return the adjusted time window
*/
- public static TimeWindowOuter adjustByTimeScalePeriod( TimeWindowOuter timeWindow, TimeScaleOuter desiredTimeScale )
+ public static TimeWindowOuter adjustTimeWindowForTimeScale( TimeWindowOuter timeWindow,
+ TimeScaleOuter timeScale )
{
- Objects.requireNonNull( timeWindow );
+ TimeWindow.Builder adjusted = timeWindow.getTimeWindow()
+ .toBuilder();
- if ( Objects.nonNull( desiredTimeScale ) && desiredTimeScale.isInstantaneous() )
+ // Earliest lead duration
+ if ( !timeWindow.getEarliestLeadDuration()
+ .equals( TimeWindowOuter.DURATION_MIN ) )
{
- LOGGER.debug( "Not adjusting the time window of {} with the time scale because the time scale is "
- + "instantaneous.", timeWindow );
- return timeWindow;
+ Duration period = Duration.ZERO;
+
+ // Adjust the lower bound of the lead duration window by the non-instantaneous desired timescale
+ if ( Objects.nonNull( timeScale )
+ && !timeScale.isInstantaneous() )
+ {
+ period = TimeScaleOuter.getOrInferPeriodFromTimeScale( timeScale );
+ }
+
+ Duration lowered = timeWindow.getEarliestLeadDuration()
+ .minus( period );
+
+ if ( Objects.nonNull( timeScale )
+ && LOGGER.isDebugEnabled() )
+ {
+ LOGGER.debug( "Adjusting the lower lead duration of time window {} from {} to {} "
+ + "in order to acquire data at the desired timescale of {}.",
+ timeWindow,
+ timeWindow.getEarliestLeadDuration(),
+ lowered,
+ timeScale );
+ }
+
+ adjusted.setEarliestLeadDuration( MessageFactory.getDuration( lowered ) );
}
- TimeWindowOuter adjustedWindow = timeWindow;
- if ( !timeWindow.getEarliestLeadDuration().equals( TimeWindowOuter.DURATION_MIN )
- && Objects.nonNull( desiredTimeScale ) )
+ // Earliest valid time
+ if ( !timeWindow.getEarliestValidTime()
+ .equals( Instant.MIN ) )
{
- Duration period = TimeScaleOuter.getOrInferPeriodFromTimeScale( desiredTimeScale );
- Duration lowerD = timeWindow.getEarliestLeadDuration()
- .minus( period );
- com.google.protobuf.Duration lower =
- com.google.protobuf.Duration.newBuilder()
- .setSeconds( lowerD.getSeconds() )
- .setNanos( lowerD.getNano() )
- .build();
+ Duration period = Duration.ZERO;
- TimeWindow inner = timeWindow.getTimeWindow()
- .toBuilder()
- .setEarliestLeadDuration( lower )
- .build();
+ // Adjust the lower bound of the lead duration window by the non-instantaneous desired timescale
+ if ( Objects.nonNull( timeScale )
+ && !timeScale.isInstantaneous() )
+ {
+ period = TimeScaleOuter.getOrInferPeriodFromTimeScale( timeScale );
+ }
- adjustedWindow = TimeWindowOuter.of( inner );
+ Instant lowered = timeWindow.getEarliestValidTime()
+ .minus( period );
- if ( LOGGER.isDebugEnabled() )
+ if ( Objects.nonNull( timeScale )
+ && LOGGER.isDebugEnabled() )
{
- LOGGER.debug(
- "Adjusted the earliest lead duration of {} by {} to {}, in order to select sufficient data "
- + "for rescaling.",
- timeWindow,
- desiredTimeScale.getPeriod(),
- adjustedWindow );
+ LOGGER.debug( "Adjusting the lower valid datetime of time window {} from {} to {} "
+ + "in order to acquire data at the desired timescale of {}.",
+ timeWindow,
+ timeWindow.getEarliestValidTime(),
+ lowered,
+ timeScale );
}
+
+ adjusted.setEarliestValidTime( MessageFactory.getTimestamp( lowered ) );
}
- return adjustedWindow;
+ return TimeWindowOuter.of( adjusted.build() );
}
/**
diff --git a/wres-datamodel/test/wres/datamodel/time/TimeSeriesSlicerTest.java b/wres-datamodel/test/wres/datamodel/time/TimeSeriesSlicerTest.java
index 2f253711f0..5b14947f95 100644
--- a/wres-datamodel/test/wres/datamodel/time/TimeSeriesSlicerTest.java
+++ b/wres-datamodel/test/wres/datamodel/time/TimeSeriesSlicerTest.java
@@ -1472,12 +1472,12 @@ void testAdjustByTimeScalePeriodWhenTimeScaleIsInstantaneous()
TimeScaleOuter timeScaleOuter = TimeScaleOuter.of( timeScale );
TimeWindowOuter timeWindowOuter = TimeWindowOuter.of( timeWindow );
- TimeWindowOuter actual = TimeSeriesSlicer.adjustByTimeScalePeriod( timeWindowOuter, timeScaleOuter );
+ TimeWindowOuter actual = TimeSeriesSlicer.adjustTimeWindowForTimeScale( timeWindowOuter, timeScaleOuter );
assertEquals( timeWindowOuter, actual );
}
@Test
- void testAdjustByTimeScalePeriod()
+ void testAdjustTimeWindowEarliestLeadDurationForTimeScale()
{
TimeWindow timeWindow = MessageFactory.getTimeWindow( Duration.ofHours( 1 ),
Duration.ofHours( 2 ) );
@@ -1488,7 +1488,7 @@ void testAdjustByTimeScalePeriod()
TimeScaleOuter timeScaleOuter = TimeScaleOuter.of( timeScale );
TimeWindowOuter timeWindowOuter = TimeWindowOuter.of( timeWindow );
- TimeWindowOuter actual = TimeSeriesSlicer.adjustByTimeScalePeriod( timeWindowOuter, timeScaleOuter );
+ TimeWindowOuter actual = TimeSeriesSlicer.adjustTimeWindowForTimeScale( timeWindowOuter, timeScaleOuter );
TimeWindow expectedInner = MessageFactory.getTimeWindow( Duration.ofMinutes( 30 ),
Duration.ofHours( 2 ) );
TimeWindowOuter expected = TimeWindowOuter.of( expectedInner );
@@ -1496,6 +1496,28 @@ void testAdjustByTimeScalePeriod()
assertEquals( expected, actual );
}
+ @Test
+ void testAdjustTimeWindowEarliestValidTimeForTimeScale()
+ {
+ Instant earliest = Instant.parse( "2055-03-23T00:00:00Z" );
+ Instant latest = Instant.parse( "2055-03-24T00:00:00Z" );
+
+ TimeWindow timeWindow = MessageFactory.getTimeWindow( earliest, latest );
+ TimeScale timeScale = TimeScale.newBuilder()
+ .setPeriod( com.google.protobuf.Duration.newBuilder()
+ .setSeconds( 86400 ) )
+ .build();
+ TimeScaleOuter timeScaleOuter = TimeScaleOuter.of( timeScale );
+ TimeWindowOuter timeWindowOuter = TimeWindowOuter.of( timeWindow );
+
+ TimeWindowOuter actual = TimeSeriesSlicer.adjustTimeWindowForTimeScale( timeWindowOuter, timeScaleOuter );
+ TimeWindow expectedInner = MessageFactory.getTimeWindow( Instant.parse( "2055-03-22T00:00:00Z" ),
+ latest );
+ TimeWindowOuter expected = TimeWindowOuter.of( expectedInner );
+
+ assertEquals( expected, actual );
+ }
+
@Test
void testGetTimesteps()
{
diff --git a/wres-io/src/wres/io/retrieving/RetrieverUtilities.java b/wres-io/src/wres/io/retrieving/RetrieverUtilities.java
index d258fcad03..724e98a3d4 100644
--- a/wres-io/src/wres/io/retrieving/RetrieverUtilities.java
+++ b/wres-io/src/wres/io/retrieving/RetrieverUtilities.java
@@ -185,8 +185,8 @@ public static TimeWindowOuter getTimeWindowWithUnconditionalLeadTimes( TimeWindo
.setLatestLeadDuration( upper )
.build();
- return TimeSeriesSlicer.adjustByTimeScalePeriod( TimeWindowOuter.of( inner ),
- timeScale );
+ return TimeSeriesSlicer.adjustTimeWindowForTimeScale( TimeWindowOuter.of( inner ),
+ timeScale );
}
/**
diff --git a/wres-io/src/wres/io/retrieving/database/TimeSeriesRetriever.java b/wres-io/src/wres/io/retrieving/database/TimeSeriesRetriever.java
index 275d6ead0f..f069ab3eab 100644
--- a/wres-io/src/wres/io/retrieving/database/TimeSeriesRetriever.java
+++ b/wres-io/src/wres/io/retrieving/database/TimeSeriesRetriever.java
@@ -38,6 +38,7 @@
import wres.datamodel.time.TimeSeries;
import wres.datamodel.time.TimeSeriesMetadata;
+import wres.datamodel.time.TimeSeriesSlicer;
import wres.datamodel.time.TimeWindowOuter;
import wres.datamodel.DataProvider;
import wres.io.database.caching.Features;
@@ -46,10 +47,8 @@
import wres.io.database.Database;
import wres.io.retrieving.Retriever;
import wres.io.retrieving.DataAccessException;
-import wres.statistics.MessageFactory;
import wres.statistics.generated.TimeScale.TimeScaleFunction;
import wres.statistics.generated.ReferenceTime.ReferenceTimeType;
-import wres.statistics.generated.TimeWindow;
/**
* Abstract base class for retrieving {@link TimeSeries} from a database.
@@ -525,7 +524,7 @@ void addTimeWindowClause( DataScripter script )
// Subtract any non-instantaneous desired timescale period from the lower bound of the lead duration and
// valid time
- filter = TimeSeriesRetriever.adjustTimeWindowForTimeScale( filter, this.desiredTimeScale );
+ filter = TimeSeriesSlicer.adjustTimeWindowForTimeScale( filter, this.desiredTimeScale );
// Forecasts?
if ( this.isForecast() )
@@ -1316,84 +1315,6 @@ private void addEventToTimeSeries( Event event, TimeSeries.Builder bui
}
}
- /**
- * Subtracts any non-instantaneous desired timescale from the lower bound of the lead duration and valid time to
- * ensure that sufficient data is retrieved for upscaling.
- *
- * @param timeWindow the time window to adjust
- * @param timeScale the timescale to use
- * @return the adjusted time window
- */
-
- private static TimeWindowOuter adjustTimeWindowForTimeScale( TimeWindowOuter timeWindow,
- TimeScaleOuter timeScale )
- {
- TimeWindow.Builder adjusted = timeWindow.getTimeWindow()
- .toBuilder();
-
- // Earliest lead duration
- if ( !timeWindow.getEarliestLeadDuration()
- .equals( TimeWindowOuter.DURATION_MIN ) )
- {
- Duration period = Duration.ZERO;
-
- // Adjust the lower bound of the lead duration window by the non-instantaneous desired timescale
- if ( Objects.nonNull( timeScale )
- && !timeScale.isInstantaneous() )
- {
- period = TimeScaleOuter.getOrInferPeriodFromTimeScale( timeScale );
- }
-
- Duration lowered = timeWindow.getEarliestLeadDuration()
- .minus( period );
-
- if ( Objects.nonNull( timeScale )
- && LOGGER.isDebugEnabled() )
- {
- LOGGER.debug( "Adjusting the lower lead duration of time window {} from {} to {} "
- + "in order to acquire data at the desired timescale of {}.",
- timeWindow,
- timeWindow.getEarliestLeadDuration(),
- lowered,
- timeScale );
- }
-
- adjusted.setEarliestLeadDuration( MessageFactory.getDuration( lowered ) );
- }
-
- // Earliest valid time
- if ( !timeWindow.getEarliestValidTime()
- .equals( Instant.MIN ) )
- {
- Duration period = Duration.ZERO;
-
- // Adjust the lower bound of the lead duration window by the non-instantaneous desired timescale
- if ( Objects.nonNull( timeScale )
- && !timeScale.isInstantaneous() )
- {
- period = TimeScaleOuter.getOrInferPeriodFromTimeScale( timeScale );
- }
-
- Instant lowered = timeWindow.getEarliestValidTime()
- .minus( period );
-
- if ( Objects.nonNull( timeScale )
- && LOGGER.isDebugEnabled() )
- {
- LOGGER.debug( "Adjusting the lower valid datetime of time window {} from {} to {} "
- + "in order to acquire data at the desired timescale of {}.",
- timeWindow,
- timeWindow.getEarliestValidTime(),
- lowered,
- timeScale );
- }
-
- adjusted.setEarliestValidTime( MessageFactory.getTimestamp( lowered ) );
- }
-
- return TimeWindowOuter.of( adjusted.build() );
- }
-
/**
* Adds the lead duration bounds (if any) to the script. The interval is left-closed.
*
@@ -1603,25 +1524,28 @@ private Instant getOrInferLowerValidTime( TimeWindowOuter timeWindow )
Instant lowerValidTime = Instant.MIN;
// Lower bound present
- if ( !timeWindow.getEarliestValidTime().equals( Instant.MIN ) )
+ if ( !timeWindow.getEarliestValidTime()
+ .equals( Instant.MIN ) )
{
lowerValidTime = timeWindow.getEarliestValidTime();
}
- // Make a best effort to infer the valid times from any forecast information
+ // Make the best effort to infer the valid times from any forecast information
else
{
// Lower reference time available?
- if ( !timeWindow.getEarliestReferenceTime().equals( Instant.MIN ) )
+ if ( !timeWindow.getEarliestReferenceTime()
+ .equals( Instant.MIN ) )
{
// Use the lower reference time
lowerValidTime = timeWindow.getEarliestReferenceTime();
// Adjust for the earliest lead duration
- if ( !timeWindow.getEarliestLeadDuration().equals( TimeWindowOuter.DURATION_MIN ) )
+ if ( !timeWindow.getEarliestLeadDuration()
+ .equals( TimeWindowOuter.DURATION_MIN ) )
{
lowerValidTime = lowerValidTime.plus( timeWindow.getEarliestLeadDuration() );
- //Adjust for the desired timescale, if available
+ // Adjust for the desired timescale, if available
Duration period = Duration.ZERO;
if ( Objects.nonNull( this.desiredTimeScale ) )
diff --git a/wres-io/src/wres/io/retrieving/memory/EnsembleRetrieverFactoryInMemory.java b/wres-io/src/wres/io/retrieving/memory/EnsembleRetrieverFactoryInMemory.java
index 26d712f400..056a6efa1c 100644
--- a/wres-io/src/wres/io/retrieving/memory/EnsembleRetrieverFactoryInMemory.java
+++ b/wres-io/src/wres/io/retrieving/memory/EnsembleRetrieverFactoryInMemory.java
@@ -126,8 +126,8 @@ public Supplier>> getRightRetriever( Set fe
Objects.requireNonNull( features );
Objects.requireNonNull( timeWindow );
- TimeWindowOuter adjustedWindow = TimeSeriesSlicer.adjustByTimeScalePeriod( timeWindow,
- this.project.getDesiredTimeScale() );
+ TimeWindowOuter adjustedWindow = TimeSeriesSlicer.adjustTimeWindowForTimeScale( timeWindow,
+ this.project.getDesiredTimeScale() );
Dataset data = DeclarationUtilities.getDeclaredDataset( this.project.getDeclaration(),
DatasetOrientation.RIGHT );
@@ -169,8 +169,8 @@ public Supplier>> getBaselineRetriever( Set
Objects.requireNonNull( features );
Objects.requireNonNull( timeWindow );
- TimeWindowOuter adjustedWindow = TimeSeriesSlicer.adjustByTimeScalePeriod( timeWindow,
- this.project.getDesiredTimeScale() );
+ TimeWindowOuter adjustedWindow = TimeSeriesSlicer.adjustTimeWindowForTimeScale( timeWindow,
+ this.project.getDesiredTimeScale() );
Dataset data = DeclarationUtilities.getDeclaredDataset( this.project.getDeclaration(),
DatasetOrientation.BASELINE );
diff --git a/wres-io/src/wres/io/retrieving/memory/EnsembleSingleValuedRetrieverFactoryInMemory.java b/wres-io/src/wres/io/retrieving/memory/EnsembleSingleValuedRetrieverFactoryInMemory.java
index 014c3b700f..bd4af6689b 100644
--- a/wres-io/src/wres/io/retrieving/memory/EnsembleSingleValuedRetrieverFactoryInMemory.java
+++ b/wres-io/src/wres/io/retrieving/memory/EnsembleSingleValuedRetrieverFactoryInMemory.java
@@ -128,8 +128,8 @@ public Supplier>> getLeftRetriever( Set featu
public Supplier>> getRightRetriever( Set features,
TimeWindowOuter timeWindow )
{
- TimeWindowOuter adjustedWindow = TimeSeriesSlicer.adjustByTimeScalePeriod( timeWindow,
- this.project.getDesiredTimeScale() );
+ TimeWindowOuter adjustedWindow = TimeSeriesSlicer.adjustTimeWindowForTimeScale( timeWindow,
+ this.project.getDesiredTimeScale() );
Dataset data = DeclarationUtilities.getDeclaredDataset( this.project.getDeclaration(),
DatasetOrientation.RIGHT );
@@ -169,8 +169,8 @@ public Supplier>> getBaselineRetriever( Set f
public Supplier>> getBaselineRetriever( Set features,
TimeWindowOuter timeWindow )
{
- TimeWindowOuter adjustedWindow = TimeSeriesSlicer.adjustByTimeScalePeriod( timeWindow,
- this.project.getDesiredTimeScale() );
+ TimeWindowOuter adjustedWindow = TimeSeriesSlicer.adjustTimeWindowForTimeScale( timeWindow,
+ this.project.getDesiredTimeScale() );
Dataset data = DeclarationUtilities.getDeclaredDataset( this.project.getDeclaration(),
DatasetOrientation.BASELINE );
diff --git a/wres-io/src/wres/io/retrieving/memory/SingleValuedRetrieverFactoryInMemory.java b/wres-io/src/wres/io/retrieving/memory/SingleValuedRetrieverFactoryInMemory.java
index ee9572a09e..5ae1c147a7 100644
--- a/wres-io/src/wres/io/retrieving/memory/SingleValuedRetrieverFactoryInMemory.java
+++ b/wres-io/src/wres/io/retrieving/memory/SingleValuedRetrieverFactoryInMemory.java
@@ -121,8 +121,8 @@ public Supplier>> getLeftRetriever( Set featu
public Supplier>> getRightRetriever( Set features,
TimeWindowOuter timeWindow )
{
- TimeWindowOuter adjustedWindow = TimeSeriesSlicer.adjustByTimeScalePeriod( timeWindow,
- this.project.getDesiredTimeScale() );
+ TimeWindowOuter adjustedWindow = TimeSeriesSlicer.adjustTimeWindowForTimeScale( timeWindow,
+ this.project.getDesiredTimeScale() );
Dataset data = DeclarationUtilities.getDeclaredDataset( this.project.getDeclaration(),
DatasetOrientation.RIGHT );
@@ -159,8 +159,8 @@ public Supplier>> getBaselineRetriever( Set f
public Supplier>> getBaselineRetriever( Set features,
TimeWindowOuter timeWindow )
{
- TimeWindowOuter adjustedWindow = TimeSeriesSlicer.adjustByTimeScalePeriod( timeWindow,
- this.project.getDesiredTimeScale() );
+ TimeWindowOuter adjustedWindow = TimeSeriesSlicer.adjustTimeWindowForTimeScale( timeWindow,
+ this.project.getDesiredTimeScale() );
Dataset data = DeclarationUtilities.getDeclaredDataset( this.project.getDeclaration(),
DatasetOrientation.BASELINE );