Allow autoplot date parsing failures to be handled downstream as HTTP 422s

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ ci:
   autoupdate_schedule: quarterly
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.5.7"
+    rev: "v0.6.0"
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

CHANGELOG.md

@@ -9,6 +9,7 @@ All notable changes to this library are documented in this file.
 
 - Discontinue persisting text product into {mcd,mpd} table storage.
 - Discontinue storage of text product into sigmets, use `product_id` instead.
+- Raise `IncompleteWebRequest` exception when autoplot datetime parsing fails.
 
 ### New Features
 

src/pyiem/util.py

@@ -53,6 +53,18 @@ def format(self, record):
         )
 
 
+def _strptime(ins: str, fmt: str, rectify: bool = False) -> datetime:
+    """Wrapper around strptime."""
+    try:
+        return datetime.strptime(ins, fmt)
+    except ValueError as exp:
+        if rectify:
+            return handle_date_err(exp, ins, fmt)
+        raise IncompleteWebRequest(
+            f"String provided: `{ins}` does not match format: `{fmt}`"
+        ) from exp
+
+
 def web2ldm(url, ldm_product_name, md5_from_name=False, pqinsert="pqinsert"):
     """Download a URL and insert into LDM.
 
@@ -321,7 +333,8 @@ def handle_date_err(exp, value, fmt):
     res = lastday.strftime("%Y-%m-%d")
     if len(tokens) > 1:
         res += " " + tokens[1]
-    return datetime.strptime(res, fmt)
+    # Careful here, we don't want a recursive loop
+    return _strptime(res, fmt, rectify=False)
 
 
 def get_autoplot_context(fdict, cfg, enforce_optional=False, **kwargs):
@@ -442,11 +455,11 @@ def _float(val):
         elif typ == "datetime":
             # tricky here, php has YYYY/mm/dd and CGI has YYYY-mm-dd
             if default is not None:
-                default = datetime.strptime(default, "%Y/%m/%d %H%M")
+                default = _strptime(default, "%Y/%m/%d %H%M")
             if minval is not None:
-                minval = datetime.strptime(minval, "%Y/%m/%d %H%M")
+                minval = _strptime(minval, "%Y/%m/%d %H%M")
             if maxval is not None:
-                maxval = datetime.strptime(maxval, "%Y/%m/%d %H%M")
+                maxval = _strptime(maxval, "%Y/%m/%d %H%M")
             if value is not None:
                 # A common problem is for the space to be missing
                 if value.find(" ") == -1:
@@ -455,50 +468,40 @@ def _float(val):
                     else:
                         value += " 0000"
                 _dtfmt = "%Y-%m-%d %H%M"
-                try:
-                    value = datetime.strptime(
-                        value[:15].replace("/", "-"), "%Y-%m-%d %H%M"
-                    )
-                except ValueError as exp:
-                    if kwargs.get("rectify_dates", False):
-                        value = handle_date_err(exp, value, _dtfmt)
-                    else:
-                        # If we are not rectifying dates, we just raise the
-                        # exception
-                        raise
+                value = _strptime(
+                    value[:15].replace("/", "-"),
+                    "%Y-%m-%d %H%M",
+                    rectify=kwargs.get("rectify_dates", False),
+                )
 
         elif typ == "sday":
             # supports legacy uris with yyyy-mm-dd, before migration to sday
             if default is not None:
-                default = datetime.strptime(f"2000{default}", "%Y%m%d").date()
+                default = _strptime(f"2000{default}", "%Y%m%d").date()
             if minval is not None:
-                minval = datetime.strptime(f"2000{minval}", "%Y%m%d").date()
+                minval = _strptime(f"2000{minval}", "%Y%m%d").date()
             if maxval is not None:
-                maxval = datetime.strptime(f"2000{maxval}", "%Y%m%d").date()
+                maxval = _strptime(f"2000{maxval}", "%Y%m%d").date()
             if value is not None:
                 if value.find("-") > -1:
-                    value = datetime.strptime(value, "%Y-%m-%d").date()
+                    value = _strptime(value, "%Y-%m-%d").date()
                 else:
-                    value = datetime.strptime(f"2000{value}", "%Y%m%d").date()
+                    value = _strptime(f"2000{value}", "%Y%m%d").date()
 
         elif typ == "date":
             # tricky here, php has YYYY/mm/dd and CGI has YYYY-mm-dd
             if default is not None:
-                default = datetime.strptime(default, "%Y/%m/%d").date()
+                default = _strptime(default, "%Y/%m/%d").date()
             if minval is not None:
-                minval = datetime.strptime(minval, "%Y/%m/%d").date()
+                minval = _strptime(minval, "%Y/%m/%d").date()
             if maxval is not None:
-                maxval = datetime.strptime(maxval, "%Y/%m/%d").date()
+                maxval = _strptime(maxval, "%Y/%m/%d").date()
             if value is not None:
-                try:
-                    value = datetime.strptime(value, "%Y-%m-%d").date()
-                except ValueError as exp:
-                    if kwargs.get("rectify_dates", False):
-                        value = handle_date_err(exp, value, "%Y-%m-%d").date()
-                    else:
-                        # If we are not rectifying dates, we just raise the
-                        # exception
-                        raise
+                value = _strptime(
+                    value,
+                    "%Y-%m-%d",
+                    rectify=kwargs.get("rectify_dates", False),
+                ).date()
         elif typ == "vtec_ps":
             # VTEC phenomena and significance
             defaults = {}

src/pyiem/webutil.py

@@ -1,13 +1,13 @@
 """Utility functions for iemwebfarm applications."""
 
-import datetime
 import random
 import re
 import string
 import sys
 import traceback
 import warnings
 from collections import namedtuple
+from datetime import datetime
 from http import HTTPStatus
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
@@ -237,7 +237,7 @@ def compute_ts_from_string(form, key):
         fmt += ".%f"
     if len(tstr.split(":")) == 2:
         fmt = "%Y-%m-%d %H:%M"
-    return datetime.datetime.strptime(tstr, fmt).replace(tzinfo=tz)
+    return datetime.strptime(tstr, fmt).replace(tzinfo=tz)
 
 
 def compute_ts(form, suffix):
@@ -260,7 +260,7 @@ def compute_ts(form, suffix):
         else:
             day = 28
 
-    return datetime.datetime(
+    return datetime(
         int(yearval),
         month,
         day,
@@ -447,7 +447,7 @@ def _handle_exp(errormsg, routine=False, code=500):
                 )
                 return [msg.encode("ascii")]
 
-            start_time = datetime.datetime.utcnow()
+            start_time = datetime.utcnow()
             status_code = 500
             try:
                 # mixed convers this to a regular dict
@@ -489,7 +489,7 @@ def _handle_exp(errormsg, routine=False, code=500):
                 )
             except Exception:
                 res = _handle_exp(traceback.format_exc())
-            end_time = datetime.datetime.utcnow()
+            end_time = datetime.utcnow()
             if kwargs.get("enable_telemetry", True):
                 write_telemetry(
                     TELEMETRY(

tests/grid/test_zs.py

@@ -3,9 +3,10 @@
 import numpy as np
 from affine import Affine
 from geopandas import GeoSeries
-from pyiem.grid import zs
 from shapely.geometry import Polygon
 
+from pyiem.grid import zs
+
 
 def test_polygon_to_left():
     """Test that we don't get an artifact in the first column."""

tests/models/test_models_shef.py

@@ -2,6 +2,7 @@
 # pylint: disable=redefined-outer-name
 
 import pytest
+
 from pyiem.models.shef import SHEFElement
 from pyiem.util import utc
 

tests/ncei/test_ds3505.py

@@ -2,6 +2,7 @@
 # pylint: disable=redefined-outer-name
 
 import numpy as np
+
 from pyiem.ncei import ds3505
 from pyiem.util import get_test_filepath, utc
 

tests/nws/products/test_cf6.py

@@ -3,6 +3,7 @@
 import datetime
 
 import pytest
+
 from pyiem.nws.products.cf6 import parser
 from pyiem.reference import TRACE_VALUE
 from pyiem.util import get_test_file, utc

tests/nws/products/test_cli.py

@@ -3,6 +3,7 @@
 import datetime
 
 import pytest
+
 from pyiem.nws.products import cli
 from pyiem.nws.products import parser as cliparser
 from pyiem.nws.products.cli import CLIException, get_number

tests/nws/products/test_cwa.py

@@ -2,11 +2,11 @@
 
 # third party
 import pytest
+from shapely.geometry import Polygon
 
 # this
 from pyiem.nws.products.cwa import parser
 from pyiem.util import get_test_file, utc
-from shapely.geometry import Polygon
 
 LOCS = {
     "AMG": {"lon": -82.51, "lat": 31.54},

tests/nws/products/test_dsm.py

@@ -4,6 +4,7 @@
 from zoneinfo import ZoneInfo
 
 import pytest
+
 from pyiem.nws.products.dsm import compute_time, parser, process
 from pyiem.util import get_test_file, utc
 

tests/nws/products/test_fd.py

@@ -1,6 +1,7 @@
 """Test FD."""
 
 import pytest
+
 from pyiem.nws.products.fd import parse_encoding, parser
 from pyiem.util import get_test_file, utc
 

tests/nws/products/test_ffg.py

@@ -1,6 +1,7 @@
 """Testing FFG parsing."""
 
 import pytest
+
 from pyiem.nws.products.ffg import parser as ffgparser
 from pyiem.util import get_test_file
 

tests/nws/products/test_hml.py

@@ -3,6 +3,7 @@
 import datetime
 
 import pytest
+
 from pyiem.nws.products.hml import parser as hmlparser
 from pyiem.util import get_test_file, utc
 

tests/nws/products/test_lsr.py

@@ -1,6 +1,7 @@
 """Test Local Storm Report parsing."""
 
 import pytest
+
 from pyiem.nws.products.lsr import parse_lsr, parser
 from pyiem.reference import TRACE_VALUE
 from pyiem.util import get_test_file

tests/nws/products/test_mcd.py

@@ -1,6 +1,7 @@
 """MCD/MPD tests."""
 
 import pytest
+
 from pyiem.exceptions import MCDException
 from pyiem.nws.products import parser
 from pyiem.util import get_test_file, utc

tests/nws/products/test_metarcollect.py

@@ -3,6 +3,7 @@
 from unittest import mock
 
 import pytest
+
 from pyiem.nws.products import metarcollect
 from pyiem.reference import TRACE_VALUE
 from pyiem.util import get_test_file, utc

tests/nws/products/test_mos.py

@@ -1,6 +1,7 @@
 """Test MOS Parsing."""
 
 import pytest
+
 from pyiem.nws.products.mos import parser as mosparser
 from pyiem.util import get_test_file, utc
 

tests/nws/products/test_nldn.py

@@ -1,6 +1,7 @@
 """Test NLDN."""
 
 import pytest
+
 from pyiem.nws.products.nldn import parser
 from pyiem.util import get_test_filepath
 

tests/nws/products/test_pirep.py

@@ -1,6 +1,7 @@
 """PIREP."""
 
 import pytest
+
 from pyiem.nws.products.pirep import parser as pirepparser
 from pyiem.util import get_test_file, utc
 

tests/nws/products/test_products_vtec.py

@@ -5,6 +5,7 @@
 
 import pandas as pd
 import pytest
+
 from pyiem.nws.nwsli import NWSLI
 from pyiem.nws.products.vtec import check_dup_ps
 from pyiem.nws.products.vtec import parser as _vtecparser

tests/nws/products/test_saw.py

@@ -1,6 +1,7 @@
 """Can we process the SAW"""
 
 import pytest
+
 from pyiem.nws.products import parser
 from pyiem.nws.products.saw import parser as sawparser
 from pyiem.util import get_test_file, utc

tests/nws/products/test_scp.py

@@ -1,6 +1,7 @@
 """Can we process the SCP"""
 
 import pytest
+
 from pyiem.nws.products.scp import parser
 from pyiem.util import get_test_file, utc
 

tests/nws/products/test_shef.py

@@ -6,6 +6,7 @@
 
 import mock
 import pytest
+
 from pyiem.exceptions import InvalidSHEFEncoding, InvalidSHEFValue
 from pyiem.nws.products.shef import (
     make_date,

tests/nws/products/test_spcpts.py

@@ -1,6 +1,7 @@
 """Unit Tests"""
 
 import pytest
+
 from pyiem.nws.products import parser
 from pyiem.nws.products._outlook_util import debug_draw
 from pyiem.nws.products.spcpts import (

tests/nws/products/test_sps.py

@@ -1,6 +1,7 @@
 """SPS Parsing"""
 
 import pytest
+
 from pyiem.nws.products import parser as spsparser
 from pyiem.nws.ugc import UGC
 from pyiem.reference import TWEET_CHARS

tests/nws/products/test_taf.py

@@ -2,6 +2,7 @@
 
 # Third Party
 import pytest
+
 from pyiem.nws.products import parser as tafparser
 from pyiem.nws.products.taf import parser as real_tafparser
 from pyiem.reference import TAF_VIS_OVER_6SM

tests/nws/test_bufkit.py

@@ -5,6 +5,7 @@
 
 # third party
 import pytest
+
 from pyiem.nws.bufkit import read_bufkit
 
 # Local

tests/nws/test_lsrclass.py

@@ -1,6 +1,7 @@
 """test lsr."""
 
 import pytest
+
 from pyiem.nws.lsr import _icestorm_remark as ir
 from pyiem.nws.products import lsr
 from pyiem.nws.products.lsr import _mylowercase

tests/nws/test_product.py

@@ -1,6 +1,7 @@
 """Testing!"""
 
 import pytest
+
 from pyiem.nws import product, ugc
 from pyiem.nws.product import (
     TextProduct,