From 1e38b9bca64f2c89e81055a5d169688a4a45ddf0 Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Wed, 29 Nov 2023 12:33:40 +0000 Subject: [PATCH] Use remote index and matrices --- earthkit/regrid/utils/caching.py | 18 ++++++++++-------- pytest.ini | 1 - tests/conftest.py | 2 +- tests/data/local/index.json | 2 +- tools/make-matrix.sh | 3 +-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/earthkit/regrid/utils/caching.py b/earthkit/regrid/utils/caching.py index be1e9cc..3bde116 100644 --- a/earthkit/regrid/utils/caching.py +++ b/earthkit/regrid/utils/caching.py @@ -95,10 +95,6 @@ def disk_usage(path): def default_serialiser(o): if isinstance(o, (datetime.date, datetime.datetime)): return o.isoformat() - if isinstance(o, (pd.Timestamp)): - return o.isoformat() - if isinstance(o, (pd.DatetimeIndex)): - return [_.isoformat() for _ in o] return json.JSONEncoder.default(o) @@ -380,13 +376,18 @@ def _delete_entry(self, entry): if entry["size"] is None: entry["size"] = 0 - path, size, owner, args = ( + path, size = ( entry["path"], entry["size"], - entry["owner"], - entry["args"], ) + # path, size, owner, args = ( + # entry["path"], + # entry["size"], + # entry["owner"], + # entry["args"], + # ) + # LOG.warning( # "Deleting entry %s", # json.dumps( @@ -538,7 +539,8 @@ def _repr_html_(self): str HTML status of the cache. """ - html = [css("table")] + # html = [css("table")] + html = [] with self.new_connection() as db: for n in db.execute("SELECT * FROM cache"): n = dict(n) diff --git a/pytest.ini b/pytest.ini index ee3b889..ddae580 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,4 +4,3 @@ testpaths = tests markers = download: a test downloading GRIB data and/or matrices from the ECMWF download server tmp_cache: a test using a freshly created temporary cache - \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 99833e7..c5910ca 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,8 +12,8 @@ def pytest_runtest_setup(item): marks_in_items = list([m.name for m in item.iter_markers()]) - from earthkit.regrid.utils.caching import CACHE, SETTINGS from earthkit.regrid.db import DB + from earthkit.regrid.utils.caching import CACHE, SETTINGS DB.clear_index() diff --git a/tests/data/local/index.json b/tests/data/local/index.json index 0f137dd..64ada8f 100644 --- a/tests/data/local/index.json +++ b/tests/data/local/index.json @@ -111,4 +111,4 @@ "global": 1 } } -} \ No newline at end of file +} diff --git a/tools/make-matrix.sh b/tools/make-matrix.sh index 5d3ef58..1fd87a4 100644 --- a/tools/make-matrix.sh +++ b/tools/make-matrix.sh @@ -24,7 +24,7 @@ input_dir="grib" [[ ! -d $output_dir ]] && mkdir $output_dir # get mir version -version=$(~/build/mir/release/bin/mir --version x x 2>/dev/null | grep mir | cut -d ' ' -f 2) +version=$(~/build/mir/release/bin/mir --version x x 2>/dev/null | grep mir | cut -d ' ' -f 2) input_grib=$input_dir/${input}.grib output_json=$output_dir/${output}-${version}.json @@ -64,4 +64,3 @@ fi python3 -c "from earthkit.regrid.utils.matrix import make_matrix; make_matrix('$output_json','$output_dir',global_input=True,global_output=True,version='$version')" #rm -f $output_json -