From 5cb4ce9970ff80088e62b197c80893b47125cb64 Mon Sep 17 00:00:00 2001 From: Philipp Geier Date: Thu, 12 Sep 2024 09:04:38 +0000 Subject: [PATCH] Add ifs-mapping action --- src/ecom/common/om_multio_utils_v3_mod.F90 | 8 +- .../multio_no_enc_output_manager_v2_mod.F90 | 52 ++++++------- src/multio/action/CMakeLists.txt | 1 + src/multio/action/ifs-mapping/IfsMapping.cc | 23 +++--- src/multio/action/ifs-mapping/IfsMapping.h | 2 +- src/multio/action/tag-filter/CMakeLists.txt | 17 +++++ src/multio/action/tag-filter/TagFilter.cc | 74 +++++++++++++++++++ src/multio/action/tag-filter/TagFilter.h | 47 ++++++++++++ src/multio/api/c/CMakeLists.txt | 2 + src/multio/message/Glossary.h | 8 +- src/multio/message/Message.cc | 20 ++++- src/multio/message/Message.h | 13 +++- src/multio/tools/multio-replay-nemo-capi.cc | 1 - .../multio-server-mpi-custom-splitting.yaml | 10 ++- .../multio-server-mpi-default-splitting.yaml | 8 ++ tests/multio/config/multio-server.yaml | 10 ++- 16 files changed, 246 insertions(+), 50 deletions(-) create mode 100644 src/multio/action/tag-filter/CMakeLists.txt create mode 100644 src/multio/action/tag-filter/TagFilter.cc create mode 100644 src/multio/action/tag-filter/TagFilter.h diff --git a/src/ecom/common/om_multio_utils_v3_mod.F90 b/src/ecom/common/om_multio_utils_v3_mod.F90 index e8cc46829..9b62f27dd 100644 --- a/src/ecom/common/om_multio_utils_v3_mod.F90 +++ b/src/ecom/common/om_multio_utils_v3_mod.F90 @@ -449,6 +449,8 @@ SUBROUTINE MULTIO_WRITE_PARAMETRIZATION( MIO_HANDLE, MODEL_PARAMS ) LON_STRET_DEG = 180._JPRD_K/(2.0_JPRD_K*ASIN(1.0_JPRD_K))*REAL(YPI%RLOCEN,JPRD_K) IF( YPI%NSTTYP .GE. 2 ) THEN + ERR = MIOMD%SET('gridType','stretched_rotated_sh') + PP_DEBUG_CRITICAL_COND_THROW( ERR.NE.MULTIO_SUCCESS, 1 ) ERR = MIOMD%SET('sphericalHarmonicsType','stretched_rotated_sh') PP_DEBUG_CRITICAL_COND_THROW( ERR.NE.MULTIO_SUCCESS, 1 ) ERR = MIOMD%SET('latitudeOfStretchingPoleInDegrees', LAT_STRET_DEG ) @@ -458,15 +460,17 @@ SUBROUTINE MULTIO_WRITE_PARAMETRIZATION( MIO_HANDLE, MODEL_PARAMS ) ERR = MIOMD%SET('stretchingFactor',YPI%RSTRET) PP_DEBUG_CRITICAL_COND_THROW( ERR.NE.MULTIO_SUCCESS, 1 ) ELSEIF( ABS(YPI%RSTRET-1.0_JPRD_K) .GE. 1.E-14_JPRD_K ) THEN + ERR = MIOMD%SET('gridType', 'stretched_sh' ) + PP_DEBUG_CRITICAL_COND_THROW( ERR.NE.MULTIO_SUCCESS, 1 ) ERR = MIOMD%SET('sphericalHarmonicsType', 'stretched_sh' ) PP_DEBUG_CRITICAL_COND_THROW( ERR.NE.MULTIO_SUCCESS, 1 ) ERR = MIOMD%SET('stretchingFactor', YPI%RSTRET ) PP_DEBUG_CRITICAL_COND_THROW( ERR.NE.MULTIO_SUCCESS, 1 ) ELSE - + ERR = MIOMD%SET('gridType','sh') + PP_DEBUG_CRITICAL_COND_THROW( ERR.NE.MULTIO_SUCCESS, 1 ) ERR = MIOMD%SET('sphericalHarmonicsType','sh') PP_DEBUG_CRITICAL_COND_THROW( ERR.NE.MULTIO_SUCCESS, 1 ) - ENDIF ERR = MIOMD%SET('pentagonalResolutionParameterJ',YPI%ISMAX) diff --git a/src/ecom/flavours/multio_no_enc_output_manager_v2_mod.F90 b/src/ecom/flavours/multio_no_enc_output_manager_v2_mod.F90 index cc000a792..23a84594c 100644 --- a/src/ecom/flavours/multio_no_enc_output_manager_v2_mod.F90 +++ b/src/ecom/flavours/multio_no_enc_output_manager_v2_mod.F90 @@ -474,28 +474,28 @@ SUBROUTINE MULTIO_NO_ENC_WRITE_ATM_DP( THIS, YDMSG, VALUES_DP ) CERR = MIOMD%SET( "paramId", INT(YDMSG%PARAM_ID_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "curr_step", INT(YDMSG%ISTEP_, C_LONG) ) + CERR = MIOMD%SET( "currentStep", INT(YDMSG%ISTEP_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "prev_pp", INT(YDMSG%IPREVPP_, C_LONG) ) + CERR = MIOMD%SET( "previousProcessingPoint", INT(YDMSG%IPREVPP_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "prefix", INT(YDMSG%IPREF_, C_LONG) ) + CERR = MIOMD%SET( "ifsPrefix", INT(YDMSG%IPREF_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "repres", INT(YDMSG%IREPRES_, C_LONG) ) + CERR = MIOMD%SET( "ifsRepres", INT(YDMSG%IREPRES_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) CERR = MIOMD%SET( "level", INT(YDMSG%ILEVG_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "nvalues", INT(YDMSG%NVALUES_, C_LONG) ) + CERR = MIOMD%SET( "numberOfValues", INT(YDMSG%NVALUES_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "nmissing", INT(YDMSG%NUNDF_, C_LONG) ) + CERR = MIOMD%SET( "numberOfMissingValues", INT(YDMSG%NUNDF_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "missing_value", REAL(YDMSG%XUNDF_, C_DOUBLE) ) + CERR = MIOMD%SET( "missingValue", REAL(YDMSG%XUNDF_, C_DOUBLE) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) ! Write to multIO @@ -659,28 +659,28 @@ SUBROUTINE MULTIO_NO_ENC_WRITE_ATM_SP( THIS, YDMSG, VALUES_SP ) CERR = MIOMD%SET( "paramId", INT(YDMSG%PARAM_ID_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "curr_step", INT(YDMSG%ISTEP_, C_LONG) ) + CERR = MIOMD%SET( "currentStep", INT(YDMSG%ISTEP_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "prev_pp", INT(YDMSG%IPREVPP_, C_LONG) ) + CERR = MIOMD%SET( "previousProcessingPoint", INT(YDMSG%IPREVPP_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "prefix", INT(YDMSG%IPREF_, C_LONG) ) + CERR = MIOMD%SET( "ifsPrefix", INT(YDMSG%IPREF_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) CERR = MIOMD%SET( "level", INT(YDMSG%ILEVG_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "repres", INT(YDMSG%IREPRES_, C_LONG) ) + CERR = MIOMD%SET( "ifsRepres", INT(YDMSG%IREPRES_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "nvalues", INT(YDMSG%NVALUES_, C_LONG) ) + CERR = MIOMD%SET( "numberOfValues", INT(YDMSG%NVALUES_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "nmissing",INT( YDMSG%NUNDF_, C_LONG) ) + CERR = MIOMD%SET( "numberOfMissingValues",INT( YDMSG%NUNDF_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "missing_value", REAL(YDMSG%XUNDF_, C_DOUBLE) ) + CERR = MIOMD%SET( "missingValue", REAL(YDMSG%XUNDF_, C_DOUBLE) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) ! Write to multIO @@ -850,22 +850,22 @@ SUBROUTINE MULTIO_NO_ENC_WRITE_WAM_DP( THIS, YDMSG, VALUES_DP ) CERR = MIOMD%SET( "paramId", INT(YDMSG%PARAM_ID_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "curr_step", INT(YDMSG%ISTEP_, C_LONG) ) + CERR = MIOMD%SET( "currentStep", INT(YDMSG%ISTEP_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "prefix", INT(YDMSG%IPREF_, C_LONG) ) + CERR = MIOMD%SET( "ifsPrefix", INT(YDMSG%IPREF_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "repres", INT(YDMSG%IREPRES_, C_LONG) ) + CERR = MIOMD%SET( "ifsRepres", INT(YDMSG%IREPRES_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "nvalues", INT(YDMSG%NVALUES_, C_LONG) ) + CERR = MIOMD%SET( "numberOfValues", INT(YDMSG%NVALUES_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "nmissing", INT(YDMSG%NUNDF_, C_LONG) ) + CERR = MIOMD%SET( "numberOfMissingValues", INT(YDMSG%NUNDF_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "missing_value", REAL(YDMSG%XUNDF_, C_DOUBLE) ) + CERR = MIOMD%SET( "missingValue", REAL(YDMSG%XUNDF_, C_DOUBLE) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) CERR = MIOMD%SET( "angle", INT(YDMSG%IANGLE, C_LONG) ) @@ -1072,22 +1072,22 @@ SUBROUTINE MULTIO_NO_ENC_WRITE_WAM_SP( THIS, YDMSG, VALUES_SP ) CERR = MIOMD%SET( "paramId", INT(YDMSG%PARAM_ID_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "curr_step", INT(YDMSG%ISTEP_, C_LONG) ) + CERR = MIOMD%SET( "currentStep", INT(YDMSG%ISTEP_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "prefix", INT(YDMSG%IPREF_, C_LONG) ) + CERR = MIOMD%SET( "ifsPrefix", INT(YDMSG%IPREF_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "repres", INT(YDMSG%IREPRES_, C_LONG) ) + CERR = MIOMD%SET( "ifsRepres", INT(YDMSG%IREPRES_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "nvalues", INT(YDMSG%NVALUES_, C_LONG) ) + CERR = MIOMD%SET( "numberOfValues", INT(YDMSG%NVALUES_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "nmissing", INT(YDMSG%NUNDF_, C_LONG) ) + CERR = MIOMD%SET( "numberOfMissingValues", INT(YDMSG%NUNDF_, C_LONG) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) - CERR = MIOMD%SET( "missing_value", REAL(YDMSG%XUNDF_, C_DOUBLE) ) + CERR = MIOMD%SET( "missingValue", REAL(YDMSG%XUNDF_, C_DOUBLE) ) PP_DEBUG_DEVELOP_COND_THROW( CERR.NE.MULTIO_SUCCESS, 2 ) CERR = MIOMD%SET( "angle", INT(YDMSG%IANGLE, C_LONG) ) diff --git a/src/multio/action/CMakeLists.txt b/src/multio/action/CMakeLists.txt index cbb04b4cd..7a26f408d 100644 --- a/src/multio/action/CMakeLists.txt +++ b/src/multio/action/CMakeLists.txt @@ -15,3 +15,4 @@ add_subdirectory(interpolate) add_subdirectory(interpolate-fesom) add_subdirectory(select) add_subdirectory(scale) +add_subdirectory(tag-filter) diff --git a/src/multio/action/ifs-mapping/IfsMapping.cc b/src/multio/action/ifs-mapping/IfsMapping.cc index 1b9494b3a..ddd4973bf 100644 --- a/src/multio/action/ifs-mapping/IfsMapping.cc +++ b/src/multio/action/ifs-mapping/IfsMapping.cc @@ -180,18 +180,20 @@ ifs::UniqueFieldId IfsMapping::fieldIdFromMetadata(const message::Metadata& md) using T = std::decay_t; if constexpr (std::is_same_v) { m2gPrefixToLevType_.insert_or_assign(eckit::Value{glossary().ifsPrefix}, eckit::Value{val}); - } + return; + } if constexpr (std::is_same_v) { m2gPrefixToLevType_.insert_or_assign(eckit::Value{glossary().ifsPrefix}, eckit::Value{val}); + return; } NOTIMP; }); m2gPrefixToLevType_.insert_or_assign(eckit::Value{glossary().paramId}, eckit::Value{res.paramId}); - m2gPrefixToLevType_.insert_or_assign(eckit::Value{glossary().ifsPrefix}, eckit::Value{res.level}); + m2gPrefixToLevType_.insert_or_assign(eckit::Value{glossary().level}, eckit::Value{res.level}); - util::Mars2GribMetadataSetter keySetter{levTypeMd_}; + util::Mars2GribMetadataSetter keySetter{levTypeMd_, true}; metkit::mars2grib::convertMars2Grib(m2gPrefixToLevType_, keySetter, metkit::mars2grib::ifsPrefixToLevTypeRuleList()); res.levtype = levTypeMd_.get("levType"); @@ -216,21 +218,21 @@ const ifs::ParamIdInfo& IfsMapping::getParamIdInfo(std::int64_t pid) { if (searchPid == paramIdInfo_.end()) { m2gPid_.insert_or_assign("paramId", pid); message::BaseMetadata md; - util::Mars2GribMetadataSetter keySetter{md}; + util::Mars2GribMetadataSetter keySetter{md, true}; metkit::mars2grib::convertMars2Grib(m2gPid_, keySetter, metkit::mars2grib::statParamRuleList()); ifs::ParamIdInfo info; - if (auto search = md.find("typeOfStatisticalProcessing"); search != md.end()) { - info.typeOfStatisticalProcessing = search->second.get(); + if (auto tosp = md.getOpt("typeOfStatisticalProcessing"); tosp) { + info.typeOfStatisticalProcessing = *tosp; info.typeOfTimeRange = ifs::TypeOfTimeRange::FromLastPP; info.emitStepZero = false; - if (auto searchLength = md.find("lengthOfTimeRange"); searchLength != md.end()) { - info.lengthOfTimeRange = searchLength->second.get(); + if (auto len = md.getOpt("lengthOfTimeRange"); len) { + info.lengthOfTimeRange = *len; info.typeOfTimeRange = ifs::TypeOfTimeRange::FixedSize; } - if (auto searchIndicator = md.find("indicatorOfUnitForTimeRange"); searchIndicator != md.end()) { - info.indicatorOfUnitForTimeRange = searchIndicator->second.get(); + if (auto ind = md.getOpt("indicatorOfUnitForTimeRange"); ind) { + info.indicatorOfUnitForTimeRange = ind; } } else { info.typeOfTimeRange = ifs::TypeOfTimeRange::Instant; @@ -257,4 +259,5 @@ void IfsMapping::print(std::ostream& os) const { static ActionBuilder IfsMappingBuilder("ifs-mapping"); + } // namespace multio::action diff --git a/src/multio/action/ifs-mapping/IfsMapping.h b/src/multio/action/ifs-mapping/IfsMapping.h index 7b297388f..6b2d21851 100644 --- a/src/multio/action/ifs-mapping/IfsMapping.h +++ b/src/multio/action/ifs-mapping/IfsMapping.h @@ -27,7 +27,7 @@ namespace multio::action::ifs { /** - * Example messag metadata={"format":"raw","precision":"double","missing_value":1.79769e+308,"nmissing":0,"nvalues":654400,"level":1,"repres":1,"prefix":1,"prev_pp":-1,"curr_step":4,"paramID":133","model":"ifsosphere"} + * Example messag metadata={"format":"raw","precision":"double","missingValue":1.79769e+308,"numberOfMissingValues":0,"numberOfValues":654400,"level":1,"repres":1,"prefix":1,"previousProcessingPoint":-1,"currentStep":4,"paramId":133","model":"ifsosphere"} */ enum class Repres: std::int64_t { diff --git a/src/multio/action/tag-filter/CMakeLists.txt b/src/multio/action/tag-filter/CMakeLists.txt new file mode 100644 index 000000000..7612b83ff --- /dev/null +++ b/src/multio/action/tag-filter/CMakeLists.txt @@ -0,0 +1,17 @@ + +ecbuild_add_library( + + TARGET multio-action-tag-filter + + SOURCES + TagFilter.cc + TagFilter.h + + PRIVATE_INCLUDES + ${ECKIT_INCLUDE_DIRS} + + CONDITION + + PUBLIC_LIBS + multio +) diff --git a/src/multio/action/tag-filter/TagFilter.cc b/src/multio/action/tag-filter/TagFilter.cc new file mode 100644 index 000000000..b69a15cfb --- /dev/null +++ b/src/multio/action/tag-filter/TagFilter.cc @@ -0,0 +1,74 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "TagFilter.h" + +#include + +#include "eckit/exception/Exceptions.h" + +#include "multio/LibMultio.h" + +using multio::message::Message; +using multio::message::match::MatchReduce; + +namespace multio::action { + +//-------------------------------------------------------------------------------------------------- + +TagFilter::TagFilter(const ComponentConfiguration& compConf) : ChainedAction{compConf}, tags_{} { + if (compConf.parsedConfig().has("tags")) { + if (compConf.parsedConfig().isString("tags")) { + tags_.emplace(message::Message::parseTag(compConf.parsedConfig().getString("tags"))); + } else if (compConf.parsedConfig().isStringList("tags")) { + for(const auto& t: compConf.parsedConfig().getStringVector("tags")) { + tags_.emplace(message::Message::parseTag(t)); + } + } else { + std::ostringstream oss; + oss << "TagFilter: Key \"tags\" must be a string or list of strings." << compConf.parsedConfig(); + throw eckit::UserError(oss.str(), Here()); + } + } +} + +void TagFilter::executeImpl(Message msg) { + // Tag filter is important for parametrization - + // otherwise parametrization is made on a custom key "category"... + // which will be read for all followup metadata that has no explicit key "Tag" + if (!tags_.empty()) { + if (auto search = tags_.find(msg.tag()); search == tags_.end()) { + // Tag not found + return; + } + } + + executeNext(std::move(msg)); +} + +void TagFilter::print(std::ostream& os) const { + os << "TagFilter("; + bool first = true; + for(auto t: tags_) { + if (!first) { + os << ", "; + first = false; + } + os << message::Message::tag2str(t); + } +} + +//-------------------------------------------------------------------------------------------------- + +static ActionBuilder TagFilterBuilder("tag-filter"); + +//-------------------------------------------------------------------------------------------------- + +} // namespace multio::action diff --git a/src/multio/action/tag-filter/TagFilter.h b/src/multio/action/tag-filter/TagFilter.h new file mode 100644 index 000000000..839ea1473 --- /dev/null +++ b/src/multio/action/tag-filter/TagFilter.h @@ -0,0 +1,47 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Domokos Sarmany +/// @author Simon Smart +/// @author Tiago Quintino + +/// @date Jan 2019 + +#pragma once + +#include +#include +#include +#include + +#include "multio/action/ChainedAction.h" +#include "multio/message/MetadataMatcher.h" + +namespace multio::action { + +//---------------------------------------------------------------------------------------------------------------------- + +class TagFilter : public ChainedAction { +public: + explicit TagFilter(const ComponentConfiguration& compConf); + +private: // methods + void print(std::ostream& os) const override; + + bool matches(const message::Message& msg) const; + + void executeImpl(message::Message msg) override; +private: // members + std::unordered_set tags_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace multio::action diff --git a/src/multio/api/c/CMakeLists.txt b/src/multio/api/c/CMakeLists.txt index 31222e982..716ee31d8 100644 --- a/src/multio/api/c/CMakeLists.txt +++ b/src/multio/api/c/CMakeLists.txt @@ -15,6 +15,7 @@ list( APPEND multio_action_plugins multio-action-null multio-action-print multio-action-select + multio-action-tag-filter multio-action-single-field-sink multio-action-sink multio-action-statistics @@ -22,6 +23,7 @@ list( APPEND multio_action_plugins multio-action-renumber-healpix multio-action-interpolate-fesom multio-action-scale + multio-action-ifs-mapping ) if( HAVE_MIR ) diff --git a/src/multio/message/Glossary.h b/src/multio/message/Glossary.h index de5669a37..dbb4d3ed2 100644 --- a/src/multio/message/Glossary.h +++ b/src/multio/message/Glossary.h @@ -246,10 +246,10 @@ struct Glossary { // IFS/ATM mapping - const KeyType ifsPrefix{"prefix"}; - const KeyType ifsRepres{"repres"}; - const KeyType ifsCurrStep{"curr_step"}; - const KeyType ifsPrevPP{"prev_pp"}; + const KeyType ifsPrefix{"ifsPrefix"}; + const KeyType ifsRepres{"ifsRepres"}; + const KeyType ifsCurrStep{"currentStep"}; + const KeyType ifsPrevPP{"previousProcessingPoint"}; static const Glossary& instance() { diff --git a/src/multio/message/Message.cc b/src/multio/message/Message.cc index 92717e28f..cf18dd671 100644 --- a/src/multio/message/Message.cc +++ b/src/multio/message/Message.cc @@ -30,7 +30,7 @@ int Message::protocolVersion() { return 1; } -std::string Message::tag2str(Tag t) { +const std::string& Message::tag2str(Tag t) { static std::map m = {{Tag::Empty, "Empty"}, {Tag::Open, "Open"}, {Tag::Close, "Close"}, @@ -49,6 +49,24 @@ std::string Message::tag2str(Tag t) { return tstr->second; } + +Message::Tag Message::parseTag(const std::string& tagStr) { + static std::map m = {{"Empty", Tag::Empty}, + {"Open", Tag::Open}, + {"Close", Tag::Close}, + {"Domain", Tag::Domain}, + {"Mask", Tag::Mask}, + {"Field", Tag::Field}, + {"Flush", Tag::Flush}, + {"Notification", Tag::Notification}, + {"Parametrization", Tag::Parametrization}}; + auto tag = m.find(tagStr); + ASSERT(tag != m.end()); + + return tag->second; +} + + Message::Message() : Message(Message::Header{Message::Tag::Empty, Peer{}, Peer{}}) {} Message::Message(Header&& header) : diff --git a/src/multio/message/Message.h b/src/multio/message/Message.h index 0bc5a050a..74f40b1a7 100644 --- a/src/multio/message/Message.h +++ b/src/multio/message/Message.h @@ -59,8 +59,7 @@ class Message { Parametrization, ENDTAG }; - - + struct LogHeader { Tag tag_; @@ -131,7 +130,9 @@ class Message { public: // methods static int protocolVersion(); - static std::string tag2str(Tag t); + static const std::string& tag2str(Tag t); + static Tag parseTag(const std::string&); + Message(const Message&) = default; Message(Message&&) = default; @@ -244,3 +245,9 @@ message::Message convert_precision(message::Message&& msg) { } // namespace multio::message + +template <> +struct std::hash { + std::size_t operator()(const multio::message::Message::Tag& t) const noexcept(true) { return (std::size_t)t; } +}; + diff --git a/src/multio/tools/multio-replay-nemo-capi.cc b/src/multio/tools/multio-replay-nemo-capi.cc index 29c46011f..d5254cba3 100644 --- a/src/multio/tools/multio-replay-nemo-capi.cc +++ b/src/multio/tools/multio-replay-nemo-capi.cc @@ -209,7 +209,6 @@ void MultioReplayNemoCApi::setDomains(bool onlyLoadDefinitions) { multio_new_metadata(&md, multio_handle); // Global size is constant for all domains, send by parametrization (mainly for testing purpose...) - multio_metadata_set_string(md, "category", "parametrization"); multio_metadata_set_bool(md, "toAllServers", true); multio_metadata_set_int(md, "globalSize", globalSize_); multio_write_parametrization(multio_handle, md); diff --git a/tests/multio/config/multio-server-mpi-custom-splitting.yaml b/tests/multio/config/multio-server-mpi-custom-splitting.yaml index 1cc1649ad..3e59cf13f 100644 --- a/tests/multio/config/multio-server-mpi-custom-splitting.yaml +++ b/tests/multio/config/multio-server-mpi-custom-splitting.yaml @@ -15,11 +15,19 @@ mpi-communicators: client: plans: + - name: ocean-replay-grid-parametrization-stream + actions: + - type: tag-filter + tags: Parametrization + + - type: transport + target: server + - name: ocean-replay-grid-info-stream actions: - type: select match: - - category: [ocean-domain-map, ocean-mask, parametrization] + - category: [ocean-domain-map, ocean-mask] - type: transport target: server diff --git a/tests/multio/config/multio-server-mpi-default-splitting.yaml b/tests/multio/config/multio-server-mpi-default-splitting.yaml index 0c5ae82c7..68dae69c2 100644 --- a/tests/multio/config/multio-server-mpi-default-splitting.yaml +++ b/tests/multio/config/multio-server-mpi-default-splitting.yaml @@ -14,6 +14,14 @@ mpi-communicators: client: plans: + - name: ocean-replay-grid-parametrization-stream + actions: + - type: tag-filter + tags: Parametrization + + - type: transport + target: server + - name: ocean-replay-grid-info-stream actions: - type: select diff --git a/tests/multio/config/multio-server.yaml b/tests/multio/config/multio-server.yaml index c65efc04f..f98dab774 100644 --- a/tests/multio/config/multio-server.yaml +++ b/tests/multio/config/multio-server.yaml @@ -8,11 +8,19 @@ run: client: on-error: abort-transport plans: + - name: ocean-replay-grid-parametrization-stream + actions: + - type: tag-filter + tags: Parametrization + + - type: transport + target: server + - name: ocean-replay-grid-info-stream actions: - type: select match: - - category: [ocean-domain-map, ocean-mask, parametrization] + - category: [ocean-domain-map, ocean-mask] - type: transport target: server