From 3ec5bce76b41804d2593fbad05256afa44b009b8 Mon Sep 17 00:00:00 2001 From: Mirco Valentini Date: Mon, 17 Jun 2024 18:00:44 +0200 Subject: [PATCH 1/8] Preliminary and untested support for mir interpolation from matrix --- src/multio/action/interpolate/CMakeLists.txt | 19 ++ .../action/interpolate/FesomCacheGenerator.cc | 252 ++++++++++++++++++ src/multio/action/interpolate/Interpolate.cc | 97 ++++++- .../interpolate/FESOM_to_HEALPix_1024.yaml | 21 ++ 4 files changed, 382 insertions(+), 7 deletions(-) create mode 100644 src/multio/action/interpolate/FesomCacheGenerator.cc create mode 100644 tests/multio/action/interpolate/FESOM_to_HEALPix_1024.yaml diff --git a/src/multio/action/interpolate/CMakeLists.txt b/src/multio/action/interpolate/CMakeLists.txt index a5a6d1cc..47006595 100644 --- a/src/multio/action/interpolate/CMakeLists.txt +++ b/src/multio/action/interpolate/CMakeLists.txt @@ -17,3 +17,22 @@ ecbuild_add_library( multio mir ) + +ecbuild_add_executable( + + TARGET + cache-generator-fesom-2-mir + + SOURCES + FesomCacheGenerator.cc + ../../tools/MultioTool.cc + + CONDITION + HAVE_ATLAS_IO + + LIBS + multio + atlas_io + eckit + mir +) \ No newline at end of file diff --git a/src/multio/action/interpolate/FesomCacheGenerator.cc b/src/multio/action/interpolate/FesomCacheGenerator.cc new file mode 100644 index 00000000..98b2d32f --- /dev/null +++ b/src/multio/action/interpolate/FesomCacheGenerator.cc @@ -0,0 +1,252 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "eckit/linalg/Triplet.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/PathName.h" +#include "eckit/log/Log.h" +#include "eckit/option/CmdArgs.h" +#include "eckit/option/SimpleOption.h" +#include "multio/tools/MultioTool.h" +#include "mir/method/WeightMatrix.h" + +namespace multio::action::interpolateFESOM2MIR { + +namespace { + +enum class orderingConvention_e : unsigned int +{ + RING, + NESTED, + UNKNOWN +}; + +orderingConvention_e orderingConvention_string2enum(const std::string& orderingConvention) { + if (orderingConvention != "ring" && orderingConvention != "nested") { + std::ostringstream os; + os << " - Unexpected value for \"orderingConvention\": " + << "\"" << orderingConvention << "\"" << std::endl; + throw eckit::SeriousBug(os.str(), Here()); + } + return (orderingConvention == "ring" ? orderingConvention_e::RING : orderingConvention_e::NESTED); +} + + +std::string orderingConvention_enum2string(orderingConvention_e orderingConvention) { + if (orderingConvention == orderingConvention_e::UNKNOWN) { + std::ostringstream os; + os << " - Unexpected value for \"orderingConvention\": " + << "\"UNKNOWN\"" << std::endl; + throw eckit::SeriousBug(os.str(), Here()); + } + return (orderingConvention == orderingConvention_e::RING ? "ring" : "nested"); +} + + +void parseInputFileName(const std::string& fname, std::string& fesomName, std::string& domain, size_t& NSide, + size_t& level, orderingConvention_e& orderingConvention) { + static const std::regex fnameGrammar("([^_]+)_([^_]+)_NSIDE([1-9][0-9]*)_([0-9][0-9]*)_([a-zA-Z]*).csv"); + std::smatch matchFName; + if (std::regex_match(fname, matchFName, fnameGrammar)) { + fesomName = matchFName[1].str(); + domain = matchFName[2].str(); + NSide = static_cast(std::stoi(matchFName[3].str())); + level = static_cast(std::stoi(matchFName[4].str())); + orderingConvention = orderingConvention_string2enum(matchFName[5].str()); + } + else { + throw eckit::SeriousBug("Unable to parse filename: " + fname, Here()); + } + return; +} + +std::string fesomCacheName(const std::string& fesomName, const std::string& domain, const std::string& precision, + size_t NSide, const std::string& orderingConvention, double level) { + std::ostringstream os; + std::string localDomain{domain}; + localDomain.erase(std::remove_if(localDomain.begin(), localDomain.end(), ::isspace), localDomain.end()); + std::transform(localDomain.begin(), localDomain.end(), localDomain.begin(), + [](unsigned char c) { return std::tolower(c); }); + os << "fesom_" << fesomName << "_" << localDomain << "_to_HEALPix_" << std::setw(6) << std::setfill('0') << NSide + << "_" << precision << "_" << orderingConvention << "_" << std::setw(8) + << std::setfill('0') << static_cast(std::fabs(level * 1000)); + return os.str(); +} + +} // namespace + + +class Fesom2mirCacheGenerator final : public multio::MultioTool { +public: // methods + Fesom2mirCacheGenerator(int argc, char** argv); + +private: + void usage(const std::string& tool) const override { + eckit::Log::info() << std::endl << "Usage: " << tool << " [options]" << std::endl; + eckit::Log::info() + << "EXAMPLE: " << std::endl + << "fesom-cache-generator --mode=fromTriplets --inputPath=. --inputFile=CORE2_ngrid_NSIDE32_0_ring.csv " + "--dumpTriplets=1" + << std::endl + << std::endl; + } + + void init(const eckit::option::CmdArgs& args) override; + + void finish(const eckit::option::CmdArgs&) override; + + void execute(const eckit::option::CmdArgs& args) override; + + int numberOfPositionalArguments() const override { return 0; } + int minimumPositionalArguments() const override { return 0; } + + std::string inputPath_; + std::string outputPath_; + std::string outputPrecision_; + std::string inputFile_; + std::string workingMode_; + bool dumpTriplets_; + + + std::string fesomName_; + std::string domain_; + size_t NSide_; + size_t level_; + size_t Nrow_; + size_t Ncol_; + orderingConvention_e orderingConvention_; + + void loadTriplets(std::vector& triplets, size_t& Nrow, size_t& Ncol ) const; +}; + + +Fesom2mirCacheGenerator::Fesom2mirCacheGenerator(int argc, char** argv) : + multio::MultioTool{argc, argv}, + inputPath_{"."}, + outputPath_{"."}, + inputFile_{"CORE2_ngrid_NSIDE32_0_ring.csv"}, + workingMode_{"fromTriplets"}, + dumpTriplets_{false}, + fesomName_{"CORE2"}, + domain_{"ngrid"}, + NSide_{0}, + level_{0}, + orderingConvention_{orderingConvention_e::RING} { + + options_.push_back(new eckit::option::SimpleOption( + "mode", "WorkingMode [fromTriplets]. Default( \"fromTriplets\" )")); + options_.push_back(new eckit::option::SimpleOption( + "inputPath", "Path of the input files with the triplets. Default( \".\" )")); + options_.push_back(new eckit::option::SimpleOption( + "outputPath", "Path of the output files with the triplets. Default( \".\" )")); + options_.push_back(new eckit::option::SimpleOption( + "inputFile", "Name of the input file. Default( \"CORE2_ngrid_NSIDE32_0_ring.csv\" )")); + options_.push_back(new eckit::option::SimpleOption("dumpTriplets", "Dump all the triplets to screen")); + + return; +} + + +void Fesom2mirCacheGenerator::loadTriplets(std::vector& triplets, size_t& Nrow, size_t& Ncol) const { + std::ifstream file(inputPath_ + "/" + inputFile_); + if (!file.good()) { + throw eckit::SeriousBug("Unable to read the input file", Here()); + } + + std::string line; + triplets.resize(0); + Nrow = 0; + Ncol = 0; + while (std::getline(file, line)) { + int32_t iHEALPix; + int32_t iFESOM; + double weight; + static const std::regex lineGrammar( + "([0-9][0-9]*)\\s+([0-9][0-9]*)\\s*([+]?([0-9]*[.])?[0-9]+([eE][-+][0-9]+)?)"); + std::smatch matchLine; + if (std::regex_match(line, matchLine, lineGrammar)) { + iHEALPix = std::stoi(matchLine[1].str()); + iFESOM = std::stoi(matchLine[2].str()); + weight = std::stod(matchLine[3].str()); + // This code is not safe, it assumes that the indices are all used + Nrow = std::max(Nrow_, static_cast(iHEALPix + 1)); + Ncol = std::max(Ncol_, static_cast(iFESOM + 1)); + } + else { + throw eckit::SeriousBug("Unable to parse line: " + line, Here()); + } + triplets.emplace_back(iHEALPix, iFESOM, weight); + } + file.close(); +} + + +void Fesom2mirCacheGenerator::init(const eckit::option::CmdArgs& args) { + + args.get("mode", workingMode_); + ASSERT(workingMode_ == "fromTriplets"); + + if (workingMode_ == "fromTriplets") { + args.get("inputPath", inputPath_); + args.get("outputPath", outputPath_); + args.get("inputFile", inputFile_); + args.get("dumpTriplets", dumpTriplets_); + + eckit::PathName inputPath_tmp{inputPath_}; + ASSERT(inputPath_tmp.exists()); + eckit::PathName inputFile_tmp{inputPath_ + "/" + inputFile_}; + ASSERT(inputFile_tmp.exists()); + eckit::PathName outputPath_tmp{outputPath_}; + outputPath_tmp.mkdir(); + parseInputFileName(inputFile_, fesomName_, domain_, NSide_, level_, orderingConvention_); + } +} + +void Fesom2mirCacheGenerator::execute(const eckit::option::CmdArgs& args) { + + size_t level; + size_t nnz; + size_t nRows; + size_t nCols; + size_t nOutRows; + std::vector landSeaMask; + std::vector rowStart; + std::vector colIdx; + std::vector valuesf; + std::vector valuesd; + + if (workingMode_ == "fromTriplets") { + std::vector triplets; + + loadTriplets(triplets, Nrow_, Ncol_); + + { + mir::method::WeightMatrix W(Nrow_, Ncol_); + W.setFromTriplets(triplets); + W.save(fesomCacheName(fesomName_, domain_, "double", + NSide_, orderingConvention_enum2string(orderingConvention_), level_)); + } + + } +}; + + +void Fesom2mirCacheGenerator::finish(const eckit::option::CmdArgs&) {} + +} // namespace multio::action::interpolateFESOM + + +int main(int argc, char** argv) { + multio::action::interpolateFESOM2MIR::Fesom2mirCacheGenerator tool(argc, argv); + return tool.start(); +} diff --git a/src/multio/action/interpolate/Interpolate.cc b/src/multio/action/interpolate/Interpolate.cc index 8f630c96..705341f0 100644 --- a/src/multio/action/interpolate/Interpolate.cc +++ b/src/multio/action/interpolate/Interpolate.cc @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include "eckit/exception/Exceptions.h" #include "eckit/mpi/Comm.h" @@ -128,6 +130,47 @@ void regularLatLongMetadata(DestType& param, std::vector grid, std::vect return; }; + + +std::string fesomCacheName(const std::string& fesomName, const std::string& domain, const std::string& precision, + size_t NSide, const std::string& orderingConvention, double level) { + std::ostringstream os; + std::string localDomain{domain}; + localDomain.erase(std::remove_if(localDomain.begin(), localDomain.end(), ::isspace), localDomain.end()); + std::transform(localDomain.begin(), localDomain.end(), localDomain.begin(), + [](unsigned char c) { return std::tolower(c); }); + os << "fesom_" << fesomName << "_" << localDomain << "_to_HEALPix_" << std::setw(6) << std::setfill('0') << NSide + << "_" << precision << "_" << orderingConvention << "_" << std::setw(8) + << std::setfill('0') << static_cast(std::fabs(level * 1000)); + return os.str(); +} + +template +std::string generateKey(const message::Message& msg, size_t NSide, const std::string& orderingConvention ) { + // TODO: Probably missing the kind of fesom grid in the name + // need to see the metadata to understand how to extract it + size_t level = static_cast(msg.metadata().getLong("level", msg.metadata().getDouble("levelist", 0))); + if ((msg.metadata().getString("category") == "ocean-3d") + && (msg.metadata().getString("fesomLevelType") == "level")) { + if (level == 0) { + std::ostringstream os; + os << " - Wrong level for the oceal level" << std::endl; + throw eckit::SeriousBug(os.str(), Here()); + } + level--; + } + if (!msg.metadata().has("unstructuredGridType")) { + std::ostringstream os; + os << " - \"unstructuredGridType\" not present in the metadata" << std::endl; + throw eckit::SeriousBug(os.str(), Here()); + } + std::string fesomGridName = msg.metadata().getString("unstructuredGridType"); + std::string key = fesomCacheName(fesomGridName, msg.domain(), (sizeof(T) == 4 ? "single" : "double"), NSide, + orderingConvention, level); + + return key; +} + } // namespace void fill_out_metadata(const message::Metadata& in_md, message::Metadata& out_md) { @@ -153,8 +196,8 @@ eckit::Value getInputGrid(const eckit::LocalConfiguration& cfg, message::Metadat } -void fill_input(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametrisation& param, std::string domain, - const eckit::Value&& inp) { +void fill_input(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametrisation& param, size_t payload_size, std::string domain, + const eckit::Value& inp) { auto regular_ll = [¶m](double west_east_increment, double south_north_increment) { regularLatLongMetadata(param, std::vector{west_east_increment, south_north_increment}, full_area); @@ -171,6 +214,7 @@ void fill_input(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametr static const std::regex gg("([FNO])([1-9][0-9]*)"); static const std::regex eORCA("^e?ORCA[0-9]+_[FTUVW]$"); static const std::regex eORCA_fromMetadata("^e?ORCA[0-9]+$"); + static const std::regex FESOM("^fesom$"); #define fp "([+]?([0-9]*[.])?[0-9]+([eE][-+][0-9]+)?)" static const std::regex ll(fp "/" fp); @@ -228,6 +272,13 @@ void fill_input(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametr param.set("gridType", "orca"); return; } + + if ( std::regex_match(input, match, eORCA_fromMetadata)) { + param.set("gridded", true); + param.set("gridType", "unstructured_grid"); + param.set("numberOfPoints", payload_size ); + return; + } } if (inp.isList()) { @@ -240,9 +291,8 @@ void fill_input(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametr NOTIMP; } - void fill_job(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametrisation& destination, - message::Metadata& md) { + message::Metadata& md, const eckit::Value& inp, const message::Message& msg) { static const struct PostProcKeys : std::vector { PostProcKeys() { @@ -267,11 +317,15 @@ void fill_job(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametris } } - + // Forward to mir the options + bool interpolationMatrix=false; if (cfg.has("options")) { const auto& options = cfg.getSubConfiguration("options"); for (const auto& key : options.keys()) { set(options, key, options.getSubConfiguration(key).get()); + if ( key == "interpolation" && options.getString(key) == "matrix" ) { + interpolationMatrix = true; + } } } @@ -308,6 +362,11 @@ void fill_job(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametris && cfg.getSubConfiguration("grid").get().head().isDouble()) { gridKind = "regular_ll"; grid = cfg.getDoubleVector("grid"); + if ( interpolationMatrix ) { + std::ostringstream os; + os << " - interpolation matrix supported only for fesom -> Healpix" << std::endl; + throw eckit::SeriousBug(os.str(), Here()); + } LOG_DEBUG_LIB(LibMultio) << " Grid is a list (" << gridKind << ")" << grid << std::endl; } // @@ -322,6 +381,12 @@ void fill_job(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametris else { regularLatLongMetadata(md, grid, full_area); } + + if ( interpolationMatrix ) { + std::ostringstream os; + os << " - interpolation matrix supported only for fesom -> Healpix" << std::endl; + throw eckit::SeriousBug(os.str(), Here()); + } } else if (gridKind == "HEALPix") { // @@ -329,6 +394,23 @@ void fill_job(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametris md.set("gridType", "HEALPix"); md.set("Nside", grid[0]); md.set("orderingConvention", "ring"); + + // If no interpolation matrix name is provided, generate one + if ( interpolationMatrix ) { + const std::string input = util::replaceCurly(inp.as(), [](std::string_view replace) { + std::string lookUpKey{replace}; + char* env = ::getenv(lookUpKey.c_str()); + return env ? std::optional{env} : std::optional{}; + }); + if ( input != "fesom" ){ + std::ostringstream os; + os << " - interpolation matrix supported only for fesom -> Healpix" << std::endl; + throw eckit::SeriousBug(os.str(), Here()); + } + const auto& options = cfg.getSubConfiguration("options"); + set(options, "interpolation-matrix", generateKey(msg, grid[0], "ring") ); + } + } else { std::cout << "Grid not implemented" << std::endl; @@ -352,7 +434,8 @@ message::Message Interpolate::InterpolateMessage(message::Message&& msg) md.set("precision", "double"); mir::param::SimpleParametrisation inputPar; - fill_input(config, inputPar, msg.domain(), getInputGrid(config, md)); + eckit::Value inp = getInputGrid(config, md); + fill_input(config, inputPar, size, msg.domain(), inp ); if (msg.metadata().has("missingValue") && msg.metadata().getBool("bitmapPresent")) { inputPar.set("missing_value", msg.metadata().getDouble("missingValue")); } @@ -360,7 +443,7 @@ message::Message Interpolate::InterpolateMessage(message::Message&& msg) mir::input::RawInput input(data, size, inputPar); mir::api::MIRJob job; - fill_job(config, job, md); + fill_job(config, job, md, inp, msg); LOG_DEBUG_LIB(LibMultio) << "Interpolate :: input :: " << std::endl << inputPar << std::endl << std::endl; diff --git a/tests/multio/action/interpolate/FESOM_to_HEALPix_1024.yaml b/tests/multio/action/interpolate/FESOM_to_HEALPix_1024.yaml new file mode 100644 index 00000000..2543318b --- /dev/null +++ b/tests/multio/action/interpolate/FESOM_to_HEALPix_1024.yaml @@ -0,0 +1,21 @@ +plans: + - name: FESOM_to_HEALPIX_1024 + actions: + - type: print + stream: cout + - type: interpolate + input: fosom + grid: H1024 + options: + interpolation: matrix + - type: print + stream: cout + - type: encode + format: grib + template: Reference_eORCA12_W_to_HEALPix_1024.grib + - type: sink + sinks: + - type: file + append: false + per-server: false # Will give you one file per server + path: MultIO_eORCA12_W_to_HEALPix_1024.grib \ No newline at end of file From f4e5e394ca603624196766c6595ce85bc501db9f Mon Sep 17 00:00:00 2001 From: Razvan Aguridan Date: Tue, 18 Jun 2024 15:10:35 +0000 Subject: [PATCH 2/8] Fixes --- .../action/interpolate/FesomCacheGenerator.cc | 79 ++++++------------- src/multio/action/interpolate/Interpolate.cc | 22 ++++-- 2 files changed, 43 insertions(+), 58 deletions(-) diff --git a/src/multio/action/interpolate/FesomCacheGenerator.cc b/src/multio/action/interpolate/FesomCacheGenerator.cc index 98b2d32f..34f890b7 100644 --- a/src/multio/action/interpolate/FesomCacheGenerator.cc +++ b/src/multio/action/interpolate/FesomCacheGenerator.cc @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include "eckit/linalg/Triplet.h" #include "eckit/exception/Exceptions.h" @@ -79,7 +81,7 @@ std::string fesomCacheName(const std::string& fesomName, const std::string& doma [](unsigned char c) { return std::tolower(c); }); os << "fesom_" << fesomName << "_" << localDomain << "_to_HEALPix_" << std::setw(6) << std::setfill('0') << NSide << "_" << precision << "_" << orderingConvention << "_" << std::setw(8) - << std::setfill('0') << static_cast(std::fabs(level * 1000)); + << std::setfill('0') << static_cast(std::fabs(level * 1000)) << ".mat"; return os.str(); } @@ -95,8 +97,7 @@ class Fesom2mirCacheGenerator final : public multio::MultioTool { eckit::Log::info() << std::endl << "Usage: " << tool << " [options]" << std::endl; eckit::Log::info() << "EXAMPLE: " << std::endl - << "fesom-cache-generator --mode=fromTriplets --inputPath=. --inputFile=CORE2_ngrid_NSIDE32_0_ring.csv " - "--dumpTriplets=1" + << "fesom-cache-generator --inputPath=. --inputFile=CORE2_ngrid_NSIDE32_0_ring.csv " << std::endl << std::endl; } @@ -114,8 +115,6 @@ class Fesom2mirCacheGenerator final : public multio::MultioTool { std::string outputPath_; std::string outputPrecision_; std::string inputFile_; - std::string workingMode_; - bool dumpTriplets_; std::string fesomName_; @@ -135,23 +134,18 @@ Fesom2mirCacheGenerator::Fesom2mirCacheGenerator(int argc, char** argv) : inputPath_{"."}, outputPath_{"."}, inputFile_{"CORE2_ngrid_NSIDE32_0_ring.csv"}, - workingMode_{"fromTriplets"}, - dumpTriplets_{false}, fesomName_{"CORE2"}, domain_{"ngrid"}, NSide_{0}, level_{0}, orderingConvention_{orderingConvention_e::RING} { - options_.push_back(new eckit::option::SimpleOption( - "mode", "WorkingMode [fromTriplets]. Default( \"fromTriplets\" )")); options_.push_back(new eckit::option::SimpleOption( "inputPath", "Path of the input files with the triplets. Default( \".\" )")); options_.push_back(new eckit::option::SimpleOption( "outputPath", "Path of the output files with the triplets. Default( \".\" )")); options_.push_back(new eckit::option::SimpleOption( "inputFile", "Name of the input file. Default( \"CORE2_ngrid_NSIDE32_0_ring.csv\" )")); - options_.push_back(new eckit::option::SimpleOption("dumpTriplets", "Dump all the triplets to screen")); return; } @@ -179,8 +173,8 @@ void Fesom2mirCacheGenerator::loadTriplets(std::vector& iFESOM = std::stoi(matchLine[2].str()); weight = std::stod(matchLine[3].str()); // This code is not safe, it assumes that the indices are all used - Nrow = std::max(Nrow_, static_cast(iHEALPix + 1)); - Ncol = std::max(Ncol_, static_cast(iFESOM + 1)); + Nrow = std::max(Nrow, static_cast(iHEALPix + 1)); + Ncol = std::max(Ncol, static_cast(iFESOM + 1)); } else { throw eckit::SeriousBug("Unable to parse line: " + line, Here()); @@ -192,52 +186,31 @@ void Fesom2mirCacheGenerator::loadTriplets(std::vector& void Fesom2mirCacheGenerator::init(const eckit::option::CmdArgs& args) { - - args.get("mode", workingMode_); - ASSERT(workingMode_ == "fromTriplets"); - - if (workingMode_ == "fromTriplets") { - args.get("inputPath", inputPath_); - args.get("outputPath", outputPath_); - args.get("inputFile", inputFile_); - args.get("dumpTriplets", dumpTriplets_); - - eckit::PathName inputPath_tmp{inputPath_}; - ASSERT(inputPath_tmp.exists()); - eckit::PathName inputFile_tmp{inputPath_ + "/" + inputFile_}; - ASSERT(inputFile_tmp.exists()); - eckit::PathName outputPath_tmp{outputPath_}; - outputPath_tmp.mkdir(); - parseInputFileName(inputFile_, fesomName_, domain_, NSide_, level_, orderingConvention_); - } + args.get("inputPath", inputPath_); + args.get("outputPath", outputPath_); + args.get("inputFile", inputFile_); + + eckit::PathName inputPath_tmp{inputPath_}; + ASSERT(inputPath_tmp.exists()); + eckit::PathName inputFile_tmp{inputPath_ + "/" + inputFile_}; + ASSERT(inputFile_tmp.exists()); + eckit::PathName outputPath_tmp{outputPath_}; + outputPath_tmp.mkdir(); + parseInputFileName(inputFile_tmp.baseName().asString(), fesomName_, + domain_, NSide_, level_, orderingConvention_); } void Fesom2mirCacheGenerator::execute(const eckit::option::CmdArgs& args) { + std::vector triplets; + loadTriplets(triplets, Nrow_, Ncol_); - size_t level; - size_t nnz; - size_t nRows; - size_t nCols; - size_t nOutRows; - std::vector landSeaMask; - std::vector rowStart; - std::vector colIdx; - std::vector valuesf; - std::vector valuesd; - - if (workingMode_ == "fromTriplets") { - std::vector triplets; - - loadTriplets(triplets, Nrow_, Ncol_); - - { - mir::method::WeightMatrix W(Nrow_, Ncol_); - W.setFromTriplets(triplets); - W.save(fesomCacheName(fesomName_, domain_, "double", - NSide_, orderingConvention_enum2string(orderingConvention_), level_)); - } + const auto orderingConvention = orderingConvention_enum2string(orderingConvention_); + const auto cacheFileName = fesomCacheName(fesomName_, domain_, "double", + NSide_, orderingConvention, level_); - } + mir::method::WeightMatrix W(Nrow_, Ncol_); + W.setFromTriplets(triplets); + W.save(cacheFileName); }; diff --git a/src/multio/action/interpolate/Interpolate.cc b/src/multio/action/interpolate/Interpolate.cc index 705341f0..ab04084c 100644 --- a/src/multio/action/interpolate/Interpolate.cc +++ b/src/multio/action/interpolate/Interpolate.cc @@ -141,12 +141,14 @@ std::string fesomCacheName(const std::string& fesomName, const std::string& doma [](unsigned char c) { return std::tolower(c); }); os << "fesom_" << fesomName << "_" << localDomain << "_to_HEALPix_" << std::setw(6) << std::setfill('0') << NSide << "_" << precision << "_" << orderingConvention << "_" << std::setw(8) - << std::setfill('0') << static_cast(std::fabs(level * 1000)); + << std::setfill('0') << static_cast(std::fabs(level * 1000)) << ".mat"; return os.str(); } template -std::string generateKey(const message::Message& msg, size_t NSide, const std::string& orderingConvention ) { +std::string generateKey(const message::Message& msg, const std::string& cache_path, + size_t NSide, const std::string& orderingConvention) +{ // TODO: Probably missing the kind of fesom grid in the name // need to see the metadata to understand how to extract it size_t level = static_cast(msg.metadata().getLong("level", msg.metadata().getDouble("levelist", 0))); @@ -168,7 +170,7 @@ std::string generateKey(const message::Message& msg, size_t NSide, const std::st std::string key = fesomCacheName(fesomGridName, msg.domain(), (sizeof(T) == 4 ? "single" : "double"), NSide, orderingConvention, level); - return key; + return cache_path + "/" + key; } } // namespace @@ -273,7 +275,7 @@ void fill_input(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametr return; } - if ( std::regex_match(input, match, eORCA_fromMetadata)) { + if ( std::regex_match(input, match, FESOM)) { param.set("gridded", true); param.set("gridType", "unstructured_grid"); param.set("numberOfPoints", payload_size ); @@ -408,7 +410,14 @@ void fill_job(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametris throw eckit::SeriousBug(os.str(), Here()); } const auto& options = cfg.getSubConfiguration("options"); - set(options, "interpolation-matrix", generateKey(msg, grid[0], "ring") ); + const auto cache_path = cfg.has("cache-path") ? cfg.getString("cache-path") : ""; + const auto expanded_cache_path = util::replaceCurly(cache_path, [](std::string_view replace) { + std::string lookUpKey{replace}; + char* env = ::getenv(lookUpKey.c_str()); + return env ? std::optional{env} : std::optional{}; + }); + const auto weights_file = generateKey(msg, expanded_cache_path, grid[0], "ring"); + destination.set("interpolation-matrix", weights_file); } } @@ -438,6 +447,9 @@ message::Message Interpolate::InterpolateMessage(message::Message&& msg) fill_input(config, inputPar, size, msg.domain(), inp ); if (msg.metadata().has("missingValue") && msg.metadata().getBool("bitmapPresent")) { inputPar.set("missing_value", msg.metadata().getDouble("missingValue")); + } else if (config.getSubConfiguration("options").has("missing_value")) { + inputPar.set("missing_value", + config.getSubConfiguration("options").getDouble("missing_value")); } mir::input::RawInput input(data, size, inputPar); From 805eef8294345dd4d6af44d4d91722ee823f2fce Mon Sep 17 00:00:00 2001 From: Razvan Aguridan Date: Thu, 27 Jun 2024 05:38:04 +0000 Subject: [PATCH 3/8] Fix FESOM interpolation cache generator --- .../generate_fesom_caches.sh | 23 +++++++++++++++++++ .../action/interpolate/FesomCacheGenerator.cc | 20 ++++++++-------- 2 files changed, 34 insertions(+), 9 deletions(-) create mode 100755 src/multio/action/interpolate-fesom/generate_fesom_caches.sh diff --git a/src/multio/action/interpolate-fesom/generate_fesom_caches.sh b/src/multio/action/interpolate-fesom/generate_fesom_caches.sh new file mode 100755 index 00000000..9db603a3 --- /dev/null +++ b/src/multio/action/interpolate-fesom/generate_fesom_caches.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +#SBATCH -p par +#SBATCH -J generate_fesom_cache +#SBATCH -o generate_fesom_cache_%j.out +#SBATCH -N 1 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=128 +#SBATCH --mem=0 +#SBATCH --time=02:00:00 + +if [[ -n $SLURM_JOB_ID ]]; then + echo "Searching in $1 for files matching $2..." + echo "Using generator tool: $BDIR/bin/cache-generator-fesom-2-mir" + files=$(find ${1} -iname "${2}") + for f in $files + do + fname=$(basename ${f}) + ${BDIR}/bin/cache-generator-fesom-2-mir --inputPath=${1} --inputFile=${fname} --nCols=${3} + done +else + sbatch --export=BDIR $0 $1 $2 $3 +fi diff --git a/src/multio/action/interpolate/FesomCacheGenerator.cc b/src/multio/action/interpolate/FesomCacheGenerator.cc index 34f890b7..b87fd68f 100644 --- a/src/multio/action/interpolate/FesomCacheGenerator.cc +++ b/src/multio/action/interpolate/FesomCacheGenerator.cc @@ -125,7 +125,7 @@ class Fesom2mirCacheGenerator final : public multio::MultioTool { size_t Ncol_; orderingConvention_e orderingConvention_; - void loadTriplets(std::vector& triplets, size_t& Nrow, size_t& Ncol ) const; + void loadTriplets(std::vector& triplets) const; }; @@ -146,12 +146,13 @@ Fesom2mirCacheGenerator::Fesom2mirCacheGenerator(int argc, char** argv) : "outputPath", "Path of the output files with the triplets. Default( \".\" )")); options_.push_back(new eckit::option::SimpleOption( "inputFile", "Name of the input file. Default( \"CORE2_ngrid_NSIDE32_0_ring.csv\" )")); + options_.push_back(new eckit::option::SimpleOption("nCols", "Size of the fesom grid.")); return; } -void Fesom2mirCacheGenerator::loadTriplets(std::vector& triplets, size_t& Nrow, size_t& Ncol) const { +void Fesom2mirCacheGenerator::loadTriplets(std::vector& triplets) const { std::ifstream file(inputPath_ + "/" + inputFile_); if (!file.good()) { throw eckit::SeriousBug("Unable to read the input file", Here()); @@ -159,8 +160,6 @@ void Fesom2mirCacheGenerator::loadTriplets(std::vector& std::string line; triplets.resize(0); - Nrow = 0; - Ncol = 0; while (std::getline(file, line)) { int32_t iHEALPix; int32_t iFESOM; @@ -172,9 +171,6 @@ void Fesom2mirCacheGenerator::loadTriplets(std::vector& iHEALPix = std::stoi(matchLine[1].str()); iFESOM = std::stoi(matchLine[2].str()); weight = std::stod(matchLine[3].str()); - // This code is not safe, it assumes that the indices are all used - Nrow = std::max(Nrow, static_cast(iHEALPix + 1)); - Ncol = std::max(Ncol, static_cast(iFESOM + 1)); } else { throw eckit::SeriousBug("Unable to parse line: " + line, Here()); @@ -196,13 +192,19 @@ void Fesom2mirCacheGenerator::init(const eckit::option::CmdArgs& args) { ASSERT(inputFile_tmp.exists()); eckit::PathName outputPath_tmp{outputPath_}; outputPath_tmp.mkdir(); - parseInputFileName(inputFile_tmp.baseName().asString(), fesomName_, + parseInputFileName(inputFile_, fesomName_, domain_, NSide_, level_, orderingConvention_); + + args.get("nCols", Ncol_); + Nrow_ = NSide_ * NSide_ * 12; } void Fesom2mirCacheGenerator::execute(const eckit::option::CmdArgs& args) { std::vector triplets; - loadTriplets(triplets, Nrow_, Ncol_); + loadTriplets(triplets); + + std::sort(begin(triplets), end(triplets), + [](const auto& a, const auto& b) { return a.row() < b.row(); }); const auto orderingConvention = orderingConvention_enum2string(orderingConvention_); const auto cacheFileName = fesomCacheName(fesomName_, domain_, "double", From ec70e164a7fb937b0e2061242271a663858138e3 Mon Sep 17 00:00:00 2001 From: Razvan Aguridan Date: Fri, 28 Jun 2024 09:29:44 +0200 Subject: [PATCH 4/8] Move script to proper directory --- .../{interpolate-fesom => interpolate}/generate_fesom_caches.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/multio/action/{interpolate-fesom => interpolate}/generate_fesom_caches.sh (100%) diff --git a/src/multio/action/interpolate-fesom/generate_fesom_caches.sh b/src/multio/action/interpolate/generate_fesom_caches.sh similarity index 100% rename from src/multio/action/interpolate-fesom/generate_fesom_caches.sh rename to src/multio/action/interpolate/generate_fesom_caches.sh From eda37c186720a4d74c2c10329b1562ddc41822f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domokos=20S=C3=A1rm=C3=A1ny?= Date: Tue, 2 Jul 2024 15:43:51 +0100 Subject: [PATCH 5/8] Remove commented-out logging --- src/multio/api/multio_c.cc | 1 - tests/multio/test_multio_conf.cc | 1 - 2 files changed, 2 deletions(-) diff --git a/src/multio/api/multio_c.cc b/src/multio/api/multio_c.cc index b44eee4a..798fa7bc 100644 --- a/src/multio/api/multio_c.cc +++ b/src/multio/api/multio_c.cc @@ -383,7 +383,6 @@ int multio_new_handle_default(multio_handle_t** mio) { int multio_delete_handle(multio_handle_t* mio) { return wrapApiFunction([mio]() { ASSERT(mio); - // std::cout << "multio_delete_handle" << std::endl; delete mio; }); } diff --git a/tests/multio/test_multio_conf.cc b/tests/multio/test_multio_conf.cc index a353d0b4..0f9cf0f0 100644 --- a/tests/multio/test_multio_conf.cc +++ b/tests/multio/test_multio_conf.cc @@ -40,7 +40,6 @@ CASE("Create default context and tranverse client plans") { int i = 0; for (ComponentConfiguration subConf : clientConf.subComponents("plans")) { - // std::cout << subConf.parsedConfig().getString("name") << std::endl; switch (i) { case 0: EXPECT(subConf.parsedConfig().getString("name") == "ocean-replay-grid-info-stream"); From 0c3b81c3eb44cee3b2a15a9c34512679d0c4a93d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domokos=20S=C3=A1rm=C3=A1ny?= Date: Tue, 2 Jul 2024 15:58:50 +0100 Subject: [PATCH 6/8] Run clang-format --- src/multio/action/encode/Encode.cc | 12 +++--- src/multio/action/encode/GribEncoder.cc | 4 +- .../interpolate-fesom/FesomCacheValidator.cc | 2 +- .../interpolate-fesom/FesomSpmvmValidator.cc | 2 +- .../action/interpolate/FesomCacheGenerator.cc | 32 ++++++-------- src/multio/action/interpolate/Interpolate.cc | 43 +++++++++---------- .../statistics/operations/OperationWithData.h | 10 +++-- src/multio/config/ComponentConfiguration.cc | 2 +- src/multio/maestro/MaestroCdo.h | 5 ++- src/multio/message/Metadata.cc | 4 +- src/multio/tools/multio-maestro-instigator.cc | 8 ++-- src/multio/tools/multio-maestro-syphon.cc | 3 +- src/multio/tools/multio-pool-manager.cc | 3 +- src/multio/util/Metadata.h | 16 ++++--- src/multio/util/Substitution.cc | 12 +++--- 15 files changed, 79 insertions(+), 79 deletions(-) diff --git a/src/multio/action/encode/Encode.cc b/src/multio/action/encode/Encode.cc index 3b7d1132..7b1a6c24 100644 --- a/src/multio/action/encode/Encode.cc +++ b/src/multio/action/encode/Encode.cc @@ -130,12 +130,12 @@ std::unique_ptr makeEncoder(const eckit::LocalConfiguration& conf, handleCodesError("eccodes error while reading the grib template: ", err, Here()); if (conf.has("atlas-named-grid")) { - const auto atlasNamedGrid = util::replaceCurly(conf.getString("atlas-named-grid"), - [](std::string_view replace) { - std::string lookUpKey{replace}; - char* env = ::getenv(lookUpKey.c_str()); - return env ? std::optional{env} : std::optional{}; - }); + const auto atlasNamedGrid + = util::replaceCurly(conf.getString("atlas-named-grid"), [](std::string_view replace) { + std::string lookUpKey{replace}; + char* env = ::getenv(lookUpKey.c_str()); + return env ? std::optional{env} : std::optional{}; + }); eckit::Log::info() << "REQUESTED ATLAS GRID DEFINITION UPDATE: " << atlasNamedGrid << std::endl; diff --git a/src/multio/action/encode/GribEncoder.cc b/src/multio/action/encode/GribEncoder.cc index 370b57d8..b7a0b1be 100644 --- a/src/multio/action/encode/GribEncoder.cc +++ b/src/multio/action/encode/GribEncoder.cc @@ -586,9 +586,9 @@ std::string getTimeReference(GribEncoder& g, const eckit::LocalConfiguration& md else { isReferingToStart = true; } - } + } else if (types_with_time_reference_offset.find(*queriedMarsFields.type) - != types_with_time_reference_offset.end()) { + != types_with_time_reference_offset.end()) { isReferingToStart = true; } } diff --git a/src/multio/action/interpolate-fesom/FesomCacheValidator.cc b/src/multio/action/interpolate-fesom/FesomCacheValidator.cc index 99e34e3c..f669a637 100644 --- a/src/multio/action/interpolate-fesom/FesomCacheValidator.cc +++ b/src/multio/action/interpolate-fesom/FesomCacheValidator.cc @@ -159,7 +159,7 @@ void FesomCacheValidator::execute(const eckit::option::CmdArgs& args) { }; -void FesomCacheValidator::finish(const eckit::option::CmdArgs&){}; +void FesomCacheValidator::finish(const eckit::option::CmdArgs&) {}; } // namespace multio::action::interpolateFESOM diff --git a/src/multio/action/interpolate-fesom/FesomSpmvmValidator.cc b/src/multio/action/interpolate-fesom/FesomSpmvmValidator.cc index 4ca583b4..0119f8fb 100644 --- a/src/multio/action/interpolate-fesom/FesomSpmvmValidator.cc +++ b/src/multio/action/interpolate-fesom/FesomSpmvmValidator.cc @@ -225,7 +225,7 @@ void FesomCacheValidator::execute(const eckit::option::CmdArgs& args) { }; -void FesomCacheValidator::finish(const eckit::option::CmdArgs&){}; +void FesomCacheValidator::finish(const eckit::option::CmdArgs&) {}; } // namespace multio::action::interpolateFESOM diff --git a/src/multio/action/interpolate/FesomCacheGenerator.cc b/src/multio/action/interpolate/FesomCacheGenerator.cc index b87fd68f..abc931af 100644 --- a/src/multio/action/interpolate/FesomCacheGenerator.cc +++ b/src/multio/action/interpolate/FesomCacheGenerator.cc @@ -2,25 +2,25 @@ #include #include +#include #include #include +#include #include #include #include #include #include #include -#include -#include -#include "eckit/linalg/Triplet.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" +#include "eckit/linalg/Triplet.h" #include "eckit/log/Log.h" #include "eckit/option/CmdArgs.h" #include "eckit/option/SimpleOption.h" -#include "multio/tools/MultioTool.h" #include "mir/method/WeightMatrix.h" +#include "multio/tools/MultioTool.h" namespace multio::action::interpolateFESOM2MIR { @@ -80,8 +80,8 @@ std::string fesomCacheName(const std::string& fesomName, const std::string& doma std::transform(localDomain.begin(), localDomain.end(), localDomain.begin(), [](unsigned char c) { return std::tolower(c); }); os << "fesom_" << fesomName << "_" << localDomain << "_to_HEALPix_" << std::setw(6) << std::setfill('0') << NSide - << "_" << precision << "_" << orderingConvention << "_" << std::setw(8) - << std::setfill('0') << static_cast(std::fabs(level * 1000)) << ".mat"; + << "_" << precision << "_" << orderingConvention << "_" << std::setw(8) << std::setfill('0') + << static_cast(std::fabs(level * 1000)) << ".mat"; return os.str(); } @@ -95,11 +95,10 @@ class Fesom2mirCacheGenerator final : public multio::MultioTool { private: void usage(const std::string& tool) const override { eckit::Log::info() << std::endl << "Usage: " << tool << " [options]" << std::endl; - eckit::Log::info() - << "EXAMPLE: " << std::endl - << "fesom-cache-generator --inputPath=. --inputFile=CORE2_ngrid_NSIDE32_0_ring.csv " - << std::endl - << std::endl; + eckit::Log::info() << "EXAMPLE: " << std::endl + << "fesom-cache-generator --inputPath=. --inputFile=CORE2_ngrid_NSIDE32_0_ring.csv " + << std::endl + << std::endl; } void init(const eckit::option::CmdArgs& args) override; @@ -192,8 +191,7 @@ void Fesom2mirCacheGenerator::init(const eckit::option::CmdArgs& args) { ASSERT(inputFile_tmp.exists()); eckit::PathName outputPath_tmp{outputPath_}; outputPath_tmp.mkdir(); - parseInputFileName(inputFile_, fesomName_, - domain_, NSide_, level_, orderingConvention_); + parseInputFileName(inputFile_, fesomName_, domain_, NSide_, level_, orderingConvention_); args.get("nCols", Ncol_); Nrow_ = NSide_ * NSide_ * 12; @@ -203,12 +201,10 @@ void Fesom2mirCacheGenerator::execute(const eckit::option::CmdArgs& args) { std::vector triplets; loadTriplets(triplets); - std::sort(begin(triplets), end(triplets), - [](const auto& a, const auto& b) { return a.row() < b.row(); }); + std::sort(begin(triplets), end(triplets), [](const auto& a, const auto& b) { return a.row() < b.row(); }); const auto orderingConvention = orderingConvention_enum2string(orderingConvention_); - const auto cacheFileName = fesomCacheName(fesomName_, domain_, "double", - NSide_, orderingConvention, level_); + const auto cacheFileName = fesomCacheName(fesomName_, domain_, "double", NSide_, orderingConvention, level_); mir::method::WeightMatrix W(Nrow_, Ncol_); W.setFromTriplets(triplets); @@ -218,7 +214,7 @@ void Fesom2mirCacheGenerator::execute(const eckit::option::CmdArgs& args) { void Fesom2mirCacheGenerator::finish(const eckit::option::CmdArgs&) {} -} // namespace multio::action::interpolateFESOM +} // namespace multio::action::interpolateFESOM2MIR int main(int argc, char** argv) { diff --git a/src/multio/action/interpolate/Interpolate.cc b/src/multio/action/interpolate/Interpolate.cc index ab04084c..b4bd0673 100644 --- a/src/multio/action/interpolate/Interpolate.cc +++ b/src/multio/action/interpolate/Interpolate.cc @@ -13,11 +13,11 @@ #include "multio/action/interpolate/Interpolate.h" #include +#include #include +#include #include #include -#include -#include #include "eckit/exception/Exceptions.h" #include "eckit/mpi/Comm.h" @@ -131,7 +131,6 @@ void regularLatLongMetadata(DestType& param, std::vector grid, std::vect }; - std::string fesomCacheName(const std::string& fesomName, const std::string& domain, const std::string& precision, size_t NSide, const std::string& orderingConvention, double level) { std::ostringstream os; @@ -140,15 +139,14 @@ std::string fesomCacheName(const std::string& fesomName, const std::string& doma std::transform(localDomain.begin(), localDomain.end(), localDomain.begin(), [](unsigned char c) { return std::tolower(c); }); os << "fesom_" << fesomName << "_" << localDomain << "_to_HEALPix_" << std::setw(6) << std::setfill('0') << NSide - << "_" << precision << "_" << orderingConvention << "_" << std::setw(8) - << std::setfill('0') << static_cast(std::fabs(level * 1000)) << ".mat"; + << "_" << precision << "_" << orderingConvention << "_" << std::setw(8) << std::setfill('0') + << static_cast(std::fabs(level * 1000)) << ".mat"; return os.str(); } template -std::string generateKey(const message::Message& msg, const std::string& cache_path, - size_t NSide, const std::string& orderingConvention) -{ +std::string generateKey(const message::Message& msg, const std::string& cache_path, size_t NSide, + const std::string& orderingConvention) { // TODO: Probably missing the kind of fesom grid in the name // need to see the metadata to understand how to extract it size_t level = static_cast(msg.metadata().getLong("level", msg.metadata().getDouble("levelist", 0))); @@ -198,8 +196,8 @@ eckit::Value getInputGrid(const eckit::LocalConfiguration& cfg, message::Metadat } -void fill_input(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametrisation& param, size_t payload_size, std::string domain, - const eckit::Value& inp) { +void fill_input(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametrisation& param, size_t payload_size, + std::string domain, const eckit::Value& inp) { auto regular_ll = [¶m](double west_east_increment, double south_north_increment) { regularLatLongMetadata(param, std::vector{west_east_increment, south_north_increment}, full_area); @@ -275,10 +273,10 @@ void fill_input(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametr return; } - if ( std::regex_match(input, match, FESOM)) { + if (std::regex_match(input, match, FESOM)) { param.set("gridded", true); param.set("gridType", "unstructured_grid"); - param.set("numberOfPoints", payload_size ); + param.set("numberOfPoints", payload_size); return; } } @@ -320,12 +318,12 @@ void fill_job(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametris } // Forward to mir the options - bool interpolationMatrix=false; + bool interpolationMatrix = false; if (cfg.has("options")) { const auto& options = cfg.getSubConfiguration("options"); for (const auto& key : options.keys()) { set(options, key, options.getSubConfiguration(key).get()); - if ( key == "interpolation" && options.getString(key) == "matrix" ) { + if (key == "interpolation" && options.getString(key) == "matrix") { interpolationMatrix = true; } } @@ -364,7 +362,7 @@ void fill_job(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametris && cfg.getSubConfiguration("grid").get().head().isDouble()) { gridKind = "regular_ll"; grid = cfg.getDoubleVector("grid"); - if ( interpolationMatrix ) { + if (interpolationMatrix) { std::ostringstream os; os << " - interpolation matrix supported only for fesom -> Healpix" << std::endl; throw eckit::SeriousBug(os.str(), Here()); @@ -384,7 +382,7 @@ void fill_job(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametris regularLatLongMetadata(md, grid, full_area); } - if ( interpolationMatrix ) { + if (interpolationMatrix) { std::ostringstream os; os << " - interpolation matrix supported only for fesom -> Healpix" << std::endl; throw eckit::SeriousBug(os.str(), Here()); @@ -398,13 +396,13 @@ void fill_job(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametris md.set("orderingConvention", "ring"); // If no interpolation matrix name is provided, generate one - if ( interpolationMatrix ) { + if (interpolationMatrix) { const std::string input = util::replaceCurly(inp.as(), [](std::string_view replace) { std::string lookUpKey{replace}; char* env = ::getenv(lookUpKey.c_str()); return env ? std::optional{env} : std::optional{}; }); - if ( input != "fesom" ){ + if (input != "fesom") { std::ostringstream os; os << " - interpolation matrix supported only for fesom -> Healpix" << std::endl; throw eckit::SeriousBug(os.str(), Here()); @@ -419,7 +417,6 @@ void fill_job(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametris const auto weights_file = generateKey(msg, expanded_cache_path, grid[0], "ring"); destination.set("interpolation-matrix", weights_file); } - } else { std::cout << "Grid not implemented" << std::endl; @@ -444,12 +441,12 @@ message::Message Interpolate::InterpolateMessage(message::Message&& msg) mir::param::SimpleParametrisation inputPar; eckit::Value inp = getInputGrid(config, md); - fill_input(config, inputPar, size, msg.domain(), inp ); + fill_input(config, inputPar, size, msg.domain(), inp); if (msg.metadata().has("missingValue") && msg.metadata().getBool("bitmapPresent")) { inputPar.set("missing_value", msg.metadata().getDouble("missingValue")); - } else if (config.getSubConfiguration("options").has("missing_value")) { - inputPar.set("missing_value", - config.getSubConfiguration("options").getDouble("missing_value")); + } + else if (config.getSubConfiguration("options").has("missing_value")) { + inputPar.set("missing_value", config.getSubConfiguration("options").getDouble("missing_value")); } mir::input::RawInput input(data, size, inputPar); diff --git a/src/multio/action/statistics/operations/OperationWithData.h b/src/multio/action/statistics/operations/OperationWithData.h index 26aff60b..880e286c 100644 --- a/src/multio/action/statistics/operations/OperationWithData.h +++ b/src/multio/action/statistics/operations/OperationWithData.h @@ -16,14 +16,16 @@ class OperationWithData : public Operation { const OperationWindow& win, const StatisticsConfiguration& cfg, T initial_value = 0.0) : Operation{name, operation, win, cfg}, values_{std::vector(sz /= sizeof(T), initial_value)}, - needRestart_{needRestart}, initialValue_{initial_value} {} + needRestart_{needRestart}, + initialValue_{initial_value} {} OperationWithData(const std::string& name, const std::string& operation, long sz, bool needRestart, const OperationWindow& win, std::shared_ptr& IOmanager, const StatisticsConfiguration& cfg, T initial_value = 0.0) : - Operation{name, operation, win, cfg}, values_{std::vector(sz /= sizeof(T), initial_value)}, needRestart_{needRestart}, - initialValue_{initial_value} - { + Operation{name, operation, win, cfg}, + values_{std::vector(sz /= sizeof(T), initial_value)}, + needRestart_{needRestart}, + initialValue_{initial_value} { load(IOmanager, cfg); } diff --git a/src/multio/config/ComponentConfiguration.cc b/src/multio/config/ComponentConfiguration.cc index 10206ee8..42cafe23 100644 --- a/src/multio/config/ComponentConfiguration.cc +++ b/src/multio/config/ComponentConfiguration.cc @@ -21,7 +21,7 @@ ComponentConfiguration SubComponentIteratorMapper::operator()(const eckit::Local ComponentConfiguration::ComponentConfiguration(const eckit::LocalConfiguration& componentConfig, const MultioConfiguration& multioConf) : - componentConf_(componentConfig), multioConf_(multioConf){}; + componentConf_(componentConfig), multioConf_(multioConf) {}; //============================================================================= diff --git a/src/multio/maestro/MaestroCdo.h b/src/multio/maestro/MaestroCdo.h index 879c371e..d16f0c8e 100644 --- a/src/multio/maestro/MaestroCdo.h +++ b/src/multio/maestro/MaestroCdo.h @@ -28,8 +28,9 @@ class MaestroCdo { void retract(); void dispose(); template - void set_attribute(const std::string& key, const T value, bool copy = true, bool user_owned_attribute_value=false) { - ASSERT(MSTRO_OK == mstro_cdo_attribute_set(cdo_, key.c_str(), (void*) value, copy, user_owned_attribute_value)); + void set_attribute(const std::string& key, const T value, bool copy = true, + bool user_owned_attribute_value = false) { + ASSERT(MSTRO_OK == mstro_cdo_attribute_set(cdo_, key.c_str(), (void*)value, copy, user_owned_attribute_value)); } template T get_attribute(const std::string& key) { diff --git a/src/multio/message/Metadata.cc b/src/multio/message/Metadata.cc index 447158c8..9ab31b77 100644 --- a/src/multio/message/Metadata.cc +++ b/src/multio/message/Metadata.cc @@ -24,10 +24,10 @@ Metadata to_metadata(const std::string& fieldId) { } MetadataException::MetadataException(const std::string& reason, const eckit::CodeLocation& l) : - eckit::Exception(std::string("MetadataException: ") + reason, l){}; + eckit::Exception(std::string("MetadataException: ") + reason, l) {}; MetadataMissingKeyException::MetadataMissingKeyException(const std::string& missingKey, const eckit::CodeLocation& l) : - MetadataException(std::string("Missing key \"") + missingKey + std::string("\""), l){}; + MetadataException(std::string("Missing key \"") + missingKey + std::string("\""), l) {}; } // namespace multio::message diff --git a/src/multio/tools/multio-maestro-instigator.cc b/src/multio/tools/multio-maestro-instigator.cc index 48f0a717..5f262b30 100644 --- a/src/multio/tools/multio-maestro-instigator.cc +++ b/src/multio/tools/multio-maestro-instigator.cc @@ -70,13 +70,13 @@ void MaestroInstigator::execute(const eckit::option::CmdArgs&) { while (tmp) { switch (tmp->kind) { case MSTRO_POOL_EVENT_APP_JOIN: - std::cout << mstro_clock() << ",JOIN," << tmp->serial << ", { " << tmp->payload.join.appid << ",\"" - << tmp->payload.join.component_name << "\" }" << std::endl; + std::cout << mstro_clock() << ",JOIN," << tmp->serial << ", { " << tmp->payload.join.appid + << ",\"" << tmp->payload.join.component_name << "\" }" << std::endl; ++joinCount; break; case MSTRO_POOL_EVENT_APP_LEAVE: - std::cout << mstro_clock() << ",LEAVE," << tmp->serial << ", { " << tmp->payload.leave.appid << " }" - << std::endl; + std::cout << mstro_clock() << ",LEAVE," << tmp->serial << ", { " << tmp->payload.leave.appid + << " }" << std::endl; ++leaveCount; break; default: diff --git a/src/multio/tools/multio-maestro-syphon.cc b/src/multio/tools/multio-maestro-syphon.cc index efa9e211..cb4bfba8 100644 --- a/src/multio/tools/multio-maestro-syphon.cc +++ b/src/multio/tools/multio-maestro-syphon.cc @@ -184,7 +184,8 @@ void MaestroSyphon::process_join_leave_events(MaestroSubscription& join_leave_su << " *** Application " << tmp->payload.join.component_name << " is joining" << std::endl; break; case MSTRO_POOL_EVENT_APP_LEAVE: - LOG_DEBUG_LIB(LibMultio) << " *** Application " << tmp->payload.leave.appid << " is leaving" << std::endl; + LOG_DEBUG_LIB(LibMultio) + << " *** Application " << tmp->payload.leave.appid << " is leaving" << std::endl; break; default: std::ostringstream os; diff --git a/src/multio/tools/multio-pool-manager.cc b/src/multio/tools/multio-pool-manager.cc index 71d330e1..3ba77c7a 100644 --- a/src/multio/tools/multio-pool-manager.cc +++ b/src/multio/tools/multio-pool-manager.cc @@ -32,7 +32,8 @@ bool process_join_leave_events(MaestroSubscription& join_leave_subscription) { break; case MSTRO_POOL_EVENT_APP_LEAVE: ++nbLeavers; - LOG_DEBUG_LIB(LibMultio) << " *** Application " << tmp->payload.leave.appid << " is leaving" << std::endl; + LOG_DEBUG_LIB(LibMultio) + << " *** Application " << tmp->payload.leave.appid << " is leaving" << std::endl; break; default: std::ostringstream os; diff --git a/src/multio/util/Metadata.h b/src/multio/util/Metadata.h index 12d42e98..c9e78a88 100644 --- a/src/multio/util/Metadata.h +++ b/src/multio/util/Metadata.h @@ -92,8 +92,8 @@ Res firstOfTyped() noexcept(noexcept(Res{})) { } template -Res firstOfTyped(T&& m, TS&&... ts) noexcept( - noexcept(evalToOptional(std::forward(m))) && noexcept(firstOfTyped(std::forward(ts)...))) { +Res firstOfTyped(T&& m, TS&&... ts) noexcept(noexcept(evalToOptional(std::forward(m))) + && noexcept(firstOfTyped(std::forward(ts)...))) { auto em = evalToOptional(std::forward(m)); if (em) { // Return with RVO @@ -117,9 +117,10 @@ auto withFirstOfTyped(Func&& func) noexcept(noexcept(makeOptional(std::forward(func)(ArgType{})); } template -auto withFirstOfTyped(Func&& func, T&& m, TS&&... ts) noexcept(noexcept(evalToOptional(std::forward(m))) && noexcept( - makeOptional(std::forward(func)(evalToOptional(std::forward( - m))))) && noexcept(withFirstOfTyped(std::forward(func), std::forward(ts)...))) { +auto withFirstOfTyped(Func&& func, T&& m, TS&&... ts) noexcept( + noexcept(evalToOptional(std::forward(m))) + && noexcept(makeOptional(std::forward(func)(evalToOptional(std::forward(m))))) + && noexcept(withFirstOfTyped(std::forward(func), std::forward(ts)...))) { auto em = evalToOptional(std::forward(m)); if (em) { return makeOptional(std::forward(func)(std::move(em))); @@ -134,7 +135,8 @@ auto withFirstOfTyped(Func&& func, T&& m, TS&&... ts) noexcept(noexcept(evalToOp // function template auto withFirstOf(OptFunc&& func, T&& m, TS&&... ts) noexcept( - noexcept(bool(func)) && noexcept(withFirstOfTyped(m)))>>( + noexcept(bool(func)) + && noexcept(withFirstOfTyped(m)))>>( (*(std::forward(func))), std::forward(m), std::forward(ts)...))) { using ArgType = std::decay_t(m)))>; using OptRetType = decltype(withFirstOfTyped((*(std::forward(func))), std::forward(m), @@ -156,7 +158,7 @@ class LookUp { std::string key_; public: - LookUp(const eckit::Configuration& c, const std::string& key) : c_(c), key_(key){}; + LookUp(const eckit::Configuration& c, const std::string& key) : c_(c), key_(key) {}; std::optional operator()() const; }; diff --git a/src/multio/util/Substitution.cc b/src/multio/util/Substitution.cc index a1902a8c..2315b562 100644 --- a/src/multio/util/Substitution.cc +++ b/src/multio/util/Substitution.cc @@ -11,12 +11,12 @@ std::optional parseBool(const eckit::LocalConfiguration& cfg, const std::s eckit::Value input = cfg.has(key) ? cfg.getSubConfiguration(key).get().isString() ? eckit::Value{eckit::Translator{}(util::replaceCurly( - cfg.getString(key), - [](std::string_view replace) { - std::string lookUpKey{replace}; - char* env = ::getenv(lookUpKey.c_str()); - return env ? std::optional{env} : std::optional{}; - }))} + cfg.getString(key), + [](std::string_view replace) { + std::string lookUpKey{replace}; + char* env = ::getenv(lookUpKey.c_str()); + return env ? std::optional{env} : std::optional{}; + }))} : cfg.getSubConfiguration(key).get() : eckit::Value{defaultValue}; return input.isBool() ? std::optional{input.as()} : std::optional{}; From 4497be504fa3777dc24483e2274e6f27329380ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domokos=20S=C3=A1rm=C3=A1ny?= Date: Wed, 3 Jul 2024 10:44:05 +0100 Subject: [PATCH 7/8] Make MIR a conditional dependency of the cache generator --- src/multio/action/interpolate/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/multio/action/interpolate/CMakeLists.txt b/src/multio/action/interpolate/CMakeLists.txt index 47006595..0131db19 100644 --- a/src/multio/action/interpolate/CMakeLists.txt +++ b/src/multio/action/interpolate/CMakeLists.txt @@ -28,11 +28,11 @@ ecbuild_add_executable( ../../tools/MultioTool.cc CONDITION - HAVE_ATLAS_IO + HAVE_MIR AND HAVE_ATLAS_IO LIBS multio atlas_io eckit mir -) \ No newline at end of file +) From f046bd9c0207d2df9b1eb7716cd6e8608a22ee93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domokos=20S=C3=A1rm=C3=A1ny?= Date: Wed, 3 Jul 2024 14:18:40 +0100 Subject: [PATCH 8/8] Disable linux-clang-12 CI job This version of the ci will be removed soon, anyway. --- .github/workflows/ci.yml | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52f4d325..78b3260f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,10 +32,7 @@ jobs: name: - linux gnu-10 - linux gnu-12 - - linux clang-12 - linux clang-14 - # - linux intel - # - macos appclang-14 include: - name: linux gnu-10 @@ -55,14 +52,6 @@ jobs: caching: true cmake_options: -DENABLE_OMP_CXX=OFF - - name: linux clang-12 - os: ubuntu-20.04 - compiler: clang-12 - compiler_cc: clang-12 - compiler_cxx: clang++-12 - compiler_fc: gfortran-10 - caching: true - - name: linux clang-14 os: ubuntu-22.04 compiler: clang-14 @@ -71,25 +60,6 @@ jobs: compiler_fc: gfortran-12 caching: true - # - name : linux intel - # os: ubuntu-20.04 - # compiler: intel - # compiler_cc: icx - # compiler_cxx: icpx - # compiler_fc: ifx - # caching: true - # coverage: false - - # - name: macos appclang-14 - # # Xcode compiler requires empty environment variables, so we pass null (~) here - # os: macos-12 - # compiler: clang-14 - # compiler_cc: ~ - # compiler_cxx: ~ - # compiler_fc: gfortran-12 - # caching: true - # cmake_options: -DMPI_SLOTS=4 - runs-on: ${{ matrix.os }} steps: - name: Checkout Repository