General improvements/bugfix to ecom:

- added tool to automatically generate the output-manager-config.yaml from a dump (ecom-conf), - added tool to inspect a dump (ecom-list) - added example configuration for ERA6 (directToFDB flag in configuration file) - added the possibility to decide which field needs to go through the multIO plans and which needs to go directly to FDB
ecmwf · Jul 15, 2024 · 52c84dd · 52c84dd
1 parent 202f9da
commit 52c84dd
Show file tree

Hide file tree

Showing 26 changed files with 8,633 additions and 381 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -107,7 +107,7 @@ ecbuild_add_option( FEATURE OUTPUT_MANAGER
 
 ecbuild_add_option( FEATURE OUTPUT_MANAGER_ENCODER_REPORT
                     CONDITION HAVE_OUTPUT_MANAGER
-                    DEFAULT OFF
+		    DEFAULT ON
                     DESCRIPTION "Enable generation of the encoding report" )
 
 ecbuild_add_option( FEATURE OUTPUT_MANAGER_TOOL

diff --git a/src/ecom/CMakeLists.txt b/src/ecom/CMakeLists.txt
@@ -122,6 +122,7 @@ list( APPEND output_manager_src
 list( APPEND output_manager_src
     grib_info/encoding_info_mod.F90
     grib_info/yaml_rules_mod.F90
+    grib_info/yaml_time_assumptions_mod.F90
     grib_info/grib_codes_mod.F90
 )
 
@@ -285,20 +286,81 @@ ecbuild_add_library(
 set_target_properties( ecom PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/ecom )
 target_include_directories( ecom PUBLIC $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/module/ecom> )
 
+ecbuild_add_library(
+  TARGET  ecom-utils
+  SOURCES
+    tool/om_tool_utils_mod.F90
+    tool/om_tool_cfg_mod.F90
+
+  DEFINITIONS ${OUTPUT_MANAGER_BUILD_FLAVOUR}
+
+  PRIVATE_INCLUDES
+    ${CMAKE_BINARY_DIR}/include
+    ${OUTPUT_MANAGER_INCLUDE_PATH}
+
+  PUBLIC_LIBS
+    fckit
+    ecom
+
+)
+
+# Module directory
+set_target_properties( ecom-utils PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/ecom )
+target_include_directories( ecom-utils PUBLIC $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/module/ecom> )
+
 
 if ( HAVE_OUTPUT_MANAGER_TOOL )
 ecbuild_add_executable(
 
   TARGET ecom-feed
 
   SOURCES
-    tool/om_tool_cfg_mod.F90
-    tool/om_tool_utils_mod.F90
-    tool/om_tool_prog.F90
+    tool/om_tool_feed_prog.F90
+
+  LIBS
+    fckit
+    ecom
+    ecom-utils
+    eccodes
+    eccodes_f90
+    multio-fapi
+
+  LINKER_LANGUAGE Fortran
+
+)
+
+
+
+ecbuild_add_executable(
+
+  TARGET ecom-list
+
+  SOURCES
+    tool/om_tool_list_prog.F90
+
+  LIBS
+    fckit
+    ecom
+    ecom-utils
+    eccodes
+    eccodes_f90
+    multio-fapi
+
+  LINKER_LANGUAGE Fortran
+
+)
+
+ecbuild_add_executable(
+
+  TARGET ecom-conf
+
+  SOURCES
+    tool/om_tool_conf_prog.F90
 
   LIBS
     fckit
     ecom
+    ecom-utils
     eccodes
     eccodes_f90
     multio-fapi

diff --git a/src/ecom/config_example/config_fc_ERA6.yaml b/src/ecom/config_example/config_fc_ERA6.yaml
@@ -0,0 +1,132 @@
+no-io-info-log:
+  encoding-info: true
+  verbose: true
+
+dump-fortran-data-reproducer:
+  path: './'
+  dump-values: true
+  verbose: true
+  profile: true
+
+grib-msg-to-file:
+  path: 'allfields.grib'
+  verbose: true
+  profile: true
+
+grib-msg-to-multio:
+  multio-plans-file: '/perm/mavm/testsData/ecom-config/multio-ifs-wam-config.yaml'
+  verbose: true
+  profile: true
+
+grib-header-to-multio:
+  multio-plans-file: '/perm/mavm/testsData/ecom-config/multio-debug-plan.yaml'
+  verbose: true
+  profile: true
+
+full-grib-header-to-multio:
+  multio-plans-file: '/perm/mavm/testsData/ecom-config/multio-debug-plan.yaml'
+  verbose: true
+  profile: true
+
+fortran-metadata-to-multio:
+  multio-plans-file: '/perm/mavm/testsData/ecom-config/multio-debug-plan.yaml'
+  verbose: true
+  profile: true
+
+
+encoding-rules:
+
+  defaults:
+    encode:
+      gribEdition: 2
+
+  rules:
+    - rule: 'grib2_gridded_ml'
+      filter:
+        levtype: ['ml']
+        repres:  ['gridded']
+        paramId:  [ 75, 76, 133, 203, 246, 247, 248, 162100, 162101, 162102, 162103, 162104, 162105, 162106, 162107, 162108, 162109, 162110, 162111, 162112, 162113 ]
+
+      encode:
+        gribEdition: 2
+        packingType: 'grid_ccsds'
+        directToFDB: true
+
+
+    - rule: 'grib2_gridded_pl'
+      filter:
+        levtype: ['pl']
+        repres:  ['gridded']
+        paramId:  [ 60, 75, 76, 133, 203, 246, 247, 248, 260290 ]
+
+      encode:
+        gribEdition: 2
+        packingType: 'grid_ccsds'
+        directToFDB: true
+
+
+    - rule: 'grib2_gridded_sfc'
+      filter:
+        levtype: ['sfc']
+        repres:  ['gridded']
+        paramId:  [ 8, 9, 20, 26, 27, 28, 29, 30, 31, 32, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 47, 49, 50, 57, 58, 59, 66, 67, 74, 78, 79, 121, 122, 123, 129, 134, 136, 137, 139, 141, 142, 143, 144, 145, 146, 147, 148, 151, 159, 160, 161, 162, 163, 164, 165, 166, 167,
+ 168, 169, 170, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 186, 187, 188, 189, 195, 196, 197, 198, 201, 202, 205, 206, 207, 208, 209, 210, 211, 212, 213, 228, 229, 230, 231, 232, 234, 235, 236, 239, 240, 243, 244, 245, 3020, 140098, 140099, 140100, 140101, 140102,
+ 140103, 140104, 140105, 140112, 140113, 140114, 140115, 140116, 140117, 140118, 140119, 140120, 140121, 140122, 140123, 140124, 140125, 140126, 140127, 140128, 140129, 140207, 140208, 140209, 140211, 140212, 140214, 140215, 140216, 140217, 140218, 140219, 140220, 140221, 140222,
+ 140223, 140224, 140225, 140226, 140227, 140228, 140229, 140230, 140231, 140232, 140233, 140234, 140235, 140236, 140237, 140238, 140239, 140244, 140245, 140249, 140251, 140252, 140253, 140254, 162059, 162060, 162061, 162062, 162063, 162071, 162072, 162093, 174096, 228003, 228007,
+ 228008, 228009, 228010, 228011, 228012, 228013, 228014, 228015, 228016, 228017, 228018, 228019, 228020, 228021, 228022, 228023, 228024, 228026, 228027, 228028, 228029, 228035, 228036, 228044, 228045, 228046, 228047, 228048, 228050, 228051, 228052, 228053, 228057, 228058, 228059,
+ 228060, 228088, 228089, 228090, 228129, 228130, 228131, 228132, 228216, 228217, 228218, 228219, 228220, 228221, 228226, 228227, 228231, 228232, 228233, 228234, 228235, 228236, 228237, 228239, 228240, 228241, 228246, 228247, 228251, 233000, 233001, 233002, 233003, 233004, 233005,
+ 233006, 233007, 233008, 233009, 233010, 233011, 233012, 233013, 233014, 233015, 233016, 233017, 233018, 233019, 233020, 233021, 233022, 233023, 233024, 233025, 233026, 233027, 233028, 233029, 233030, 233031, 260015, 260048, 260109, 260121, 260123, 260318, 260319, 260320, 260321,
+ 260338, 260339, 262139, 262140 ]
+
+      encode:
+        gribEdition: 2
+        packingType: 'grid_ccsds'
+        directToFDB: true
+
+
+    - rule: 'grib2_gridded_sfc_to_plan'
+      filter:
+        levtype: ['sfc']
+        repres:  ['gridded']
+        paramId:  [ 183 ]
+
+      encode:
+        gribEdition: 2
+        packingType: 'grid_ccsds'
+        directToFDB: false
+
+    - rule: 'grib2_gridded_sol'
+      filter:
+        levtype: ['sol']
+        repres:  ['gridded']
+        paramId:  [ 33, 238, 228038, 228141 ]
+
+      encode:
+        gribEdition: 2
+        packingType: 'grid_ccsds'
+        directToFDB: true
+
+
+    - rule: 'grib2_spherical_harmonics_ml'
+      filter:
+        levtype: ['ml']
+        repres:  ['spherical_harmonics']
+        paramId:  [ 77, 129, 130, 135, 138, 152, 155 ]
+
+      encode:
+        gribEdition: 2
+        packingType: 'spectral_complex'
+        directToFDB: true
+
+
+    - rule: 'grib2_spherical_harmonics_pl'
+      filter:
+        levtype: ['pl']
+        repres:  ['spherical_harmonics']
+        paramId:  [ 129, 130, 135, 138, 152, 155, 157 ]
+
+      encode:
+        gribEdition: 2
+        packingType: 'spectral_complex'
+        directToFDB: true
+