Prometheus metrics library upgrade

.circleci/config.yml

@@ -372,7 +372,7 @@ jobs:
           at: ~/project
       - run:
           name: AcceptanceTests
-          no_output_timeout: 20m
+          no_output_timeout: 30m
           command: |
             CLASSNAMES=$(circleci tests glob "**/src/acceptance-test/java/**/*.java" \
               | sed 's@.*/src/acceptance-test/java/@@' \

CHANGELOG.md

@@ -7,6 +7,21 @@
 ## Unreleased Changes
 
 ### Breaking Changes
+- With the upgrade of the Prometheus Java Metrics library, there are the following changes:
+  - Gauge names are not allowed to end with `total`, therefore metrics as `beacon_proposers_data_total` and `beacon_eth1_current_period_votes_total` are dropping the `_total` suffix
+  - The `_created` timestamps are not returned by default.
+  - Some JVM metrics have changed name to adhere to the OTEL standard (see the table below), [Teku Detailed Dashboard](https://grafana.com/grafana/dashboards/16737-teku-detailed/) is updated to support both names
+
+    | Old Name                        | New Name                        |
+    |---------------------------------|---------------------------------|
+    | jvm_memory_bytes_committed      | jvm_memory_committed_bytes      |
+    | jvm_memory_bytes_init           | jvm_memory_init_bytes           |
+    | jvm_memory_bytes_max            | jvm_memory_max_bytes            |
+    | jvm_memory_bytes_used           | jvm_memory_used_bytes           |
+    | jvm_memory_pool_bytes_committed | jvm_memory_pool_committed_bytes |
+    | jvm_memory_pool_bytes_init      | jvm_memory_pool_init_bytes      |
+    | jvm_memory_pool_bytes_max       | jvm_memory_pool_max_bytes       |
+    | jvm_memory_pool_bytes_used      | jvm_memory_pool_used_bytes      |
 
 ### Additions and Improvements
 - Optimized blobs validation pipeline

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/metrics/MetricFetcher.java

@@ -97,7 +97,7 @@ public static MetricValue parse(final String line) {
 
       final Map<String, String> metricLabels = new HashMap<>();
       final String labelsString = line.substring(line.indexOf("{") + 1, line.indexOf("}"));
-      final Pattern p = Pattern.compile("(.*?)=\"(.*?)\",");
+      final Pattern p = Pattern.compile("(.*?)=\"(.*?)\",?");
       final Matcher m = p.matcher(labelsString);
       while (m.find()) {
         metricLabels.put(m.group(1), m.group(2));

beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/Eth1DataCache.java

@@ -35,7 +35,7 @@
 public class Eth1DataCache {
   static final String CACHE_SIZE_METRIC_NAME = "eth1_block_cache_size";
   static final String VOTES_MAX_METRIC_NAME = "eth1_current_period_votes_max";
-  static final String VOTES_TOTAL_METRIC_NAME = "eth1_current_period_votes_total";
+  static final String VOTES_TOTAL_METRIC_NAME = "eth1_current_period_votes";
   static final String VOTES_UNKNOWN_METRIC_NAME = "eth1_current_period_votes_unknown";
   static final String VOTES_CURRENT_METRIC_NAME = "eth1_current_period_votes_current";
   static final String VOTES_BEST_METRIC_NAME = "eth1_current_period_votes_best";

data/publisher/src/main/java/tech/pegasys/teku/data/publisher/PrometheusMetricsPublisherSource.java

@@ -99,7 +99,7 @@ public long getGossipBytesTotalReceived() {
   }
 
   private void storeObservationIfNeeded(final Observation observation) {
-    MetricCategory category = observation.getCategory();
+    MetricCategory category = observation.category();
     if (category.equals(PROCESS)) {
       readProcessCategoryItem(observation);
     } else if (category.equals(VALIDATOR)) {
@@ -113,37 +113,37 @@ private void storeObservationIfNeeded(final Observation observation) {
 
   private void readBeaconCategoryItem(final Observation observation) {
     isBeaconNodePresent = true;
-    switch (observation.getMetricName()) {
+    switch (observation.metricName()) {
       case "head_slot":
-        headSlot = getLongValue(observation.getValue());
+        headSlot = getLongValue(observation.value());
         break;
       case "eth1_request_queue_size":
         isEth1Connected = true;
         break;
       case "peer_count":
-        peerCount = getIntValue(observation.getValue());
+        peerCount = getIntValue(observation.value());
         break;
       case "node_syncing_active":
-        isEth2Synced = getIntValue(observation.getValue()) == 0;
+        isEth2Synced = getIntValue(observation.value()) == 0;
         break;
     }
   }
 
   private void readProcessCategoryItem(final Observation observation) {
-    if ("cpu_seconds_total".equals(observation.getMetricName())) {
-      cpuSecondsTotal = getLongValue(observation.getValue());
+    if ("cpu_seconds_total".equals(observation.metricName())) {
+      cpuSecondsTotal = getLongValue(observation.value());
     }
   }
 
   private void readJvmCategoryItem(final Observation observation) {
-    if ("memory_pool_bytes_used".equals(observation.getMetricName())) {
-      addToMemoryPoolBytesUsed((Double) observation.getValue());
+    if ("memory_pool_bytes_used".equals(observation.metricName())) {
+      addToMemoryPoolBytesUsed((Double) observation.value());
     }
   }
 
   private void readValidatorCategoryItem(final Observation observation) {
-    if ("local_validator_counts".equals(observation.getMetricName())) {
-      addToLocalValidators(observation.getLabels(), (Double) observation.getValue());
+    if ("local_validator_counts".equals(observation.metricName())) {
+      addToLocalValidators(observation.labels(), (Double) observation.value());
     }
   }
 

ethereum/dataproviders/src/main/java/tech/pegasys/teku/dataproviders/generators/CachingTaskQueue.java

@@ -26,8 +26,8 @@
 import java.util.stream.Stream;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 import org.hyperledger.besu.plugin.services.metrics.Counter;
-import org.hyperledger.besu.plugin.services.metrics.LabelledGauge;
 import org.hyperledger.besu.plugin.services.metrics.LabelledMetric;
+import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedMetric;
 import tech.pegasys.teku.infrastructure.async.AsyncRunner;
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
 import tech.pegasys.teku.infrastructure.collections.LimitedMap;
@@ -88,8 +88,8 @@ public static <K, V> CachingTaskQueue<K, V> create(
   }
 
   public void startMetrics() {
-    final LabelledGauge taskQueueMetrics =
-        metricsSystem.createLabelledGauge(
+    final LabelledSuppliedMetric taskQueueMetrics =
+        metricsSystem.createLabelledSuppliedGauge(
             TekuMetricCategory.STORAGE,
             metricsPrefix + "_tasks",
             "Labelled task queue metrics",

ethereum/dataproviders/src/test/java/tech/pegasys/teku/dataproviders/generators/CachingTaskQueueTest.java

@@ -272,33 +272,29 @@ private void assertCacheSizeMetric(final int expectedSize) {
 
   private void assertCacheHitCount(final int expectedCount) {
     final double value =
-        metricsSystem
-            .getCounter(TekuMetricCategory.STORAGE, METRICS_PREFIX + "_tasks_total")
-            .getValue("cached");
+        metricsSystem.getCounterValue(
+            TekuMetricCategory.STORAGE, METRICS_PREFIX + "_tasks_total", "cached");
     assertThat(value).isEqualTo(expectedCount);
   }
 
   private void assertNewTaskCount(final int expectedCount) {
     final double value =
-        metricsSystem
-            .getCounter(TekuMetricCategory.STORAGE, METRICS_PREFIX + "_tasks_total")
-            .getValue("new");
+        metricsSystem.getCounterValue(
+            TekuMetricCategory.STORAGE, METRICS_PREFIX + "_tasks_total", "new");
     assertThat(value).isEqualTo(expectedCount);
   }
 
   private void assertDuplicateTaskCount(final int expectedCount) {
     final double value =
-        metricsSystem
-            .getCounter(TekuMetricCategory.STORAGE, METRICS_PREFIX + "_tasks_total")
-            .getValue("duplicate");
+        metricsSystem.getCounterValue(
+            TekuMetricCategory.STORAGE, METRICS_PREFIX + "_tasks_total", "duplicate");
     assertThat(value).isEqualTo(expectedCount);
   }
 
   private void assertRebasedTaskCount(final int expectedCount) {
     final double value =
-        metricsSystem
-            .getCounter(TekuMetricCategory.STORAGE, METRICS_PREFIX + "_tasks_total")
-            .getValue("rebase");
+        metricsSystem.getCounterValue(
+            TekuMetricCategory.STORAGE, METRICS_PREFIX + "_tasks_total", "rebase");
     assertThat(value).isEqualTo(expectedCount);
   }
 

ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerBlockProductionManagerImplTest.java

@@ -567,9 +567,11 @@ private void verifyEngineCalled(
 
   private void verifySourceCounter(final Source source, final FallbackReason reason) {
     final long actualCount =
-        stubMetricsSystem
-            .getCounter(TekuMetricCategory.BEACON, "execution_payload_source_total")
-            .getValue(source.toString(), reason.toString());
+        stubMetricsSystem.getCounterValue(
+            TekuMetricCategory.BEACON,
+            "execution_payload_source_total",
+            source.toString(),
+            reason.toString());
     assertThat(actualCount).isOne();
   }
 }

ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImplTest.java

@@ -887,9 +887,11 @@ private void verifyEngineCalled(
 
   private void verifySourceCounter(final Source source, final FallbackReason reason) {
     final long actualCount =
-        stubMetricsSystem
-            .getCounter(TekuMetricCategory.BEACON, "execution_payload_source_total")
-            .getValue(source.toString(), reason.toString());
+        stubMetricsSystem.getCounterValue(
+            TekuMetricCategory.BEACON,
+            "execution_payload_source_total",
+            source.toString(),
+            reason.toString());
     assertThat(actualCount).isOne();
   }
 }

ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/datastructures/interop/MergedGenesisTestBuilder.java

@@ -20,7 +20,7 @@
 import org.hyperledger.besu.ethereum.chain.GenesisState;
 import org.hyperledger.besu.ethereum.core.Block;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
-import org.hyperledger.besu.ethereum.core.MiningParameters;
+import org.hyperledger.besu.ethereum.core.MiningConfiguration;
 import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
@@ -40,7 +40,7 @@ public static ExecutionPayloadHeader createPayloadForBesuGenesis(
     final ProtocolSchedule protocolSchedule =
         MainnetProtocolSchedule.fromConfig(
             genesisConfigOptions,
-            MiningParameters.MINING_DISABLED,
+            MiningConfiguration.MINING_DISABLED,
             badBlockManager,
             false,
             new NoOpMetricsSystem());

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ProposersDataManager.java

@@ -23,7 +23,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.tuweni.bytes.Bytes32;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
-import org.hyperledger.besu.plugin.services.metrics.LabelledGauge;
+import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedMetric;
 import tech.pegasys.teku.ethereum.events.SlotEventsChannel;
 import tech.pegasys.teku.ethereum.execution.types.Eth1Address;
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
@@ -66,10 +66,10 @@ public ProposersDataManager(
       final RecentChainData recentChainData,
       final Optional<Eth1Address> proposerDefaultFeeRecipient,
       final boolean forkChoiceUpdatedAlwaysSendPayloadAttribute) {
-    final LabelledGauge labelledGauge =
-        metricsSystem.createLabelledGauge(
+    final LabelledSuppliedMetric labelledGauge =
+        metricsSystem.createLabelledSuppliedGauge(
             TekuMetricCategory.BEACON,
-            "proposers_data_total",
+            "proposers_data",
             "Total number proposers/validators under management",
             "type");
 

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/signatures/AggregatingSignatureVerificationService.java

@@ -27,6 +27,7 @@
 import org.apache.tuweni.bytes.Bytes;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 import org.hyperledger.besu.plugin.services.metrics.Counter;
+import org.hyperledger.besu.plugin.services.metrics.Histogram;
 import tech.pegasys.teku.bls.BLS;
 import tech.pegasys.teku.bls.BLSPublicKey;
 import tech.pegasys.teku.bls.BLSSignature;
@@ -53,7 +54,7 @@ public class AggregatingSignatureVerificationService extends SignatureVerificati
   private final AsyncRunner asyncRunner;
   private final Counter batchCounter;
   private final Counter taskCounter;
-  private final MetricsHistogram batchSizeHistogram;
+  private final Histogram batchSizeHistogram;
 
   @VisibleForTesting
   AggregatingSignatureVerificationService(
@@ -89,13 +90,11 @@ public class AggregatingSignatureVerificationService extends SignatureVerificati
             "signature_verifications_task_count_total",
             "Reports the number of individual verification tasks processed");
     batchSizeHistogram =
-        MetricsHistogram.create(
+        metricsSystem.createHistogram(
             TekuMetricCategory.EXECUTOR,
-            metricsSystem,
             "signature_verifications_batch_size",
             "Histogram of signature verification batch sizes",
-            3,
-            List.of());
+            MetricsHistogram.getDefaultBuckets());
   }
 
   public AggregatingSignatureVerificationService(
@@ -188,7 +187,7 @@ private List<SignatureTask> waitForBatch() {
   void batchVerifySignatures(final List<SignatureTask> tasks) {
     batchCounter.inc();
     taskCounter.inc(tasks.size());
-    batchSizeHistogram.recordValue(tasks.size());
+    batchSizeHistogram.observe(tasks.size());
     final List<List<BLSPublicKey>> allKeys = new ArrayList<>();
     final List<Bytes> allMessages = new ArrayList<>();
     final List<BLSSignature> allSignatures = new ArrayList<>();

ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ProposerDataManagerTest.java

@@ -155,15 +155,15 @@ private void prepareRegistrations() {
   private void assertPreparedProposersCount(final int expectedCount) {
     final OptionalDouble optionalValue =
         metricsSystem
-            .getLabelledGauge(TekuMetricCategory.BEACON, "proposers_data_total")
+            .getLabelledGauge(TekuMetricCategory.BEACON, "proposers_data")
             .getValue("prepared_proposers");
     assertThat(optionalValue).hasValue(expectedCount);
   }
 
   private void assertRegisteredValidatorsCount(final int expectedCount) {
     final OptionalDouble optionalValue =
         metricsSystem
-            .getLabelledGauge(TekuMetricCategory.BEACON, "proposers_data_total")
+            .getLabelledGauge(TekuMetricCategory.BEACON, "proposers_data")
             .getValue("registered_validators");
     assertThat(optionalValue).hasValue(expectedCount);
   }

ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java

@@ -42,6 +42,7 @@
 import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 import org.hyperledger.besu.metrics.Observation;
 import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import tech.pegasys.infrastructure.logging.LogCaptor;
@@ -79,6 +80,7 @@ public class BlockBlobSidecarsTrackersPoolImplTest {
   private final StubAsyncRunner asyncRunner = new StubAsyncRunner();
   private final RecentChainData recentChainData = mock(RecentChainData.class);
   private final ExecutionLayerChannel executionLayer = mock(ExecutionLayerChannel.class);
+  BlockBlobSidecarsTrackersPoolImpl blockBlobSidecarsTrackersPool;
 
   @SuppressWarnings("unchecked")
   private final Function<BlobSidecar, SafeFuture<Void>> blobSidecarPublisher = mock(Function.class);
@@ -88,21 +90,6 @@ public class BlockBlobSidecarsTrackersPoolImplTest {
 
   private final BlockImportChannel blockImportChannel = mock(BlockImportChannel.class);
   private final int maxItems = 15;
-  private final BlockBlobSidecarsTrackersPoolImpl blockBlobSidecarsTrackersPool =
-      new PoolFactory(metricsSystem)
-          .createPoolForBlockBlobSidecarsTrackers(
-              blockImportChannel,
-              spec,
-              timeProvider,
-              asyncRunner,
-              recentChainData,
-              executionLayer,
-              () -> blobSidecarGossipValidator,
-              blobSidecarPublisher,
-              historicalTolerance,
-              futureTolerance,
-              maxItems,
-              this::trackerFactory);
 
   private UInt64 currentSlot = historicalTolerance.times(2);
   private final List<Bytes32> requiredBlockRootEvents = new ArrayList<>();
@@ -116,6 +103,21 @@ public class BlockBlobSidecarsTrackersPoolImplTest {
 
   @BeforeEach
   public void setup() {
+    blockBlobSidecarsTrackersPool =
+        new PoolFactory(metricsSystem)
+            .createPoolForBlockBlobSidecarsTrackers(
+                blockImportChannel,
+                spec,
+                timeProvider,
+                asyncRunner,
+                recentChainData,
+                executionLayer,
+                () -> blobSidecarGossipValidator,
+                blobSidecarPublisher,
+                historicalTolerance,
+                futureTolerance,
+                maxItems,
+                this::trackerFactory);
     // Set up slot
     blockBlobSidecarsTrackersPool.subscribeRequiredBlockRoot(requiredBlockRootEvents::add);
     blockBlobSidecarsTrackersPool.subscribeRequiredBlockRootDropped(
@@ -130,6 +132,11 @@ public void setup() {
     setSlot(currentSlot);
   }
 
+  @AfterEach
+  public void tearDown() {
+    metricsSystem.shutdown();
+  }
+
   private void setSlot(final long slot) {
     setSlot(UInt64.valueOf(slot));
   }
@@ -1298,7 +1305,7 @@ private static BlobIdentifier blobIdentifierFromBlobSidecar(final BlobSidecar bl
 
   private void assertStats(final String type, final String subType, final double count) {
     assertThat(
-            getMetricsValues("block_blobs_trackers_pool_stats_total").get(List.of(type, subType)))
+            getMetricsValues("block_blobs_trackers_pool_stats_total").get(List.of(subType, type)))
         .isEqualTo(count);
   }
 
@@ -1321,8 +1328,8 @@ private void assertBlobSidecarsTrackersCount(final int count) {
   private Map<List<String>, Object> getMetricsValues(final String metricName) {
     return metricsSystem
         .streamObservations(TekuMetricCategory.BEACON)
-        .filter(ob -> ob.getMetricName().equals(metricName))
-        .collect(Collectors.toMap(Observation::getLabels, Observation::getValue));
+        .filter(ob -> ob.metricName().equals(metricName))
+        .collect(Collectors.toMap(Observation::labels, Observation::value));
   }
 
   private BlockBlobSidecarsTracker trackerFactory(final SlotAndBlockRoot slotAndBlockRoot) {