add core_block_time_latency metric

zetaclient/metrics/metrics.go

@@ -85,6 +85,14 @@ var (
 		Help:      "Last core block number",
 	})
 
+	// CoreBlockLatency is a gauge that measures the difference between system time and
+	// block time from zetacore
+	CoreBlockLatency = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: ZetaClientNamespace,
+		Name:      "core_block_latency",
+		Help:      "Difference between system time and block time from zetacore",
+	})
+
 	// Info is a gauge that contains information about the zetaclient environment
 	Info = promauto.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: ZetaClientNamespace,

zetaclient/orchestrator/orchestrator.go

@@ -307,6 +307,9 @@ func (oc *Orchestrator) runScheduler(ctx context.Context) error {
 		case newBlock := <-newBlockChan:
 			bn := newBlock.Block.Height
 
+			blockTimeLatency := time.Since(newBlock.Block.Time)
+			metrics.CoreBlockLatency.Set(blockTimeLatency.Seconds())
+
 			balance, err := oc.zetacoreClient.GetZetaHotKeyBalance(ctx)
 			if err != nil {
 				oc.logger.Error().Err(err).Msgf("couldn't get operator balance")