From 886f88dd753e09512a70cb6d6a6dd08322da736a Mon Sep 17 00:00:00 2001 From: Joe Bowman Date: Wed, 15 May 2024 10:06:42 +0100 Subject: [PATCH] add network wide metrics collection (#1606) * add network wide metrics collection * lint and test fix --------- Co-authored-by: Jacob Gadikian --- app/app.go | 7 +++ app/app_test.go | 2 + app/config.go | 1 + app/metrics.go | 78 ++++++++++++++++++++++++ app/test_helpers.go | 2 + cmd/quicksilverd/root.go | 4 ++ server/config/config.go | 29 ++++++--- server/config/template.go | 3 + test/simulation/sim_test.go | 2 + x/interchainstaking/keeper/zones_test.go | 1 + 10 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 app/metrics.go diff --git a/app/app.go b/app/app.go index 8fa8e9d9d..f8fb9a25a 100644 --- a/app/app.go +++ b/app/app.go @@ -95,6 +95,8 @@ type Quicksilver struct { configurator module.Configurator tpsCounter *tpsCounter + + metricsURL string } // NewQuicksilver returns a reference to a new initialized Quicksilver application. @@ -110,6 +112,7 @@ func NewQuicksilver( appOpts servertypes.AppOptions, mock bool, enableSupplyEndpoint bool, + metricsURL string, baseAppOptions ...func(*baseapp.BaseApp), ) *Quicksilver { appCodec := encodingConfig.Marshaler @@ -134,6 +137,7 @@ func NewQuicksilver( appCodec: appCodec, interfaceRegistry: interfaceRegistry, invCheckPeriod: invCheckPeriod, + metricsURL: metricsURL, } app.AppKeepers = keepers.NewAppKeepers( @@ -237,6 +241,9 @@ func (app *Quicksilver) BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock // EndBlocker updates every end block. func (app *Quicksilver) EndBlocker(ctx sdk.Context, req abci.RequestEndBlock) abci.ResponseEndBlock { + if app.metricsURL != "" { + go app.ShipMetrics(ctx) + } return app.mm.EndBlock(ctx, req) } diff --git a/app/app_test.go b/app/app_test.go index 70b9912ad..93997cebc 100644 --- a/app/app_test.go +++ b/app/app_test.go @@ -54,6 +54,7 @@ func TestQuicksilverExport(t *testing.T) { app.EmptyAppOptions{}, false, false, + "", ) genesisState := app.NewDefaultGenesisState() @@ -84,6 +85,7 @@ func TestQuicksilverExport(t *testing.T) { app.EmptyAppOptions{}, false, false, + "", ) _, err = app2.ExportAppStateAndValidators(false, []string{}) require.NoError(t, err, "ExportAppStateAndValidators should not have an error") diff --git a/app/config.go b/app/config.go index 7f108b2b2..f8b991865 100644 --- a/app/config.go +++ b/app/config.go @@ -57,6 +57,7 @@ func NewAppConstructor(encCfg EncodingConfig) network.AppConstructor { EmptyAppOptions{}, false, false, + "", baseapp.SetPruning(purningtypes.NewPruningOptionsFromString(val.AppConfig.Pruning)), // baseapp.SetMinGasPrices(val.AppConfig.MinGasPrices), ) diff --git a/app/metrics.go b/app/metrics.go new file mode 100644 index 000000000..b4470c42e --- /dev/null +++ b/app/metrics.go @@ -0,0 +1,78 @@ +package app + +import ( + "bytes" + "context" + "fmt" + "net/http" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/expfmt" + "github.com/tendermint/tendermint/abci/types" + + "github.com/cosmos/cosmos-sdk/client/grpc/tmservice" + sdk "github.com/cosmos/cosmos-sdk/types" +) + +var ( + nodeID string + nodeIDMutex sync.Mutex +) + +func (app *Quicksilver) InitNodeID() (string, error) { + nodeIDMutex.Lock() + defer nodeIDMutex.Unlock() + if nodeID == "" { + query := tmservice.GetNodeInfoRequest{} + queryRes := app.Query(types.RequestQuery{Data: app.appCodec.MustMarshal(&query), Path: "/cosmos.base.tendermint.v1beta1.Service/GetNodeInfo", Height: 0, Prove: false}) + nodeInfo := queryRes.GetValue() + result := tmservice.GetNodeInfoResponse{} + err := app.appCodec.Unmarshal(nodeInfo, &result) + if err != nil { + return "", err + } + nodeID = result.DefaultNodeInfo.DefaultNodeID + } + return nodeID, nil +} + +func (app *Quicksilver) ShipMetrics(ctx sdk.Context) { + metricsFamilies, err := prometheus.DefaultGatherer.Gather() + if err != nil { + ctx.Logger().Error("Error gathering metrics", "error", err) + } + + buf := &bytes.Buffer{} + defer buf.Reset() + + e := expfmt.NewEncoder(buf, expfmt.FmtText) + for _, mf := range metricsFamilies { + if err := e.Encode(mf); err != nil { + ctx.Logger().Error("Error encoding metrics", "error", err) + return + } + } + + nodeID, err := app.InitNodeID() + if err != nil { + ctx.Logger().Error("Error getting node ID", "error", err) + return + } + c, cancel := context.WithTimeout(ctx.Context(), time.Second*10) + defer cancel() + req, err := http.NewRequestWithContext(c, "POST", fmt.Sprintf("%s/chain_id/%s/node_id/%s", app.metricsURL, ctx.ChainID(), nodeID), bytes.NewReader(buf.Bytes())) + if err != nil { + ctx.Logger().Error("Error creating metrics request", "error", err) + return + } + req.Header.Set("Content-Type", string(expfmt.FmtText)) + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + ctx.Logger().Error("Error sending metrics", "error", err) + return + } + defer resp.Body.Close() +} diff --git a/app/test_helpers.go b/app/test_helpers.go index fee3174cd..1f64e468f 100644 --- a/app/test_helpers.go +++ b/app/test_helpers.go @@ -91,6 +91,7 @@ func Setup(t *testing.T, isCheckTx bool) *Quicksilver { EmptyAppOptions{}, false, false, + "", ) genesisState := NewDefaultGenesisState() @@ -139,6 +140,7 @@ func SetupTestingApp() (testApp ibctesting.TestingApp, genesisState map[string]j EmptyAppOptions{}, true, // set mock state to true false, + "", ) return app, NewDefaultGenesisState() } diff --git a/cmd/quicksilverd/root.go b/cmd/quicksilverd/root.go index b4f7e60fb..2105489ca 100644 --- a/cmd/quicksilverd/root.go +++ b/cmd/quicksilverd/root.go @@ -41,6 +41,7 @@ import ( const ( EnvPrefix = "QUICK" FlagSupplyEnabled = "supply.enabled" + FlagMetricsURL = "metrics.url" ) type appCreator struct { @@ -257,6 +258,7 @@ func (ac appCreator) newApp( appOpts, false, cast.ToBool(appOpts.Get(FlagSupplyEnabled)), + cast.ToString(appOpts.Get(FlagMetricsURL)), baseapp.SetPruning(pruningOpts), baseapp.SetMinGasPrices(cast.ToString(appOpts.Get(server.FlagMinGasPrices))), baseapp.SetHaltHeight(cast.ToUint64(appOpts.Get(server.FlagHaltHeight))), @@ -272,6 +274,7 @@ func (ac appCreator) newApp( func addModuleInitFlags(startCmd *cobra.Command) { crisis.AddModuleInitFlags(startCmd) startCmd.Flags().Bool(FlagSupplyEnabled, false, "Enable supply module endpoint") + startCmd.Flags().Bool(FlagMetricsURL, false, "Enable supply module endpoint") } func (ac appCreator) appExport( @@ -305,6 +308,7 @@ func (ac appCreator) appExport( appOpts, false, false, + "", ) if height != -1 { diff --git a/server/config/config.go b/server/config/config.go index a0bd84d6f..647bbde67 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -21,8 +21,9 @@ const ( // from the SDK as well as the TLS configuration. type Config struct { config.Config - TLS TLSConfig `mapstructure:"tls"` - Supply SupplyConfig `mapstructure:"supply"` + TLS TLSConfig `mapstructure:"tls"` + Supply SupplyConfig `mapstructure:"supply"` + Metrics MetricsConfig `mapstructure:"metrics"` } // TLSConfig defines the certificate and matching private key for the server. @@ -38,6 +39,10 @@ type SupplyConfig struct { Enabled bool `mapstructure:"enabled"` } +type MetricsConfig struct { + URL string `mapstructure:"url"` +} + // AppConfig helps to override default appConfig template and configs. // return "", nil if no custom configuration is required for the application. func AppConfig(denom string) (customAppTemplate string, customAppConfig interface{}) { @@ -62,9 +67,10 @@ func AppConfig(denom string) (customAppTemplate string, customAppConfig interfac } customAppConfig = Config{ - Config: *srvCfg, - TLS: *DefaultTLSConfig(), - Supply: *DefaultSupplyConfig(), + Config: *srvCfg, + TLS: *DefaultTLSConfig(), + Supply: *DefaultSupplyConfig(), + Metrics: *DefaultMetricsConfig(), } customAppTemplate = config.DefaultConfigTemplate + DefaultConfigTemplate @@ -75,9 +81,10 @@ func AppConfig(denom string) (customAppTemplate string, customAppConfig interfac // DefaultConfig returns server's default configuration. func DefaultConfig() *Config { return &Config{ - Config: *config.DefaultConfig(), - TLS: *DefaultTLSConfig(), - Supply: *DefaultSupplyConfig(), + Config: *config.DefaultConfig(), + TLS: *DefaultTLSConfig(), + Supply: *DefaultSupplyConfig(), + Metrics: *DefaultMetricsConfig(), } } @@ -88,6 +95,12 @@ func DefaultSupplyConfig() *SupplyConfig { } } +func DefaultMetricsConfig() *MetricsConfig { + return &MetricsConfig{ + URL: "", + } +} + // DefaultTLSConfig returns the default TLS configuration. func DefaultTLSConfig() *TLSConfig { return &TLSConfig{ diff --git a/server/config/template.go b/server/config/template.go index 54c1a75d4..4e5765339 100644 --- a/server/config/template.go +++ b/server/config/template.go @@ -14,4 +14,7 @@ key-path = "{{ .TLS.KeyPath }}" [supply] # The supply module endpoint is resource intensive, and should never be opened publicly. enabled = "{{ .Supply.Enabled }}" + +[metrics] +url = "{{ .Metrics.URL }}" ` diff --git a/test/simulation/sim_test.go b/test/simulation/sim_test.go index 36d6353aa..7d7544608 100644 --- a/test/simulation/sim_test.go +++ b/test/simulation/sim_test.go @@ -58,6 +58,7 @@ func BenchmarkSimulation(b *testing.B) { app.EmptyAppOptions{}, false, false, + "", ) // Run randomized simulations @@ -117,6 +118,7 @@ func TestAppStateDeterminism(t *testing.T) { app.EmptyAppOptions{}, false, false, + "", interBlockCacheOpt(), ) diff --git a/x/interchainstaking/keeper/zones_test.go b/x/interchainstaking/keeper/zones_test.go index a15f5032f..51691c66e 100644 --- a/x/interchainstaking/keeper/zones_test.go +++ b/x/interchainstaking/keeper/zones_test.go @@ -38,6 +38,7 @@ func newQuicksilver(t *testing.T) *app.Quicksilver { app.EmptyAppOptions{}, true, false, + "", ) }