add network wide metrics collection (#1606)

* add network wide metrics collection

* lint and test fix

---------

Co-authored-by: Jacob Gadikian <[email protected]>

app/app.go

@@ -95,6 +95,8 @@ type Quicksilver struct {
 	configurator module.Configurator
 
 	tpsCounter *tpsCounter
+
+	metricsURL string
 }
 
 // NewQuicksilver returns a reference to a new initialized Quicksilver application.
@@ -110,6 +112,7 @@ func NewQuicksilver(
 	appOpts servertypes.AppOptions,
 	mock bool,
 	enableSupplyEndpoint bool,
+	metricsURL string,
 	baseAppOptions ...func(*baseapp.BaseApp),
 ) *Quicksilver {
 	appCodec := encodingConfig.Marshaler
@@ -134,6 +137,7 @@ func NewQuicksilver(
 		appCodec:          appCodec,
 		interfaceRegistry: interfaceRegistry,
 		invCheckPeriod:    invCheckPeriod,
+		metricsURL:        metricsURL,
 	}
 
 	app.AppKeepers = keepers.NewAppKeepers(
@@ -237,6 +241,9 @@ func (app *Quicksilver) BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock
 
 // EndBlocker updates every end block.
 func (app *Quicksilver) EndBlocker(ctx sdk.Context, req abci.RequestEndBlock) abci.ResponseEndBlock {
+	if app.metricsURL != "" {
+		go app.ShipMetrics(ctx)
+	}
 	return app.mm.EndBlock(ctx, req)
 }
 

app/app_test.go

@@ -54,6 +54,7 @@ func TestQuicksilverExport(t *testing.T) {
 		app.EmptyAppOptions{},
 		false,
 		false,
+		"",
 	)
 
 	genesisState := app.NewDefaultGenesisState()
@@ -84,6 +85,7 @@ func TestQuicksilverExport(t *testing.T) {
 		app.EmptyAppOptions{},
 		false,
 		false,
+		"",
 	)
 	_, err = app2.ExportAppStateAndValidators(false, []string{})
 	require.NoError(t, err, "ExportAppStateAndValidators should not have an error")

app/config.go

@@ -57,6 +57,7 @@ func NewAppConstructor(encCfg EncodingConfig) network.AppConstructor {
 			EmptyAppOptions{},
 			false,
 			false,
+			"",
 			baseapp.SetPruning(purningtypes.NewPruningOptionsFromString(val.AppConfig.Pruning)),
 			// baseapp.SetMinGasPrices(val.AppConfig.MinGasPrices),
 		)

app/metrics.go

@@ -0,0 +1,78 @@
+package app
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/expfmt"
+	"github.com/tendermint/tendermint/abci/types"
+
+	"github.com/cosmos/cosmos-sdk/client/grpc/tmservice"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+)
+
+var (
+	nodeID      string
+	nodeIDMutex sync.Mutex
+)
+
+func (app *Quicksilver) InitNodeID() (string, error) {
+	nodeIDMutex.Lock()
+	defer nodeIDMutex.Unlock()
+	if nodeID == "" {
+		query := tmservice.GetNodeInfoRequest{}
+		queryRes := app.Query(types.RequestQuery{Data: app.appCodec.MustMarshal(&query), Path: "/cosmos.base.tendermint.v1beta1.Service/GetNodeInfo", Height: 0, Prove: false})
+		nodeInfo := queryRes.GetValue()
+		result := tmservice.GetNodeInfoResponse{}
+		err := app.appCodec.Unmarshal(nodeInfo, &result)
+		if err != nil {
+			return "", err
+		}
+		nodeID = result.DefaultNodeInfo.DefaultNodeID
+	}
+	return nodeID, nil
+}
+
+func (app *Quicksilver) ShipMetrics(ctx sdk.Context) {
+	metricsFamilies, err := prometheus.DefaultGatherer.Gather()
+	if err != nil {
+		ctx.Logger().Error("Error gathering metrics", "error", err)
+	}
+
+	buf := &bytes.Buffer{}
+	defer buf.Reset()
+
+	e := expfmt.NewEncoder(buf, expfmt.FmtText)
+	for _, mf := range metricsFamilies {
+		if err := e.Encode(mf); err != nil {
+			ctx.Logger().Error("Error encoding metrics", "error", err)
+			return
+		}
+	}
+
+	nodeID, err := app.InitNodeID()
+	if err != nil {
+		ctx.Logger().Error("Error getting node ID", "error", err)
+		return
+	}
+	c, cancel := context.WithTimeout(ctx.Context(), time.Second*10)
+	defer cancel()
+	req, err := http.NewRequestWithContext(c, "POST", fmt.Sprintf("%s/chain_id/%s/node_id/%s", app.metricsURL, ctx.ChainID(), nodeID), bytes.NewReader(buf.Bytes()))
+	if err != nil {
+		ctx.Logger().Error("Error creating metrics request", "error", err)
+		return
+	}
+	req.Header.Set("Content-Type", string(expfmt.FmtText))
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		ctx.Logger().Error("Error sending metrics", "error", err)
+		return
+	}
+	defer resp.Body.Close()
+}

app/test_helpers.go

@@ -91,6 +91,7 @@ func Setup(t *testing.T, isCheckTx bool) *Quicksilver {
 		EmptyAppOptions{},
 		false,
 		false,
+		"",
 	)
 
 	genesisState := NewDefaultGenesisState()
@@ -139,6 +140,7 @@ func SetupTestingApp() (testApp ibctesting.TestingApp, genesisState map[string]j
 		EmptyAppOptions{},
 		true, // set mock state to true
 		false,
+		"",
 	)
 	return app, NewDefaultGenesisState()
 }

cmd/quicksilverd/root.go

@@ -41,6 +41,7 @@ import (
 const (
 	EnvPrefix         = "QUICK"
 	FlagSupplyEnabled = "supply.enabled"
+	FlagMetricsURL    = "metrics.url"
 )
 
 type appCreator struct {
@@ -257,6 +258,7 @@ func (ac appCreator) newApp(
 		appOpts,
 		false,
 		cast.ToBool(appOpts.Get(FlagSupplyEnabled)),
+		cast.ToString(appOpts.Get(FlagMetricsURL)),
 		baseapp.SetPruning(pruningOpts),
 		baseapp.SetMinGasPrices(cast.ToString(appOpts.Get(server.FlagMinGasPrices))),
 		baseapp.SetHaltHeight(cast.ToUint64(appOpts.Get(server.FlagHaltHeight))),
@@ -272,6 +274,7 @@ func (ac appCreator) newApp(
 func addModuleInitFlags(startCmd *cobra.Command) {
 	crisis.AddModuleInitFlags(startCmd)
 	startCmd.Flags().Bool(FlagSupplyEnabled, false, "Enable supply module endpoint")
+	startCmd.Flags().Bool(FlagMetricsURL, false, "Enable supply module endpoint")
 }
 
 func (ac appCreator) appExport(
@@ -305,6 +308,7 @@ func (ac appCreator) appExport(
 		appOpts,
 		false,
 		false,
+		"",
 	)
 
 	if height != -1 {

server/config/config.go

@@ -21,8 +21,9 @@ const (
 // from the SDK as well as the TLS configuration.
 type Config struct {
 	config.Config
-	TLS    TLSConfig    `mapstructure:"tls"`
-	Supply SupplyConfig `mapstructure:"supply"`
+	TLS     TLSConfig     `mapstructure:"tls"`
+	Supply  SupplyConfig  `mapstructure:"supply"`
+	Metrics MetricsConfig `mapstructure:"metrics"`
 }
 
 // TLSConfig defines the certificate and matching private key for the server.
@@ -38,6 +39,10 @@ type SupplyConfig struct {
 	Enabled bool `mapstructure:"enabled"`
 }
 
+type MetricsConfig struct {
+	URL string `mapstructure:"url"`
+}
+
 // AppConfig helps to override default appConfig template and configs.
 // return "", nil if no custom configuration is required for the application.
 func AppConfig(denom string) (customAppTemplate string, customAppConfig interface{}) {
@@ -62,9 +67,10 @@ func AppConfig(denom string) (customAppTemplate string, customAppConfig interfac
 	}
 
 	customAppConfig = Config{
-		Config: *srvCfg,
-		TLS:    *DefaultTLSConfig(),
-		Supply: *DefaultSupplyConfig(),
+		Config:  *srvCfg,
+		TLS:     *DefaultTLSConfig(),
+		Supply:  *DefaultSupplyConfig(),
+		Metrics: *DefaultMetricsConfig(),
 	}
 
 	customAppTemplate = config.DefaultConfigTemplate + DefaultConfigTemplate
@@ -75,9 +81,10 @@ func AppConfig(denom string) (customAppTemplate string, customAppConfig interfac
 // DefaultConfig returns server's default configuration.
 func DefaultConfig() *Config {
 	return &Config{
-		Config: *config.DefaultConfig(),
-		TLS:    *DefaultTLSConfig(),
-		Supply: *DefaultSupplyConfig(),
+		Config:  *config.DefaultConfig(),
+		TLS:     *DefaultTLSConfig(),
+		Supply:  *DefaultSupplyConfig(),
+		Metrics: *DefaultMetricsConfig(),
 	}
 }
 
@@ -88,6 +95,12 @@ func DefaultSupplyConfig() *SupplyConfig {
 	}
 }
 
+func DefaultMetricsConfig() *MetricsConfig {
+	return &MetricsConfig{
+		URL: "",
+	}
+}
+
 // DefaultTLSConfig returns the default TLS configuration.
 func DefaultTLSConfig() *TLSConfig {
 	return &TLSConfig{

server/config/template.go

@@ -14,4 +14,7 @@ key-path = "{{ .TLS.KeyPath }}"
 [supply]
 # The supply module endpoint is resource intensive, and should never be opened publicly.
 enabled = "{{ .Supply.Enabled }}"
+
+[metrics]
+url = "{{ .Metrics.URL }}"
 `

test/simulation/sim_test.go

@@ -58,6 +58,7 @@ func BenchmarkSimulation(b *testing.B) {
 		app.EmptyAppOptions{},
 		false,
 		false,
+		"",
 	)
 
 	// Run randomized simulations
@@ -117,6 +118,7 @@ func TestAppStateDeterminism(t *testing.T) {
 				app.EmptyAppOptions{},
 				false,
 				false,
+				"",
 				interBlockCacheOpt(),
 			)
 

x/interchainstaking/keeper/zones_test.go

@@ -38,6 +38,7 @@ func newQuicksilver(t *testing.T) *app.Quicksilver {
 		app.EmptyAppOptions{},
 		true,
 		false,
+		"",
 	)
 }