From 4ef1e5fbb182cff8ca3e48d23920aa125ed35981 Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Sat, 12 Sep 2020 20:10:08 -0400 Subject: [PATCH 1/5] Add metrics to track the number of vtx/blocks put into MultiPut msgs --- network/network.go | 4 ++++ snow/engine/avalanche/metrics.go | 17 +++++++++++++++++ snow/engine/avalanche/transitive.go | 1 + snow/engine/snowman/metrics.go | 17 +++++++++++++++++ snow/engine/snowman/transitive.go | 1 + 5 files changed, 40 insertions(+) diff --git a/network/network.go b/network/network.go index a4867ff51081..e72d4ea4fb50 100644 --- a/network/network.go +++ b/network/network.go @@ -288,6 +288,10 @@ func (n *network) GetAcceptedFrontier(validatorIDs ids.ShortSet, chainID ids.ID, sent = peer.send(msg) } if !sent { + n.log.Debug("failed to send GetAcceptedFrontier(%s, %s, %d)", + vID, + chainID, + requestID) n.executor.Add(func() { n.router.GetAcceptedFrontierFailed(vID, chainID, requestID) }) n.getAcceptedFrontier.numFailed.Inc() } else { diff --git a/snow/engine/avalanche/metrics.go b/snow/engine/avalanche/metrics.go index 212474a7200c..3624bf211713 100644 --- a/snow/engine/avalanche/metrics.go +++ b/snow/engine/avalanche/metrics.go @@ -11,6 +11,7 @@ import ( type metrics struct { numVtxRequests, numPendingVts, numMissingTxs prometheus.Gauge + getAncestorsVtxs prometheus.Histogram } // Initialize implements the Engine interface @@ -30,6 +31,22 @@ func (m *metrics) Initialize(namespace string, registerer prometheus.Registerer) Name: "missing_txs", Help: "Number of missing transactions", }) + m.getAncestorsVtxs = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespace, + Name: "get_ancestors_vtxs", + Help: "The number of vertices fetched in a call to GetAncestors", + Buckets: []float64{ + 0, + 1, + 5, + 10, + 100, + 500, + 1000, + 1500, + 2000, + }, + }) errs := wrappers.Errs{} errs.Add( diff --git a/snow/engine/avalanche/transitive.go b/snow/engine/avalanche/transitive.go index b73d51ae75fa..d69f219d2957 100644 --- a/snow/engine/avalanche/transitive.go +++ b/snow/engine/avalanche/transitive.go @@ -186,6 +186,7 @@ func (t *Transitive) GetAncestors(vdr ids.ShortID, requestID uint32, vtxID ids.I } } + t.metrics.getAncestorsVtxs.Observe(float64(len(ancestorsBytes))) t.Sender.MultiPut(vdr, requestID, ancestorsBytes) return nil } diff --git a/snow/engine/snowman/metrics.go b/snow/engine/snowman/metrics.go index 5ab5fb520320..6b7485b043b9 100644 --- a/snow/engine/snowman/metrics.go +++ b/snow/engine/snowman/metrics.go @@ -11,6 +11,7 @@ import ( type metrics struct { numRequests, numBlocked prometheus.Gauge + getAncestorsBlks prometheus.Histogram } // Initialize the metrics @@ -25,6 +26,22 @@ func (m *metrics) Initialize(namespace string, registerer prometheus.Registerer) Name: "blocked", Help: "Number of blocks that are pending issuance", }) + m.getAncestorsBlks = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespace, + Name: "get_ancestors_blks", + Help: "The number of blocks fetched in a call to GetAncestors", + Buckets: []float64{ + 0, + 1, + 5, + 10, + 100, + 500, + 1000, + 1500, + 2000, + }, + }) errs := wrappers.Errs{} errs.Add( diff --git a/snow/engine/snowman/transitive.go b/snow/engine/snowman/transitive.go index 2093ed6ef54f..38839132dd24 100644 --- a/snow/engine/snowman/transitive.go +++ b/snow/engine/snowman/transitive.go @@ -179,6 +179,7 @@ func (t *Transitive) GetAncestors(vdr ids.ShortID, requestID uint32, blkID ids.I } } + t.metrics.getAncestorsBlks.Observe(float64(len(ancestorsBytes))) t.Sender.MultiPut(vdr, requestID, ancestorsBytes) return nil } From ec154d9824013f3755fa0657eb9d08f2e77d0486 Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Sat, 12 Sep 2020 20:27:30 -0400 Subject: [PATCH 2/5] Change block to container --- snow/networking/router/handler.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/snow/networking/router/handler.go b/snow/networking/router/handler.go index 90905cefa0ce..9f84d99ac687 100644 --- a/snow/networking/router/handler.go +++ b/snow/networking/router/handler.go @@ -394,26 +394,26 @@ func (h *Handler) GetFailed(validatorID ids.ShortID, requestID uint32) { } // PushQuery passes a PushQuery message received from the network to the consensus engine. -func (h *Handler) PushQuery(validatorID ids.ShortID, requestID uint32, deadline time.Time, blockID ids.ID, block []byte) bool { +func (h *Handler) PushQuery(validatorID ids.ShortID, requestID uint32, deadline time.Time, containerID ids.ID, container []byte) bool { return h.serviceQueue.PushMessage(message{ messageType: pushQueryMsg, validatorID: validatorID, requestID: requestID, deadline: deadline, - containerID: blockID, - container: block, + containerID: containerID, + container: container, received: h.clock.Time(), }) } // PullQuery passes a PullQuery message received from the network to the consensus engine. -func (h *Handler) PullQuery(validatorID ids.ShortID, requestID uint32, deadline time.Time, blockID ids.ID) bool { +func (h *Handler) PullQuery(validatorID ids.ShortID, requestID uint32, deadline time.Time, containerID ids.ID) bool { return h.serviceQueue.PushMessage(message{ messageType: pullQueryMsg, validatorID: validatorID, requestID: requestID, deadline: deadline, - containerID: blockID, + containerID: containerID, received: h.clock.Time(), }) } From aab065b5b97837e8864160fd3144b336131e214a Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Sat, 12 Sep 2020 20:27:51 -0400 Subject: [PATCH 3/5] Improve message string function --- snow/networking/router/message.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/snow/networking/router/message.go b/snow/networking/router/message.go index 12b613339700..8d6a41bf0e71 100644 --- a/snow/networking/router/message.go +++ b/snow/networking/router/message.go @@ -60,9 +60,26 @@ func (m message) String() string { sb.WriteString(fmt.Sprintf("\n messageType: %s", m.messageType)) sb.WriteString(fmt.Sprintf("\n validatorID: %s", m.validatorID)) sb.WriteString(fmt.Sprintf("\n requestID: %d", m.requestID)) - sb.WriteString(fmt.Sprintf("\n containerID: %s", m.containerID)) - sb.WriteString(fmt.Sprintf("\n containerIDs: %s", m.containerIDs)) - if m.messageType == notifyMsg { + switch m.messageType { + case getAcceptedMsg: + sb.WriteString(fmt.Sprintf("\n containerIDs: %s", m.containerIDs)) + case acceptedMsg: + sb.WriteString(fmt.Sprintf("\n containerIDs: %s", m.containerIDs)) + case getMsg: + sb.WriteString(fmt.Sprintf("\n containerID: %s", m.containerID)) + case getAncestorsMsg: + sb.WriteString(fmt.Sprintf("\n containerID: %s", m.containerID)) + case putMsg: + sb.WriteString(fmt.Sprintf("\n containerID: %s", m.containerID)) + case multiPutMsg: + sb.WriteString(fmt.Sprintf("\n numContainers: %d", len(m.containers))) + case pushQueryMsg: + sb.WriteString(fmt.Sprintf("\n containerID: %s", m.containerID)) + case pullQueryMsg: + sb.WriteString(fmt.Sprintf("\n containerID: %s", m.containerID)) + case chitsMsg: + sb.WriteString(fmt.Sprintf("\n containerIDs: %s", m.containerIDs)) + case notifyMsg: sb.WriteString(fmt.Sprintf("\n notification: %s", m.notification)) } if !m.deadline.IsZero() { From 85b37bdc439bd0b64509e72f35afa5c72bb3c499 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Sat, 12 Sep 2020 21:03:16 -0400 Subject: [PATCH 4/5] cleaned up switch case --- snow/networking/router/message.go | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/snow/networking/router/message.go b/snow/networking/router/message.go index 8d6a41bf0e71..4f55fa22eae8 100644 --- a/snow/networking/router/message.go +++ b/snow/networking/router/message.go @@ -61,24 +61,12 @@ func (m message) String() string { sb.WriteString(fmt.Sprintf("\n validatorID: %s", m.validatorID)) sb.WriteString(fmt.Sprintf("\n requestID: %d", m.requestID)) switch m.messageType { - case getAcceptedMsg: - sb.WriteString(fmt.Sprintf("\n containerIDs: %s", m.containerIDs)) - case acceptedMsg: + case getAcceptedMsg, acceptedMsg, chitsMsg: sb.WriteString(fmt.Sprintf("\n containerIDs: %s", m.containerIDs)) - case getMsg: - sb.WriteString(fmt.Sprintf("\n containerID: %s", m.containerID)) - case getAncestorsMsg: - sb.WriteString(fmt.Sprintf("\n containerID: %s", m.containerID)) - case putMsg: + case getMsg, getAncestorsMsg, putMsg, pushQueryMsg, pullQueryMsg: sb.WriteString(fmt.Sprintf("\n containerID: %s", m.containerID)) case multiPutMsg: sb.WriteString(fmt.Sprintf("\n numContainers: %d", len(m.containers))) - case pushQueryMsg: - sb.WriteString(fmt.Sprintf("\n containerID: %s", m.containerID)) - case pullQueryMsg: - sb.WriteString(fmt.Sprintf("\n containerID: %s", m.containerID)) - case chitsMsg: - sb.WriteString(fmt.Sprintf("\n containerIDs: %s", m.containerIDs)) case notifyMsg: sb.WriteString(fmt.Sprintf("\n notification: %s", m.notification)) } From c25dc794cf15feac39499b70c71990af6fe48d26 Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Sat, 12 Sep 2020 22:02:18 -0400 Subject: [PATCH 5/5] Register new metrics --- snow/engine/avalanche/metrics.go | 1 + snow/engine/snowman/metrics.go | 1 + 2 files changed, 2 insertions(+) diff --git a/snow/engine/avalanche/metrics.go b/snow/engine/avalanche/metrics.go index 3624bf211713..a0bda312a015 100644 --- a/snow/engine/avalanche/metrics.go +++ b/snow/engine/avalanche/metrics.go @@ -53,6 +53,7 @@ func (m *metrics) Initialize(namespace string, registerer prometheus.Registerer) registerer.Register(m.numVtxRequests), registerer.Register(m.numPendingVts), registerer.Register(m.numMissingTxs), + registerer.Register(m.getAncestorsVtxs), ) return errs.Err } diff --git a/snow/engine/snowman/metrics.go b/snow/engine/snowman/metrics.go index 6b7485b043b9..77fb9af1385f 100644 --- a/snow/engine/snowman/metrics.go +++ b/snow/engine/snowman/metrics.go @@ -47,6 +47,7 @@ func (m *metrics) Initialize(namespace string, registerer prometheus.Registerer) errs.Add( registerer.Register(m.numRequests), registerer.Register(m.numBlocked), + registerer.Register(m.getAncestorsBlks), ) return errs.Err }