From 1ddbb2146e7e920356d1c216d4757075187030db Mon Sep 17 00:00:00 2001
From: Christopher Li <christopher.li@databricks.com>
Date: Sun, 17 Mar 2024 21:54:31 -0400
Subject: [PATCH] flush thanos retry state once no connection issue

---
 pkg/receive/handler.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/pkg/receive/handler.go b/pkg/receive/handler.go
index c0c0fea21f..bca22a6dc3 100644
--- a/pkg/receive/handler.go
+++ b/pkg/receive/handler.go
@@ -812,6 +812,12 @@ func (h *Handler) fanoutForward(pctx context.Context, tenant string, wreqs map[e
 							h.peerStates[writeTarget.endpoint] = &retryState{nextAllowed: time.Now().Add(h.expBackoff.ForAttempt(0))}
 						}
 						h.mtx.Unlock()
+						// For other error codes, we want to flush out the endpoint since it means connection is working.
+					} else {
+						h.mtx.Lock()
+						delete(h.peerStates, writeTarget.endpoint)
+						level.Info(tLogger).Log("msg", "flushing retry endpoint", "endpoint", writeTarget.endpoint)
+						h.mtx.Unlock()
 					}
 				}
 				werr := errors.Wrapf(err, "forwarding request to endpoint %v", writeTarget.endpoint)