Make activate-client command retryable

go.mod

@@ -6,6 +6,7 @@ require (
 	cosmossdk.io/core v0.11.0
 	cosmossdk.io/errors v1.0.1
 	cosmossdk.io/store v1.0.2
+	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/cosmos/cosmos-sdk v0.50.5
 	github.com/cosmos/gogoproto v1.4.11
 	github.com/cosmos/ibc-go/v8 v8.2.0
@@ -39,7 +40,6 @@ require (
 	github.com/99designs/keyring v1.2.1 // indirect
 	github.com/DataDog/datadog-go v3.2.0+incompatible // indirect
 	github.com/DataDog/zstd v1.5.5 // indirect
-	github.com/avast/retry-go v3.0.0+incompatible // indirect
 	github.com/aws/aws-sdk-go v1.44.224 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect

relay/cmd.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"time"
 
 	"github.com/cosmos/cosmos-sdk/client/flags"
 	"github.com/ethereum/go-ethereum/common"
@@ -16,6 +17,8 @@ import (
 const (
 	flagSrc                     = "src"
 	flagHeight                  = "height"
+	flagRetryInterval           = "retry_interval"
+	flagRetryMaxAttempts        = "retry_max_attempts"
 	flagELCClientID             = "elc_client_id"
 	flagNewOperators            = "new_operators"
 	flagNonce                   = "nonce"
@@ -99,10 +102,10 @@ func activateClientCmd(ctx *config.Context) *cobra.Command {
 				pathEnd = path.Dst
 				target, counterparty = c[dst], c[src]
 			}
-			return activateClient(pathEnd, target, counterparty)
+			return activateClient(pathEnd, target, counterparty, viper.GetDuration(flagRetryInterval), viper.GetUint(flagRetryMaxAttempts))
 		},
 	}
-	return srcFlag(cmd)
+	return retryMaxAttemptsFlag(retryIntervalFlag(srcFlag(cmd)))
 }
 
 func createELCCmd(ctx *config.Context) *cobra.Command {
@@ -359,6 +362,22 @@ func heightFlag(cmd *cobra.Command) *cobra.Command {
 	return cmd
 }
 
+func retryIntervalFlag(cmd *cobra.Command) *cobra.Command {
+	cmd.Flags().DurationP(flagRetryInterval, "", time.Second, "a retry interval duration")
+	if err := viper.BindPFlag(flagRetryInterval, cmd.Flags().Lookup(flagRetryInterval)); err != nil {
+		panic(err)
+	}
+	return cmd
+}
+
+func retryMaxAttemptsFlag(cmd *cobra.Command) *cobra.Command {
+	cmd.Flags().IntP(flagRetryMaxAttempts, "", 0, "a maximum number of retry attempts")
+	if err := viper.BindPFlag(flagRetryMaxAttempts, cmd.Flags().Lookup(flagRetryMaxAttempts)); err != nil {
+		panic(err)
+	}
+	return cmd
+}
+
 func elcClientIDFlag(cmd *cobra.Command) *cobra.Command {
 	cmd.Flags().StringP(flagELCClientID, "", "", "a client ID of the ELC client")
 	if err := viper.BindPFlag(flagELCClientID, cmd.Flags().Lookup(flagELCClientID)); err != nil {

relay/lcp.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/avast/retry-go"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	clienttypes "github.com/cosmos/ibc-go/v8/modules/core/02-client/types"
 	ibcexported "github.com/cosmos/ibc-go/v8/modules/core/exported"
@@ -290,10 +291,11 @@ func (pr *Prover) updateELC(elcClientID string, includeState bool) ([]*elc.MsgUp
 		return nil, err
 	}
 	if clientState.GetLatestHeight().GTE(latestHeader.GetHeight()) {
+		pr.getLogger().Info("no need to update the client", "elc_client_id", elcClientID, "client_state.latest_height", clientState.GetLatestHeight(), "latest", latestHeader.GetHeight())
 		return nil, nil
 	}
 
-	pr.getLogger().Info("try to setup headers", "elc_client_id", elcClientID, "current", clientState.GetLatestHeight(), "latest", latestHeader.GetHeight())
+	pr.getLogger().Info("try to setup headers", "elc_client_id", elcClientID, "client_state.latest_height", clientState.GetLatestHeight(), "latest", latestHeader.GetHeight())
 
 	// 2. query the header from the upstream chain
 
@@ -636,15 +638,26 @@ func (pr *Prover) createELC(elcClientID string, height ibcexported.Height) (*elc
 	})
 }
 
-func activateClient(pathEnd *core.PathEnd, src, dst *core.ProvableChain) error {
+func activateClient(pathEnd *core.PathEnd, src, dst *core.ProvableChain, retryInterval time.Duration, retryMaxAttempts uint) error {
 	srcProver := src.Prover.(*Prover)
 	if err := srcProver.UpdateEKIfNeeded(context.TODO(), dst); err != nil {
 		return err
 	}
 
-	// 1. LCP synchronises with the latest header of the upstream chain
-	updates, err := srcProver.updateELC(srcProver.config.ElcClientId, true)
-	if err != nil {
+	srcProver.getLogger().Info("try to activate the LCP client", "elc_client_id", srcProver.config.ElcClientId)
+
+	// 1. LCP client synchronises with the latest header of the upstream chain
+	var updates []*elc.MsgUpdateClientResponse
+	if err := retry.Do(func() error {
+		var err error
+		updates, err = srcProver.updateELC(srcProver.config.ElcClientId, true)
+		if err != nil {
+			return err
+		} else if len(updates) == 0 {
+			return fmt.Errorf("no available updates: elc_client_id=%v", srcProver.config.ElcClientId)
+		}
+		return nil
+	}, retry.Attempts(retryMaxAttempts+1), retry.Delay(retryInterval)); err != nil {
 		return err
 	}