check-deals: add whitelist of providers

cmd-check-deals.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/anjor/carlet"
@@ -16,9 +17,39 @@ import (
 	"k8s.io/klog/v2"
 )
 
+type commaSeparatedStringSliceFlag struct {
+	slice []string
+}
+
+func (f *commaSeparatedStringSliceFlag) String() string {
+	return fmt.Sprintf("%v", f.slice)
+}
+
+func (f *commaSeparatedStringSliceFlag) Set(value string) error {
+	// split by ",":
+	split := strings.Split(value, ",")
+	for _, item := range split {
+		// trim spaces:
+		item = strings.TrimSpace(item)
+		f.slice = append(f.slice, item)
+	}
+	return nil
+}
+
+// Has
+func (f *commaSeparatedStringSliceFlag) Has(value string) bool {
+	for _, item := range f.slice {
+		if item == value {
+			return true
+		}
+	}
+	return false
+}
+
 func newCmd_check_deals() *cli.Command {
 	var includePatterns cli.StringSlice
 	var excludePatterns cli.StringSlice
+	var providerWhitelist commaSeparatedStringSliceFlag
 	return &cli.Command{
 		Name:        "check-deals",
 		Description: "Validate remote split car retrieval for the given config files",
@@ -39,6 +70,12 @@ func newCmd_check_deals() *cli.Command {
 				Value:       cli.NewStringSlice(".git"),
 				Destination: &excludePatterns,
 			},
+			// provider-whitelist
+			&cli.GenericFlag{
+				Name:  "provider-whitelist",
+				Usage: "Whitelist of providers to check",
+				Value: &providerWhitelist,
+			},
 		},
 		Action: func(c *cli.Context) error {
 			src := c.Args().Slice()
@@ -69,6 +106,13 @@ func newCmd_check_deals() *cli.Command {
 			klog.Infof("Loaded %d epoch configs (NO VALIDATION)", len(configs))
 			klog.Info("Will check remote storage pieces for each epoch config")
 
+			// Check provider whitelist:
+			if len(providerWhitelist.slice) > 0 {
+				klog.Infof("Provider whitelist: %v", providerWhitelist.slice)
+			} else {
+				klog.Infof("Provider whitelist: <empty>")
+			}
+
 			// Check deals:
 			for _, config := range configs {
 				epoch := *config.Epoch
@@ -100,6 +144,7 @@ func newCmd_check_deals() *cli.Command {
 						epoch,
 						metadata,
 						dealRegistry,
+						providerWhitelist,
 						&dm,
 					)
 					if err != nil {
@@ -127,6 +172,7 @@ func checkAllPieces(
 	epoch uint64,
 	meta *splitcarfetcher.Metadata,
 	dealRegistry *splitcarfetcher.DealRegistry,
+	providerWhitelist commaSeparatedStringSliceFlag,
 	dm *splitcarfetcher.MinerInfoCache,
 ) error {
 	errs := make([]error, 0)
@@ -145,6 +191,12 @@ func checkAllPieces(
 				piece.CommP,
 				minerID,
 			)
+			if len(providerWhitelist.slice) > 0 {
+				if !providerWhitelist.Has(minerID.String()) {
+					klog.Infof("skipping piece %d/%d with CID %s, because miner %s is not in the whitelist", pieceIndex+1, numPieces, piece.CommP, minerID)
+					return nil
+				}
+			}
 			minerInfo, err := dm.GetProviderInfo(ctx, minerID)
 			if err != nil {
 				return fmt.Errorf("failed to get miner info for miner %s, for piece %s: %w", minerID, piece.CommP, err)

split-car-fetcher/deals.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/ipfs/go-cid"
+	"k8s.io/klog/v2"
 )
 
 // provider,deal_uuid,file_name,url,commp_piece_cid,file_size,padded_size,payload_cid
@@ -99,6 +100,11 @@ func DealsFromCSV(path string) (*DealRegistry, error) {
 			PayloadCID:     record[7],
 		}
 
+		// if the same piece CID is associated with multiple deals, the last one wins, but print a warning
+		if _, ok := registry.pieceToDeal[deal.CommpPieceCID]; ok {
+			klog.Warningf("WARNING: piece CID %s is associated with multiple deals, the last one wins\n", deal.CommpPieceCID)
+		}
+
 		registry.pieceToDeal[deal.CommpPieceCID] = deal
 	}