From 8a3447cc97cf276a8e674ee57e6070a8743729c4 Mon Sep 17 00:00:00 2001 From: Shiming Zhang Date: Mon, 9 Dec 2024 15:43:22 +0800 Subject: [PATCH] Refactor sync --- cmd/crproxy/main.go | 28 +++-- cmd/crproxy/sync/sync.go | 162 ++++++++++++++++++++++++++ go.mod | 33 +++--- go.sum | 75 ++++++------ sync.go => sync/sync.go | 241 ++++++++++++++++++++++----------------- 5 files changed, 380 insertions(+), 159 deletions(-) create mode 100644 cmd/crproxy/sync/sync.go rename sync.go => sync/sync.go (50%) diff --git a/cmd/crproxy/main.go b/cmd/crproxy/main.go index 189abfb..270ac8d 100644 --- a/cmd/crproxy/main.go +++ b/cmd/crproxy/main.go @@ -21,9 +21,10 @@ import ( "github.com/daocloud/crproxy/cache" "github.com/daocloud/crproxy/clientset" + csync "github.com/daocloud/crproxy/cmd/crproxy/sync" "github.com/docker/distribution/registry/storage/driver/factory" "github.com/gorilla/handlers" - "github.com/spf13/pflag" + "github.com/spf13/cobra" "github.com/wzshiming/geario" "github.com/wzshiming/hostmatcher" @@ -139,7 +140,23 @@ func init() { pflag.StringVar(&tokenPrivateKeyFile, "token-private-key-file", "", "private key file") pflag.StringVar(&tokenPublicKeyFile, "token-public-key-file", "", "public key file") - pflag.Parse() + + cmd.AddCommand(csync.NewCommand()) +} + +var ( + cmd = &cobra.Command{ + Use: "crproxy", + Short: "crproxy", + Run: func(cmd *cobra.Command, args []string) { + run(cmd.Context()) + }, + } + pflag = cmd.Flags() +) + +func main() { + cmd.Execute() } func toUserAndPass(userpass []string) (map[string]clientset.Userpass, error) { @@ -165,8 +182,7 @@ func toUserAndPass(userpass []string) (map[string]clientset.Userpass, error) { return bc, nil } -func main() { - ctx := context.Background() +func run(ctx context.Context) { logger := slog.New(slog.NewJSONHandler(os.Stderr, nil)) mux := http.NewServeMux() @@ -591,10 +607,6 @@ func main() { mux.Handle("/v2/", crp) - if enableInternalAPI { - mux.HandleFunc("/internal/api/image/sync", crp.Sync) - } - if enablePprof { mux.HandleFunc("/debug/pprof/", pprof.Index) mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) diff --git a/cmd/crproxy/sync/sync.go b/cmd/crproxy/sync/sync.go new file mode 100644 index 0000000..7ba86a5 --- /dev/null +++ b/cmd/crproxy/sync/sync.go @@ -0,0 +1,162 @@ +package sync + +import ( + "bufio" + "context" + "fmt" + "log/slog" + "os" + "strings" + + "github.com/daocloud/crproxy/cache" + "github.com/daocloud/crproxy/clientset" + csync "github.com/daocloud/crproxy/sync" + "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/registry/storage/driver/factory" + "github.com/spf13/cobra" +) + +type flagpole struct { + StorageDriver string + StorageParameters map[string]string + List []string + ListFromFile string + Platform []string + MaxWarn int +} + +func NewCommand() *cobra.Command { + flags := &flagpole{ + Platform: []string{ + "linux/amd64", + "linux/arm64", + }, + MaxWarn: -1, + } + + cmd := &cobra.Command{ + Args: cobra.NoArgs, + Use: "sync", + Short: "Sync images", + RunE: func(cmd *cobra.Command, args []string) error { + return runE(cmd.Context(), flags) + }, + } + cmd.Flags().StringVar(&flags.StorageDriver, "storage-driver", flags.StorageDriver, "Storage driver") + cmd.Flags().StringToStringVar(&flags.StorageParameters, "storage-parameters", flags.StorageParameters, "Storage parameters") + cmd.Flags().StringSliceVar(&flags.List, "list", flags.List, "List") + cmd.Flags().StringVar(&flags.ListFromFile, "list-from-file", flags.ListFromFile, "List from file") + cmd.Flags().StringSliceVar(&flags.Platform, "platform", flags.Platform, "Platform") + cmd.Flags().IntVar(&flags.MaxWarn, "max-warn", flags.MaxWarn, "Max warn") + return cmd +} + +func runE(ctx context.Context, flags *flagpole) error { + + opts := []csync.Option{} + + logger := slog.New(slog.NewJSONHandler(os.Stderr, nil)) + + cacheOpts := []cache.Option{} + + parameters := map[string]interface{}{} + for k, v := range flags.StorageParameters { + parameters[k] = v + } + sd, err := factory.Create(flags.StorageDriver, parameters) + if err != nil { + return fmt.Errorf("create storage driver failed: %w", err) + } + cacheOpts = append(cacheOpts, cache.WithStorageDriver(sd)) + + cache, err := cache.NewCache(cacheOpts...) + if err != nil { + return fmt.Errorf("create cache failed: %w", err) + } + + clientOpts := []clientset.Option{ + clientset.WithLogger(logger), + clientset.WithMaxClientSizeForEachRegistry(16), + } + + client, err := clientset.NewClientset(clientOpts...) + if err != nil { + return fmt.Errorf("create clientset failed: %w", err) + } + + opts = append(opts, + csync.WithCache(cache), + csync.WithDomainAlias(map[string]string{ + "docker.io": "registry-1.docker.io", + "ollama.ai": "registry.ollama.ai", + }), + csync.WithClient(client), + csync.WithLogger(logger), + ) + + sm, err := csync.NewSyncManager(opts...) + if err != nil { + return fmt.Errorf("create sync manager failed: %w", err) + } + + warnCount := 0 + for _, item := range flags.List { + err = sm.Image(ctx, item, platformFilter(flags.Platform), func(sp csync.Progress) error { + logger.Info("Sync", "progress", sp) + return nil + }) + if err != nil { + logger.Warn("Sync failed", "error ", err) + warnCount++ + if flags.MaxWarn > -1 && warnCount >= flags.MaxWarn { + return fmt.Errorf("max warn reached") + } + } + } + + if flags.ListFromFile != "" { + f, err := os.Open(flags.ListFromFile) + if err != nil { + return fmt.Errorf("read list from file failed: %w", err) + } + defer f.Close() + reader := bufio.NewReader(f) + for { + line, err := reader.ReadString('\n') + if err != nil { + break + } + line = strings.TrimSpace(line) + if line == "" { + continue + } + err = sm.Image(ctx, line, platformFilter(flags.Platform), func(sp csync.Progress) error { + logger.Info("Sync", "progress", sp) + return nil + }) + if err != nil { + logger.Warn("Sync failed", "error ", err) + warnCount++ + if flags.MaxWarn > -1 && warnCount >= flags.MaxWarn { + return fmt.Errorf("max warn reached") + } + } + } + } + return nil +} + +func platformFilter(ps []string) func(pf manifestlist.PlatformSpec) bool { + platforms := map[string]struct{}{} + for _, p := range ps { + platforms[p] = struct{}{} + } + return func(pf manifestlist.PlatformSpec) bool { + p := fmt.Sprintf("%s/%s", pf.OS, pf.Architecture) + + if _, ok := platforms[p]; ok { + return true + } + return false + } +} diff --git a/go.mod b/go.mod index bf38b2a..f4187ab 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/gorilla/handlers v1.5.2 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.6+incompatible github.com/opencontainers/go-digest v1.0.0 - github.com/spf13/pflag v1.0.5 + github.com/spf13/cobra v1.8.1 github.com/wzshiming/cmux v0.4.2 github.com/wzshiming/geario v0.0.0-20240308093553-a996e3817533 github.com/wzshiming/hostmatcher v0.0.3 @@ -35,7 +35,8 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/aws/aws-sdk-go v1.48.10 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dnaeon/go-vcr v1.2.0 // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 // indirect @@ -43,24 +44,26 @@ require ( github.com/gofrs/uuid v4.0.0+incompatible // indirect github.com/golang-jwt/jwt/v4 v4.5.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/go-cmp v0.6.0 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/s2a-go v0.1.4 // indirect - github.com/google/uuid v1.3.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/googleapis/gax-go/v2 v2.11.0 // indirect github.com/gorilla/mux v1.8.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kr/text v0.2.0 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/opencontainers/image-spec v1.1.0-rc3 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/prometheus/client_golang v1.17.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.19.1 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/stretchr/testify v1.8.4 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/wzshiming/trie v0.3.1 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/net v0.30.0 // indirect @@ -70,7 +73,7 @@ require ( google.golang.org/api v0.126.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/grpc v1.59.0 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/grpc v1.65.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/go.sum b/go.sum index a9e5f85..0f062ea 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -41,10 +41,12 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/distribution v2.8.3+incompatible h1:RlpEXBLq/WPXYvBYMDAmBX/SnhD67qwtvW/DzKc8pAo= github.com/distribution/distribution v2.8.3+incompatible/go.mod h1:EgLm2NgWtdKgzF9NpMzUKgzmR7AMmb0VQi2B+ZzDRjc= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= @@ -94,8 +96,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -109,8 +111,8 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= @@ -122,6 +124,8 @@ github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWS github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.6+incompatible h1:/2MdLc7zHJqzV7J2uVGaoGymVobB/OHC8wmEyWRaK68= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.6+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -137,50 +141,54 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec= -github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= +github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -194,8 +202,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= @@ -309,12 +317,11 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= @@ -323,8 +330,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -336,8 +343,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/sync.go b/sync/sync.go similarity index 50% rename from sync.go rename to sync/sync.go index bdf8400..0d2fb46 100644 --- a/sync.go +++ b/sync/sync.go @@ -2,17 +2,17 @@ package crproxy import ( "context" - "encoding/json" "fmt" - "net/http" + "log/slog" + "github.com/daocloud/crproxy/cache" + "github.com/daocloud/crproxy/clientset" "github.com/distribution/reference" "github.com/docker/distribution" "github.com/docker/distribution/manifest/manifestlist" "github.com/docker/distribution/manifest/ocischema" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" - "github.com/docker/distribution/registry/api/errcode" "github.com/docker/distribution/registry/client" "github.com/opencontainers/go-digest" ) @@ -33,55 +33,80 @@ func newNameWithoutDomain(named reference.Named, name string) reference.Named { } } -func (c *CRProxy) Sync(rw http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPut || c.cache == nil { - errcode.ServeJSON(rw, errcode.ErrorCodeUnsupported) - return - } +type Progress struct { + Digest string `json:"digest,omitempty"` + Size int64 `json:"size,omitempty"` + Status string `json:"status,omitempty"` + Platform *manifestlist.PlatformSpec `json:"platform,omitempty"` + Name string `json:"name,omitempty"` +} - query := r.URL.Query() +type SyncManager struct { + client *clientset.Clientset + cache *cache.Cache + logger *slog.Logger + domainAlias map[string]string +} - rw.Header().Set("Content-Type", "application/json") +func (c *SyncManager) getDomainAlias(host string) string { + if c.domainAlias == nil { + return host + } + h, ok := c.domainAlias[host] + if !ok { + return host + } + return h +} - images := query["image"] +type Option func(*SyncManager) - flusher, _ := rw.(http.Flusher) +func WithDomainAlias(domainAlias map[string]string) Option { + return func(c *SyncManager) { + c.domainAlias = domainAlias + } +} - encoder := json.NewEncoder(rw) - for _, image := range images { - if image == "" { - continue - } - err := c.SyncImageLayer(r.Context(), r.RemoteAddr, image, nil, func(sp SyncProgress) error { - err := encoder.Encode(sp) - if err != nil { - return err - } +func WithLogger(logger *slog.Logger) Option { + return func(c *SyncManager) { + c.logger = logger + } +} - if flusher != nil { - flusher.Flush() - } +func WithCache(cache *cache.Cache) Option { + return func(c *SyncManager) { + c.cache = cache + } +} - return nil - }) - if err != nil { - c.errorResponse(rw, r, err) - return - } +func WithClient(client *clientset.Clientset) Option { + return func(c *SyncManager) { + c.client = client } } -type SyncProgress struct { - Digest string `json:"digest,omitempty"` - Size int64 `json:"size,omitempty"` - Status string `json:"status,omitempty"` - Platform *manifestlist.PlatformSpec `json:"platform,omitempty"` +func NewSyncManager(opts ...Option) (*SyncManager, error) { + c := &SyncManager{ + logger: slog.Default(), + } + for _, opt := range opts { + opt(c) + } + if c.client == nil { + return nil, fmt.Errorf("client is required") + } + + if c.cache == nil { + return nil, fmt.Errorf("cache is required") + } + + return c, nil } -func (c *CRProxy) SyncImageLayer(ctx context.Context, ip, image string, filter func(pf manifestlist.PlatformSpec) bool, cb func(sp SyncProgress) error) error { +func (c *SyncManager) Image(ctx context.Context, image string, filter func(pf manifestlist.PlatformSpec) bool, cb func(sp Progress) error) error { ref, err := reference.Parse(image) if err != nil { - return err + return fmt.Errorf("parse image failed: %w", err) } named, ok := ref.(reference.Named) @@ -91,66 +116,42 @@ func (c *CRProxy) SyncImageLayer(ctx context.Context, ip, image string, filter f host := reference.Domain(named) - var name reference.Named - - info := &ImageInfo{ - Host: host, - Name: reference.Path(named), - } - if c.modify != nil { - info = c.modify(info) - name = newNameWithoutDomain(named, info.Name) - } else { - name = newNameWithoutDomain(named, info.Name) - } - - if c.blockFunc != nil { - blockMessage, block := c.block(&BlockInfo{ - IP: ip, - Host: info.Host, - Name: info.Name, - }) - if block { - if blockMessage != "" { - return errcode.ErrorCodeDenied.WithMessage(blockMessage) - } else { - return errcode.ErrorCodeDenied - } - } - } + path := reference.Path(named) host = c.getDomainAlias(host) - info.Host = host + + name := newNameWithoutDomain(named, path) err = c.client.Ping(host) if err != nil { - return err + return fmt.Errorf("ping registry failed: %w", err) } - cli := c.client.GetClientset(host, name.Name()) + cli := c.client.GetClientset(host, path) repo, err := client.NewRepository(name, c.client.HostURL(host), cli.Transport) if err != nil { - return err + return fmt.Errorf("create repository failed: %w", err) } ms, err := repo.Manifests(ctx) if err != nil { - return err + return fmt.Errorf("get manifests failed: %w", err) } bs := repo.Blobs(ctx) uniq := map[digest.Digest]struct{}{} - blobCallback := func(dgst digest.Digest, size int64, pf *manifestlist.PlatformSpec) error { + blobCallback := func(dgst digest.Digest, size int64, pf *manifestlist.PlatformSpec, name string) error { _, ok := uniq[dgst] if ok { if cb != nil { - err = cb(SyncProgress{ + err = cb(Progress{ Digest: dgst.String(), Size: size, Status: "SKIP", Platform: pf, + Name: name, }) if err != nil { return err @@ -169,11 +170,12 @@ func (c *CRProxy) SyncImageLayer(ctx context.Context, ip, image string, filter f c.logger.Info("skip blob", "digest", dgst) if cb != nil { - err = cb(SyncProgress{ + err = cb(Progress{ Digest: blob, Size: size, Status: "SKIP", Platform: pf, + Name: name, }) if err != nil { return err @@ -185,11 +187,12 @@ func (c *CRProxy) SyncImageLayer(ctx context.Context, ip, image string, filter f } else { c.logger.Info("skip blob", "digest", dgst) if cb != nil { - err = cb(SyncProgress{ + err = cb(Progress{ Digest: dgst.String(), Size: -1, Status: "SKIP", Platform: pf, + Name: name, }) if err != nil { return err @@ -201,23 +204,24 @@ func (c *CRProxy) SyncImageLayer(ctx context.Context, ip, image string, filter f f, err := bs.Open(ctx, dgst) if err != nil { - return err + return fmt.Errorf("open blob failed: %w", err) } defer f.Close() n, err := c.cache.PutBlob(ctx, blob, f) if err != nil { - return err + return fmt.Errorf("put blob failed: %w", err) } c.logger.Info("sync blob", "digest", dgst) if cb != nil { - err = cb(SyncProgress{ + err = cb(Progress{ Digest: dgst.String(), Size: n, Status: "CACHE", Platform: pf, + Name: name, }) if err != nil { return err @@ -229,26 +233,47 @@ func (c *CRProxy) SyncImageLayer(ctx context.Context, ip, image string, filter f manifestCallback := func(tagOrHash string, m distribution.Manifest) error { _, playload, err := m.Payload() if err != nil { - return err + return fmt.Errorf("get manifest payload failed: %w", err) } - _, _, err = c.cache.PutManifestContent(ctx, info.Host, info.Name, tagOrHash, playload) + _, _, err = c.cache.PutManifestContent(ctx, host, path, tagOrHash, playload) if err != nil { - return err + return fmt.Errorf("put manifest content failed: %w", err) } return nil } - err = getLayerFromManifestList(ctx, ms, ref, filter, blobCallback, manifestCallback) - if err != nil { - return err + switch ref.(type) { + case reference.Digested, reference.Tagged: + err = c.syncLayerFromManifestList(ctx, ms, ref, filter, blobCallback, manifestCallback, host+"/"+ref.String()) + if err != nil { + return fmt.Errorf("sync layer from manifest list failed: %w", err) + } + default: + t := repo.Tags(ctx) + tags, err := t.All(ctx) + if err != nil { + return fmt.Errorf("get tags failed: %w", err) + } + + for _, tag := range tags { + t, err := reference.WithTag(name, tag) + if err != nil { + return fmt.Errorf("with tag failed: %w", err) + } + err = c.syncLayerFromManifestList(ctx, ms, t, filter, blobCallback, manifestCallback, host+"/"+t.String()) + if err != nil { + return fmt.Errorf("sync layer from manifest list failed: %w", err) + } + } } + return nil } -func getLayerFromManifestList(ctx context.Context, ms distribution.ManifestService, ref reference.Reference, filter func(pf manifestlist.PlatformSpec) bool, - digestCallback func(dgst digest.Digest, size int64, pf *manifestlist.PlatformSpec) error, - manifestCallback func(tagOrHash string, m distribution.Manifest) error) error { +func (c *SyncManager) syncLayerFromManifestList(ctx context.Context, ms distribution.ManifestService, ref reference.Reference, filter func(pf manifestlist.PlatformSpec) bool, + digestCallback func(dgst digest.Digest, size int64, pf *manifestlist.PlatformSpec, name string) error, + manifestCallback func(tagOrHash string, m distribution.Manifest) error, name string) error { var ( m distribution.Manifest err error @@ -257,21 +282,21 @@ func getLayerFromManifestList(ctx context.Context, ms distribution.ManifestServi case reference.Digested: m, err = ms.Get(ctx, r.Digest()) if err != nil { - return err + return fmt.Errorf("get manifest digest failed: %w", err) } err = manifestCallback(r.Digest().String(), m) if err != nil { - return err + return fmt.Errorf("manifest callback failed: %w", err) } case reference.Tagged: tag := r.Tag() - m, err = ms.Get(ctx, "", distribution.WithTag(r.Tag())) + m, err = ms.Get(ctx, "", distribution.WithTag(tag)) if err != nil { - return err + return fmt.Errorf("get manifest tag failed: %w", err) } err = manifestCallback(tag, m) if err != nil { - return err + return fmt.Errorf("manifest callback failed: %w", err) } default: return fmt.Errorf("%s no reference to any source", ref) @@ -286,54 +311,66 @@ func getLayerFromManifestList(ctx context.Context, ms distribution.ManifestServi m0, err := ms.Get(ctx, mfest.Digest) if err != nil { - return err + return fmt.Errorf("get manifest failed: %w", err) } err = manifestCallback(mfest.Digest.String(), m0) if err != nil { - return err + return fmt.Errorf("manifest callback failed: %w", err) } - err = getLayerFromManifest(m0, func(dgst digest.Digest, size int64) error { - return digestCallback(dgst, size, &mfest.Platform) + err = c.syncLayerFromManifest(m0, func(dgst digest.Digest, size int64) error { + return digestCallback(dgst, size, &mfest.Platform, name) }) if err != nil { - return err + return fmt.Errorf("get layer from manifest failed: %w", err) } } return nil default: - return getLayerFromManifest(m, func(dgst digest.Digest, size int64) error { - return digestCallback(dgst, size, nil) + return c.syncLayerFromManifest(m, func(dgst digest.Digest, size int64) error { + return digestCallback(dgst, size, nil, name) }) } } -func getLayerFromManifest(m distribution.Manifest, cb func(dgst digest.Digest, size int64) error) error { +func (c *SyncManager) syncLayerFromManifest(m distribution.Manifest, cb func(dgst digest.Digest, size int64) error) error { switch m := m.(type) { case *ocischema.DeserializedManifest: + if m.Config.Size != 0 { + err := cb(m.Config.Digest, m.Config.Size) + if err != nil { + return fmt.Errorf("digest callback failed: %w", err) + } + } for _, layer := range m.Layers { if layer.Size == 0 { continue } err := cb(layer.Digest, layer.Size) if err != nil { - return err + return fmt.Errorf("digest callback failed: %w", err) } } case *schema2.DeserializedManifest: + if m.Config.Size != 0 { + err := cb(m.Config.Digest, m.Config.Size) + if err != nil { + return fmt.Errorf("digest callback failed: %w", err) + } + } for _, layer := range m.Layers { if layer.Size == 0 { continue } err := cb(layer.Digest, layer.Size) if err != nil { - return err + return fmt.Errorf("digest callback failed: %w", err) } } case *schema1.SignedManifest: for _, layer := range m.FSLayers { err := cb(layer.BlobSum, -1) if err != nil { - return err + return fmt.Errorf("digest callback failed: %w", err) } } }