From 48f8c4d3868d4100e5a672d15b001a7b0d5003ec Mon Sep 17 00:00:00 2001 From: Tomoya Amachi Date: Wed, 26 Feb 2020 03:14:25 +0900 Subject: [PATCH] Add ratelimit to dockerhub client (#16) * dockerhub returns uploadedAt * dockerhub only set uploadedat * add ratelimit to dockerhub * fix for test * add makefile and lint checks --- Makefile | 25 +++++++++ cmd/dockertags/main.go | 2 +- internal/log/log.go | 2 + internal/report/json.go | 6 +- internal/report/table.go | 2 + internal/report/writer.go | 1 + internal/types/error.go | 1 + internal/types/types.go | 1 + pkg/provider/dockerhub/auth.go | 2 +- pkg/provider/dockerhub/dockerhub.go | 71 ++++++++++++++---------- pkg/provider/dockerhub/dockerhub_test.go | 18 +++--- pkg/provider/dockerhub/transport.go | 10 ++-- pkg/provider/ecr/aws.go | 3 +- pkg/provider/gcr/gcr.go | 10 ++-- pkg/provider/provider.go | 6 +- pkg/registry/registry.go | 6 +- pkg/run.go | 2 +- 17 files changed, 111 insertions(+), 57 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a87509a --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +.PHONY: \ + lint \ + vet \ + fmt \ + pretest \ + test \ + +PKGS = $(shell go list ./... | grep -v /vendor/) +SRCS = $(shell git ls-files '*.go') +GO := GO111MODULE=on go +GO_OFF := GO111MODULE=off go + +lint: + $(GO_OFF) get -u golang.org/x/lint/golint + $(foreach file,$(PKGS),golint -set_exit_status $(file) || exit;) +vet: + echo $(PKGS) | xargs env $(GO) vet || exit; + +fmt: + $(foreach file,$(SRCS),gofmt -s -d $(file);) + +pretest: lint vet fmt + +test: + $(foreach file,$(PKGS),$(GO) test -v $(file) || exit;) \ No newline at end of file diff --git a/cmd/dockertags/main.go b/cmd/dockertags/main.go index 6af1640..f7c37e8 100644 --- a/cmd/dockertags/main.go +++ b/cmd/dockertags/main.go @@ -58,7 +58,7 @@ OPTIONS: }, cli.StringFlag{ Name: "authurl, auth", - Usage: "Url when fetch authentication", + Usage: "GetURL when fetch authentication", }, cli.DurationFlag{ Name: "timeout, t", diff --git a/internal/log/log.go b/internal/log/log.go index a38e233..6d8175c 100644 --- a/internal/log/log.go +++ b/internal/log/log.go @@ -8,10 +8,12 @@ import ( ) var ( + // Logger is Logger *zap.SugaredLogger debugOption bool ) +// InitLogger set SugaredLogger to log.Logger func InitLogger(debug bool) (err error) { debugOption = debug Logger, err = newLogger(debug) diff --git a/internal/report/json.go b/internal/report/json.go index f064c33..5ea595a 100644 --- a/internal/report/json.go +++ b/internal/report/json.go @@ -8,11 +8,13 @@ import ( "github.com/goodwithtech/dockertags/internal/types" ) -type JsonWriter struct { +// JSONWriter create json output +type JSONWriter struct { Output io.Writer } -func (jw JsonWriter) Write(tags types.ImageTags) (err error) { +// Write is +func (jw JSONWriter) Write(tags types.ImageTags) (err error) { output, err := json.MarshalIndent(tags, "", " ") if err != nil { return fmt.Errorf("failed to marshal json: %w", err) diff --git a/internal/report/table.go b/internal/report/table.go index a5bec16..6a4a336 100644 --- a/internal/report/table.go +++ b/internal/report/table.go @@ -13,10 +13,12 @@ import ( "github.com/goodwithtech/dockertags/internal/utils" ) +// TableWriter output table format type TableWriter struct { Output io.Writer } +// Write is func (w TableWriter) Write(tags types.ImageTags) (err error) { table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"Tag", "Size", "Created At", "Uploaded At"}) diff --git a/internal/report/writer.go b/internal/report/writer.go index 22fe206..7e3ba68 100644 --- a/internal/report/writer.go +++ b/internal/report/writer.go @@ -2,6 +2,7 @@ package report import "github.com/goodwithtech/dockertags/internal/types" +// Writer is type Writer interface { Write(types.ImageTags) error } diff --git a/internal/types/error.go b/internal/types/error.go index a47730f..6b52e42 100644 --- a/internal/types/error.go +++ b/internal/types/error.go @@ -4,6 +4,7 @@ import ( "errors" ) +// errors var ( ErrBasicAuth = errors.New("basic auth required") ErrInvalidURL = errors.New("invalid url") diff --git a/internal/types/types.go b/internal/types/types.go index 52f166e..3fef0c0 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -35,6 +35,7 @@ type ImageTag struct { // ImageTags : tag information slice type ImageTags []ImageTag +// Len interface method of sort func (t ImageTags) Len() int { return len(t) } func (t ImageTags) Swap(i, j int) { t[i], t[j] = t[j], t[i] } func (t ImageTags) Less(i, j int) bool { diff --git a/pkg/provider/dockerhub/auth.go b/pkg/provider/dockerhub/auth.go index 3574268..3d38b8b 100644 --- a/pkg/provider/dockerhub/auth.go +++ b/pkg/provider/dockerhub/auth.go @@ -30,7 +30,7 @@ func getJSON(ctx context.Context, url string, auth types.AuthConfig, timeout tim func new(auth types.AuthConfig, timeout time.Duration) (*http.Client, error) { transport := http.DefaultTransport - tokenTransport := &DockerhubTokenTransport{ + tokenTransport := &tokenTransport{ Transport: transport, Username: auth.Username, Password: auth.Password, diff --git a/pkg/provider/dockerhub/dockerhub.go b/pkg/provider/dockerhub/dockerhub.go index fafc137..4951cd8 100644 --- a/pkg/provider/dockerhub/dockerhub.go +++ b/pkg/provider/dockerhub/dockerhub.go @@ -16,23 +16,30 @@ import ( dockertypes "github.com/docker/docker/api/types" ) -const registryURL = "https://registry.hub.docker.com/" +const ( + registryURL = "https://registry.hub.docker.com/" + rateLimit = 64 +) // DockerHub implements Run type DockerHub struct { - filterOpt *types.FilterOption + filterOpt *types.FilterOption + requestOpt *types.RequestOption + authCfg dockertypes.AuthConfig } // Run returns tag list func (p *DockerHub) Run(ctx context.Context, domain, repository string, reqOpt *types.RequestOption, filterOpt *types.FilterOption) (types.ImageTags, error) { p.filterOpt = filterOpt - auth := dockertypes.AuthConfig{ + p.requestOpt = reqOpt + p.authCfg = dockertypes.AuthConfig{ ServerAddress: "registry.hub.docker.com", Username: reqOpt.UserName, Password: reqOpt.Password, } + // fetch page 1 for check max item count. - tagResp, err := getTagResponse(ctx, auth, reqOpt.Timeout, repository, 1) + tagResp, err := getTagResponse(ctx, p.authCfg, reqOpt.Timeout, repository, 1) if err != nil { return nil, err } @@ -42,11 +49,29 @@ func (p *DockerHub) Run(ctx context.Context, domain, repository string, reqOpt * lastPage := calcMaxRequestPage(tagResp.Count, reqOpt.MaxCount, filterOpt) // create ch (page - 1), already fetched first page, tagsPerPage := make(chan []ImageSummary, lastPage-1) - eg := errgroup.Group{} + if err = p.controlGetTags(ctx, tagsPerPage, repository, 2, lastPage); err != nil { + return nil, err + } for page := 2; page <= lastPage; page++ { + select { + case tags := <-tagsPerPage: + totalTagSummary = append(totalTagSummary, tags...) + } + } + close(tagsPerPage) + return p.convertResultToTag(totalTagSummary), nil +} + +// rate limit for socket: too many open files +func (p *DockerHub) controlGetTags(ctx context.Context, tagsPerPage chan []ImageSummary, repository string, from, to int) error { + slots := make(chan struct{}, rateLimit) + eg := errgroup.Group{} + for page := from; page <= to; page++ { page := page + slots <- struct{}{} eg.Go(func() error { - tagResp, err := getTagResponse(ctx, auth, reqOpt.Timeout, repository, page) + defer func() { <-slots }() + tagResp, err := getTagResponse(ctx, p.authCfg, p.requestOpt.Timeout, repository, page) if err != nil { return err } @@ -54,17 +79,7 @@ func (p *DockerHub) Run(ctx context.Context, domain, repository string, reqOpt * return nil }) } - if err := eg.Wait(); err != nil { - return nil, err - } - - for page := 2; page <= lastPage; page++ { - select { - case tags := <-tagsPerPage: - totalTagSummary = append(totalTagSummary, tags...) - } - } - return p.convertResultToTag(totalTagSummary), nil + return eg.Wait() } func summarizeByHash(summaries []ImageSummary) map[string]types.ImageTag { @@ -81,16 +96,16 @@ func summarizeByHash(summaries []ImageSummary) map[string]types.ImageTag { sort.Sort(imageSummary.Images) firstHash := imageSummary.Images[0].Digest target, ok := pools[firstHash] - // create first one if not exist + // create first hash key if not exist if !ok { - pools[firstHash] = createImageTag(imageSummary) + pools[firstHash] = convertUploadImageTag(imageSummary) continue } - // set newer CreatedAt + // set newer uploaded at target.Tags = append(target.Tags, imageSummary.Name) - createdAt, _ := time.Parse(time.RFC3339Nano, imageSummary.LastUpdated) - if createdAt.After(target.CreatedAt) { - target.CreatedAt = createdAt + uploadedAt, _ := time.Parse(time.RFC3339Nano, imageSummary.LastUpdated) + if uploadedAt.After(target.UploadedAt) { + target.UploadedAt = uploadedAt } pools[firstHash] = target } @@ -110,13 +125,13 @@ func (p *DockerHub) convertResultToTag(summaries []ImageSummary) types.ImageTags return tags } -func createImageTag(is ImageSummary) types.ImageTag { - createdAt, _ := time.Parse(time.RFC3339Nano, is.LastUpdated) +func convertUploadImageTag(is ImageSummary) types.ImageTag { + uploadedAt, _ := time.Parse(time.RFC3339Nano, is.LastUpdated) tagNames := []string{is.Name} return types.ImageTag{ - Tags: tagNames, - Byte: is.FullSize, - CreatedAt: createdAt, + Tags: tagNames, + Byte: is.FullSize, + UploadedAt: uploadedAt, } } diff --git a/pkg/provider/dockerhub/dockerhub_test.go b/pkg/provider/dockerhub/dockerhub_test.go index 03b6adf..27ec6a2 100644 --- a/pkg/provider/dockerhub/dockerhub_test.go +++ b/pkg/provider/dockerhub/dockerhub_test.go @@ -76,7 +76,7 @@ func TestScanImage(t *testing.T) { sort.Strings(out) return out }), - cmpopts.IgnoreFields(types.ImageTag{}, "Byte", "CreatedAt"), + cmpopts.IgnoreFields(types.ImageTag{}, "Byte", "UploadedAt"), } sort.Sort(actual) if diff := cmp.Diff(v.expected, actual, opts...); diff != "" { @@ -128,12 +128,12 @@ func TestSummarizeByHash(t *testing.T) { }, expected: map[string]types.ImageTag{ "400": { - Tags: []string{"a", "c"}, - CreatedAt: time.Date(2019, time.December, 3, 0, 0, 0, 0, time.UTC), + Tags: []string{"a", "c"}, + UploadedAt: time.Date(2019, time.December, 3, 0, 0, 0, 0, time.UTC), }, "400b": { - Tags: []string{"b"}, - CreatedAt: time.Date(2019, time.December, 1, 0, 0, 0, 0, time.UTC), + Tags: []string{"b"}, + UploadedAt: time.Date(2019, time.December, 1, 0, 0, 0, 0, time.UTC), }, }, }, @@ -175,12 +175,12 @@ func TestSummarizeByHash(t *testing.T) { }, expected: map[string]types.ImageTag{ "400": { - Tags: []string{"a", "c"}, - CreatedAt: time.Date(2019, time.December, 2, 0, 0, 0, 0, time.UTC), + Tags: []string{"a", "c"}, + UploadedAt: time.Date(2019, time.December, 2, 0, 0, 0, 0, time.UTC), }, "400b": { - Tags: []string{"b"}, - CreatedAt: time.Date(2019, time.December, 1, 0, 0, 0, 0, time.UTC), + Tags: []string{"b"}, + UploadedAt: time.Date(2019, time.December, 1, 0, 0, 0, 0, time.UTC), }, }, }, diff --git a/pkg/provider/dockerhub/transport.go b/pkg/provider/dockerhub/transport.go index 62bc5ec..0000323 100644 --- a/pkg/provider/dockerhub/transport.go +++ b/pkg/provider/dockerhub/transport.go @@ -14,14 +14,14 @@ type authToken struct { Token string `json:"token"` } -type DockerhubTokenTransport struct { +type tokenTransport struct { Transport http.RoundTripper Username string Password string } // RoundTrip defines the round tripper for token transport. -func (t *DockerhubTokenTransport) RoundTrip(req *http.Request) (*http.Response, error) { +func (t *tokenTransport) RoundTrip(req *http.Request) (*http.Response, error) { resp, err := t.Transport.RoundTrip(req) if err != nil { return resp, err @@ -46,7 +46,7 @@ func isTokenDemand(resp *http.Response) bool { return false } -func (t *DockerhubTokenTransport) authAndRetry(req *http.Request) (*http.Response, error) { +func (t *tokenTransport) authAndRetry(req *http.Request) (*http.Response, error) { token, authResp, err := t.auth(req.Context()) if err != nil { return authResp, err @@ -59,12 +59,12 @@ func (t *DockerhubTokenTransport) authAndRetry(req *http.Request) (*http.Respons return response, err } -func (t *DockerhubTokenTransport) retry(req *http.Request, token string) (*http.Response, error) { +func (t *tokenTransport) retry(req *http.Request, token string) (*http.Response, error) { req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) return t.Transport.RoundTrip(req) } -func (t *DockerhubTokenTransport) auth(ctx context.Context) (string, *http.Response, error) { +func (t *tokenTransport) auth(ctx context.Context) (string, *http.Response, error) { jsonStr := []byte(fmt.Sprintf(`{"username": "%s","password": "%s"}`, t.Username, t.Password)) req, err := http.NewRequest("POST", authURL, bytes.NewBuffer(jsonStr)) req.Header.Set("Content-Type", "application/json") diff --git a/pkg/provider/ecr/aws.go b/pkg/provider/ecr/aws.go index 542eaee..c63ea25 100644 --- a/pkg/provider/ecr/aws.go +++ b/pkg/provider/ecr/aws.go @@ -19,12 +19,14 @@ import ( service "github.com/aws/aws-sdk-go/service/ecr" ) +// ECR : type ECR struct{} var _ time.Duration var _ strings.Reader var _ aws.Config +// Run : interface method func (p *ECR) Run(ctx context.Context, domain, repository string, reqOpt *types.RequestOption, filterOpt *types.FilterOption) (types.ImageTags, error) { sess, err := getSession(reqOpt) if err != nil { @@ -83,7 +85,6 @@ func (p *ECR) Run(ctx context.Context, domain, repository string, reqOpt *types. imageTags = append(imageTags, types.ImageTag{ Tags: tags, Byte: getIntByte(detail.ImageSizeInBytes), - CreatedAt: time.Time{}, UploadedAt: pushedAt, }) } diff --git a/pkg/provider/gcr/gcr.go b/pkg/provider/gcr/gcr.go index b4223b1..b11c52d 100644 --- a/pkg/provider/gcr/gcr.go +++ b/pkg/provider/gcr/gcr.go @@ -20,6 +20,7 @@ import ( "github.com/goodwithtech/dockertags/pkg/registry" ) +// GCR : type GCR struct { registry *registry.Registry domain string @@ -30,16 +31,17 @@ type GCR struct { type tagsResponse struct { Next string `json:"next"` Previous string `json:"previous"` - Manifest map[string]ManifestSummary `json:"manifest"` + Manifest map[string]manifestSummary `json:"manifest"` } -type ManifestSummary struct { +type manifestSummary struct { Tag []string `json:"tag"` ImageSizeBytes string `json:"imageSizeBytes"` CreatedMS string `json:"timeCreatedMs"` UploadedMS string `json:"timeUploadedMs"` } +// Run : interface method func (p *GCR) Run(ctx context.Context, domain, repository string, reqOpt *types.RequestOption, filterOpt *types.FilterOption) (imageTags types.ImageTags, err error) { p.domain = domain p.reqOpt = reqOpt @@ -93,8 +95,8 @@ func stringMStoTime(msstring string) (time.Time, error) { } // getTags returns the tags -func (p *GCR) getTags(ctx context.Context, repository string) (map[string]ManifestSummary, error) { - url := p.registry.Url("/v2/%s/tags/list", repository) +func (p *GCR) getTags(ctx context.Context, repository string) (map[string]manifestSummary, error) { + url := p.registry.GetURL("/v2/%s/tags/list", repository) log.Logger.Debugf("url=%s,repository=%s", url, repository) var response tagsResponse diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 2c0a20b..829fa6e 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -21,17 +21,19 @@ const ( gcrURL = "gcr.io" ) +// Provider : type Provider interface { Run(ctx context.Context, domain, repository string, reqOpt *types.RequestOption, filterOpt *types.FilterOption) (types.ImageTags, error) } +// Exec execute run in each provider func Exec(imageName string, reqOpt *types.RequestOption, filterOpt *types.FilterOption) (types.ImageTags, error) { image, err := image.ParseImage(imageName) if err != nil { return nil, err } - p := NewProvider(image.Domain) + p := newProvider(image.Domain) ctx, cancel := context.WithTimeout(context.Background(), reqOpt.Timeout) defer cancel() imageTags, err := p.Run(ctx, image.Domain, image.Path, reqOpt, filterOpt) @@ -42,7 +44,7 @@ func Exec(imageName string, reqOpt *types.RequestOption, filterOpt *types.Filter return imageTags, nil } -func NewProvider(domain string) Provider { +func newProvider(domain string) Provider { if strings.HasSuffix(domain, ecrURL) { return &ecr.ECR{} } diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index cf50e16..a65c44a 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -108,14 +108,14 @@ func newFromTransport(ctx context.Context, auth types.AuthConfig, transport http return registry, nil } -// Url : url returns a registry URL with the passed arguements concatenated. -func (r *Registry) Url(pathTemplate string, args ...interface{}) string { +// GetURL returns a registry URL with the passed arguements concatenated. +func (r *Registry) GetURL(pathTemplate string, args ...interface{}) string { pathSuffix := fmt.Sprintf(pathTemplate, args...) url := fmt.Sprintf("%s%s", r.URL, pathSuffix) return url } -// GetJSON +// GetJSON returns api func (r *Registry) GetJSON(ctx context.Context, url string, response interface{}) (http.Header, error) { req, err := http.NewRequest("GET", url, nil) if err != nil { diff --git a/pkg/run.go b/pkg/run.go index 0e5b692..b9c823c 100644 --- a/pkg/run.go +++ b/pkg/run.go @@ -56,7 +56,7 @@ func Run(c *cli.Context) (err error) { var writer report.Writer switch format := c.String("format"); format { case "json": - writer = &report.JsonWriter{Output: output} + writer = &report.JSONWriter{Output: output} default: writer = &report.TableWriter{Output: output} }