Add ratelimit to dockerhub client (#16)

* dockerhub returns uploadedAt

* dockerhub only set uploadedat

* add ratelimit to dockerhub

* fix for test

* add makefile and lint checks

Makefile

@@ -0,0 +1,25 @@
+.PHONY: \
+  lint \
+  vet \
+  fmt \
+  pretest \
+  test \
+
+PKGS = $(shell go list ./... | grep -v /vendor/)
+SRCS = $(shell git ls-files '*.go')
+GO := GO111MODULE=on go
+GO_OFF := GO111MODULE=off go
+
+lint:
+	$(GO_OFF) get -u golang.org/x/lint/golint
+	$(foreach file,$(PKGS),golint -set_exit_status $(file) || exit;)
+vet:
+	echo $(PKGS) | xargs env $(GO) vet || exit;
+
+fmt:
+	$(foreach file,$(SRCS),gofmt -s -d $(file);)
+
+pretest: lint vet fmt
+
+test:
+	$(foreach file,$(PKGS),$(GO) test -v $(file) || exit;)

cmd/dockertags/main.go

@@ -58,7 +58,7 @@ OPTIONS:
 		},
 		cli.StringFlag{
 			Name:  "authurl, auth",
-			Usage: "Url when fetch authentication",
+			Usage: "GetURL when fetch authentication",
 		},
 		cli.DurationFlag{
 			Name:  "timeout, t",

internal/log/log.go

@@ -8,10 +8,12 @@ import (
 )
 
 var (
+	// Logger is
 	Logger      *zap.SugaredLogger
 	debugOption bool
 )
 
+// InitLogger set SugaredLogger to log.Logger
 func InitLogger(debug bool) (err error) {
 	debugOption = debug
 	Logger, err = newLogger(debug)

internal/report/json.go

@@ -8,11 +8,13 @@ import (
 	"github.com/goodwithtech/dockertags/internal/types"
 )
 
-type JsonWriter struct {
+// JSONWriter create json output
+type JSONWriter struct {
 	Output io.Writer
 }
 
-func (jw JsonWriter) Write(tags types.ImageTags) (err error) {
+// Write is
+func (jw JSONWriter) Write(tags types.ImageTags) (err error) {
 	output, err := json.MarshalIndent(tags, "", "  ")
 	if err != nil {
 		return fmt.Errorf("failed to marshal json: %w", err)

internal/report/table.go

@@ -13,10 +13,12 @@ import (
 	"github.com/goodwithtech/dockertags/internal/utils"
 )
 
+// TableWriter output table format
 type TableWriter struct {
 	Output io.Writer
 }
 
+// Write is
 func (w TableWriter) Write(tags types.ImageTags) (err error) {
 	table := tablewriter.NewWriter(os.Stdout)
 	table.SetHeader([]string{"Tag", "Size", "Created At", "Uploaded At"})

internal/report/writer.go

@@ -2,6 +2,7 @@ package report
 
 import "github.com/goodwithtech/dockertags/internal/types"
 
+// Writer is
 type Writer interface {
 	Write(types.ImageTags) error
 }

internal/types/error.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 )
 
+// errors
 var (
 	ErrBasicAuth  = errors.New("basic auth required")
 	ErrInvalidURL = errors.New("invalid url")

internal/types/types.go

@@ -35,6 +35,7 @@ type ImageTag struct {
 // ImageTags : tag information slice
 type ImageTags []ImageTag
 
+// Len interface method of sort
 func (t ImageTags) Len() int      { return len(t) }
 func (t ImageTags) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
 func (t ImageTags) Less(i, j int) bool {

pkg/provider/dockerhub/auth.go

@@ -30,7 +30,7 @@ func getJSON(ctx context.Context, url string, auth types.AuthConfig, timeout tim
 
 func new(auth types.AuthConfig, timeout time.Duration) (*http.Client, error) {
 	transport := http.DefaultTransport
-	tokenTransport := &DockerhubTokenTransport{
+	tokenTransport := &tokenTransport{
 		Transport: transport,
 		Username:  auth.Username,
 		Password:  auth.Password,

pkg/provider/dockerhub/dockerhub.go

@@ -16,23 +16,30 @@ import (
 	dockertypes "github.com/docker/docker/api/types"
 )
 
-const registryURL = "https://registry.hub.docker.com/"
+const (
+	registryURL = "https://registry.hub.docker.com/"
+	rateLimit   = 64
+)
 
 // DockerHub implements Run
 type DockerHub struct {
-	filterOpt *types.FilterOption
+	filterOpt  *types.FilterOption
+	requestOpt *types.RequestOption
+	authCfg    dockertypes.AuthConfig
 }
 
 // Run returns tag list
 func (p *DockerHub) Run(ctx context.Context, domain, repository string, reqOpt *types.RequestOption, filterOpt *types.FilterOption) (types.ImageTags, error) {
 	p.filterOpt = filterOpt
-	auth := dockertypes.AuthConfig{
+	p.requestOpt = reqOpt
+	p.authCfg = dockertypes.AuthConfig{
 		ServerAddress: "registry.hub.docker.com",
 		Username:      reqOpt.UserName,
 		Password:      reqOpt.Password,
 	}
+
 	// fetch page 1 for check max item count.
-	tagResp, err := getTagResponse(ctx, auth, reqOpt.Timeout, repository, 1)
+	tagResp, err := getTagResponse(ctx, p.authCfg, reqOpt.Timeout, repository, 1)
 	if err != nil {
 		return nil, err
 	}
@@ -42,29 +49,37 @@ func (p *DockerHub) Run(ctx context.Context, domain, repository string, reqOpt *
 	lastPage := calcMaxRequestPage(tagResp.Count, reqOpt.MaxCount, filterOpt)
 	// create ch (page - 1), already fetched first page,
 	tagsPerPage := make(chan []ImageSummary, lastPage-1)
-	eg := errgroup.Group{}
+	if err = p.controlGetTags(ctx, tagsPerPage, repository, 2, lastPage); err != nil {
+		return nil, err
+	}
 	for page := 2; page <= lastPage; page++ {
+		select {
+		case tags := <-tagsPerPage:
+			totalTagSummary = append(totalTagSummary, tags...)
+		}
+	}
+	close(tagsPerPage)
+	return p.convertResultToTag(totalTagSummary), nil
+}
+
+// rate limit for socket: too many open files
+func (p *DockerHub) controlGetTags(ctx context.Context, tagsPerPage chan []ImageSummary, repository string, from, to int) error {
+	slots := make(chan struct{}, rateLimit)
+	eg := errgroup.Group{}
+	for page := from; page <= to; page++ {
 		page := page
+		slots <- struct{}{}
 		eg.Go(func() error {
-			tagResp, err := getTagResponse(ctx, auth, reqOpt.Timeout, repository, page)
+			defer func() { <-slots }()
+			tagResp, err := getTagResponse(ctx, p.authCfg, p.requestOpt.Timeout, repository, page)
 			if err != nil {
 				return err
 			}
 			tagsPerPage <- tagResp.Results
 			return nil
 		})
 	}
-	if err := eg.Wait(); err != nil {
-		return nil, err
-	}
-
-	for page := 2; page <= lastPage; page++ {
-		select {
-		case tags := <-tagsPerPage:
-			totalTagSummary = append(totalTagSummary, tags...)
-		}
-	}
-	return p.convertResultToTag(totalTagSummary), nil
+	return eg.Wait()
 }
 
 func summarizeByHash(summaries []ImageSummary) map[string]types.ImageTag {
@@ -81,16 +96,16 @@ func summarizeByHash(summaries []ImageSummary) map[string]types.ImageTag {
 		sort.Sort(imageSummary.Images)
 		firstHash := imageSummary.Images[0].Digest
 		target, ok := pools[firstHash]
-		// create first one if not exist
+		// create first hash key if not exist
 		if !ok {
-			pools[firstHash] = createImageTag(imageSummary)
+			pools[firstHash] = convertUploadImageTag(imageSummary)
 			continue
 		}
-		// set newer CreatedAt
+		// set newer uploaded at
 		target.Tags = append(target.Tags, imageSummary.Name)
-		createdAt, _ := time.Parse(time.RFC3339Nano, imageSummary.LastUpdated)
-		if createdAt.After(target.CreatedAt) {
-			target.CreatedAt = createdAt
+		uploadedAt, _ := time.Parse(time.RFC3339Nano, imageSummary.LastUpdated)
+		if uploadedAt.After(target.UploadedAt) {
+			target.UploadedAt = uploadedAt
 		}
 		pools[firstHash] = target
 	}
@@ -110,13 +125,13 @@ func (p *DockerHub) convertResultToTag(summaries []ImageSummary) types.ImageTags
 	return tags
 }
 
-func createImageTag(is ImageSummary) types.ImageTag {
-	createdAt, _ := time.Parse(time.RFC3339Nano, is.LastUpdated)
+func convertUploadImageTag(is ImageSummary) types.ImageTag {
+	uploadedAt, _ := time.Parse(time.RFC3339Nano, is.LastUpdated)
 	tagNames := []string{is.Name}
 	return types.ImageTag{
-		Tags:      tagNames,
-		Byte:      is.FullSize,
-		CreatedAt: createdAt,
+		Tags:       tagNames,
+		Byte:       is.FullSize,
+		UploadedAt: uploadedAt,
 	}
 }
 

pkg/provider/dockerhub/dockerhub_test.go

@@ -76,7 +76,7 @@ func TestScanImage(t *testing.T) {
 				sort.Strings(out)
 				return out
 			}),
-			cmpopts.IgnoreFields(types.ImageTag{}, "Byte", "CreatedAt"),
+			cmpopts.IgnoreFields(types.ImageTag{}, "Byte", "UploadedAt"),
 		}
 		sort.Sort(actual)
 		if diff := cmp.Diff(v.expected, actual, opts...); diff != "" {
@@ -128,12 +128,12 @@ func TestSummarizeByHash(t *testing.T) {
 			},
 			expected: map[string]types.ImageTag{
 				"400": {
-					Tags:      []string{"a", "c"},
-					CreatedAt: time.Date(2019, time.December, 3, 0, 0, 0, 0, time.UTC),
+					Tags:       []string{"a", "c"},
+					UploadedAt: time.Date(2019, time.December, 3, 0, 0, 0, 0, time.UTC),
 				},
 				"400b": {
-					Tags:      []string{"b"},
-					CreatedAt: time.Date(2019, time.December, 1, 0, 0, 0, 0, time.UTC),
+					Tags:       []string{"b"},
+					UploadedAt: time.Date(2019, time.December, 1, 0, 0, 0, 0, time.UTC),
 				},
 			},
 		},
@@ -175,12 +175,12 @@ func TestSummarizeByHash(t *testing.T) {
 			},
 			expected: map[string]types.ImageTag{
 				"400": {
-					Tags:      []string{"a", "c"},
-					CreatedAt: time.Date(2019, time.December, 2, 0, 0, 0, 0, time.UTC),
+					Tags:       []string{"a", "c"},
+					UploadedAt: time.Date(2019, time.December, 2, 0, 0, 0, 0, time.UTC),
 				},
 				"400b": {
-					Tags:      []string{"b"},
-					CreatedAt: time.Date(2019, time.December, 1, 0, 0, 0, 0, time.UTC),
+					Tags:       []string{"b"},
+					UploadedAt: time.Date(2019, time.December, 1, 0, 0, 0, 0, time.UTC),
 				},
 			},
 		},

pkg/provider/dockerhub/transport.go

@@ -14,14 +14,14 @@ type authToken struct {
 	Token string `json:"token"`
 }
 
-type DockerhubTokenTransport struct {
+type tokenTransport struct {
 	Transport http.RoundTripper
 	Username  string
 	Password  string
 }
 
 // RoundTrip defines the round tripper for token transport.
-func (t *DockerhubTokenTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+func (t *tokenTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 	resp, err := t.Transport.RoundTrip(req)
 	if err != nil {
 		return resp, err
@@ -46,7 +46,7 @@ func isTokenDemand(resp *http.Response) bool {
 	return false
 }
 
-func (t *DockerhubTokenTransport) authAndRetry(req *http.Request) (*http.Response, error) {
+func (t *tokenTransport) authAndRetry(req *http.Request) (*http.Response, error) {
 	token, authResp, err := t.auth(req.Context())
 	if err != nil {
 		return authResp, err
@@ -59,12 +59,12 @@ func (t *DockerhubTokenTransport) authAndRetry(req *http.Request) (*http.Respons
 	return response, err
 }
 
-func (t *DockerhubTokenTransport) retry(req *http.Request, token string) (*http.Response, error) {
+func (t *tokenTransport) retry(req *http.Request, token string) (*http.Response, error) {
 	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
 	return t.Transport.RoundTrip(req)
 }
 
-func (t *DockerhubTokenTransport) auth(ctx context.Context) (string, *http.Response, error) {
+func (t *tokenTransport) auth(ctx context.Context) (string, *http.Response, error) {
 	jsonStr := []byte(fmt.Sprintf(`{"username": "%s","password": "%s"}`, t.Username, t.Password))
 	req, err := http.NewRequest("POST", authURL, bytes.NewBuffer(jsonStr))
 	req.Header.Set("Content-Type", "application/json")

pkg/provider/ecr/aws.go

@@ -19,12 +19,14 @@ import (
 	service "github.com/aws/aws-sdk-go/service/ecr"
 )
 
+// ECR :
 type ECR struct{}
 
 var _ time.Duration
 var _ strings.Reader
 var _ aws.Config
 
+// Run : interface method
 func (p *ECR) Run(ctx context.Context, domain, repository string, reqOpt *types.RequestOption, filterOpt *types.FilterOption) (types.ImageTags, error) {
 	sess, err := getSession(reqOpt)
 	if err != nil {
@@ -83,7 +85,6 @@ func (p *ECR) Run(ctx context.Context, domain, repository string, reqOpt *types.
 		imageTags = append(imageTags, types.ImageTag{
 			Tags:       tags,
 			Byte:       getIntByte(detail.ImageSizeInBytes),
-			CreatedAt:  time.Time{},
 			UploadedAt: pushedAt,
 		})
 	}

pkg/provider/gcr/gcr.go

@@ -20,6 +20,7 @@ import (
 	"github.com/goodwithtech/dockertags/pkg/registry"
 )
 
+// GCR :
 type GCR struct {
 	registry *registry.Registry
 	domain   string
@@ -30,16 +31,17 @@ type GCR struct {
 type tagsResponse struct {
 	Next     string                     `json:"next"`
 	Previous string                     `json:"previous"`
-	Manifest map[string]ManifestSummary `json:"manifest"`
+	Manifest map[string]manifestSummary `json:"manifest"`
 }
 
-type ManifestSummary struct {
+type manifestSummary struct {
 	Tag            []string `json:"tag"`
 	ImageSizeBytes string   `json:"imageSizeBytes"`
 	CreatedMS      string   `json:"timeCreatedMs"`
 	UploadedMS     string   `json:"timeUploadedMs"`
 }
 
+// Run : interface method
 func (p *GCR) Run(ctx context.Context, domain, repository string, reqOpt *types.RequestOption, filterOpt *types.FilterOption) (imageTags types.ImageTags, err error) {
 	p.domain = domain
 	p.reqOpt = reqOpt
@@ -93,8 +95,8 @@ func stringMStoTime(msstring string) (time.Time, error) {
 }
 
 // getTags returns the tags
-func (p *GCR) getTags(ctx context.Context, repository string) (map[string]ManifestSummary, error) {
-	url := p.registry.Url("/v2/%s/tags/list", repository)
+func (p *GCR) getTags(ctx context.Context, repository string) (map[string]manifestSummary, error) {
+	url := p.registry.GetURL("/v2/%s/tags/list", repository)
 	log.Logger.Debugf("url=%s,repository=%s", url, repository)
 
 	var response tagsResponse