fix limit option logic

goodwithtech · Dec 22, 2019 · e52cb5d · e52cb5d
1 parent f4f2910
commit e52cb5d
Show file tree

Hide file tree

Showing 8 changed files with 162 additions and 139 deletions.
diff --git a/README.md b/README.md
@@ -11,68 +11,56 @@ $ brew install goodwithtech/r/dockertags
 $ dockertags [IMAGE_NAME]
 ```
 
+## When to Use
+
+Make easy to fetch target tag in scheduled operation.
+
+```.env
+$ dockertags -limit 1 -format json <imagename> | jq -r .[0].tags[0]
+...output tag...
+
+# Scan a latest container image with https://github.com/aquasecurity/trivy
+$ export IMAGENAME=<imagename>
+$ trivy $IMAGENAME:$(dockertags -limit 1 -format json $IMAGENAME | jq -r .[0].tags[0])
+```
+
 ## Examples
 
 ```bash
 $ dockertags goodwithtech/dockle
-+-----------------------------+---------+-------+----------------------+-------------+
-|            FULL             |   TAG   | SIZE  |      CREATED AT      | UPLOADED AT |
-+-----------------------------+---------+-------+----------------------+-------------+
-| goodwithtech/dockle:v0.1.16 | v0.1.16 | 20.5M | 2019-08-25T06:35:40Z | NULL        |
-| goodwithtech/dockle:latest  | latest  | 20.5M | 2019-08-25T06:15:39Z | NULL        |
-| goodwithtech/dockle:v0.1.15 | v0.1.15 | 20.5M | 2019-08-16T00:01:43Z | NULL        |
-| goodwithtech/dockle:v0.1.14 | v0.1.14 | 20.5M | 2019-07-11T15:11:08Z | NULL        |
-| goodwithtech/dockle:v0.1.13 | v0.1.13 | 20.5M | 2019-06-19T13:32:17Z | NULL        |
-| goodwithtech/dockle:v0.1.12 | v0.1.12 | 20.5M | 2019-06-16T18:49:37Z | NULL        |
-| goodwithtech/dockle:v0.1.11 | v0.1.11 | 20.5M | 2019-06-16T17:58:23Z | NULL        |
-| goodwithtech/dockle:v0.1.10 | v0.1.10 | 20.5M | 2019-06-15T14:38:20Z | NULL        |
-| goodwithtech/dockle:v0.1.9  | v0.1.9  | 20.5M | 2019-06-15T14:11:43Z | NULL        |
-| goodwithtech/dockle:v0.1.8  | v0.1.8  | 20.5M | 2019-06-14T19:20:04Z | NULL        |
-+-----------------------------+---------+-------+----------------------+-------------+
-
-$ dockertags --limit 20 debian
-+-------------------------------+------------------------+-------+----------------------+-------------+
-|             FULL              |          TAG           | SIZE  |      CREATED AT      | UPLOADED AT |
-+-------------------------------+------------------------+-------+----------------------+-------------+
-| debian:rc-buggy               | rc-buggy               | 43.3M | 2019-09-12T01:07:58Z | NULL        |
-| debian:experimental-20190910  | experimental-20190910  | 43.3M | 2019-09-12T01:07:36Z | NULL        |
-| debian:experimental           | experimental           | 43.3M | 2019-09-12T01:07:32Z | NULL        |
-| debian:bullseye-20190910-slim | bullseye-20190910-slim | 43.3M | 2019-09-12T01:07:12Z | NULL        |
-| debian:bullseye-20190910      | bullseye-20190910      | 43.3M | 2019-09-12T01:07:07Z | NULL        |
-| debian:bullseye               | bullseye               | 43.3M | 2019-09-12T01:07:03Z | NULL        |
-| debian:9.11-slim              | 9.11-slim              | 43.3M | 2019-09-12T01:06:52Z | NULL        |
-| debian:9.11                   | 9.11                   | 43.3M | 2019-09-12T01:06:48Z | NULL        |
-| debian:9-slim                 | 9-slim                 | 43.3M | 2019-09-12T01:06:44Z | NULL        |
-| debian:9                      | 9                      | 43.3M | 2019-09-12T01:06:39Z | NULL        |
-| debian:stable-20190910-slim   | stable-20190910-slim   | 43.3M | 2019-09-11T23:56:41Z | NULL        |
-| debian:stable-20190910        | stable-20190910        | 43.3M | 2019-09-11T23:56:09Z | NULL        |
-| debian:stable                 | stable                 | 43.3M | 2019-09-11T23:56:02Z | NULL        |
-| debian:sid-slim               | sid-slim               | 43.3M | 2019-09-11T23:55:29Z | NULL        |
-| debian:sid-20190910-slim      | sid-20190910-slim      | 43.3M | 2019-09-11T23:55:24Z | NULL        |
-| debian:sid-20190910           | sid-20190910           | 43.3M | 2019-09-11T23:54:50Z | NULL        |
-| debian:sid                    | sid                    | 43.3M | 2019-09-11T23:54:45Z | NULL        |
-| debian:rc-buggy-20190910      | rc-buggy-20190910      | 43.3M | 2019-09-11T23:54:20Z | NULL        |
-| debian:oldstable-slim         | oldstable-slim         | 43.3M | 2019-09-11T23:53:32Z | NULL        |
-| debian:oldstable-backports    | oldstable-backports    | 43.3M | 2019-09-11T23:53:28Z | NULL        |
-+-------------------------------+------------------------+-------+----------------------+-------------+
++---------+-------+----------------------+-------------+
+|   TAG   | SIZE  |      CREATED AT      | UPLOADED AT |
++---------+-------+----------------------+-------------+
+| latest  | 21.1M | 2019-12-16T14:05:18Z | NULL        |
+| v0.2.4  | 21.1M | 2019-12-05T05:18:04Z | NULL        |
+| v0.2.3  | 21.1M | 2019-11-17T15:03:10Z | NULL        |
+| v0.2.2  | 21.1M | 2019-11-17T14:45:53Z | NULL        |
+...... 
+| v0.0.18 | 20.4M | 2019-06-10T18:31:45Z | NULL        |
++---------+-------+----------------------+-------------+
+
+# You can set limit, filter and format
+$ dockertags -limit 2 -contain v0.2 -format json goodwithtech/dockle
+[
+  {
+    "tags": [
+      "v0.2.4"
+    ],
+    "byte": 22154435,
+    "created_at": "2019-12-05T05:18:04.174078Z",
+    "uploaded_at": null
+  },
+  {
+    "tags": [
+      "v0.2.3"
+    ],
+    "byte": 22154435,
+    "created_at": "2019-11-17T15:03:10.914092Z",
+    "uploaded_at": null
+  }
+]
 ```
 
-## Options
-
-```
-USAGE:
-  dockertags [options] image_name
-OPTIONS:
-  --all                          fetch all tagged image information
-  --limit value, -l value        Set max fetch count (default: 50)
-  --timeout value, -t value      e.g)5s, 1m (default: 10s)
-  --username value, -u value     Username
-  --password value, -p value     Using -password via CLI is insecure. Be careful.
-  --authurl value, --auth value  Url when fetch authentication
-  --debug, -d                    Show debug logs
-  --help, -h                     show help
-  --version, -v                  print the version
-```
 
 ## Authentication
 
@@ -89,17 +77,15 @@ dockertags -u goodwithtech -p xxxx goodwithtech/privateimage
 Use [AWS CLI's ENVIRONMENT variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html).
 
 ```bash
-export AWS_ACCESS_KEY_ID={AWS ACCESS KEY}
-export AWS_SECRET_ACCESS_KEY={SECRET KEY}
-export AWS_DEFAULT_REGION={AWS REGION}
+AWS_PROFILE={PROFILE_NAME}
+AWS_DEFAULT_REGION={REGION}
 ```
 
 ### GCR (Google Container Registry)
 
 If you'd like to use the target project's repository, you can settle via `GOOGLE_APPLICATION_CREDENTIAL`.
 
 ```bash
-# must set DOCKLE_USERNAME empty char
-export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credential.json
+GOOGLE_APPLICATION_CREDENTIALS=/path/to/credential.json
 ```
 
diff --git a/cmd/dockertags/main.go b/cmd/dockertags/main.go
@@ -36,12 +36,11 @@ OPTIONS:
 	app.Version = version
 	app.ArgsUsage = "image_name"
 
-	app.Usage = "Fetch docker tags informations"
+	app.Usage = "fetch docker image tags"
 
 	app.Flags = []cli.Flag{
 		cli.IntFlag{
 			Name:  "limit, l",
-			Value: 0,
 			Usage: "set max tags count. if exist no tag image will be short numbers. limit=0 means fetch all tags",
 		},
 		cli.StringFlag{

diff --git a/internal/utils/tag.go b/internal/utils/tag.go
@@ -7,7 +7,7 @@ import (
 )
 
 // MatchConditionTags retunrn matched option
-func MatchConditionTags(opt types.FilterOption, tagNames []string) (contained bool) {
+func MatchConditionTags(opt *types.FilterOption, tagNames []string) (contained bool) {
 	if opt.Contain == "" {
 		return true
 	}

diff --git a/pkg/provider/dockerhub/dockerhub.go b/pkg/provider/dockerhub/dockerhub.go
@@ -13,15 +13,12 @@ import (
 	"github.com/goodwithtech/dockertags/internal/utils"
 
 	dockertypes "github.com/docker/docker/api/types"
-
-	"github.com/goodwithtech/dockertags/pkg/registry"
 )
 
 const registryURL = "https://registry.hub.docker.com/"
 
 type DockerHub struct {
-	registry  *registry.Registry
-	filterOpt types.FilterOption
+	filterOpt *types.FilterOption
 }
 
 type tagsResponse struct {
@@ -37,43 +34,45 @@ type ImageSummary struct {
 	LastUpdated string `json:"last_updated"`
 }
 
-// curl 'https://registry.hub.docker.com/v2/repositories/library/debian/tags/'
-func (p *DockerHub) Run(ctx context.Context, domain, repository string, reqOpt types.RequestOption, filterOpt types.FilterOption) (types.ImageTags, error) {
+func (p *DockerHub) Run(ctx context.Context, domain, repository string, reqOpt *types.RequestOption, filterOpt *types.FilterOption) (types.ImageTags, error) {
 	p.filterOpt = filterOpt
 	auth := dockertypes.AuthConfig{
 		ServerAddress: "registry.hub.docker.com",
 		Username:      reqOpt.UserName,
 		Password:      reqOpt.Password,
 	}
-	// 1ページ目は普通に取得
+	// fetch page 1 for check max item count.
 	tagResp, err := getTagResponse(ctx, auth, reqOpt.Timeout, repository, 1)
 	if err != nil {
 		return nil, err
 	}
 	imageTags := p.convertResultToTag(tagResp.Results)
+	if reqOpt.MaxCount > 0 && len(imageTags) > reqOpt.MaxCount {
+		return imageTags, nil
+	}
 
-	// 2ページ目以降はgoroutine
-	maxPage := tagResp.Count/types.ITEM_PER_PAGE + 1
-	tagCh := make(chan types.ImageTags, maxPage-1)
+	lastPage := calcMaxRequestPage(tagResp.Count, reqOpt.MaxCount, filterOpt)
+	// create ch (page - 1), already fetched first page,
+	tagsPerPage := make(chan types.ImageTags, lastPage-1)
 	eg := errgroup.Group{}
-	for page := 2; page < maxPage; page++ {
+	for page := 2; page <= lastPage; page++ {
 		page := page
 		eg.Go(func() error {
 			tagResp, err := getTagResponse(ctx, auth, reqOpt.Timeout, repository, page)
 			if err != nil {
 				return err
 			}
-			tagCh <- p.convertResultToTag(tagResp.Results)
+			tagsPerPage <- p.convertResultToTag(tagResp.Results)
 			return nil
 		})
 	}
 	if err := eg.Wait(); err != nil {
 		return nil, err
 	}
 
-	for page := 2; page < maxPage; page++ {
+	for page := 2; page <= lastPage; page++ {
 		select {
-		case tags := <-tagCh:
+		case tags := <-tagsPerPage:
 			imageTags = append(imageTags, tags...)
 		}
 	}
@@ -91,7 +90,6 @@ func (p *DockerHub) convertResultToTag(summaries []ImageSummary) types.ImageTags
 		if !utils.MatchConditionTags(p.filterOpt, tagNames) {
 			continue
 		}
-
 		tags = append(tags, types.ImageTag{
 			Tags:      tagNames,
 			Byte:      &detail.FullSize,
@@ -102,6 +100,7 @@ func (p *DockerHub) convertResultToTag(summaries []ImageSummary) types.ImageTags
 }
 
 // getTagResponse returns the tags for a specific repository.
+// curl 'https://registry.hub.docker.com/v2/repositories/library/debian/tags/'
 func getTagResponse(ctx context.Context, auth dockertypes.AuthConfig, timeout time.Duration, repository string, page int) (tagsResponse, error) {
 	url := fmt.Sprintf("%s/v2/repositories/%s/tags/?page=%d", registryURL, repository, page)
 	log.Logger.Debugf("url=%s,repository=%s", url, repository)
@@ -112,3 +111,15 @@ func getTagResponse(ctx context.Context, auth dockertypes.AuthConfig, timeout ti
 
 	return response, nil
 }
+
+func calcMaxRequestPage(totalCnt, needCnt int, option *types.FilterOption) int {
+	maxPage := totalCnt/types.ITEM_PER_PAGE + 1
+	if needCnt == 0 || option.Contain != "" {
+		return maxPage
+	}
+	needPage := needCnt/types.ITEM_PER_PAGE + 1
+	if needPage >= maxPage {
+		return maxPage
+	}
+	return needPage
+}
diff --git a/pkg/provider/ecr/aws.go b/pkg/provider/ecr/aws.go
@@ -25,30 +25,7 @@ var _ time.Duration
 var _ strings.Reader
 var _ aws.Config
 
-func getSession(option types.RequestOption) (*session.Session, error) {
-	// create custom credential information if option is valid
-	if option.AwsSecretKey != "" && option.AwsAccessKey != "" && option.AwsRegion != "" {
-		return session.NewSessionWithOptions(
-			session.Options{
-				Config: aws.Config{
-					Region: aws.String(option.AwsRegion),
-					Credentials: credentials.NewStaticCredentialsFromCreds(
-						credentials.Value{
-							AccessKeyID:     option.AwsAccessKey,
-							SecretAccessKey: option.AwsSecretKey,
-							SessionToken:    option.AwsSessionToken,
-						},
-					),
-				},
-			})
-	}
-	// use shared configuration normally
-	return session.NewSessionWithOptions(session.Options{
-		SharedConfigState: session.SharedConfigEnable,
-	})
-}
-
-func (p *ECR) Run(ctx context.Context, domain, repository string, reqOpt types.RequestOption, filterOpt types.FilterOption) (types.ImageTags, error) {
+func (p *ECR) Run(ctx context.Context, domain, repository string, reqOpt *types.RequestOption, filterOpt *types.FilterOption) (types.ImageTags, error) {
 	sess, err := getSession(reqOpt)
 	if err != nil {
 		return nil, err
@@ -59,6 +36,11 @@ func (p *ECR) Run(ctx context.Context, domain, repository string, reqOpt types.R
 		// Only show tagged image
 		Filter: &service.DescribeImagesFilter{TagStatus: aws.String("TAGGED")},
 	}
+	if reqOpt.MaxCount > 0 {
+		//var maxResults *int64
+		maxResults := int64(reqOpt.MaxCount)
+		input.MaxResults = &maxResults
+	}
 
 	result, err := svc.DescribeImages(input)
 	if err != nil {
@@ -105,6 +87,29 @@ func (p *ECR) Run(ctx context.Context, domain, repository string, reqOpt types.R
 	return imageTags, nil
 }
 
+func getSession(option *types.RequestOption) (*session.Session, error) {
+	// create custom credential information if option is valid
+	if option.AwsSecretKey != "" && option.AwsAccessKey != "" && option.AwsRegion != "" {
+		return session.NewSessionWithOptions(
+			session.Options{
+				Config: aws.Config{
+					Region: aws.String(option.AwsRegion),
+					Credentials: credentials.NewStaticCredentialsFromCreds(
+						credentials.Value{
+							AccessKeyID:     option.AwsAccessKey,
+							SecretAccessKey: option.AwsSecretKey,
+							SessionToken:    option.AwsSessionToken,
+						},
+					),
+				},
+			})
+	}
+	// use shared configuration normally
+	return session.NewSessionWithOptions(session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+	})
+}
+
 func getIntByte(b *int64) *int {
 	if b == nil {
 		return nil

diff --git a/pkg/provider/gcr/gcr.go b/pkg/provider/gcr/gcr.go
@@ -23,6 +23,7 @@ import (
 type GCR struct {
 	registry *registry.Registry
 	domain   string
+	reqOpt   *types.RequestOption
 	Store    store.GCRCredStore
 }
 
@@ -39,9 +40,10 @@ type ManifestSummary struct {
 	UploadedMS     string   `json:"timeUploadedMs"`
 }
 
-func (p *GCR) Run(ctx context.Context, domain, repository string, reqOpt types.RequestOption, filterOpt types.FilterOption) (imageTags types.ImageTags, err error) {
+func (p *GCR) Run(ctx context.Context, domain, repository string, reqOpt *types.RequestOption, filterOpt *types.FilterOption) (imageTags types.ImageTags, err error) {
 	p.domain = domain
-	authconfig, err := p.getAuthConfig(ctx, domain, reqOpt)
+	p.reqOpt = reqOpt
+	authconfig, err := p.getAuthConfig(ctx, domain)
 	if err != nil {
 		log.Logger.Debugf("Fail to get gcp credential : %s", err)
 	}
@@ -102,19 +104,19 @@ func (p *GCR) getTags(ctx context.Context, repository string) (map[string]Manife
 	return response.Manifest, nil
 }
 
-func (p *GCR) getAuthConfig(ctx context.Context, domain string, opt types.RequestOption) (authconfig dockertypes.AuthConfig, err error) {
-	if opt.GcpCredPath != "" {
-		p.Store = store.NewGCRCredStore(opt.GcpCredPath)
+func (p *GCR) getAuthConfig(ctx context.Context, domain string) (authconfig dockertypes.AuthConfig, err error) {
+	if p.reqOpt.GcpCredPath != "" {
+		p.Store = store.NewGCRCredStore(p.reqOpt.GcpCredPath)
 	}
-	authDomain := opt.AuthURL
+	authDomain := p.reqOpt.AuthURL
 	if authDomain == "" {
 		authDomain = domain
 	}
 	authconfig.ServerAddress = authDomain
 	// check registry which particular to get credential
 	authconfig.Username, authconfig.Password, err = p.getCredential(ctx)
 	if err != nil {
-		return auth.GetAuthConfig(opt.UserName, opt.Password, authDomain)
+		return auth.GetAuthConfig(p.reqOpt.UserName, p.reqOpt.Password, authDomain)
 	}
 	return authconfig, nil
 }

diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go
@@ -22,10 +22,10 @@ const (
 )
 
 type Provider interface {
-	Run(ctx context.Context, domain, repository string, reqOpt types.RequestOption, filterOpt types.FilterOption) (types.ImageTags, error)
+	Run(ctx context.Context, domain, repository string, reqOpt *types.RequestOption, filterOpt *types.FilterOption) (types.ImageTags, error)
 }
 
-func Exec(imageName string, reqOpt types.RequestOption, filterOpt types.FilterOption) (types.ImageTags, error) {
+func Exec(imageName string, reqOpt *types.RequestOption, filterOpt *types.FilterOption) (types.ImageTags, error) {
 	image, err := image.ParseImage(imageName)
 	if err != nil {
 		return nil, err