From 8513d91ea9bb2a60d996e6d1333367b1a3ccf49b Mon Sep 17 00:00:00 2001 From: Tore Martin Hagen <93583343+ToreMerkely@users.noreply.github.com> Date: Tue, 10 Dec 2024 12:58:34 +0100 Subject: [PATCH] Deprecated registryProviderFlag and updated docs (#382) * Deprecated registryProviderFlag and updated docs * fix linting and test errors --------- Co-authored-by: Sami Alajrami --- cmd/kosli/cli_utils.go | 135 +------------------------ cmd/kosli/cli_utils_test.go | 70 +------------ cmd/kosli/fingerprint.go | 14 ++- cmd/kosli/flags.go | 9 ++ cmd/kosli/root.go | 26 ++--- cmd/kosli/testdata/output/docs/snyk.md | 7 +- 6 files changed, 42 insertions(+), 219 deletions(-) diff --git a/cmd/kosli/cli_utils.go b/cmd/kosli/cli_utils.go index 09c61bd5a..fa233f7b4 100644 --- a/cmd/kosli/cli_utils.go +++ b/cmd/kosli/cli_utils.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "io" - "net/http" urlPackage "net/url" "os" "path/filepath" @@ -19,7 +18,6 @@ import ( "github.com/kosli-dev/cli/internal/digest" "github.com/kosli-dev/cli/internal/gitview" log "github.com/kosli-dev/cli/internal/logger" - "github.com/kosli-dev/cli/internal/requests" "github.com/kosli-dev/cli/internal/utils" cp "github.com/otiai10/copy" "github.com/spf13/cobra" @@ -311,92 +309,6 @@ func GetFlagFromVarName(varName string) string { return result } -type registryProviderEndpoints struct { - mainApi string - authApi string - service string -} - -func getRegistryEndpointForProvider(provider string) (*registryProviderEndpoints, error) { - switch provider { - case "dockerhub": - return ®istryProviderEndpoints{ - mainApi: "https://registry-1.docker.io/v2", - authApi: "https://auth.docker.io", - service: "registry.docker.io", - }, nil - case "github": - return ®istryProviderEndpoints{ - mainApi: "https://ghcr.io/v2", - authApi: "https://ghcr.io", - service: "ghcr.io", - }, nil - - default: - return getRegistryEndpoint(provider) - } -} - -func getRegistryEndpoint(url string) (*registryProviderEndpoints, error) { - url = strings.TrimPrefix(url, "https://") - url = strings.Split(url, "/")[0] - - return ®istryProviderEndpoints{ - mainApi: "https://" + url + "/v2", - authApi: "https://" + url + "/oauth2", - service: url, - }, nil -} - -// getDockerRegistryAPIToken returns a short-lived read-only api token for a docker registry api -func getDockerRegistryAPIToken(providerInfo *registryProviderEndpoints, username, password, imageName string) (string, error) { - var res *requests.HTTPResponse - var err error - - if strings.Contains(providerInfo.service, "jfrog") { - url := "https://" + providerInfo.service + "/artifactory/api/security/token" - - form := urlPackage.Values{} - form.Add("username", username) - form.Add("scope", "member-of-groups:readers") - form.Add("expires_in", "60") - - reqParams := &requests.RequestParams{ - Method: http.MethodPost, - URL: url, - Payload: form.Encode(), - Username: username, - Password: password, - AdditionalHeaders: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, - } - res, err = kosliClient.Do(reqParams) - } else { - url := fmt.Sprintf("%s/token?scope=repository:%s:pull&service=%s", providerInfo.authApi, imageName, providerInfo.service) - reqParams := &requests.RequestParams{ - Method: http.MethodGet, - URL: url, - Username: username, - Password: password, - } - res, err = kosliClient.Do(reqParams) - } - - if err != nil { - return "", fmt.Errorf("failed to create an authentication token for the docker registry: %v %v", err, res) - } - - var responseData map[string]interface{} - err = json.Unmarshal([]byte(res.Body), &responseData) - if err != nil { - return "", err - } - token := responseData["token"] - if token == nil { - token = responseData["access_token"] - } - return token.(string), nil -} - // GetSha256Digest calculates the sha256 digest of an artifact. // Supported artifact types are: dir, file, docker func GetSha256Digest(artifactName string, o *fingerprintOptions, logger *log.Logger) (string, error) { @@ -410,42 +322,8 @@ func GetSha256Digest(artifactName string, o *fingerprintOptions, logger *log.Log case "oci": fingerprint, err = digest.OciSha256(artifactName, o.registryUsername, o.registryPassword) case "docker": - if o.registryProvider != "" { - var providerInfo *registryProviderEndpoints - providerInfo, err = getRegistryEndpointForProvider(o.registryProvider) - if err != nil { - return "", err - } - - nameSlice := strings.Split(artifactName, ":") - if len(nameSlice) < 2 { - nameSlice = append(nameSlice, "latest") - } - imageName := nameSlice[0] - imageTag := nameSlice[1] - - if strings.Contains(nameSlice[0], "/") { - strSlice := strings.Split(nameSlice[0], "/") - urlOrRepo := strSlice[0] - if strings.Contains(urlOrRepo, ".") { - imageName = strings.TrimPrefix(nameSlice[0], urlOrRepo+"/") - } - } - - if !strings.Contains(imageName, "/") && o.registryProvider == "dockerhub" { - imageName = fmt.Sprintf("library/%s", imageName) - } - - token, err := getDockerRegistryAPIToken(providerInfo, o.registryUsername, o.registryPassword, imageName) - if err != nil { - return "", err - } - - fingerprint, err = digest.RemoteDockerImageSha256(imageName, imageTag, providerInfo.mainApi, token, logger) - if err != nil { - return "", err - } - + if o.registryUsername != "" { + fingerprint, err = digest.OciSha256(artifactName, o.registryUsername, o.registryPassword) } else { fingerprint, err = digest.DockerImageSha256(artifactName) } @@ -540,13 +418,10 @@ func ValidateAttestationArtifactArg(args []string, artifactType, inputSha256 str // remote digest. func ValidateRegistryFlags(cmd *cobra.Command, o *fingerprintOptions) error { if o.artifactType != "docker" && o.artifactType != "oci" && (o.registryPassword != "" || o.registryUsername != "") { - return ErrorBeforePrintingUsage(cmd, "--registry-provider, --registry-username and registry-password are only applicable when --artifact-type is 'docker'") - } - if o.registryProvider != "" && (o.registryPassword == "" || o.registryUsername == "") { - return ErrorBeforePrintingUsage(cmd, "both --registry-username and registry-password are required when --registry-provider is used") + return ErrorBeforePrintingUsage(cmd, "--registry-username and registry-password are only applicable when --artifact-type is 'docker' or 'oci'") } - if o.registryProvider == "" && o.artifactType != "oci" && (o.registryPassword != "" || o.registryUsername != "") { - return ErrorBeforePrintingUsage(cmd, "--registry-username and registry-password are only used when --registry-provider is used") + if (o.registryPassword == "" && o.registryUsername != "") || (o.registryPassword != "" && o.registryUsername == "") { + return ErrorBeforePrintingUsage(cmd, "--registry-username and registry-password must both be set") } return nil } diff --git a/cmd/kosli/cli_utils_test.go b/cmd/kosli/cli_utils_test.go index ad1d6e36d..8fc5a0598 100644 --- a/cmd/kosli/cli_utils_test.go +++ b/cmd/kosli/cli_utils_test.go @@ -561,45 +561,6 @@ func (suite *CliUtilsTestSuite) TestValidateArtifactArg() { } } -func (suite *CliUtilsTestSuite) TestGetRegistryEndpointForProvider() { - for _, t := range []struct { - name string - provider string - want *registryProviderEndpoints - expectError bool - }{ - { - name: "github provider returns expected endpoints", - provider: "github", - want: ®istryProviderEndpoints{ - mainApi: "https://ghcr.io/v2", - authApi: "https://ghcr.io", - service: "ghcr.io", - }, - }, - { - name: "dockerhub provider returns expected endpoints", - provider: "dockerhub", - want: ®istryProviderEndpoints{ - mainApi: "https://registry-1.docker.io/v2", - authApi: "https://auth.docker.io", - service: "registry.docker.io", - }, - }, - } { - suite.Run(t.name, func() { - endpoints, err := getRegistryEndpointForProvider(t.provider) - if t.expectError { - require.Errorf(suite.T(), err, "error was expected but got none") - } else { - require.NoErrorf(suite.T(), err, "error was NOT expected but got %v", err) - require.Equalf(suite.T(), t.want, endpoints, - "TestGetRegistryEndpointForProvider: got %v -- want %v", t.want, endpoints) - } - }) - } -} - func (suite *CliUtilsTestSuite) TestValidateRegistryFlags() { for _, t := range []struct { name string @@ -610,16 +571,14 @@ func (suite *CliUtilsTestSuite) TestValidateRegistryFlags() { name: "registry flags are valid", options: &fingerprintOptions{ artifactType: "docker", - registryProvider: "dockerhub", registryUsername: "user", registryPassword: "pass", }, }, { - name: "non-docker type with registry flags set casues an error", + name: "non-docker type with registry flags set causes an error", options: &fingerprintOptions{ artifactType: "file", - registryProvider: "dockerhub", registryUsername: "user", registryPassword: "pass", }, @@ -629,7 +588,6 @@ func (suite *CliUtilsTestSuite) TestValidateRegistryFlags() { name: "missing username causes an error", options: &fingerprintOptions{ artifactType: "docker", - registryProvider: "dockerhub", registryPassword: "pass", }, expectError: true, @@ -638,36 +596,10 @@ func (suite *CliUtilsTestSuite) TestValidateRegistryFlags() { name: "missing password causes an error", options: &fingerprintOptions{ artifactType: "docker", - registryProvider: "dockerhub", registryUsername: "user", }, expectError: true, }, - { - name: "missing provider causes an error 1", - options: &fingerprintOptions{ - artifactType: "docker", - registryUsername: "user", - registryPassword: "pass", - }, - expectError: true, - }, - { - name: "missing provider causes an error 2", - options: &fingerprintOptions{ - artifactType: "docker", - registryUsername: "user", - }, - expectError: true, - }, - { - name: "missing provider causes an error 3", - options: &fingerprintOptions{ - artifactType: "docker", - registryPassword: "pass", - }, - expectError: true, - }, } { suite.Run(t.name, func() { err := ValidateRegistryFlags(&cobra.Command{}, t.options) diff --git a/cmd/kosli/fingerprint.go b/cmd/kosli/fingerprint.go index 20fc52e53..6a0c4ff92 100644 --- a/cmd/kosli/fingerprint.go +++ b/cmd/kosli/fingerprint.go @@ -19,9 +19,10 @@ plus the ability to use recursive globs "**" const fingerprintLongDesc = fingerprintShortDesc + ` Requires ^--artifact-type^ flag to be set. -Artifact type can be one of: "file" for files, "dir" for directories, "docker" for docker images. +Artifact type can be one of: "file" for files, "dir" for directories, "oci" for container +images in registries or "docker" for local docker images. -Fingerprinting docker images can be done using the local docker daemon or the fingerprint can be fetched +Fingerprinting container images can be done using the local docker daemon or the fingerprint can be fetched from a remote registry. ` + fingerprintDirSynopsis @@ -36,8 +37,14 @@ kosli fingerprint --artifact-type dir mydir # fingerprint a dir while excluding paths kosli fingerprint --artifact-type dir --exclude logs --exclude *.exe mydir -# fingerprint a locally available docker image +# fingerprint a locally available docker image (requires docker daemon running) kosli fingerprint --artifact-type docker nginx:latest + +# fingerprint a public image from a remote registry +kosli fingerprint --artifact-type oci nginx:latest + +# fingerprint a private image from a remote registry +kosli fingerprint --artifact-type oci private:latest --registry-username YourUsername --registry-password YourPassword ` type fingerprintOptions struct { @@ -74,6 +81,7 @@ func newFingerprintCmd(out io.Writer) *cobra.Command { err = DeprecateFlags(cmd, map[string]string{ "e": "use -x instead", }) + if err != nil { logger.Error("failed to configure deprecated flags: %v", err) } diff --git a/cmd/kosli/flags.go b/cmd/kosli/flags.go index 08d6c8beb..95a0bf952 100644 --- a/cmd/kosli/flags.go +++ b/cmd/kosli/flags.go @@ -7,6 +7,7 @@ import ( ghUtils "github.com/kosli-dev/cli/internal/github" gitlabUtils "github.com/kosli-dev/cli/internal/gitlab" "github.com/spf13/cobra" + "log" ) // allowed commit redaction values @@ -22,6 +23,14 @@ func addFingerprintFlags(cmd *cobra.Command, o *fingerprintOptions) { cmd.Flags().StringVar(&o.registryUsername, "registry-username", "", registryUsernameFlag) cmd.Flags().StringVar(&o.registryPassword, "registry-password", "", registryPasswordFlag) cmd.Flags().StringSliceVarP(&o.excludePaths, "exclude", "x", []string{}, excludePathsFlag) + + err := DeprecateFlags(cmd, map[string]string{ + "registry-provider": "no longer used", + }) + + if err != nil { + log.Fatalf("failed to configure deprecated flags: %v", err) + } } func addAWSAuthFlags(cmd *cobra.Command, o *aws.AWSStaticCreds) { diff --git a/cmd/kosli/root.go b/cmd/kosli/root.go index e65f84edf..f666e72ce 100644 --- a/cmd/kosli/root.go +++ b/cmd/kosli/root.go @@ -40,7 +40,15 @@ const ( credentialsStoreKeySecretName = "kosli-encryption-key" // the following constants are used in the docs/help - fingerprintDesc = "The artifact SHA256 fingerprint is calculated (based on the ^--artifact-type^ flag and the artifact name/path argument) or can be provided directly (with the ^--fingerprint^ flag)." + fingerprintDesc = ` +The artifact fingerprint can be provided directly with the ^--fingerprint^ flag, or +calculated based on ^--artifact-type^ flag. + +Artifact type can be one of: "file" for files, "dir" for directories, "oci" for container +images in registries or "docker" for local docker images. + +` + attestationBindingDesc = ` The attestation can be bound to a trail using the trail name. @@ -82,20 +90,17 @@ The ^.kosli_ignore^ will be treated as part of the artifact like any other file, maxAPIRetryFlag = "[defaulted] How many times should API calls be retried when the API host is not reachable." configFileFlag = "[optional] The Kosli config file path." debugFlag = "[optional] Print debug logs to stdout. A boolean flag https://docs.kosli.com/faq/#boolean-flags (default false)" - artifactTypeFlag = "The type of the artifact to calculate its SHA256 fingerprint. One of: [docker, file, dir]. Only required if you want Kosli to calculate the fingerprint for you (i.e. when you don't specify '--fingerprint' on commands that allow it)." + artifactTypeFlag = "The type of the artifact to calculate its SHA256 fingerprint. One of: [oci, docker, file, dir]. Only required if you want Kosli to calculate the fingerprint for you (i.e. when you don't specify '--fingerprint' on commands that allow it)." flowNameFlag = "The Kosli flow name." trailNameFlag = "The Kosli trail name." trailNameFlagOptional = "[optional] The Kosli trail name." templateArtifactName = "The name of the artifact in the yml template file." flowNamesFlag = "[defaulted] The comma separated list of Kosli flows. Defaults to all flows of the org." - newFlowFlag = "The name of the flow to be created or updated." outputFlag = "[defaulted] The format of the output. Valid formats are: [table, json]." - pipefileFlag = "[deprecated] The path to the JSON pipefile." environmentNameFlag = "The environment name." approvalEnvironmentNameFlag = "[defaulted] The environment the artifact is approved for. (defaults to all environments)" pageNumberFlag = "[defaulted] The page number of a response." pageLimitFlag = "[defaulted] The number of elements per page." - newEnvNameFlag = "The name of environment to be created." newEnvTypeFlag = "The type of environment. Valid types are: [K8S, ECS, server, S3, lambda, docker, azure-apps, logical]." envAllowListFlag = "The environment name for which the artifact is allowlisted." reasonFlag = "The reason why this artifact is allowlisted." @@ -103,7 +108,6 @@ The ^.kosli_ignore^ will be treated as part of the artifact like any other file, newestCommitFlag = "[defaulted] The source commit sha for the newest change in the deployment. Can be any commit-ish." repoRootFlag = "[defaulted] The directory where the source git repository is available." approvalDescriptionFlag = "[optional] The approval description." - artifactDescriptionFlag = "[optional] The artifact description." deploymentDescriptionFlag = "[optional] The deployment description." evidenceDescriptionFlag = "[optional] The evidence description." jiraBaseUrlFlag = "The base url for the jira project, e.g. 'https://kosli.atlassian.net'" @@ -128,7 +132,6 @@ The ^.kosli_ignore^ will be treated as part of the artifact like any other file, buildUrlFlag = "The url of CI pipeline that built the artifact. (defaulted in some CIs: https://docs.kosli.com/ci-defaults )." commitUrlFlag = "The url for the git commit that created the artifact. (defaulted in some CIs: https://docs.kosli.com/ci-defaults )." evidenceCompliantFlag = "[defaulted] Whether the evidence is compliant or not. A boolean flag https://docs.kosli.com/faq/#boolean-flags" - evidenceTypeFlag = "The type of evidence being reported." bbUsernameFlag = "Bitbucket username." bbPasswordFlag = "Bitbucket App password. See https://developer.atlassian.com/cloud/bitbucket/rest/intro/#authentication for more details." bbWorkspaceFlag = "Bitbucket workspace ID." @@ -141,7 +144,6 @@ The ^.kosli_ignore^ will be treated as part of the artifact like any other file, azureTokenFlag = "Azure Personal Access token." azureProjectFlag = "Azure project.(defaulted if you are running in Azure Devops pipelines: https://docs.kosli.com/ci-defaults )." azureOrgUrlFlag = "Azure organization url. E.g. \"https://dev.azure.com/myOrg\" (defaulted if you are running in Azure Devops pipelines: https://docs.kosli.com/ci-defaults )." - azureBaseURLFlag = "[optional] Azure Devops base URL." azureClientIdFlag = "Azure client ID." azureClientSecretFlag = "Azure client secret." azureTenantIdFlag = "Azure tenant ID." @@ -154,9 +156,9 @@ The ^.kosli_ignore^ will be treated as part of the artifact like any other file, gitlabTokenFlag = "Gitlab token." gitlabOrgFlag = "Gitlab organization. (defaulted if you are running in Gitlab Pipelines: https://docs.kosli.com/ci-defaults )." gitlabBaseURLFlag = "[optional] Gitlab base URL (only needed for on-prem Gitlab installations)." - registryProviderFlag = "[conditional] The docker registry provider or url. Only required if you want to read docker image SHA256 digest from a remote docker registry." - registryUsernameFlag = "[conditional] The docker registry username. Only required if you want to read docker image SHA256 digest from a remote docker registry." - registryPasswordFlag = "[conditional] The docker registry password or access token. Only required if you want to read docker image SHA256 digest from a remote docker registry." + registryProviderFlag = "[deprecated] The docker registry provider or url. Only required if you want to read docker image SHA256 digest from a remote docker registry." + registryUsernameFlag = "[conditional] The container registry username. Only required if you want to read container image SHA256 digest from a remote container registry." + registryPasswordFlag = "[conditional] The container registry password or access token. Only required if you want to read container image SHA256 digest from a remote container registry." resultsDirFlag = "[defaulted] The path to a directory with JUnit test results. By default, the directory will be uploaded to Kosli's evidence vault." snykJsonResultsFileFlag = "The path to Snyk SARIF or JSON scan results file from 'snyk test' and 'snyk container test'. By default, the Snyk results will be uploaded to Kosli's evidence vault." snykSarifResultsFileFlag = "The path to Snyk scan SARIF results file from 'snyk test' and 'snyk container test'. By default, the Snyk results will be uploaded to Kosli's evidence vault." @@ -187,14 +189,12 @@ The ^.kosli_ignore^ will be treated as part of the artifact like any other file, excludePathsFlag = "[optional] The comma separated list of directories and files to exclude from fingerprinting. Can take glob patterns. Only applicable for --artifact-type dir." serverExcludePathsFlag = "[optional] The comma separated list of directories and files to exclude from fingerprinting. Can take glob patterns." shortFlag = "[optional] Print only the Kosli CLI version number." - longFlag = "[optional] Print detailed output." reverseFlag = "[defaulted] Reverse the order of output list." evidenceNameFlag = "The name of the evidence." evidenceFingerprintFlag = "[optional] The SHA256 fingerprint of the evidence file or dir." evidenceURLFlag = "[optional] The external URL where the evidence file or dir is stored." evidencePathsFlag = "[optional] The comma-separated list of paths containing supporting proof for the reported evidence. Paths can be for files or directories. All provided proofs will be uploaded to Kosli's evidence vault." fingerprintFlag = "[conditional] The SHA256 fingerprint of the artifact. Only required if you don't specify '--artifact-type'." - evidenceCommitFlag = "The git commit SHA1 for which the evidence belongs. (defaulted in some CIs: https://docs.kosli.com/ci-defaults )." intervalFlag = "[optional] Expression to define specified snapshots range." showUnchangedArtifactsFlag = "[defaulted] Show the unchanged artifacts present in both snapshots within the diff output." approverFlag = "[optional] The user approving an approval." diff --git a/cmd/kosli/testdata/output/docs/snyk.md b/cmd/kosli/testdata/output/docs/snyk.md index e5dc43a1f..bc1f614f7 100644 --- a/cmd/kosli/testdata/output/docs/snyk.md +++ b/cmd/kosli/testdata/output/docs/snyk.md @@ -37,7 +37,7 @@ snyk [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags] | Flag | Description | | :--- | :--- | | --annotate stringToString | [optional] Annotate the attestation with data using key=value. | -| -t, --artifact-type string | The type of the artifact to calculate its SHA256 fingerprint. One of: [docker, file, dir]. Only required if you want Kosli to calculate the fingerprint for you (i.e. when you don't specify '--fingerprint' on commands that allow it). | +| -t, --artifact-type string | The type of the artifact to calculate its SHA256 fingerprint. One of: [oci, docker, file, dir]. Only required if you want Kosli to calculate the fingerprint for you (i.e. when you don't specify '--fingerprint' on commands that allow it). | | --attachments strings | [optional] The comma-separated list of paths of attachments for the reported attestation. Attachments can be files or directories. All attachments are compressed and uploaded to Kosli's evidence vault. | | -g, --commit string | [conditional] The git commit for which the attestation is associated to. Becomes required when reporting an attestation for an artifact before reporting it to Kosli. (defaulted in some CIs: https://docs.kosli.com/ci-defaults ). | | --description string | [optional] attestation description | @@ -51,9 +51,8 @@ snyk [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags] | -n, --name string | The name of the attestation as declared in the flow or trail yaml template. | | -o, --origin-url string | [optional] The url pointing to where the attestation came from or is related. (defaulted to the CI url in some CIs: https://docs.kosli.com/ci-defaults ). | | --redact-commit-info strings | [optional] The list of commit info to be redacted before sending to Kosli. Allowed values are one or more of [author, message, branch]. | -| --registry-password string | [conditional] The docker registry password or access token. Only required if you want to read docker image SHA256 digest from a remote docker registry. | -| --registry-provider string | [conditional] The docker registry provider or url. Only required if you want to read docker image SHA256 digest from a remote docker registry. | -| --registry-username string | [conditional] The docker registry username. Only required if you want to read docker image SHA256 digest from a remote docker registry. | +| --registry-password string | [conditional] The container registry password or access token. Only required if you want to read container image SHA256 digest from a remote container registry. | +| --registry-username string | [conditional] The container registry username. Only required if you want to read container image SHA256 digest from a remote container registry. | | --repo-root string | [defaulted] The directory where the source git repository is available. Only used if --commit is used. (default ".") | | -R, --scan-results string | The path to Snyk scan SARIF results file from 'snyk test' and 'snyk container test'. By default, the Snyk results will be uploaded to Kosli's evidence vault. | | -T, --trail string | The Kosli trail name. |