From 2db6e56eae2c3f1984b2fa50ab444d84fad59ffc Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Fri, 18 Mar 2022 18:07:35 +1100 Subject: [PATCH 01/20] Initial pass at the terrform promotion script. Still a WIP. --- Makefile | 9 +- tooling/cmd/promote-terraform/args.go | 112 ++++++++ tooling/cmd/promote-terraform/main.go | 252 ++++++++++++++++++ tooling/go.mod | 45 ++++ tooling/go.sum | 203 ++++++++++++++ tooling/internal/filename/parse.go | 40 +++ tooling/internal/filename/parse_test.go | 39 +++ tooling/internal/staging/staging.go | 127 +++++++++ tooling/internal/staging/staging_test.go | 76 ++++++ .../internal/terraform/registry/archives.go | 62 +++++ .../terraform/registry/archives_test.go | 7 + .../internal/terraform/registry/protocol.go | 160 +++++++++++ .../terraform/registry/protocol_test.go | 158 +++++++++++ tooling/internal/terraform/registry/sha.go | 18 ++ tooling/internal/terraform/registry/sign.go | 147 ++++++++++ 15 files changed, 1453 insertions(+), 2 deletions(-) create mode 100644 tooling/cmd/promote-terraform/args.go create mode 100644 tooling/cmd/promote-terraform/main.go create mode 100644 tooling/go.mod create mode 100644 tooling/go.sum create mode 100644 tooling/internal/filename/parse.go create mode 100644 tooling/internal/filename/parse_test.go create mode 100644 tooling/internal/staging/staging.go create mode 100644 tooling/internal/staging/staging_test.go create mode 100644 tooling/internal/terraform/registry/archives.go create mode 100644 tooling/internal/terraform/registry/archives_test.go create mode 100644 tooling/internal/terraform/registry/protocol.go create mode 100644 tooling/internal/terraform/registry/protocol_test.go create mode 100644 tooling/internal/terraform/registry/sha.go create mode 100644 tooling/internal/terraform/registry/sign.go diff --git a/Makefile b/Makefile index f8c5e938e..7d037ab2d 100644 --- a/Makefile +++ b/Makefile @@ -55,9 +55,14 @@ event-handler: # Run all tests .PHONY: test -test: +test: test-tooling @echo Testing plugins against Teleport $(TELEPORT_GET_VERSION) - go test -race -count 1 ./... + go test -race -count 1 $(shell go list ./...) + + +.PHONY: test-tooling +test-tooling: + (cd tooling; go test -v -race ./...) # Individual releases .PHONY: release/access-slack diff --git a/tooling/cmd/promote-terraform/args.go b/tooling/cmd/promote-terraform/args.go new file mode 100644 index 000000000..9c22fdb48 --- /dev/null +++ b/tooling/cmd/promote-terraform/args.go @@ -0,0 +1,112 @@ +package main + +import ( + "os" + "strings" + + "github.com/gravitational/kingpin" +) + +type bucketConfig struct { + region string + bucketName string + accessKeyID string + secretAccessKey string +} + +type args struct { + providerTag string + workingDir string + registryURL string + staging bucketConfig + production bucketConfig + signingKeyText string + protocolVersions []string + providerNamespace string + providerName string +} + +func parseCommandLine() *args { + app := kingpin.New("update-registry", "Adds files to a terraform registry") + result := &args{} + + app.Flag("tag", "The version tag identifying version of provider to promote"). + Required(). + StringVar(&result.providerTag) + + app.Flag("staging-bucket", "S3 Staging bucket url (where to fetch tarballs for promotion)"). + Envar("STAGING_BUCKET"). + Required(). + StringVar(&result.staging.bucketName) + + app.Flag("staging-region", "AWS region the staging bucket is in"). + Default("us-west-2"). + StringVar(&result.staging.region) + + app.Flag("staging-access-key-id", "AWS access key id for staging bucket"). + Envar("STAGING_ACCESS_KEY_ID"). + Required(). + StringVar(&result.staging.accessKeyID) + + app.Flag("staging-secret-access-key", "AWS secret access key for staging bucket"). + Envar("STAGING_SECRET_ACCESS_KEY"). + Required(). + StringVar(&result.staging.secretAccessKey) + + app.Flag("prod-bucket", "S3 production bucket url (where to push the resulting registry)"). + Envar("PROD_BUCKET"). + StringVar(&result.production.bucketName) + + app.Flag("prod-region", "AWS region the production bucket is in"). + Default("us-east-1"). + StringVar(&result.production.region) + + app.Flag("prod-access-key-id", "AWS access key id for production bucket"). + Envar("PROD_ACCESS_KEY_ID"). + Required(). + StringVar(&result.production.accessKeyID) + + app.Flag("prod-secret-access-key", "AWS secret access key for production bucket"). + Envar("PROD_SECRET_ACCESS_KEY"). + Required(). + StringVar(&result.production.secretAccessKey) + + app.Flag("working-dir", "Working directory to store generated files"). + Short('d'). + Default("./workspace"). + StringVar(&result.workingDir) + + app.Flag("signing-key", "GPG signing key in ASCII armor format"). + Short('k'). + Envar("SIGNING_KEY"). + StringVar(&result.signingKeyText) + + app.Flag("protocol", "Terraform protocol supported by files"). + Short('p'). + Default("4.0", "5.1"). + StringsVar(&result.protocolVersions) + + app.Flag("registry-url", "Address where registry objects will be served."). + Default("https://terraform.releases.teleport.dev/"). + StringVar(&result.registryURL) + + app.Flag("namespace", "Terraform provider namespace"). + Default("gravitational"). + StringVar(&result.providerNamespace) + + app.Flag("name", "Terraform provider name"). + Default("teleport"). + StringVar(&result.providerName) + + kingpin.MustParse(app.Parse(os.Args[1:])) + + // Marshal the arguments into a canonical format here, so we don't have to + // second guess the format later on when we're in the thick of doing the + // actual work... + + if !strings.HasSuffix(result.registryURL, "/") { + result.registryURL = result.registryURL + "/" + } + + return result +} diff --git a/tooling/cmd/promote-terraform/main.go b/tooling/cmd/promote-terraform/main.go new file mode 100644 index 000000000..361d9bf17 --- /dev/null +++ b/tooling/cmd/promote-terraform/main.go @@ -0,0 +1,252 @@ +package main + +import ( + "context" + "errors" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/armor" + "github.com/ProtonMail/go-crypto/openpgp/packet" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go/aws" + "github.com/gravitational/teleport-plugins/tooling/internal/staging" + "github.com/gravitational/teleport-plugins/tooling/internal/terraform/registry" + "github.com/gravitational/trace" + + log "github.com/sirupsen/logrus" +) + +const ( + objectStoreKeyPrefix = "store" + registryKeyPrefix = "registry" +) + +func main() { + args := parseCommandLine() + log.Infof("Version tag is %s\n", args.providerTag) + + workspace, err := ensureWorkspaceExists(args.workingDir) + if err != nil { + log.WithError(err).Fatalf("Failed setting up workspace") + } + + log.StandardLogger().SetLevel(log.DebugLevel) + + signingKey, err := loadSigningKey(args.signingKeyText) + if err != nil { + log.WithError(err).Fatalf("Failed decosing signing key") + } + + files, err := downloadStagedArtifacts(context.Background(), args.providerTag, workspace.stagingDir, &args.staging) + if err != nil { + log.WithError(err).Fatalf("Failed fetching artifacts") + } + + objectStoreUrl := args.registryURL + "store/" + + versionRecord := registry.Version{ + Protocols: args.protocolVersions, + } + + for _, fn := range files { + if !registry.IsProviderTarball(fn) { + continue + } + + log.Infof("Found provider tarball %s", fn) + + registryInfo, err := registry.RepackProvider(workspace.objectStoreDir, fn, signingKey) + if err != nil { + log.WithError(err).Fatalf("Failed repacking provider") + } + + log.Infof("Provider repacked to %s/%s", workspace.objectStoreDir, registryInfo.Zip) + + if versionRecord.Version == "" { + versionRecord.Version = registryInfo.Version + } else if versionRecord.Version != registryInfo.Version { + log.Fatalf("Version mismatch. Expected %s, got %s", versionRecord.Version, registryInfo.Version) + } + + downloadInfo, err := registry.NewDownloadFromRepackResult(registryInfo, args.protocolVersions, objectStoreUrl) + if err != nil { + log.WithError(err).Fatalf("Failed creating download info record") + } + + err = downloadInfo.Save(workspace.registryDir, args.providerNamespace, args.providerName, registryInfo.Version) + if err != nil { + log.WithError(err).Fatalf("Failed saving download info record") + } + + versionRecord.Platforms = append(versionRecord.Platforms, registry.Platform{ + OS: registryInfo.OS, + Arch: registryInfo.Arch, + }) + } + + err = updateRegistry(context.Background(), &args.production, workspace.registryDir, args.providerNamespace, args.providerName, &versionRecord) + if err != nil { + log.WithError(err).Fatal("Failed updating registry") + } +} + +func updateRegistry(ctx context.Context, prodBucket *bucketConfig, registryDir, namespace, provider string, newVersion *registry.Version) error { + s3client, err := newS3ClientFromBucketConfig(ctx, prodBucket) + if err != nil { + return trace.Wrap(err) + } + + versionsFileKey, versionsFilePath := makeVersionsFilePaths(registryDir, namespace, provider) + log.Infof("Downloading version index for %s/%s from %s", namespace, provider, versionsFileKey) + + // Try downloading the versions file. This may not exist in an empty/new + // registry, so if the download fails with a NotFound error we ignore it + // and use an empty index. + versions := registry.NewVersions() + err = download(ctx, s3client, versionsFilePath, prodBucket.bucketName, versionsFileKey) + switch { + case err == nil: + versions, err = registry.LoadVersionsFile(versionsFilePath) + if err != nil { + return trace.Wrap(err) + } + + case isNotFound(err): + log.Info("No index found. Using empty index.") + + default: + return trace.Wrap(err, "failed downloading index") + } + + versions.Versions = append(versions.Versions, newVersion) + + if err = versions.Save(versionsFilePath); err != nil { + trace.Wrap(err, "failed saving index file") + } + + return nil +} + +func download(ctx context.Context, client *s3.Client, dstFileName, bucket, key string) error { + + err := os.MkdirAll(filepath.Dir(dstFileName), 0700) + if err != nil { + return trace.Wrap(err, "failed creating destination dir for download") + } + + dst, err := os.Create(dstFileName) + if err != nil { + return trace.Wrap(err, "failed creating destination file download") + } + defer dst.Close() + + downloader := manager.NewDownloader(client) + + _, err = downloader.Download(ctx, dst, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + + return trace.Wrap(err, "failed downloading object") +} + +func isNotFound(err error) bool { + var responseError *awshttp.ResponseError + if !errors.As(err, &responseError) { + return false + } + return responseError.ResponseError.HTTPStatusCode() == http.StatusNotFound +} + +func makeVersionsFilePaths(registryDir, namespace, provider string) (string, string) { + key := registry.VersionsFilePath(registryKeyPrefix, namespace, provider) + path := registry.VersionsFilePath(registryDir, namespace, provider) + return key, path +} + +type workspacePaths struct { + stagingDir string + productionDir string + registryDir string + objectStoreDir string +} + +func ensureWorkspaceExists(workspaceDir string) (*workspacePaths, error) { + // Ensure that the working dir and so on exist + stagingDir := filepath.Join(workspaceDir, "staging") + err := os.MkdirAll(stagingDir, 0700) + if err != nil { + return nil, trace.Wrap(err, "failed ensuring staging dir %s exists", stagingDir) + } + + productionDir := filepath.Join(workspaceDir, "production") + + registryDir := filepath.Join(productionDir, "registry") + err = os.MkdirAll(registryDir, 0700) + if err != nil { + return nil, trace.Wrap(err, "failed ensuring registry output dir %s exists", registryDir) + } + + objectStoreDir := filepath.Join(productionDir, "store") + err = os.MkdirAll(objectStoreDir, 0700) + if err != nil { + return nil, trace.Wrap(err, "failed ensuring registry output dir %s exists", objectStoreDir) + } + + return &workspacePaths{ + stagingDir: stagingDir, + productionDir: productionDir, + registryDir: registryDir, + objectStoreDir: objectStoreDir, + }, nil +} + +func downloadStagedArtifacts(ctx context.Context, tag string, dstDir string, stagingBucket *bucketConfig) ([]string, error) { + + client, err := newS3ClientFromBucketConfig(ctx, stagingBucket) + if err != nil { + return nil, trace.Wrap(err) + } + + return staging.FetchByTag(ctx, client, dstDir, stagingBucket.bucketName, tag) +} + +func newS3ClientFromBucketConfig(ctx context.Context, bucket *bucketConfig) (*s3.Client, error) { + creds := credentials.NewStaticCredentialsProvider( + bucket.accessKeyID, bucket.secretAccessKey, "") + + cfg, err := config.LoadDefaultConfig(ctx, + config.WithRegion(bucket.region), + config.WithCredentialsProvider(creds)) + if err != nil { + return nil, trace.Wrap(err) + } + + return s3.NewFromConfig(cfg), nil +} + +func loadSigningKey(keyText string) (*openpgp.Entity, error) { + log.Info("Decoding signing key") + + strings.NewReader(keyText) + + block, err := armor.Decode(strings.NewReader(keyText)) + if err != nil { + return nil, trace.Wrap(err) + } + + entity, err := openpgp.ReadEntity(packet.NewReader(block.Body)) + if err != nil { + return nil, trace.Wrap(err, "failed loading entity from private key") + } + + return entity, nil +} diff --git a/tooling/go.mod b/tooling/go.mod new file mode 100644 index 000000000..d733ebb8f --- /dev/null +++ b/tooling/go.mod @@ -0,0 +1,45 @@ +module github.com/gravitational/teleport-plugins/tooling + +go 1.17 + +require ( + github.com/ProtonMail/go-crypto v0.0.0-20220113124808-70ae35bab23f + github.com/aws/aws-sdk-go v1.43.19 + github.com/aws/aws-sdk-go-v2/config v1.15.0 + github.com/aws/aws-sdk-go-v2/credentials v1.10.0 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.26.0 + github.com/gravitational/kingpin v2.1.10+incompatible + github.com/gravitational/trace v1.1.18 + github.com/sirupsen/logrus v1.8.1 + github.com/stretchr/testify v1.7.1 +) + +require ( + github.com/alecthomas/assert v1.0.0 // indirect + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/aws/aws-sdk-go-v2 v1.15.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.16.0 // indirect + github.com/aws/smithy-go v1.11.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/jonboulle/clockwork v0.2.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/objx v0.1.0 // indirect + golang.org/x/crypto v0.0.0-20220126234351-aa10faf2a1f8 // indirect + golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect + golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect + golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect + gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect +) diff --git a/tooling/go.sum b/tooling/go.sum new file mode 100644 index 000000000..088ffd1e0 --- /dev/null +++ b/tooling/go.sum @@ -0,0 +1,203 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/ProtonMail/go-crypto v0.0.0-20220113124808-70ae35bab23f h1:J2FzIrXN82q5uyUraeJpLIm7U6PffRwje2ORho5yIik= +github.com/ProtonMail/go-crypto v0.0.0-20220113124808-70ae35bab23f/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= +github.com/alecthomas/assert v1.0.0 h1:3XmGh/PSuLzDbK3W2gUbRXwgW5lqPkuqvRgeQ30FI5o= +github.com/alecthomas/assert v1.0.0/go.mod h1:va/d2JC+M7F6s+80kl/R3G7FUiW6JzUO+hPhLyJ36ZY= +github.com/alecthomas/colour v0.1.0 h1:nOE9rJm6dsZ66RGWYSFrXw461ZIt9A6+nHgL7FRrDUk= +github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/aws/aws-sdk-go v1.43.19 h1:n7YAreaCpcstusW7F0+XiocZxh7rwmcAPO4HTEPJ6mE= +github.com/aws/aws-sdk-go v1.43.19/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go-v2 v1.15.0 h1:f9kWLNfyCzCB43eupDAk3/XgJ2EpgktiySD6leqs0js= +github.com/aws/aws-sdk-go-v2 v1.15.0/go.mod h1:lJYcuZZEHWNIb6ugJjbQY1fykdoobWbOS7kJYb4APoI= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.0 h1:J/tiyHbl07LL4/1i0rFrW5pbLMvo7M6JrekBUNpLeT4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.0/go.mod h1:ohZjRmiToJ4NybwWTGOCbzlUQU8dxSHxYKzuX7k5l6Y= +github.com/aws/aws-sdk-go-v2/config v1.15.0 h1:cibCYF2c2uq0lsbu0Ggbg8RuGeiHCmXwUlTMS77CiK4= +github.com/aws/aws-sdk-go-v2/config v1.15.0/go.mod h1:NccaLq2Z9doMmeQXHQRrt2rm+2FbkrcPvfdbCaQn5hY= +github.com/aws/aws-sdk-go-v2/credentials v1.10.0 h1:M/FFpf2w31F7xqJqJLgiM0mFpLOtBvwZggORr6QCpo8= +github.com/aws/aws-sdk-go-v2/credentials v1.10.0/go.mod h1:HWJMr4ut5X+Lt/7epc7I6Llg5QIcoFHKAeIzw32t6EE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.0 h1:gUlb+I7NwDtqJUIRcFYDiheYa97PdVHG/5Iz+SwdoHE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.0/go.mod h1:prX26x9rmLwkEE1VVCelQOQgRN9sOVIssgowIJ270SE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.0 h1:G/5sApTwgC9qCw1TTtrVsZyZjgNIvo0rl9jjGEICcoY= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.0/go.mod h1:1vV+vjdjBD9ZzATKf7rlze/RwvjvluywiMzY12sNGo4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.6 h1:xiGjGVQsem2cxoIX61uRGy+Jux2s9C/kKbTrWLdrU54= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.6/go.mod h1:SSPEdf9spsFgJyhjrXvawfpyzrXHBCUe+2eQ1CjC1Ak= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.0 h1:bt3zw79tm209glISdMRCIVRCwvSDXxgAxh5KWe2qHkY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.0/go.mod h1:viTrxhAuejD+LszDahzAE2x40YjYWhMqzHxv2ZiWaME= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.7 h1:QOMEP8jnO8sm0SX/4G7dbaIq2eEP2wcWEsF0jzrXLJc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.7/go.mod h1:P5sjYYf2nc5dE6cZIzEMsVtq6XeLD7c4rM+kQJPrByA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.0 h1:uhb7moM7VjqIEpWzTpCvceLDSwrWpaleXm39OnVjuLE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.0/go.mod h1:pA2St3Pu2Ldy6fBPY45Azoh1WBG4oS7eIKOd4XN7Meg= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.0 h1:IhiVUezzcKlszx6wXSDQYDjEn/bIO6Mc73uNQ1YfTmA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.0/go.mod h1:kLKc4lo+XKlMhENIpKbp7dCePpyUqUG1PqGIAXoxwNE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.0 h1:YQ3fTXACo7xeAqg0NiqcCmBOXJruUfh+4+O2qxF2EjQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.0/go.mod h1:R31ot6BgESRCIoxwfKtIHzZMo/vsZn2un81g9BJ4nmo= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.0 h1:i+7ve93k5G0S2xWBu60CKtmzU5RjBj9g7fcSypQNLR0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.0/go.mod h1:L8EoTDLnnN2zL7MQPhyfCbmiZqEs8Cw7+1d9RlLXT5s= +github.com/aws/aws-sdk-go-v2/service/s3 v1.26.0 h1:6IdBZVY8zod9umkwWrtbH2opcM00eKEmIfZKGUg5ywI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.26.0/go.mod h1:WJzrjAFxq82Hl42oh8HuvwpugTgxmoiJBBX8SLwVs74= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.0 h1:gZLEXLH6NiU8Y52nRhK1jA+9oz7LZzBK242fi/ziXa4= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.0/go.mod h1:d1WcT0OjggjQCAdOkph8ijkr5sUwk1IH/VenOn7W1PU= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.0 h1:0+X/rJ2+DTBKWbUsn7WtF0JvNk/fRf928vkFsXkbbZs= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.0/go.mod h1:+8k4H2ASUZZXmjx/s3DFLo9tGBb44lkz3XcgfypJY7s= +github.com/aws/smithy-go v1.11.1 h1:IQ+lPZVkSM3FRtyaDox41R8YS6iwPMYIreejOgPW49g= +github.com/aws/smithy-go v1.11.1/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gravitational/kingpin v2.1.10+incompatible h1:QCidzEf5zBD+IGoqytoLP10XizbqcdecKyIjhHXeVDw= +github.com/gravitational/kingpin v2.1.10+incompatible/go.mod h1:LWxG30M3FcrjhOn3T4zz7JmBoQJ45MWZmOXgy9Ganoc= +github.com/gravitational/trace v1.1.18 h1:Ulobib6xd5g1ct+ZC01HPAEvODws7QerjuTY9L4U8pY= +github.com/gravitational/trace v1.1.18/go.mod h1:n0ijrq6psJY0sOI/NzLp+xdd8xl79jjwzVOFHDY6+kQ= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20220126234351-aa10faf2a1f8 h1:kACShD3qhmr/3rLmg1yXyt+N4HcwutKyPRB93s54TIU= +golang.org/x/crypto v0.0.0-20220126234351-aa10faf2a1f8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/tooling/internal/filename/parse.go b/tooling/internal/filename/parse.go new file mode 100644 index 000000000..f3e9452a3 --- /dev/null +++ b/tooling/internal/filename/parse.go @@ -0,0 +1,40 @@ +package filename + +import ( + "fmt" + "path/filepath" + "regexp" + + "github.com/gravitational/trace" +) + +var ( + filenamePattern *regexp.Regexp = regexp.MustCompile(`^(?P.*)-teleport-v(?P.*)-(?Plinux|darwin|windows)-(?Pamd64|arm|aarch)-bin.tar.gz$`) +) + +type Info struct { + Type string + Version string + OS string + Arch string +} + +func Parse(filename string) (Info, error) { + filename = filepath.Base(filename) + + matches := filenamePattern.FindStringSubmatch(filename) + if len(matches) == 0 { + return Info{}, trace.Errorf("Filename %q does not match required pattern", filename) + } + + return Info{ + Type: matches[1], + Version: matches[2], + OS: matches[3], + Arch: matches[4], + }, nil +} + +func (info *Info) Filename(extension string) string { + return fmt.Sprintf("%s-teleport-v%s-%s-%s-bin%s", info.Type, info.Version, info.OS, info.Arch, extension) +} diff --git a/tooling/internal/filename/parse_test.go b/tooling/internal/filename/parse_test.go new file mode 100644 index 000000000..2707e4656 --- /dev/null +++ b/tooling/internal/filename/parse_test.go @@ -0,0 +1,39 @@ +package filename + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseFilename(t *testing.T) { + t.Run("WithLeadingPath", func(t *testing.T) { + info, err := Parse("/some/bath/to/file/terraform-provider-teleport-v7.0.0-darwin-amd64-bin.tar.gz") + require.NoError(t, err) + + require.Equal(t, "terraform-provider", info.Type) + require.Equal(t, "7.0.0", info.Version) + require.Equal(t, "darwin", info.OS) + require.Equal(t, "amd64", info.Arch) + }) + + t.Run("WithoutLeadingPath", func(t *testing.T) { + info, err := Parse("terraform-provider-teleport-v1.2.3-linux-arm-bin.tar.gz") + require.NoError(t, err) + require.Equal(t, "terraform-provider", info.Type) + require.Equal(t, "1.2.3", info.Version) + require.Equal(t, "linux", info.OS) + require.Equal(t, "arm", info.Arch) + }) + + t.Run("Random Junk", func(t *testing.T) { + _, err := Parse("blahblahblah") + require.Error(t, err) + }) +} + +func TestGenerateFilename(t *testing.T) { + info := Info{Type: "some-plugin", Version: "1.2.3", OS: "darwin", Arch: "amd64"} + fn := info.Filename(".banana") + require.Equal(t, "some-plugin-teleport-v1.2.3-darwin-amd64-bin.banana", fn) +} diff --git a/tooling/internal/staging/staging.go b/tooling/internal/staging/staging.go new file mode 100644 index 000000000..60ea76af7 --- /dev/null +++ b/tooling/internal/staging/staging.go @@ -0,0 +1,127 @@ +// Package staging contains tools for manipulating the plugin artifacts +// stored in the staging bucket. +package staging + +import ( + "context" + "io" + "os" + "path" + "path/filepath" + "strings" + + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go/aws" + "github.com/gravitational/trace" + log "github.com/sirupsen/logrus" +) + +const ( + keyPrefix = "teleport-plugins/tag/" +) + +// FetchByTag pulls all of the staging artifacts out of the staging storage +// bucket based on the tag naming scheme +func FetchByTag(ctx context.Context, client *s3.Client, dstDir string, bucket, tag string) ([]string, error) { + taggedPrefix := tagPrefix(tag) + + objectKeys, err := listKeysWithPrefix(ctx, client, bucket, taggedPrefix) + if err != nil { + return nil, trace.Wrap(err) + } + + if len(objectKeys) == 0 { + return nil, nil + } + + downloader := manager.NewDownloader(client) + result := make([]string, 0, len(objectKeys)) + + for _, key := range objectKeys { + filename, err := fetchObject(ctx, downloader, dstDir, bucket, key, taggedPrefix) + if err != nil { + return nil, trace.Wrap(err) + } + result = append(result, filename) + } + + return result, nil +} + +type downloader interface { + Download(ctx context.Context, w io.WriterAt, input *s3.GetObjectInput, options ...func(*manager.Downloader)) (n int64, err error) +} + +func tagPrefix(tag string) string { + return path.Join(keyPrefix, tag) +} + +func fetchObject(ctx context.Context, client downloader, dstDirRoot string, bucket, key string, keyPrefix string) (string, error) { + dstFilename, err := filenameForKey(key, keyPrefix) + if err != nil { + return "", trace.Wrap(err) + } + dstFilename = filepath.Join(dstDirRoot, dstFilename) + + log.Infof("Fetching %s", key) + log.Debugf("into %s", dstFilename) + dstDir := filepath.Dir(dstFilename) + err = os.MkdirAll(dstDir, 0700) + if err != nil { + return "", trace.Wrap(err, "failed ensuring dst dir %q exists", dstDir) + } + + dstFile, err := os.Create(dstFilename) + if err != nil { + return "", trace.Wrap(err, "failed creating destination file") + } + defer dstFile.Close() + + _, err = client.Download(ctx, dstFile, &s3.GetObjectInput{ + Key: aws.String(key), + Bucket: aws.String(bucket), + }) + + if err != nil { + return "", trace.Wrap(err, "download failed") + } + + return dstFilename, nil +} + +func filenameForKey(key, pfx string) (string, error) { + if !strings.HasPrefix(key, pfx) { + return "", trace.Errorf("Key missing staging prefix: %q", key) + } + return key[len(pfx):], nil +} + +func listKeysWithPrefix(ctx context.Context, client *s3.Client, bucket, pfx string) ([]string, error) { + log.Infof("Listing objects in %s with key prefix %s", bucket, pfx) + objectKeys := []string{} + + input := &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + Prefix: aws.String(pfx), + } + + for { + output, err := client.ListObjectsV2(ctx, input) + if err != nil { + return nil, err + } + + for _, object := range output.Contents { + objectKeys = append(objectKeys, *object.Key) + } + + if output.ContinuationToken == nil { + break + } + + input.ContinuationToken = output.ContinuationToken + } + + return objectKeys, nil +} diff --git a/tooling/internal/staging/staging_test.go b/tooling/internal/staging/staging_test.go new file mode 100644 index 000000000..6b1ac4a80 --- /dev/null +++ b/tooling/internal/staging/staging_test.go @@ -0,0 +1,76 @@ +package staging + +import ( + "context" + "io" + "path/filepath" + "testing" + + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +type mockDownloader struct { + mock.Mock +} + +func (m *mockDownloader) Download(ctx context.Context, w io.WriterAt, input *s3.GetObjectInput, options ...func(*manager.Downloader)) (n int64, err error) { + result := m.Called(ctx, w, input, options) + return int64(result.Int(0)), result.Error(1) +} + +func TestNonNestedFetch(t *testing.T) { + const ( + bucket = "some-random-bucket" + prefix = "teleport-plugins/tag/" + key = prefix + "toplevel-key" + ) + + tmpDir := t.TempDir() + + dl := mockDownloader{} + dl.On("Download", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + input := args.Get(2).(*s3.GetObjectInput) + require.Equal(t, bucket, *input.Bucket) + require.Equal(t, key, *input.Key) + }). + Return(0, nil) + + dstfile, err := fetchObject(context.Background(), &dl, tmpDir, bucket, key, prefix) + dl.AssertCalled(t, "Download", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + require.NoError(t, err) + + expected := filepath.Join(tmpDir, "toplevel-key") + require.Equal(t, expected, dstfile) + require.FileExists(t, expected) +} + +func TestNestedFetch(t *testing.T) { + const ( + bucket = "some-random-bucket" + prefix = "teleport-plugins/tag/some/" + key = prefix + "nested/object/key" + ) + + tmpDir := t.TempDir() + + dl := mockDownloader{} + dl.On("Download", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + input := args.Get(2).(*s3.GetObjectInput) + require.Equal(t, bucket, *input.Bucket) + require.Equal(t, key, *input.Key) + }). + Return(0, nil) + + dstfile, err := fetchObject(context.Background(), &dl, tmpDir, bucket, key, prefix) + dl.AssertCalled(t, "Download", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + require.NoError(t, err) + + expected := filepath.Join(tmpDir, "nested", "object", "key") + require.Equal(t, expected, dstfile) + require.FileExists(t, expected) +} diff --git a/tooling/internal/terraform/registry/archives.go b/tooling/internal/terraform/registry/archives.go new file mode 100644 index 000000000..00b672621 --- /dev/null +++ b/tooling/internal/terraform/registry/archives.go @@ -0,0 +1,62 @@ +package registry + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "io" + + "github.com/gravitational/trace" +) + +func repack(dst io.Writer, src io.Reader) error { + uncompressedReader, err := gzip.NewReader(src) + if err != nil { + return trace.Wrap(err) + } + tarReader := tar.NewReader(uncompressedReader) + + zipWriter := zip.NewWriter(dst) + defer zipWriter.Close() + + for { + header, err := tarReader.Next() + if err != nil { + if err == io.EOF { + break + } + return trace.Wrap(err) + } + + if header.Typeflag == tar.TypeReg { + if err = copyfile(zipWriter, header, tarReader); err != nil { + return trace.Wrap(err, "failed repacking tar file item") + } + } + } + + return nil +} + +// copyfile copies a file from a tar archive into a zip archive, preserving +// the file attributes as far as possible. +func copyfile(zipfile *zip.Writer, header *tar.Header, src io.Reader) error { + zipHeader, err := zip.FileInfoHeader(header.FileInfo()) + if err != nil { + return trace.Wrap(err, "failed initialising zipfile header") + } + zipHeader.Name = header.Name + zipHeader.Method = zip.Deflate + + dst, err := zipfile.CreateHeader(zipHeader) + if err != nil { + return trace.Wrap(err, "failed writing zipfile header") + } + + _, err = io.Copy(dst, src) + if err != nil { + return trace.Wrap(err, "failed adding data to zipfile") + } + + return nil +} diff --git a/tooling/internal/terraform/registry/archives_test.go b/tooling/internal/terraform/registry/archives_test.go new file mode 100644 index 000000000..41519954f --- /dev/null +++ b/tooling/internal/terraform/registry/archives_test.go @@ -0,0 +1,7 @@ +package registry + +import "testing" + +func TestRepack(t *testing.T) { + +} diff --git a/tooling/internal/terraform/registry/protocol.go b/tooling/internal/terraform/registry/protocol.go new file mode 100644 index 000000000..02981f8cc --- /dev/null +++ b/tooling/internal/terraform/registry/protocol.go @@ -0,0 +1,160 @@ +package registry + +import ( + "bytes" + "encoding/json" + "os" + "path" + "path/filepath" + + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/gravitational/trace" + "golang.org/x/crypto/openpgp/armor" +) + +const ( + versionsFilename = "versions" +) + +func VersionsFilePath(prefix, namespace, name string) string { + return path.Join(prefix, namespace, name, versionsFilename) +} + +// Versions is a Go representation of the Terraform Registry Protocol +// `versions` file +type Versions struct { + Versions []*Version `json:"versions"` +} + +func NewVersions() Versions { + return Versions{Versions: []*Version{}} +} + +func (idx *Versions) Save(filename string) error { + indexFile, err := os.Create(filename) + if err != nil { + return trace.Wrap(err) + } + defer indexFile.Close() + + encoder := json.NewEncoder(indexFile) + encoder.SetIndent("", " ") + + return encoder.Encode(idx) +} + +func LoadVersionsFile(filename string) (Versions, error) { + f, err := os.Open(filename) + if err != nil { + return Versions{}, trace.Wrap(err, "failed opening versions file") + } + defer f.Close() + + decoder := json.NewDecoder(f) + var idx Versions + if err = decoder.Decode(&idx); err != nil { + return Versions{}, trace.Wrap(err, "failed decoding versions file") + } + + return idx, nil +} + +type Platform struct { + OS string `json:"os"` + Arch string `json:"arch"` +} + +type Version struct { + Version string `json:"version"` + Protocols []string `json:"protocols"` + Platforms []Platform `json:"platforms"` +} + +type GpgPublicKey struct { + KeyID string `json:"key_id"` + ASCIIArmor string `json:"ascii_armor"` + TrustSignature string `json:"trust_signature"` + Source string `json:"source,omitempty"` + SourceURL string `json:"source_url,omitempty"` +} + +type SigningKeys struct { + GpgPublicKeys []GpgPublicKey `json:"gpg_public_keys"` +} + +type Download struct { + Protocols []string `json:"protocols"` + OS string `json:"os"` + Arch string `json:"arch"` + Filename string `json:"filename"` + DownloadURL string `json:"download_url"` + ShaURL string `json:"shasums_url"` + SignatureURL string `json:"shasums_signature_url"` + Sha string `json:"shasum"` + SigningKeys SigningKeys `json:"signing_keys"` +} + +func (dl *Download) Save(indexDir, namespace, name, version string) error { + filename := filepath.Join(indexDir, namespace, name, version, "download", dl.OS, dl.Arch) + + if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil { + return trace.Wrap(err, "Creating download dir") + } + + f, err := os.Create(filename) + if err != nil { + return trace.Wrap(err, "Failed opening file") + } + defer f.Close() + + encoder := json.NewEncoder(f) + encoder.SetIndent("", " ") + + return encoder.Encode(dl) +} + +func NewDownloadFromRepackResult(info *RepackResult, protocols []string, objectStoreURL string) (*Download, error) { + + keyText, err := formatPublicKey(info.SigningEntity) + if err != nil { + return nil, trace.Wrap(err, "failed formatting public key") + } + + return &Download{ + Protocols: protocols, + OS: info.OS, + Arch: info.Arch, + Filename: info.Zip, + Sha: info.Sha256String(), + DownloadURL: objectStoreURL + info.Zip, + ShaURL: objectStoreURL + info.Sum, + SignatureURL: objectStoreURL + info.Sig, + SigningKeys: SigningKeys{ + GpgPublicKeys: []GpgPublicKey{ + { + KeyID: info.SigningEntity.PrivateKey.PublicKey.KeyIdString(), + ASCIIArmor: keyText, + }, + }, + }, + }, nil +} + +func formatPublicKey(signingKey *openpgp.Entity) (string, error) { + var text bytes.Buffer + writer, err := armor.Encode(&text, openpgp.PublicKeyType, nil) + if err != nil { + return "", trace.Wrap(err) + } + + err = signingKey.Serialize(writer) + if err != nil { + return "", trace.Wrap(err) + } + + if err = writer.Close(); err != nil { + return "", trace.Wrap(err) + } + + return text.String(), nil +} diff --git a/tooling/internal/terraform/registry/protocol_test.go b/tooling/internal/terraform/registry/protocol_test.go new file mode 100644 index 000000000..fc42308a2 --- /dev/null +++ b/tooling/internal/terraform/registry/protocol_test.go @@ -0,0 +1,158 @@ +package registry + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +const validIndex = ` +{ + "versions": [ + { + "version": "2.0.0", + "protocols": ["4.0", "5.1"], + "platforms": [ + {"os": "darwin", "arch": "amd64"}, + {"os": "linux", "arch": "amd64"}, + {"os": "linux", "arch": "arm"}, + {"os": "windows", "arch": "amd64"} + ] + }, + { + "version": "2.0.1", + "protocols": ["5.2"], + "platforms": [ + {"os": "darwin", "arch": "amd64"}, + {"os": "linux", "arch": "amd64"}, + {"os": "linux", "arch": "arm"}, + {"os": "windows", "arch": "amd64"} + ] + } + ] + +} +` + +func TestIndexJson(t *testing.T) { + uut := Index{ + Versions: []*Version{ + { + Version: "2.0.0", + Protocols: []string{"4.0", "5.1"}, + Platforms: []Platform{ + {OS: "darwin", Arch: "amd64"}, + {OS: "linux", Arch: "amd64"}, + {OS: "linux", Arch: "arm"}, + {OS: "windows", Arch: "amd64"}, + }, + }, + { + Version: "2.0.1", + Protocols: []string{"5.2"}, + Platforms: []Platform{ + {OS: "darwin", Arch: "amd64"}, + {OS: "linux", Arch: "amd64"}, + {OS: "linux", Arch: "arm"}, + {OS: "windows", Arch: "amd64"}, + }, + }, + }, + } + + t.Run("Formatting", func(t *testing.T) { + formattedUut, err := json.Marshal(&uut) + require.NoError(t, err) + + // To avoid having to deal with trivial formatting & whitespace + // differences we will parse the generated JSON into a generic + // nested key-value map, and compare it against a similar map + // generated from the expected JSON. + + var expected map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(validIndex), &expected)) + + var actual map[string]interface{} + require.NoError(t, json.Unmarshal(formattedUut, &actual)) + + require.Equal(t, expected, actual) + }) + + t.Run("Parsing", func(t *testing.T) { + var parsedIndex Index + require.NoError(t, json.Unmarshal([]byte(validIndex), &parsedIndex)) + require.Equal(t, uut, parsedIndex) + }) +} + +const validDownloadText = ` +{ + "protocols": ["4.0", "5.1"], + "os": "linux", + "arch": "amd64", + "filename": "terraform-provider-random_2.0.0_linux_amd64.zip", + "download_url": "https://releases.hashicorp.com/terraform-provider-random/2.0.0/terraform-provider-random_2.0.0_linux_amd64.zip", + "shasums_url": "https://releases.hashicorp.com/terraform-provider-random/2.0.0/terraform-provider-random_2.0.0_SHA256SUMS", + "shasums_signature_url": "https://releases.hashicorp.com/terraform-provider-random/2.0.0/terraform-provider-random_2.0.0_SHA256SUMS.sig", + "shasum": "5f9c7aa76b7c34d722fc9123208e26b22d60440cb47150dd04733b9b94f4541a", + "signing_keys": { + "gpg_public_keys": [ + { + "key_id": "51852D87348FFC4C", + "ascii_armor": "-----BEGIN PGP PUBLIC KEY BLOCK-----\nVersion: GnuPG v1\n\nmQENBFMORM0BCADBRyKO1MhCirazOSVwcfTr1xUxjPvfxD3hjUwHtjsOy/bT6p9f\nW2mRPfwnq2JB5As+paL3UGDsSRDnK9KAxQb0NNF4+eVhr/EJ18s3wwXXDMjpIifq\nfIm2WyH3G+aRLTLPIpscUNKDyxFOUbsmgXAmJ46Re1fn8uKxKRHbfa39aeuEYWFA\n3drdL1WoUngvED7f+RnKBK2G6ZEpO+LDovQk19xGjiMTtPJrjMjZJ3QXqPvx5wca\nKSZLr4lMTuoTI/ZXyZy5bD4tShiZz6KcyX27cD70q2iRcEZ0poLKHyEIDAi3TM5k\nSwbbWBFd5RNPOR0qzrb/0p9ksKK48IIfH2FvABEBAAG0K0hhc2hpQ29ycCBTZWN1\ncml0eSA8c2VjdXJpdHlAaGFzaGljb3JwLmNvbT6JATgEEwECACIFAlMORM0CGwMG\nCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEFGFLYc0j/xMyWIIAIPhcVqiQ59n\nJc07gjUX0SWBJAxEG1lKxfzS4Xp+57h2xxTpdotGQ1fZwsihaIqow337YHQI3q0i\nSqV534Ms+j/tU7X8sq11xFJIeEVG8PASRCwmryUwghFKPlHETQ8jJ+Y8+1asRydi\npsP3B/5Mjhqv/uOK+Vy3zAyIpyDOMtIpOVfjSpCplVRdtSTFWBu9Em7j5I2HMn1w\nsJZnJgXKpybpibGiiTtmnFLOwibmprSu04rsnP4ncdC2XRD4wIjoyA+4PKgX3sCO\nklEzKryWYBmLkJOMDdo52LttP3279s7XrkLEE7ia0fXa2c12EQ0f0DQ1tGUvyVEW\nWmJVccm5bq25AQ0EUw5EzQEIANaPUY04/g7AmYkOMjaCZ6iTp9hB5Rsj/4ee/ln9\nwArzRO9+3eejLWh53FoN1rO+su7tiXJA5YAzVy6tuolrqjM8DBztPxdLBbEi4V+j\n2tK0dATdBQBHEh3OJApO2UBtcjaZBT31zrG9K55D+CrcgIVEHAKY8Cb4kLBkb5wM\nskn+DrASKU0BNIV1qRsxfiUdQHZfSqtp004nrql1lbFMLFEuiY8FZrkkQ9qduixo\nmTT6f34/oiY+Jam3zCK7RDN/OjuWheIPGj/Qbx9JuNiwgX6yRj7OE1tjUx6d8g9y\n0H1fmLJbb3WZZbuuGFnK6qrE3bGeY8+AWaJAZ37wpWh1p0cAEQEAAYkBHwQYAQIA\nCQUCUw5EzQIbDAAKCRBRhS2HNI/8TJntCAClU7TOO/X053eKF1jqNW4A1qpxctVc\nz8eTcY8Om5O4f6a/rfxfNFKn9Qyja/OG1xWNobETy7MiMXYjaa8uUx5iFy6kMVaP\n0BXJ59NLZjMARGw6lVTYDTIvzqqqwLxgliSDfSnqUhubGwvykANPO+93BBx89MRG\nunNoYGXtPlhNFrAsB1VR8+EyKLv2HQtGCPSFBhrjuzH3gxGibNDDdFQLxxuJWepJ\nEK1UbTS4ms0NgZ2Uknqn1WRU1Ki7rE4sTy68iZtWpKQXZEJa0IGnuI2sSINGcXCJ\noEIgXTMyCILo34Fa/C6VCm2WBgz9zZO8/rHIiQm1J5zqz0DrDwKBUM9C\n=LYpS\n-----END PGP PUBLIC KEY BLOCK-----", + "trust_signature": "", + "source": "HashiCorp", + "source_url": "https://www.hashicorp.com/security.html" + } + ] + } + } +` + +func TestDownloadJson(t *testing.T) { + uut := Download{ + Protocols: []string{"4.0", "5.1"}, + OS: "linux", + Arch: "amd64", + Filename: "terraform-provider-random_2.0.0_linux_amd64.zip", + DownloadURL: "https://releases.hashicorp.com/terraform-provider-random/2.0.0/terraform-provider-random_2.0.0_linux_amd64.zip", + ShaURL: "https://releases.hashicorp.com/terraform-provider-random/2.0.0/terraform-provider-random_2.0.0_SHA256SUMS", + SignatureURL: "https://releases.hashicorp.com/terraform-provider-random/2.0.0/terraform-provider-random_2.0.0_SHA256SUMS.sig", + Sha: "5f9c7aa76b7c34d722fc9123208e26b22d60440cb47150dd04733b9b94f4541a", + SigningKeys: SigningKeys{ + GpgPublicKeys: []GpgPublicKey{ + { + KeyID: "51852D87348FFC4C", + ASCIIArmor: "-----BEGIN PGP PUBLIC KEY BLOCK-----\nVersion: GnuPG v1\n\nmQENBFMORM0BCADBRyKO1MhCirazOSVwcfTr1xUxjPvfxD3hjUwHtjsOy/bT6p9f\nW2mRPfwnq2JB5As+paL3UGDsSRDnK9KAxQb0NNF4+eVhr/EJ18s3wwXXDMjpIifq\nfIm2WyH3G+aRLTLPIpscUNKDyxFOUbsmgXAmJ46Re1fn8uKxKRHbfa39aeuEYWFA\n3drdL1WoUngvED7f+RnKBK2G6ZEpO+LDovQk19xGjiMTtPJrjMjZJ3QXqPvx5wca\nKSZLr4lMTuoTI/ZXyZy5bD4tShiZz6KcyX27cD70q2iRcEZ0poLKHyEIDAi3TM5k\nSwbbWBFd5RNPOR0qzrb/0p9ksKK48IIfH2FvABEBAAG0K0hhc2hpQ29ycCBTZWN1\ncml0eSA8c2VjdXJpdHlAaGFzaGljb3JwLmNvbT6JATgEEwECACIFAlMORM0CGwMG\nCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEFGFLYc0j/xMyWIIAIPhcVqiQ59n\nJc07gjUX0SWBJAxEG1lKxfzS4Xp+57h2xxTpdotGQ1fZwsihaIqow337YHQI3q0i\nSqV534Ms+j/tU7X8sq11xFJIeEVG8PASRCwmryUwghFKPlHETQ8jJ+Y8+1asRydi\npsP3B/5Mjhqv/uOK+Vy3zAyIpyDOMtIpOVfjSpCplVRdtSTFWBu9Em7j5I2HMn1w\nsJZnJgXKpybpibGiiTtmnFLOwibmprSu04rsnP4ncdC2XRD4wIjoyA+4PKgX3sCO\nklEzKryWYBmLkJOMDdo52LttP3279s7XrkLEE7ia0fXa2c12EQ0f0DQ1tGUvyVEW\nWmJVccm5bq25AQ0EUw5EzQEIANaPUY04/g7AmYkOMjaCZ6iTp9hB5Rsj/4ee/ln9\nwArzRO9+3eejLWh53FoN1rO+su7tiXJA5YAzVy6tuolrqjM8DBztPxdLBbEi4V+j\n2tK0dATdBQBHEh3OJApO2UBtcjaZBT31zrG9K55D+CrcgIVEHAKY8Cb4kLBkb5wM\nskn+DrASKU0BNIV1qRsxfiUdQHZfSqtp004nrql1lbFMLFEuiY8FZrkkQ9qduixo\nmTT6f34/oiY+Jam3zCK7RDN/OjuWheIPGj/Qbx9JuNiwgX6yRj7OE1tjUx6d8g9y\n0H1fmLJbb3WZZbuuGFnK6qrE3bGeY8+AWaJAZ37wpWh1p0cAEQEAAYkBHwQYAQIA\nCQUCUw5EzQIbDAAKCRBRhS2HNI/8TJntCAClU7TOO/X053eKF1jqNW4A1qpxctVc\nz8eTcY8Om5O4f6a/rfxfNFKn9Qyja/OG1xWNobETy7MiMXYjaa8uUx5iFy6kMVaP\n0BXJ59NLZjMARGw6lVTYDTIvzqqqwLxgliSDfSnqUhubGwvykANPO+93BBx89MRG\nunNoYGXtPlhNFrAsB1VR8+EyKLv2HQtGCPSFBhrjuzH3gxGibNDDdFQLxxuJWepJ\nEK1UbTS4ms0NgZ2Uknqn1WRU1Ki7rE4sTy68iZtWpKQXZEJa0IGnuI2sSINGcXCJ\noEIgXTMyCILo34Fa/C6VCm2WBgz9zZO8/rHIiQm1J5zqz0DrDwKBUM9C\n=LYpS\n-----END PGP PUBLIC KEY BLOCK-----", + Source: "HashiCorp", + SourceURL: "https://www.hashicorp.com/security.html", + }, + }, + }, + } + + t.Run("Format", func(t *testing.T) { + formattedUut, err := json.Marshal(&uut) + require.NoError(t, err) + + // To avoid having to deal with trivial formatting & whitespace + // differences we will parse the generated JSON into a generic + // nested key-value map, and compare it against a similar map + // generated from the expected JSON. + + var expected map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(validDownloadText), &expected)) + + var actual map[string]interface{} + require.NoError(t, json.Unmarshal(formattedUut, &actual)) + + require.Equal(t, expected, actual) + }) + + t.Run("Parsing", func(t *testing.T) { + var parsedDownload Download + require.NoError(t, json.Unmarshal([]byte(validDownloadText), &parsedDownload)) + require.Equal(t, uut, parsedDownload) + }) +} diff --git a/tooling/internal/terraform/registry/sha.go b/tooling/internal/terraform/registry/sha.go new file mode 100644 index 000000000..9e7d9aba1 --- /dev/null +++ b/tooling/internal/terraform/registry/sha.go @@ -0,0 +1,18 @@ +package registry + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + + "github.com/gravitational/trace" +) + +// Sha256Sum generates the equivalent contents of running `sha256sum` om some +// data and writes it to the supplied stream, returning the sha bytes. +func sha256Sum(dst io.Writer, filename string, data []byte) ([]byte, error) { + sha := sha256.Sum256(data) + _, err := fmt.Fprintf(dst, "%s %s\n", hex.EncodeToString(sha[:]), filename) + return sha[:], trace.Wrap(err, "failed generating sum file") +} diff --git a/tooling/internal/terraform/registry/sign.go b/tooling/internal/terraform/registry/sign.go new file mode 100644 index 000000000..37ec7997f --- /dev/null +++ b/tooling/internal/terraform/registry/sign.go @@ -0,0 +1,147 @@ +package registry + +import ( + "bytes" + "encoding/hex" + "os" + "path/filepath" + + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/gravitational/teleport-plugins/tooling/internal/filename" + "github.com/gravitational/trace" + log "github.com/sirupsen/logrus" +) + +// FileNames describes the location of a registry-compatible zipfile and its +// associated sidecar files +type FileNames struct { + Zip string + Sum string + Sig string +} + +func IsProviderTarball(fn string) bool { + info, err := filename.Parse(fn) + if err != nil { + return false + } + + return info.Type == "terraform-provider" +} + +func makeFileNames(info filename.Info) FileNames { + zipFileName := info.Filename(".zip") + return FileNames{ + Zip: zipFileName, + Sum: zipFileName + ".sums", + Sig: zipFileName + ".sums.sig", + } +} + +// RepackResult describes a fully-repacked provider and all of its sidecar files +type RepackResult struct { + filename.Info + FileNames + Sha256 []byte + SigningEntity *openpgp.Entity +} + +func (r *RepackResult) Sha256String() string { + return hex.EncodeToString(r.Sha256) +} + +// RepackProvider takes a provider tarball and repacks it as a zipfile compatible +// with a terraform provider registry, generating all the required sidecar files +// as well. Returns a `RepackResult` instance containing the location of the +// generated files and information about the packed plugin +func RepackProvider(dstDir string, srcFileName string, signingEntity *openpgp.Entity) (*RepackResult, error) { + info, err := filename.Parse(srcFileName) + if err != nil { + return nil, trace.Wrap(err, "Bad filename") + } + + log.Infof("Provider platform: %s/%s/%s\n", info.Version, info.OS, info.Arch) + + src, err := os.Open(srcFileName) + if err != nil { + return nil, trace.Wrap(err, "failed opening source file") + } + defer src.Close() + + // Create the zip archive in memory in order to make it easier to + // hash and sign + var zipArchive bytes.Buffer + + err = repack(&zipArchive, src) + if err != nil { + return nil, trace.Wrap(err, "failed repacking provider") + } + + result := &RepackResult{ + Info: info, + FileNames: makeFileNames(info), + SigningEntity: signingEntity, + } + + // compute sha256 and format the sha file as per sha256sum + var sums bytes.Buffer + result.Sha256, err = sha256Sum(&sums, result.Zip, zipArchive.Bytes()) + if err != nil { + return nil, trace.Wrap(err) + } + + // sign the sums with our private key and generate a signature file + var sig bytes.Buffer + err = openpgp.DetachSign(&sig, signingEntity, bytes.NewReader(sums.Bytes()), nil) + if err != nil { + return nil, trace.Wrap(err) + } + + // Write everything out to the dstdir + err = writeOutput(dstDir, result, zipArchive.Bytes(), sums.Bytes(), sig.Bytes()) + if err != nil { + return nil, trace.Wrap(err) + } + + return result, nil +} + +func writeOutput(dstDir string, entry *RepackResult, zip, sums, sig []byte) error { + zipFileName := filepath.Join(dstDir, entry.Zip) + zipFile, err := os.Create(zipFileName) + if err != nil { + return trace.Wrap(err, "opening zipfile failed") + } + defer zipFile.Close() + + _, err = zipFile.Write(zip) + if err != nil { + return trace.Wrap(err, "writing zipfile failed") + } + + sumFileName := filepath.Join(dstDir, entry.Sum) + sumFile, err := os.Create(sumFileName) + if err != nil { + return trace.Wrap(err, "opening sum file failed") + } + defer sumFile.Close() + + _, err = sumFile.Write(sums) + if err != nil { + return trace.Wrap(err, "writing sumfile failed") + } + + sigFileName := filepath.Join(dstDir, entry.Sig) + sigFile, err := os.Create(sigFileName) + if err != nil { + return trace.Wrap(err, "opening sig file failed") + } + defer sigFile.Close() + + _, err = sigFile.Write(sig) + if err != nil { + return trace.Wrap(err, "writing sigFile failed") + } + + return nil +} From 4db91936764630993b23a01a19373036b810e176 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Mon, 21 Mar 2022 11:14:24 +1100 Subject: [PATCH 02/20] Additional semver tests --- tooling/internal/filename/parse_test.go | 36 +++++++++++++++++-- .../terraform/registry/protocol_test.go | 4 +-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/tooling/internal/filename/parse_test.go b/tooling/internal/filename/parse_test.go index 2707e4656..69039d7ff 100644 --- a/tooling/internal/filename/parse_test.go +++ b/tooling/internal/filename/parse_test.go @@ -8,7 +8,7 @@ import ( func TestParseFilename(t *testing.T) { t.Run("WithLeadingPath", func(t *testing.T) { - info, err := Parse("/some/bath/to/file/terraform-provider-teleport-v7.0.0-darwin-amd64-bin.tar.gz") + info, err := Parse("/some/path/to/file/terraform-provider-teleport-v7.0.0-darwin-amd64-bin.tar.gz") require.NoError(t, err) require.Equal(t, "terraform-provider", info.Type) @@ -26,10 +26,42 @@ func TestParseFilename(t *testing.T) { require.Equal(t, "arm", info.Arch) }) - t.Run("Random Junk", func(t *testing.T) { + t.Run("RandomJunk", func(t *testing.T) { _, err := Parse("blahblahblah") require.Error(t, err) }) + + t.Run("WithPreRelease", func(t *testing.T) { + info, err := Parse("terraform-provider-teleport-v1.2.3-beta.1-linux-arm-bin.tar.gz") + require.NoError(t, err) + require.Equal(t, "terraform-provider", info.Type) + require.Equal(t, "1.2.3-beta.1", info.Version) + require.Equal(t, "linux", info.OS) + require.Equal(t, "arm", info.Arch) + }) + + t.Run("WithBuild", func(t *testing.T) { + info, err := Parse("terraform-provider-teleport-v1.2.3+1-linux-arm-bin.tar.gz") + require.NoError(t, err) + require.Equal(t, "terraform-provider", info.Type) + require.Equal(t, "1.2.3+1", info.Version) + require.Equal(t, "linux", info.OS) + require.Equal(t, "arm", info.Arch) + }) + + t.Run("WithPreReleaseAndBuild", func(t *testing.T) { + info, err := Parse("terraform-provider-teleport-v1.2.3-beta.1+42-linux-arm-bin.tar.gz") + require.NoError(t, err) + require.Equal(t, "terraform-provider", info.Type) + require.Equal(t, "1.2.3-beta.1+42", info.Version) + require.Equal(t, "linux", info.OS) + require.Equal(t, "arm", info.Arch) + }) + + t.Run("UnsupportedOS", func(t *testing.T) { + _, err := Parse("terraform-provider-teleport-v1.2.3-beos-arm-bin.tar.gz") + require.Error(t, err) + }) } func TestGenerateFilename(t *testing.T) { diff --git a/tooling/internal/terraform/registry/protocol_test.go b/tooling/internal/terraform/registry/protocol_test.go index fc42308a2..c0dc147f7 100644 --- a/tooling/internal/terraform/registry/protocol_test.go +++ b/tooling/internal/terraform/registry/protocol_test.go @@ -36,7 +36,7 @@ const validIndex = ` ` func TestIndexJson(t *testing.T) { - uut := Index{ + uut := Versions{ Versions: []*Version{ { Version: "2.0.0", @@ -80,7 +80,7 @@ func TestIndexJson(t *testing.T) { }) t.Run("Parsing", func(t *testing.T) { - var parsedIndex Index + var parsedIndex Versions require.NoError(t, json.Unmarshal([]byte(validIndex), &parsedIndex)) require.Equal(t, uut, parsedIndex) }) From a6c2bc3ad4c47e1fa1f5b55a70e5a0b4bc41b0a4 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Tue, 22 Mar 2022 09:17:39 +1100 Subject: [PATCH 03/20] Upload & Review fixups --- tooling/cmd/promote-terraform/args.go | 5 + tooling/cmd/promote-terraform/main.go | 202 +++++++++++++----- tooling/go.mod | 3 +- tooling/go.sum | 2 + tooling/internal/filename/parse.go | 12 +- tooling/internal/staging/staging.go | 2 +- .../internal/terraform/registry/protocol.go | 27 +-- tooling/internal/terraform/registry/sha.go | 3 +- tooling/internal/terraform/registry/sign.go | 19 +- 9 files changed, 197 insertions(+), 78 deletions(-) diff --git a/tooling/cmd/promote-terraform/args.go b/tooling/cmd/promote-terraform/args.go index 9c22fdb48..1d137587d 100644 --- a/tooling/cmd/promote-terraform/args.go +++ b/tooling/cmd/promote-terraform/args.go @@ -24,6 +24,7 @@ type args struct { protocolVersions []string providerNamespace string providerName string + verbosity int } func parseCommandLine() *args { @@ -98,6 +99,10 @@ func parseCommandLine() *args { Default("teleport"). StringVar(&result.providerName) + app.Flag("verbose", "Output more trace output"). + Short('v'). + CounterVar(&result.verbosity) + kingpin.MustParse(app.Parse(os.Args[1:])) // Marshal the arguments into a canonical format here, so we don't have to diff --git a/tooling/cmd/promote-terraform/main.go b/tooling/cmd/promote-terraform/main.go index 361d9bf17..f57390403 100644 --- a/tooling/cmd/promote-terraform/main.go +++ b/tooling/cmd/promote-terraform/main.go @@ -6,6 +6,7 @@ import ( "net/http" "os" "path/filepath" + "sort" "strings" "github.com/ProtonMail/go-crypto/openpgp" @@ -17,6 +18,7 @@ import ( "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go/aws" + "github.com/coreos/go-semver/semver" "github.com/gravitational/teleport-plugins/tooling/internal/staging" "github.com/gravitational/teleport-plugins/tooling/internal/terraform/registry" "github.com/gravitational/trace" @@ -38,11 +40,11 @@ func main() { log.WithError(err).Fatalf("Failed setting up workspace") } - log.StandardLogger().SetLevel(log.DebugLevel) + log.StandardLogger().SetLevel(logLevel(args)) - signingKey, err := loadSigningKey(args.signingKeyText) + signingEntity, err := loadSigningEntity(args.signingKeyText) if err != nil { - log.WithError(err).Fatalf("Failed decosing signing key") + log.WithError(err).Fatalf("Failed decoding signing key") } files, err := downloadStagedArtifacts(context.Background(), args.providerTag, workspace.stagingDir, &args.staging) @@ -52,59 +54,37 @@ func main() { objectStoreUrl := args.registryURL + "store/" - versionRecord := registry.Version{ - Protocols: args.protocolVersions, + versionRecord, newFiles, err := repackProviders(files, workspace, objectStoreUrl, signingEntity, args.protocolVersions, args.providerNamespace, args.providerName) + if err != nil { + log.WithError(err).Fatalf("Failed repacking artifacts") } - for _, fn := range files { - if !registry.IsProviderTarball(fn) { - continue - } - - log.Infof("Found provider tarball %s", fn) - - registryInfo, err := registry.RepackProvider(workspace.objectStoreDir, fn, signingKey) - if err != nil { - log.WithError(err).Fatalf("Failed repacking provider") - } - - log.Infof("Provider repacked to %s/%s", workspace.objectStoreDir, registryInfo.Zip) - - if versionRecord.Version == "" { - versionRecord.Version = registryInfo.Version - } else if versionRecord.Version != registryInfo.Version { - log.Fatalf("Version mismatch. Expected %s, got %s", versionRecord.Version, registryInfo.Version) - } + err = updateRegistry(context.Background(), &args.production, workspace, args.providerNamespace, args.providerName, &versionRecord, newFiles) + if err != nil { + log.WithError(err).Fatal("Failed updating registry") + } +} - downloadInfo, err := registry.NewDownloadFromRepackResult(registryInfo, args.protocolVersions, objectStoreUrl) - if err != nil { - log.WithError(err).Fatalf("Failed creating download info record") - } +func logLevel(args *args) log.Level { + switch { + case args.verbosity >= 2: + return log.TraceLevel - err = downloadInfo.Save(workspace.registryDir, args.providerNamespace, args.providerName, registryInfo.Version) - if err != nil { - log.WithError(err).Fatalf("Failed saving download info record") - } + case args.verbosity == 1: + return log.DebugLevel - versionRecord.Platforms = append(versionRecord.Platforms, registry.Platform{ - OS: registryInfo.OS, - Arch: registryInfo.Arch, - }) - } - - err = updateRegistry(context.Background(), &args.production, workspace.registryDir, args.providerNamespace, args.providerName, &versionRecord) - if err != nil { - log.WithError(err).Fatal("Failed updating registry") + default: + return log.InfoLevel } } -func updateRegistry(ctx context.Context, prodBucket *bucketConfig, registryDir, namespace, provider string, newVersion *registry.Version) error { +func updateRegistry(ctx context.Context, prodBucket *bucketConfig, workspace *workspacePaths, namespace, provider string, newVersion *registry.Version, files []string) error { s3client, err := newS3ClientFromBucketConfig(ctx, prodBucket) if err != nil { return trace.Wrap(err) } - versionsFileKey, versionsFilePath := makeVersionsFilePaths(registryDir, namespace, provider) + versionsFileKey, versionsFilePath := makeVersionsFilePaths(workspace.registryDir, namespace, provider) log.Infof("Downloading version index for %s/%s from %s", namespace, provider, versionsFileKey) // Try downloading the versions file. This may not exist in an empty/new @@ -126,15 +106,143 @@ func updateRegistry(ctx context.Context, prodBucket *bucketConfig, registryDir, return trace.Wrap(err, "failed downloading index") } - versions.Versions = append(versions.Versions, newVersion) + // Index the available version by their semver version, so that we can find the + // appropriate release if we're overwriting an existing version + versionIndex := map[semver.Version]*registry.Version{} + for _, v := range versions.Versions { + versionIndex[v.Version] = v + } + + // add/overwrite the existing version entry + versionIndex[newVersion.Version] = newVersion + versions.Versions = flattenVersionIndex(versionIndex) if err = versions.Save(versionsFilePath); err != nil { - trace.Wrap(err, "failed saving index file") + return trace.Wrap(err, "failed saving index file") + } + files = append(files, versionsFilePath) + + // Finally, push the new index files to the production bucket + err = uploadRegistry(ctx, s3client, prodBucket.bucketName, workspace.productionDir, files) + if err != nil { + return trace.Wrap(err, "failed uploading new files") } return nil } +func uploadRegistry(ctx aws.Context, s3Client *s3.Client, bucketName string, productionDir string, files []string) error { + uploader := manager.NewUploader(s3Client) + log.Infof("Production dir: %s", productionDir) + for _, f := range files { + log.Infof("Uploading %s", f) + + if !strings.HasPrefix(f, productionDir) { + return trace.Errorf("file outside of registry dir") + } + + key, err := filepath.Rel(productionDir, f) + if err != nil { + return trace.Wrap(err, "failed to extract key") + } + + log.Tracef("... to %s", key) + + doUpload := func() error { + f, err := os.Open(f) + if err != nil { + return err + } + defer f.Close() + + _, err = uploader.Upload(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Body: f, + }) + + return err + } + + if err = doUpload(); err != nil { + return trace.Wrap(err, "failed uploading registry file") + } + } + + return nil +} + +func flattenVersionIndex(versionIndex map[semver.Version]*registry.Version) []*registry.Version { + + // We want to output a list of versions with semver ordering, so first we + // generate a sorted list of keys + keys := make([]semver.Version, 0, len(versionIndex)) + for k, _ := range versionIndex { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { return keys[i].LessThan(keys[j]) }) + + // Now we can simply walk the index using the sorted key list and we have + // our sorted output list + result := make([]*registry.Version, 0, len(keys)) + for _, k := range keys { + result = append(result, versionIndex[k]) + } + + return result +} + +func repackProviders(candidateFilenames []string, workspace *workspacePaths, objectStoreUrl string, signingEntity *openpgp.Entity, protocolVersions []string, providerNamespace, providerName string) (registry.Version, []string, error) { + + versionRecord := registry.Version{ + Protocols: protocolVersions, + } + + newFiles := []string{} + + unsetVersion := semver.Version{} + + for _, fn := range candidateFilenames { + if !registry.IsProviderTarball(fn) { + continue + } + + log.Infof("Found provider tarball %s", fn) + + registryInfo, err := registry.RepackProvider(workspace.objectStoreDir, fn, signingEntity) + if err != nil { + return registry.Version{}, nil, trace.Wrap(err, "failed repacking provider") + } + + log.Infof("Provider repacked to %s/%s", workspace.objectStoreDir, registryInfo.Zip) + newFiles = append(newFiles, registryInfo.Zip, registryInfo.Sum, registryInfo.Sig) + + if versionRecord.Version == unsetVersion { + versionRecord.Version = registryInfo.Version + } else if !versionRecord.Version.Equal(registryInfo.Version) { + return registry.Version{}, nil, trace.Wrap(err, "version mismatch. Expected %s, got %s", versionRecord.Version, registryInfo.Version) + } + + downloadInfo, err := registry.NewDownloadFromRepackResult(registryInfo, protocolVersions, objectStoreUrl) + if err != nil { + return registry.Version{}, nil, trace.Wrap(err, "failed creating download info record") + } + + filename, err := downloadInfo.Save(workspace.registryDir, providerNamespace, providerName, registryInfo.Version) + if err != nil { + return registry.Version{}, nil, trace.Wrap(err, "Failed saving download info record") + } + newFiles = append(newFiles, filename) + + versionRecord.Platforms = append(versionRecord.Platforms, registry.Platform{ + OS: registryInfo.OS, + Arch: registryInfo.Arch, + }) + } + + return versionRecord, newFiles, nil +} + func download(ctx context.Context, client *s3.Client, dstFileName, bucket, key string) error { err := os.MkdirAll(filepath.Dir(dstFileName), 0700) @@ -233,11 +341,9 @@ func newS3ClientFromBucketConfig(ctx context.Context, bucket *bucketConfig) (*s3 return s3.NewFromConfig(cfg), nil } -func loadSigningKey(keyText string) (*openpgp.Entity, error) { +func loadSigningEntity(keyText string) (*openpgp.Entity, error) { log.Info("Decoding signing key") - strings.NewReader(keyText) - block, err := armor.Decode(strings.NewReader(keyText)) if err != nil { return nil, trace.Wrap(err) diff --git a/tooling/go.mod b/tooling/go.mod index d733ebb8f..a53fb3cda 100644 --- a/tooling/go.mod +++ b/tooling/go.mod @@ -5,10 +5,12 @@ go 1.17 require ( github.com/ProtonMail/go-crypto v0.0.0-20220113124808-70ae35bab23f github.com/aws/aws-sdk-go v1.43.19 + github.com/aws/aws-sdk-go-v2 v1.15.0 github.com/aws/aws-sdk-go-v2/config v1.15.0 github.com/aws/aws-sdk-go-v2/credentials v1.10.0 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.26.0 + github.com/coreos/go-semver v0.3.0 github.com/gravitational/kingpin v2.1.10+incompatible github.com/gravitational/trace v1.1.18 github.com/sirupsen/logrus v1.8.1 @@ -19,7 +21,6 @@ require ( github.com/alecthomas/assert v1.0.0 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect - github.com/aws/aws-sdk-go-v2 v1.15.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.0 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.0 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.6 // indirect diff --git a/tooling/go.sum b/tooling/go.sum index 088ffd1e0..20846c675 100644 --- a/tooling/go.sum +++ b/tooling/go.sum @@ -57,6 +57,8 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/tooling/internal/filename/parse.go b/tooling/internal/filename/parse.go index f3e9452a3..0cfff52b1 100644 --- a/tooling/internal/filename/parse.go +++ b/tooling/internal/filename/parse.go @@ -5,6 +5,7 @@ import ( "path/filepath" "regexp" + "github.com/coreos/go-semver/semver" "github.com/gravitational/trace" ) @@ -14,7 +15,7 @@ var ( type Info struct { Type string - Version string + Version semver.Version OS string Arch string } @@ -24,12 +25,17 @@ func Parse(filename string) (Info, error) { matches := filenamePattern.FindStringSubmatch(filename) if len(matches) == 0 { - return Info{}, trace.Errorf("Filename %q does not match required pattern", filename) + return Info{}, trace.Errorf("filename %q does not match required pattern", filename) + } + + version, err := semver.NewVersion(matches[2]) + if err != nil { + return Info{}, trace.Wrap(err, "failed parsing version as semver") } return Info{ Type: matches[1], - Version: matches[2], + Version: *version, OS: matches[3], Arch: matches[4], }, nil diff --git a/tooling/internal/staging/staging.go b/tooling/internal/staging/staging.go index 60ea76af7..345e3a0cd 100644 --- a/tooling/internal/staging/staging.go +++ b/tooling/internal/staging/staging.go @@ -65,7 +65,7 @@ func fetchObject(ctx context.Context, client downloader, dstDirRoot string, buck dstFilename = filepath.Join(dstDirRoot, dstFilename) log.Infof("Fetching %s", key) - log.Debugf("into %s", dstFilename) + log.Tracef("... into %s", dstFilename) dstDir := filepath.Dir(dstFilename) err = os.MkdirAll(dstDir, 0700) if err != nil { diff --git a/tooling/internal/terraform/registry/protocol.go b/tooling/internal/terraform/registry/protocol.go index 02981f8cc..ae9393b99 100644 --- a/tooling/internal/terraform/registry/protocol.go +++ b/tooling/internal/terraform/registry/protocol.go @@ -8,8 +8,9 @@ import ( "path/filepath" "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/armor" + "github.com/coreos/go-semver/semver" "github.com/gravitational/trace" - "golang.org/x/crypto/openpgp/armor" ) const ( @@ -65,9 +66,9 @@ type Platform struct { } type Version struct { - Version string `json:"version"` - Protocols []string `json:"protocols"` - Platforms []Platform `json:"platforms"` + Version semver.Version `json:"version"` + Protocols []string `json:"protocols"` + Platforms []Platform `json:"platforms"` } type GpgPublicKey struct { @@ -94,23 +95,23 @@ type Download struct { SigningKeys SigningKeys `json:"signing_keys"` } -func (dl *Download) Save(indexDir, namespace, name, version string) error { - filename := filepath.Join(indexDir, namespace, name, version, "download", dl.OS, dl.Arch) +func (dl *Download) Save(indexDir, namespace, name string, version semver.Version) (string, error) { + filename := filepath.Join(indexDir, namespace, name, version.String(), "download", dl.OS, dl.Arch) if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil { - return trace.Wrap(err, "Creating download dir") + return "", trace.Wrap(err, "Creating download dir") } f, err := os.Create(filename) if err != nil { - return trace.Wrap(err, "Failed opening file") + return "", trace.Wrap(err, "Failed opening file") } defer f.Close() encoder := json.NewEncoder(f) encoder.SetIndent("", " ") - return encoder.Encode(dl) + return filename, encoder.Encode(dl) } func NewDownloadFromRepackResult(info *RepackResult, protocols []string, objectStoreURL string) (*Download, error) { @@ -124,11 +125,11 @@ func NewDownloadFromRepackResult(info *RepackResult, protocols []string, objectS Protocols: protocols, OS: info.OS, Arch: info.Arch, - Filename: info.Zip, + Filename: filepath.Base(info.Zip), Sha: info.Sha256String(), - DownloadURL: objectStoreURL + info.Zip, - ShaURL: objectStoreURL + info.Sum, - SignatureURL: objectStoreURL + info.Sig, + DownloadURL: objectStoreURL + filepath.Base(info.Zip), + ShaURL: objectStoreURL + filepath.Base(info.Sum), + SignatureURL: objectStoreURL + filepath.Base(info.Sig), SigningKeys: SigningKeys{ GpgPublicKeys: []GpgPublicKey{ { diff --git a/tooling/internal/terraform/registry/sha.go b/tooling/internal/terraform/registry/sha.go index 9e7d9aba1..11c8e2703 100644 --- a/tooling/internal/terraform/registry/sha.go +++ b/tooling/internal/terraform/registry/sha.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "io" + "path/filepath" "github.com/gravitational/trace" ) @@ -13,6 +14,6 @@ import ( // data and writes it to the supplied stream, returning the sha bytes. func sha256Sum(dst io.Writer, filename string, data []byte) ([]byte, error) { sha := sha256.Sum256(data) - _, err := fmt.Fprintf(dst, "%s %s\n", hex.EncodeToString(sha[:]), filename) + _, err := fmt.Fprintf(dst, "%s %s\n", hex.EncodeToString(sha[:]), filepath.Base(filename)) return sha[:], trace.Wrap(err, "failed generating sum file") } diff --git a/tooling/internal/terraform/registry/sign.go b/tooling/internal/terraform/registry/sign.go index 37ec7997f..3b03e43f9 100644 --- a/tooling/internal/terraform/registry/sign.go +++ b/tooling/internal/terraform/registry/sign.go @@ -29,8 +29,8 @@ func IsProviderTarball(fn string) bool { return info.Type == "terraform-provider" } -func makeFileNames(info filename.Info) FileNames { - zipFileName := info.Filename(".zip") +func makeFileNames(dstDir string, info filename.Info) FileNames { + zipFileName := filepath.Join(dstDir, info.Filename(".zip")) return FileNames{ Zip: zipFileName, Sum: zipFileName + ".sums", @@ -79,7 +79,7 @@ func RepackProvider(dstDir string, srcFileName string, signingEntity *openpgp.En result := &RepackResult{ Info: info, - FileNames: makeFileNames(info), + FileNames: makeFileNames(dstDir, info), SigningEntity: signingEntity, } @@ -98,7 +98,7 @@ func RepackProvider(dstDir string, srcFileName string, signingEntity *openpgp.En } // Write everything out to the dstdir - err = writeOutput(dstDir, result, zipArchive.Bytes(), sums.Bytes(), sig.Bytes()) + err = writeOutput(result, zipArchive.Bytes(), sums.Bytes(), sig.Bytes()) if err != nil { return nil, trace.Wrap(err) } @@ -106,9 +106,8 @@ func RepackProvider(dstDir string, srcFileName string, signingEntity *openpgp.En return result, nil } -func writeOutput(dstDir string, entry *RepackResult, zip, sums, sig []byte) error { - zipFileName := filepath.Join(dstDir, entry.Zip) - zipFile, err := os.Create(zipFileName) +func writeOutput(entry *RepackResult, zip, sums, sig []byte) error { + zipFile, err := os.Create(entry.Zip) if err != nil { return trace.Wrap(err, "opening zipfile failed") } @@ -119,8 +118,7 @@ func writeOutput(dstDir string, entry *RepackResult, zip, sums, sig []byte) erro return trace.Wrap(err, "writing zipfile failed") } - sumFileName := filepath.Join(dstDir, entry.Sum) - sumFile, err := os.Create(sumFileName) + sumFile, err := os.Create(entry.Sum) if err != nil { return trace.Wrap(err, "opening sum file failed") } @@ -131,8 +129,7 @@ func writeOutput(dstDir string, entry *RepackResult, zip, sums, sig []byte) erro return trace.Wrap(err, "writing sumfile failed") } - sigFileName := filepath.Join(dstDir, entry.Sig) - sigFile, err := os.Create(sigFileName) + sigFile, err := os.Create(entry.Sig) if err != nil { return trace.Wrap(err, "opening sig file failed") } From 3d082cb79b7507c08b751263a8923e8526406930 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Tue, 22 Mar 2022 15:51:55 +1100 Subject: [PATCH 04/20] Drone --- .drone.yml | 330 ++------------------------ tooling/cmd/promote-terraform/args.go | 4 +- 2 files changed, 28 insertions(+), 306 deletions(-) diff --git a/.drone.yml b/.drone.yml index 149030793..f019c5cba 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,206 +1,3 @@ ---- -kind: pipeline -type: kubernetes -name: test-linux - -trigger: - branch: - - master - - branch/* - event: - include: - - push - repo: - include: - - gravitational/* - -workspace: - path: /go/src/github.com/gravitational/teleport-plugins - -steps: - - name: Run linter - image: golangci/golangci-lint:v1.39.0 - commands: - - make lint - - - name: Run tests - image: golang:1.17.5 - environment: - TELEPORT_ENTERPRISE_LICENSE: - from_secret: TELEPORT_ENTERPRISE_LICENSE - TELEPORT_GET_VERSION: v8.0.5 - commands: - - echo Testing plugins against Teleport $TELEPORT_GET_VERSION - - make test - ---- -kind: pipeline -type: exec -name: test-darwin - -concurrency: - limit: 1 - -platform: - os: darwin - arch: amd64 - -trigger: - branch: - - master - - branch/* - event: - include: - - push - repo: - include: - - gravitational/* - -workspace: - path: /tmp/teleport-plugins/test-darwin - -steps: - - name: Clean up exec runner storage - commands: - # This will remove subdirectories under pkg/mod which 0400 permissions. - # See for more details: https://github.com/golang/go/issues/27455 - - go clean -modcache - - rm -rf /tmp/teleport-plugins/test-darwin/go - - mkdir -p /tmp/teleport-plugins/test-darwin/go - - - name: Run tests - environment: - TELEPORT_ENTERPRISE_LICENSE: - from_secret: TELEPORT_ENTERPRISE_LICENSE - TELEPORT_GET_VERSION: v8.0.5 - GOPATH: /tmp/teleport-plugins/test-darwin/go - GOCACHE: /tmp/teleport-plugins/test-darwin/go/cache - commands: - - go version - - make test - ---- -kind: pipeline -type: kubernetes -name: build-on-push-linux - -trigger: - branch: - - master - - branch/* - event: - include: - - push - repo: - include: - - gravitational/* - -depends_on: - - test-linux - -workspace: - path: /go/src/github.com/gravitational/teleport-plugins - -steps: - - name: Build artifacts - image: golang:1.17.5 - commands: - - make build-all - ---- -kind: pipeline -type: exec -name: build-on-push-darwin - -concurrency: - limit: 1 - -platform: - os: darwin - arch: amd64 - -trigger: - branch: - - master - - branch/* - event: - include: - - push - repo: - include: - - gravitational/* - -depends_on: - - test-darwin - -workspace: - path: /tmp/teleport-plugins/build-darwin - -steps: - - name: Clean up exec runner storage (pre) - commands: - # This will remove subdirectories under pkg/mod which 0400 permissions. - # See for more details: https://github.com/golang/go/issues/27455 - - go clean -modcache - - rm -rf /tmp/teleport-plugins/build-darwin/go - - mkdir -p /tmp/teleport-plugins/build-darwin/go/cache - - - name: Build artifacts (darwin) - environment: - GOPATH: /tmp/teleport-plugins/build-darwin/go - GOCACHE: /tmp/teleport-plugins/build-darwin/go/cache - commands: - - make build-all - ---- -kind: pipeline -type: kubernetes -name: tag-build-plugins-linux - -trigger: - event: - - tag - ref: - include: - - refs/tags/teleport-gitlab-v* - - refs/tags/teleport-jira-v* - - refs/tags/teleport-mattermost-v* - - refs/tags/teleport-pagerduty-v* - - refs/tags/teleport-slack-v* - - refs/tags/teleport-email-v* - -depends_on: - - test-linux - -workspace: - path: /go/src/github.com/gravitational/teleport-plugins - -steps: - - name: Build artifacts - image: golang:1.17.5 - commands: - - mkdir -p build/ - - export PLUGIN_TYPE=$(echo ${DRONE_TAG} | cut -d- -f2) - - make release/access-$PLUGIN_TYPE - - find access/ -iname "*.tar.gz" -print -exec cp {} build/ \; - - cd build - - for FILE in *.tar.gz; do sha256sum $FILE > $FILE.sha256; done - - ls -l . - - - name: Upload to S3 - image: plugins/s3 - settings: - bucket: - from_secret: AWS_S3_BUCKET - access_key: - from_secret: AWS_ACCESS_KEY_ID - secret_key: - from_secret: AWS_SECRET_ACCESS_KEY - region: us-west-2 - source: /go/src/github.com/gravitational/teleport-plugins/build/* - target: teleport-plugins/tag/${DRONE_TAG} - strip_prefix: /go/src/github.com/gravitational/teleport-plugins/build - --- kind: pipeline type: kubernetes @@ -286,130 +83,53 @@ steps: --- kind: pipeline type: kubernetes -name: tag-build-event-handler-linux +name: stage-terraform-provider-registry trigger: event: - tag ref: include: - - refs/tags/teleport-event-handler-v* - -workspace: - path: /go/src/github.com/gravitational/teleport-plugins - -steps: - - name: Build artifacts - image: golang:1.17.5 - commands: - - mkdir -p build/ - - make release/event-handler - - find event-handler/ -iname "*.tar.gz" -print -exec cp {} build/ \; - - cd build - - for FILE in *.tar.gz; do sha256sum $FILE > $FILE.sha256; done - - ls -l . - - - name: Upload to S3 - image: plugins/s3 - settings: - bucket: - from_secret: AWS_S3_BUCKET - access_key: - from_secret: AWS_ACCESS_KEY_ID - secret_key: - from_secret: AWS_SECRET_ACCESS_KEY - region: us-west-2 - source: /go/src/github.com/gravitational/teleport-plugins/build/* - target: teleport-plugins/tag/${DRONE_TAG} - strip_prefix: /go/src/github.com/gravitational/teleport-plugins/build + - refs/tags/terraform-provider-teleport-v* ---- -kind: pipeline -type: exec -name: tag-build-event-handler-darwin +depends_on: + - tag-build-terraform-linux + - tag-build-terraform-darwin concurrency: limit: 1 -platform: - os: darwin - arch: amd64 - -trigger: - event: - - tag - ref: - include: - - refs/tags/teleport-event-handler-v* - steps: - - name: Build artifacts + - name: Promote terraform provider to staging registry + image: golang:1.17.5 commands: - - mkdir -p build/ - - make release/event-handler - - find event-handler/ -iname "*.tar.gz" -print -exec cp {} build/ \; - - cd build - - for FILE in *.tar.gz; do shasum -a 256 $FILE > $FILE.sha256; done - - ls -l . + - cd tooling + - go run ./cmd-promote-terraform \ + --tag ${DRONE_TAG} \ + -p 4.0 -p 5.1 \ + --namespace gravitational \ + --name teleport - - name: Upload to S3 environment: - AWS_S3_BUCKET: + STAGING_REGION: us-west-2 + STAGING_BUCKET: from_secret: AWS_S3_BUCKET - AWS_ACCESS_KEY_ID: + STAGING_ACCESS_KEY_ID: from_secret: AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: + STAGING_SECRET_ACCESS_KEY: from_secret: AWS_SECRET_ACCESS_KEY - AWS_REGION: us-west-2 - commands: - - cd build - - aws s3 sync . s3://$AWS_S3_BUCKET/teleport-plugins/tag/${DRONE_TAG}/ - ---- -kind: pipeline -type: kubernetes -name: promote-artifact - -trigger: - event: - - promote - target: - - production - -workspace: - path: /go/src/github.com/gravitational/teleport-plugins - -clone: - disable: true - -steps: - - name: Download artifact from S3 artifact publishing bucket - image: amazon/aws-cli - environment: - AWS_S3_BUCKET: + + PRODUCTION_REGION: us-west-2 + PROD_BUCKET: from_secret: AWS_S3_BUCKET - AWS_ACCESS_KEY_ID: + PROD_ACCESS_KEY_ID: from_secret: AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: + PROD_SECRET_ACCESS_KEY: from_secret: AWS_SECRET_ACCESS_KEY - AWS_REGION: us-west-2 - commands: - - aws s3 sync s3://$AWS_S3_BUCKET/teleport-plugins/tag/${DRONE_TAG}/ . + + SIGNING_KEY: + from_secret: TERRAFORM_REGISTRY_SIGNING_KEY - - name: Upload artifact to production S3 bucket with public read access - image: plugins/s3 - settings: - bucket: - from_secret: PRODUCTION_AWS_S3_BUCKET - access_key: - from_secret: PRODUCTION_AWS_ACCESS_KEY_ID - secret_key: - from_secret: PRODUCTION_AWS_SECRET_ACCESS_KEY - region: us-east-1 - acl: public-read - source: /go/src/github.com/gravitational/teleport-plugins/* - target: teleport-plugins/${DRONE_TAG##*-v}/ - strip_prefix: /go/src/github.com/gravitational/teleport-plugins/ --- kind: signature diff --git a/tooling/cmd/promote-terraform/args.go b/tooling/cmd/promote-terraform/args.go index 1d137587d..2405b59ef 100644 --- a/tooling/cmd/promote-terraform/args.go +++ b/tooling/cmd/promote-terraform/args.go @@ -28,7 +28,7 @@ type args struct { } func parseCommandLine() *args { - app := kingpin.New("update-registry", "Adds files to a terraform registry") + app := kingpin.New("promote-terraform", "Adds files to a terraform registry") result := &args{} app.Flag("tag", "The version tag identifying version of provider to promote"). @@ -41,6 +41,7 @@ func parseCommandLine() *args { StringVar(&result.staging.bucketName) app.Flag("staging-region", "AWS region the staging bucket is in"). + Envar("STAGING_REGION"). Default("us-west-2"). StringVar(&result.staging.region) @@ -59,6 +60,7 @@ func parseCommandLine() *args { StringVar(&result.production.bucketName) app.Flag("prod-region", "AWS region the production bucket is in"). + Envar("PRODUCTION_REGION"). Default("us-east-1"). StringVar(&result.production.region) From c0ebff9674b8eeeaeea9591e9193f40b7aedc354 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Tue, 22 Mar 2022 16:52:02 +1100 Subject: [PATCH 05/20] Sign drone --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index f019c5cba..c5e91f4d6 100644 --- a/.drone.yml +++ b/.drone.yml @@ -133,6 +133,6 @@ steps: --- kind: signature -hmac: c427752b53eaa9eb47bf95e1c3aed04b4c8273f930ba4ab83c67f12052d37821 +hmac: c411568d2cdc1b25c2d17dac15f7e9c09bebf4363bd3faa4f05ed335b8786fca ... From c84015df115c02cbfd3b578d74a000ba65f970f2 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Wed, 23 Mar 2022 10:54:18 +1100 Subject: [PATCH 06/20] More drone tweaks --- .drone.yml | 46 ++-------------------------------------------- 1 file changed, 2 insertions(+), 44 deletions(-) diff --git a/.drone.yml b/.drone.yml index c5e91f4d6..9ce076cb5 100644 --- a/.drone.yml +++ b/.drone.yml @@ -38,48 +38,6 @@ steps: target: teleport-plugins/tag/${DRONE_TAG} strip_prefix: /go/src/github.com/gravitational/teleport-plugins/build ---- -kind: pipeline -type: exec -name: tag-build-terraform-darwin - -concurrency: - limit: 1 - -platform: - os: darwin - arch: amd64 - -trigger: - event: - - tag - ref: - include: - - refs/tags/terraform-provider-teleport-v* - -steps: - - name: Build artifacts - commands: - - mkdir -p build/ - - make release/terraform - - find terraform/ -iname "*.tar.gz" -print -exec cp {} build/ \; - - cd build - - for FILE in *.tar.gz; do shasum -a 256 $FILE > $FILE.sha256; done - - ls -l . - - - name: Upload to S3 - environment: - AWS_S3_BUCKET: - from_secret: AWS_S3_BUCKET - AWS_ACCESS_KEY_ID: - from_secret: AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: - from_secret: AWS_SECRET_ACCESS_KEY - AWS_REGION: us-west-2 - commands: - - cd build - - aws s3 sync . s3://$AWS_S3_BUCKET/teleport-plugins/tag/${DRONE_TAG}/ - --- kind: pipeline type: kubernetes @@ -94,7 +52,7 @@ trigger: depends_on: - tag-build-terraform-linux - - tag-build-terraform-darwin +# - tag-build-terraform-darwin concurrency: limit: 1 @@ -133,6 +91,6 @@ steps: --- kind: signature -hmac: c411568d2cdc1b25c2d17dac15f7e9c09bebf4363bd3faa4f05ed335b8786fca +hmac: a309996cfeafb57a6f1c7b377163f18d880ee45a013462736c35257fb64a7b36 ... From f4f612b97b425a614de4dfefc7157fa03227e35b Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Wed, 23 Mar 2022 11:02:14 +1100 Subject: [PATCH 07/20] drone tweaks --- .drone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 9ce076cb5..4fe26dfc6 100644 --- a/.drone.yml +++ b/.drone.yml @@ -33,7 +33,7 @@ steps: from_secret: AWS_ACCESS_KEY_ID secret_key: from_secret: AWS_SECRET_ACCESS_KEY - region: us-west-2 + region: us-east-2 source: /go/src/github.com/gravitational/teleport-plugins/build/* target: teleport-plugins/tag/${DRONE_TAG} strip_prefix: /go/src/github.com/gravitational/teleport-plugins/build @@ -91,6 +91,6 @@ steps: --- kind: signature -hmac: a309996cfeafb57a6f1c7b377163f18d880ee45a013462736c35257fb64a7b36 +hmac: 5d193c5f00757e5d35c147696da8b09e6c09cd6012d080c1df1a2e1f6b91463a ... From ad72f29d59e6399c24e6ae9067b9633c27544079 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Wed, 23 Mar 2022 11:12:06 +1100 Subject: [PATCH 08/20] fix inviocation --- .drone.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 4fe26dfc6..64e62c00d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -62,7 +62,8 @@ steps: image: golang:1.17.5 commands: - cd tooling - - go run ./cmd-promote-terraform \ + - | + go run ./cmd/promote-terraform \ --tag ${DRONE_TAG} \ -p 4.0 -p 5.1 \ --namespace gravitational \ From ef065d33df8ec661d7af75541fa8d43bca14e662 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Wed, 23 Mar 2022 11:29:25 +1100 Subject: [PATCH 09/20] AWS tweak --- .drone.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.drone.yml b/.drone.yml index 64e62c00d..7eeaec02d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -70,7 +70,7 @@ steps: --name teleport environment: - STAGING_REGION: us-west-2 + STAGING_REGION: us-east-2 STAGING_BUCKET: from_secret: AWS_S3_BUCKET STAGING_ACCESS_KEY_ID: @@ -78,13 +78,13 @@ steps: STAGING_SECRET_ACCESS_KEY: from_secret: AWS_SECRET_ACCESS_KEY - PRODUCTION_REGION: us-west-2 + PRODUCTION_REGION: us-east-2 PROD_BUCKET: - from_secret: AWS_S3_BUCKET + from_secret: AWS_TERRAFORM_STAGING_BUCKET PROD_ACCESS_KEY_ID: - from_secret: AWS_ACCESS_KEY_ID + from_secret: AWS_TERRAFORM_STAGING_ACCESS_KEY_ID PROD_SECRET_ACCESS_KEY: - from_secret: AWS_SECRET_ACCESS_KEY + from_secret: AWS_TERRAFORM_SECRET_ACCESS_KEY # rename to AWS_TERRAFORM_STAGING_SECRET_ACCESS_KEY SIGNING_KEY: from_secret: TERRAFORM_REGISTRY_SIGNING_KEY @@ -92,6 +92,6 @@ steps: --- kind: signature -hmac: 5d193c5f00757e5d35c147696da8b09e6c09cd6012d080c1df1a2e1f6b91463a +hmac: 212e3c224f4d500ce35063370bdf6adaf350dff5008ae56b6a566c05044ac43e ... From d92ddc52d9b85d118af95697e5b110e960637538 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Wed, 23 Mar 2022 17:33:09 +1100 Subject: [PATCH 10/20] promotion --- .drone.yml | 60 +++++++++++++++---- tooling/cmd/promote-terraform/main.go | 1 - .../terraform/registry/protocol_test.go | 6 +- tooling/internal/terraform/registry/sign.go | 2 +- 4 files changed, 53 insertions(+), 16 deletions(-) diff --git a/.drone.yml b/.drone.yml index 7eeaec02d..4f79074ac 100644 --- a/.drone.yml +++ b/.drone.yml @@ -58,16 +58,16 @@ concurrency: limit: 1 steps: - - name: Promote terraform provider to staging registry + - name: Upload terraform provider to staging registry image: golang:1.17.5 commands: - cd tooling - | - go run ./cmd/promote-terraform \ - --tag ${DRONE_TAG} \ - -p 4.0 -p 5.1 \ - --namespace gravitational \ - --name teleport + go run ./cmd/promote-terraform \ + --tag ${DRONE_TAG} \ + -p 4.0 -p 5.1 \ + --namespace gravitational \ + --name teleport environment: STAGING_REGION: us-east-2 @@ -80,18 +80,56 @@ steps: PRODUCTION_REGION: us-east-2 PROD_BUCKET: - from_secret: AWS_TERRAFORM_STAGING_BUCKET + from_secret: STAGING_AWS_TERRAFORM_BUCKET PROD_ACCESS_KEY_ID: - from_secret: AWS_TERRAFORM_STAGING_ACCESS_KEY_ID + from_secret: STAGING_AWS_TERRAFORM_ACCESS_KEY_ID PROD_SECRET_ACCESS_KEY: - from_secret: AWS_TERRAFORM_SECRET_ACCESS_KEY # rename to AWS_TERRAFORM_STAGING_SECRET_ACCESS_KEY + from_secret: STAGING_AWS_TERRAFORM_SECRET_ACCESS_KEY SIGNING_KEY: - from_secret: TERRAFORM_REGISTRY_SIGNING_KEY + from_secret: STAGING_TERRAFORM_REGISTRY_SIGNING_KEY +--- +kind: pipeline +type: kubernetes +name: promote-terraform-provider + +concurrency: + limit: 1 + +steps: + - name: Promote terraform provider to public registry + image: golang:1.17.5 + commands: + - cd tooling + - | + go run ./cmd/promote-terraform \ + --tag ${DRONE_TAG} \ + -p 4.0 -p 5.1 \ + --namespace gravitational \ + --name teleport + + environment: + STAGING_REGION: us-east-2 + STAGING_BUCKET: + from_secret: AWS_S3_BUCKET + STAGING_ACCESS_KEY_ID: + from_secret: AWS_ACCESS_KEY_ID + STAGING_SECRET_ACCESS_KEY: + from_secret: AWS_SECRET_ACCESS_KEY + + PRODUCTION_REGION: ap-southeast-2 + PROD_BUCKET: + from_secret: PRODUCTION_AWS_TERRAFORM_BUCKET + PROD_ACCESS_KEY_ID: + from_secret: PRODUCTION_AWS_TERRAFORM_ACCESS_KEY_ID + PROD_SECRET_ACCESS_KEY: + from_secret: PRODUCTION_AWS_TERRAFORM_SECRET_ACCESS_KEY + SIGNING_KEY: + from_secret: PRODUCTION_TERRAFORM_REGISTRY_SIGNING_KEY --- kind: signature -hmac: 212e3c224f4d500ce35063370bdf6adaf350dff5008ae56b6a566c05044ac43e +hmac: 0da8f5b80ad340c206367ddc7b64f44b4c22ee0b81e4fe4179491c04754bb82a ... diff --git a/tooling/cmd/promote-terraform/main.go b/tooling/cmd/promote-terraform/main.go index f57390403..d85b6ec2b 100644 --- a/tooling/cmd/promote-terraform/main.go +++ b/tooling/cmd/promote-terraform/main.go @@ -33,7 +33,6 @@ const ( func main() { args := parseCommandLine() - log.Infof("Version tag is %s\n", args.providerTag) workspace, err := ensureWorkspaceExists(args.workingDir) if err != nil { diff --git a/tooling/internal/terraform/registry/protocol_test.go b/tooling/internal/terraform/registry/protocol_test.go index c0dc147f7..ed5422c23 100644 --- a/tooling/internal/terraform/registry/protocol_test.go +++ b/tooling/internal/terraform/registry/protocol_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "testing" + "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/require" ) @@ -31,7 +32,6 @@ const validIndex = ` ] } ] - } ` @@ -39,7 +39,7 @@ func TestIndexJson(t *testing.T) { uut := Versions{ Versions: []*Version{ { - Version: "2.0.0", + Version: semver.Version{Major: 2, Minor: 0, Patch: 0}, Protocols: []string{"4.0", "5.1"}, Platforms: []Platform{ {OS: "darwin", Arch: "amd64"}, @@ -49,7 +49,7 @@ func TestIndexJson(t *testing.T) { }, }, { - Version: "2.0.1", + Version: semver.Version{Major: 2, Minor: 0, Patch: 1}, Protocols: []string{"5.2"}, Platforms: []Platform{ {OS: "darwin", Arch: "amd64"}, diff --git a/tooling/internal/terraform/registry/sign.go b/tooling/internal/terraform/registry/sign.go index 3b03e43f9..53b032b46 100644 --- a/tooling/internal/terraform/registry/sign.go +++ b/tooling/internal/terraform/registry/sign.go @@ -60,7 +60,7 @@ func RepackProvider(dstDir string, srcFileName string, signingEntity *openpgp.En return nil, trace.Wrap(err, "Bad filename") } - log.Infof("Provider platform: %s/%s/%s\n", info.Version, info.OS, info.Arch) + log.Infof("Provider platform: %s/%s/%s", info.Version, info.OS, info.Arch) src, err := os.Open(srcFileName) if err != nil { From 660f90c74973c6d2b59e018e3907e976440f72b2 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Wed, 23 Mar 2022 17:35:03 +1100 Subject: [PATCH 11/20] promotion --- .drone.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.drone.yml b/.drone.yml index 4f79074ac..d4f563c7f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -93,6 +93,12 @@ kind: pipeline type: kubernetes name: promote-terraform-provider +trigger: + event: + - promote + target: + - production + concurrency: limit: 1 From e489ef6e371a36bb59d124b785a4e794bf08b678 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Wed, 23 Mar 2022 18:33:31 +1100 Subject: [PATCH 12/20] Restore full drone --- .drone.yml | 380 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 377 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index d4f563c7f..62670cd24 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,3 +1,206 @@ +--- +kind: pipeline +type: kubernetes +name: test-linux + +trigger: + branch: + - master + - branch/* + event: + include: + - push + repo: + include: + - gravitational/* + +workspace: + path: /go/src/github.com/gravitational/teleport-plugins + +steps: + - name: Run linter + image: golangci/golangci-lint:v1.39.0 + commands: + - make lint + + - name: Run tests + image: golang:1.17.5 + environment: + TELEPORT_ENTERPRISE_LICENSE: + from_secret: TELEPORT_ENTERPRISE_LICENSE + TELEPORT_GET_VERSION: v8.0.5 + commands: + - echo Testing plugins against Teleport $TELEPORT_GET_VERSION + - make test + +--- +kind: pipeline +type: exec +name: test-darwin + +concurrency: + limit: 1 + +platform: + os: darwin + arch: amd64 + +trigger: + branch: + - master + - branch/* + event: + include: + - push + repo: + include: + - gravitational/* + +workspace: + path: /tmp/teleport-plugins/test-darwin + +steps: + - name: Clean up exec runner storage + commands: + # This will remove subdirectories under pkg/mod which 0400 permissions. + # See for more details: https://github.com/golang/go/issues/27455 + - go clean -modcache + - rm -rf /tmp/teleport-plugins/test-darwin/go + - mkdir -p /tmp/teleport-plugins/test-darwin/go + + - name: Run tests + environment: + TELEPORT_ENTERPRISE_LICENSE: + from_secret: TELEPORT_ENTERPRISE_LICENSE + TELEPORT_GET_VERSION: v8.0.5 + GOPATH: /tmp/teleport-plugins/test-darwin/go + GOCACHE: /tmp/teleport-plugins/test-darwin/go/cache + commands: + - go version + - make test + +--- +kind: pipeline +type: kubernetes +name: build-on-push-linux + +trigger: + branch: + - master + - branch/* + event: + include: + - push + repo: + include: + - gravitational/* + +depends_on: + - test-linux + +workspace: + path: /go/src/github.com/gravitational/teleport-plugins + +steps: + - name: Build artifacts + image: golang:1.17.5 + commands: + - make build-all + +--- +kind: pipeline +type: exec +name: build-on-push-darwin + +concurrency: + limit: 1 + +platform: + os: darwin + arch: amd64 + +trigger: + branch: + - master + - branch/* + event: + include: + - push + repo: + include: + - gravitational/* + +depends_on: + - test-darwin + +workspace: + path: /tmp/teleport-plugins/build-darwin + +steps: + - name: Clean up exec runner storage (pre) + commands: + # This will remove subdirectories under pkg/mod which 0400 permissions. + # See for more details: https://github.com/golang/go/issues/27455 + - go clean -modcache + - rm -rf /tmp/teleport-plugins/build-darwin/go + - mkdir -p /tmp/teleport-plugins/build-darwin/go/cache + + - name: Build artifacts (darwin) + environment: + GOPATH: /tmp/teleport-plugins/build-darwin/go + GOCACHE: /tmp/teleport-plugins/build-darwin/go/cache + commands: + - make build-all + +--- +kind: pipeline +type: kubernetes +name: tag-build-plugins-linux + +trigger: + event: + - tag + ref: + include: + - refs/tags/teleport-gitlab-v* + - refs/tags/teleport-jira-v* + - refs/tags/teleport-mattermost-v* + - refs/tags/teleport-pagerduty-v* + - refs/tags/teleport-slack-v* + - refs/tags/teleport-email-v* + +depends_on: + - test-linux + +workspace: + path: /go/src/github.com/gravitational/teleport-plugins + +steps: + - name: Build artifacts + image: golang:1.17.5 + commands: + - mkdir -p build/ + - export PLUGIN_TYPE=$(echo ${DRONE_TAG} | cut -d- -f2) + - make release/access-$PLUGIN_TYPE + - find access/ -iname "*.tar.gz" -print -exec cp {} build/ \; + - cd build + - for FILE in *.tar.gz; do sha256sum $FILE > $FILE.sha256; done + - ls -l . + + - name: Upload to S3 + image: plugins/s3 + settings: + bucket: + from_secret: AWS_S3_BUCKET + access_key: + from_secret: AWS_ACCESS_KEY_ID + secret_key: + from_secret: AWS_SECRET_ACCESS_KEY + region: us-west-2 + source: /go/src/github.com/gravitational/teleport-plugins/build/* + target: teleport-plugins/tag/${DRONE_TAG} + strip_prefix: /go/src/github.com/gravitational/teleport-plugins/build + --- kind: pipeline type: kubernetes @@ -33,11 +236,182 @@ steps: from_secret: AWS_ACCESS_KEY_ID secret_key: from_secret: AWS_SECRET_ACCESS_KEY - region: us-east-2 + region: us-west-2 source: /go/src/github.com/gravitational/teleport-plugins/build/* target: teleport-plugins/tag/${DRONE_TAG} strip_prefix: /go/src/github.com/gravitational/teleport-plugins/build +--- +kind: pipeline +type: exec +name: tag-build-terraform-darwin + +concurrency: + limit: 1 + +platform: + os: darwin + arch: amd64 + +trigger: + event: + - tag + ref: + include: + - refs/tags/terraform-provider-teleport-v* + +steps: + - name: Build artifacts + commands: + - mkdir -p build/ + - make release/terraform + - find terraform/ -iname "*.tar.gz" -print -exec cp {} build/ \; + - cd build + - for FILE in *.tar.gz; do shasum -a 256 $FILE > $FILE.sha256; done + - ls -l . + + - name: Upload to S3 + environment: + AWS_S3_BUCKET: + from_secret: AWS_S3_BUCKET + AWS_ACCESS_KEY_ID: + from_secret: AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: + from_secret: AWS_SECRET_ACCESS_KEY + AWS_REGION: us-west-2 + commands: + - cd build + - aws s3 sync . s3://$AWS_S3_BUCKET/teleport-plugins/tag/${DRONE_TAG}/ + +--- +kind: pipeline +type: kubernetes +name: tag-build-event-handler-linux + +trigger: + event: + - tag + ref: + include: + - refs/tags/teleport-event-handler-v* + +workspace: + path: /go/src/github.com/gravitational/teleport-plugins + +steps: + - name: Build artifacts + image: golang:1.17.5 + commands: + - mkdir -p build/ + - make release/event-handler + - find event-handler/ -iname "*.tar.gz" -print -exec cp {} build/ \; + - cd build + - for FILE in *.tar.gz; do sha256sum $FILE > $FILE.sha256; done + - ls -l . + + - name: Upload to S3 + image: plugins/s3 + settings: + bucket: + from_secret: AWS_S3_BUCKET + access_key: + from_secret: AWS_ACCESS_KEY_ID + secret_key: + from_secret: AWS_SECRET_ACCESS_KEY + region: us-west-2 + source: /go/src/github.com/gravitational/teleport-plugins/build/* + target: teleport-plugins/tag/${DRONE_TAG} + strip_prefix: /go/src/github.com/gravitational/teleport-plugins/build + +--- +kind: pipeline +type: exec +name: tag-build-event-handler-darwin + +concurrency: + limit: 1 + +platform: + os: darwin + arch: amd64 + +trigger: + event: + - tag + ref: + include: + - refs/tags/teleport-event-handler-v* + +steps: + - name: Build artifacts + commands: + - mkdir -p build/ + - make release/event-handler + - find event-handler/ -iname "*.tar.gz" -print -exec cp {} build/ \; + - cd build + - for FILE in *.tar.gz; do shasum -a 256 $FILE > $FILE.sha256; done + - ls -l . + + - name: Upload to S3 + environment: + AWS_S3_BUCKET: + from_secret: AWS_S3_BUCKET + AWS_ACCESS_KEY_ID: + from_secret: AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: + from_secret: AWS_SECRET_ACCESS_KEY + AWS_REGION: us-west-2 + commands: + - cd build + - aws s3 sync . s3://$AWS_S3_BUCKET/teleport-plugins/tag/${DRONE_TAG}/ + +--- +kind: pipeline +type: kubernetes +name: promote-artifact + +trigger: + event: + - promote + target: + - production + +workspace: + path: /go/src/github.com/gravitational/teleport-plugins + +clone: + disable: true + +steps: + - name: Download artifact from S3 artifact publishing bucket + image: amazon/aws-cli + environment: + AWS_S3_BUCKET: + from_secret: AWS_S3_BUCKET + AWS_ACCESS_KEY_ID: + from_secret: AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: + from_secret: AWS_SECRET_ACCESS_KEY + AWS_REGION: us-west-2 + commands: + - aws s3 sync s3://$AWS_S3_BUCKET/teleport-plugins/tag/${DRONE_TAG}/ . + + - name: Upload artifact to production S3 bucket with public read access + image: plugins/s3 + settings: + bucket: + from_secret: PRODUCTION_AWS_S3_BUCKET + access_key: + from_secret: PRODUCTION_AWS_ACCESS_KEY_ID + secret_key: + from_secret: PRODUCTION_AWS_SECRET_ACCESS_KEY + region: us-east-1 + acl: public-read + source: /go/src/github.com/gravitational/teleport-plugins/* + target: teleport-plugins/${DRONE_TAG##*-v}/ + strip_prefix: /go/src/github.com/gravitational/teleport-plugins/ + + --- kind: pipeline type: kubernetes @@ -52,7 +426,7 @@ trigger: depends_on: - tag-build-terraform-linux -# - tag-build-terraform-darwin + - tag-build-terraform-darwin concurrency: limit: 1 @@ -136,6 +510,6 @@ steps: --- kind: signature -hmac: 0da8f5b80ad340c206367ddc7b64f44b4c22ee0b81e4fe4179491c04754bb82a +hmac: 3e9993ff153de8b48ae73f7a556bcf5890832e0c34aeece005da84827a06f1e9 ... From 115619dc93990dae082f34af3b3e1c1444641318 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Mon, 28 Mar 2022 11:10:07 +1100 Subject: [PATCH 13/20] Repack test --- .../terraform/registry/archives_test.go | 111 +++++++++++++++++- 1 file changed, 110 insertions(+), 1 deletion(-) diff --git a/tooling/internal/terraform/registry/archives_test.go b/tooling/internal/terraform/registry/archives_test.go index 41519954f..845323227 100644 --- a/tooling/internal/terraform/registry/archives_test.go +++ b/tooling/internal/terraform/registry/archives_test.go @@ -1,7 +1,116 @@ package registry -import "testing" +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "io" + "io/fs" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/coreos/go-semver/semver" + "github.com/gravitational/teleport-plugins/tooling/internal/filename" + "github.com/stretchr/testify/require" +) + +const ( + contentFilename = "major-generals-song" + content = "I am the very model of a modern Major-General\nI've information Animal, Vegetable and Mineral." +) + +func newPackage(t *testing.T, timestamp time.Time, version, system, arch string) string { + v, err := semver.NewVersion(version) + require.NoError(t, err) + + info := filename.Info{ + Type: "terraform-provider", + Version: *v, + OS: system, + Arch: arch, + } + filename := filepath.Join(t.TempDir(), info.Filename(".tar.gz")) + + f, err := os.Create(filename) + require.NoError(t, err) + defer f.Close() + + compressor := gzip.NewWriter(f) + defer compressor.Close() + + tarwriter := tar.NewWriter(compressor) + defer tarwriter.Close() + + err = tarwriter.WriteHeader(&tar.Header{ + Name: contentFilename, + Size: int64(len(content)), + Mode: 0755, + ModTime: timestamp, + }) + require.NoError(t, err) + + _, err = tarwriter.Write([]byte(content)) + require.NoError(t, err) + + return filename +} + +func newKey(t *testing.T) *openpgp.Entity { + entity, err := openpgp.NewEntity("testing", "test key", "root@example.com", nil) + require.NoError(t, err) + return entity +} func TestRepack(t *testing.T) { + signer := newKey(t) + timestamp := time.Now() + srcPkg := newPackage(t, timestamp, "1.2.3", "linux", "arm") + dstDir := t.TempDir() + + result, err := RepackProvider(dstDir, srcPkg, signer) + require.NoError(t, err) + require.Equal(t, semver.Version{Major: 1, Minor: 2, Patch: 3}, result.Version) + require.Equal(t, "linux", result.OS) + require.Equal(t, "arm", result.Arch) + + t.Run("Signature", func(t *testing.T) { + keyring := openpgp.EntityList{signer} + shafile, err := os.Open(result.Sum) + require.NoError(t, err) + defer shafile.Close() + + sigfile, err := os.Open(result.Sig) + require.NoError(t, err) + defer sigfile.Close() + + actualSigner, err := openpgp.CheckDetachedSignature(keyring, shafile, sigfile, nil) + require.NoError(t, err) + require.Equal(t, signer.PrivateKey.KeyId, actualSigner.PrivateKey.KeyId) + }) + + t.Run("Content", func(t *testing.T) { + zipFile, err := zip.OpenReader(result.Zip) + require.NoError(t, err) + defer zipFile.Close() + + require.Len(t, zipFile.File, 1) + f := zipFile.File[0] + require.Equal(t, contentFilename, f.Name) + require.Equal(t, fs.FileMode(0755), f.Mode()) + require.Equal(t, uint64(len(content)), f.UncompressedSize64) + + expectedTimestamp := timestamp.Round(time.Second) + require.True(t, expectedTimestamp.Equal(f.Modified), "Expected %s == %s", expectedTimestamp, f.Modified) + + body, err := f.Open() + require.NoError(t, err) + defer body.Close() + actualContent, err := io.ReadAll(body) + require.NoError(t, err) + require.Equal(t, content, string(actualContent)) + }) } From d78e5111c2aeffbe8b7261fb549009079419af2d Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Mon, 28 Mar 2022 11:25:05 +1100 Subject: [PATCH 14/20] Fix filename tests --- tooling/internal/filename/parse_test.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tooling/internal/filename/parse_test.go b/tooling/internal/filename/parse_test.go index 69039d7ff..69021f99f 100644 --- a/tooling/internal/filename/parse_test.go +++ b/tooling/internal/filename/parse_test.go @@ -3,6 +3,7 @@ package filename import ( "testing" + "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/require" ) @@ -12,7 +13,7 @@ func TestParseFilename(t *testing.T) { require.NoError(t, err) require.Equal(t, "terraform-provider", info.Type) - require.Equal(t, "7.0.0", info.Version) + require.Equal(t, *semver.New("7.0.0"), info.Version) require.Equal(t, "darwin", info.OS) require.Equal(t, "amd64", info.Arch) }) @@ -21,7 +22,7 @@ func TestParseFilename(t *testing.T) { info, err := Parse("terraform-provider-teleport-v1.2.3-linux-arm-bin.tar.gz") require.NoError(t, err) require.Equal(t, "terraform-provider", info.Type) - require.Equal(t, "1.2.3", info.Version) + require.Equal(t, *semver.New("1.2.3"), info.Version) require.Equal(t, "linux", info.OS) require.Equal(t, "arm", info.Arch) }) @@ -35,7 +36,7 @@ func TestParseFilename(t *testing.T) { info, err := Parse("terraform-provider-teleport-v1.2.3-beta.1-linux-arm-bin.tar.gz") require.NoError(t, err) require.Equal(t, "terraform-provider", info.Type) - require.Equal(t, "1.2.3-beta.1", info.Version) + require.Equal(t, *semver.New("1.2.3-beta.1"), info.Version) require.Equal(t, "linux", info.OS) require.Equal(t, "arm", info.Arch) }) @@ -44,7 +45,7 @@ func TestParseFilename(t *testing.T) { info, err := Parse("terraform-provider-teleport-v1.2.3+1-linux-arm-bin.tar.gz") require.NoError(t, err) require.Equal(t, "terraform-provider", info.Type) - require.Equal(t, "1.2.3+1", info.Version) + require.Equal(t, *semver.New("1.2.3+1"), info.Version) require.Equal(t, "linux", info.OS) require.Equal(t, "arm", info.Arch) }) @@ -53,7 +54,7 @@ func TestParseFilename(t *testing.T) { info, err := Parse("terraform-provider-teleport-v1.2.3-beta.1+42-linux-arm-bin.tar.gz") require.NoError(t, err) require.Equal(t, "terraform-provider", info.Type) - require.Equal(t, "1.2.3-beta.1+42", info.Version) + require.Equal(t, *semver.New("1.2.3-beta.1+42"), info.Version) require.Equal(t, "linux", info.OS) require.Equal(t, "arm", info.Arch) }) @@ -65,7 +66,13 @@ func TestParseFilename(t *testing.T) { } func TestGenerateFilename(t *testing.T) { - info := Info{Type: "some-plugin", Version: "1.2.3", OS: "darwin", Arch: "amd64"} + + info := Info{ + Type: "some-plugin", + Version: *semver.New("1.2.3"), + OS: "darwin", + Arch: "amd64", + } fn := info.Filename(".banana") require.Equal(t, "some-plugin-teleport-v1.2.3-darwin-amd64-bin.banana", fn) } From 0c168a8eba1a094e30ea6456c285aab81b216207 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Wed, 30 Mar 2022 16:57:25 +1100 Subject: [PATCH 15/20] Update tooling/internal/terraform/registry/sha.go Co-authored-by: Roman Tkachenko --- tooling/internal/terraform/registry/sha.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tooling/internal/terraform/registry/sha.go b/tooling/internal/terraform/registry/sha.go index 11c8e2703..8a7d233eb 100644 --- a/tooling/internal/terraform/registry/sha.go +++ b/tooling/internal/terraform/registry/sha.go @@ -10,7 +10,7 @@ import ( "github.com/gravitational/trace" ) -// Sha256Sum generates the equivalent contents of running `sha256sum` om some +// sha256Sum generates the equivalent contents of running `sha256sum` on some // data and writes it to the supplied stream, returning the sha bytes. func sha256Sum(dst io.Writer, filename string, data []byte) ([]byte, error) { sha := sha256.Sum256(data) From 55271c8af75d8cef41c247a573821b61cfb8ca36 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Thu, 31 Mar 2022 18:42:50 +1100 Subject: [PATCH 16/20] Review concerns --- tooling/cmd/promote-terraform/main.go | 16 ++-- tooling/internal/filename/parse.go | 8 ++ .../internal/terraform/registry/archives.go | 7 +- .../internal/terraform/registry/protocol.go | 28 +++++- .../terraform/registry/protocol_test.go | 2 +- tooling/internal/terraform/registry/sha.go | 15 ++- tooling/internal/terraform/registry/sign.go | 91 ++++++++++++------- .../{archives_test.go => sign_test.go} | 2 +- 8 files changed, 114 insertions(+), 55 deletions(-) rename tooling/internal/terraform/registry/{archives_test.go => sign_test.go} (98%) diff --git a/tooling/cmd/promote-terraform/main.go b/tooling/cmd/promote-terraform/main.go index d85b6ec2b..bb5ac261c 100644 --- a/tooling/cmd/promote-terraform/main.go +++ b/tooling/cmd/promote-terraform/main.go @@ -58,7 +58,7 @@ func main() { log.WithError(err).Fatalf("Failed repacking artifacts") } - err = updateRegistry(context.Background(), &args.production, workspace, args.providerNamespace, args.providerName, &versionRecord, newFiles) + err = updateRegistry(context.Background(), &args.production, workspace, args.providerNamespace, args.providerName, versionRecord, newFiles) if err != nil { log.WithError(err).Fatal("Failed updating registry") } @@ -77,7 +77,7 @@ func logLevel(args *args) log.Level { } } -func updateRegistry(ctx context.Context, prodBucket *bucketConfig, workspace *workspacePaths, namespace, provider string, newVersion *registry.Version, files []string) error { +func updateRegistry(ctx context.Context, prodBucket *bucketConfig, workspace *workspacePaths, namespace, provider string, newVersion registry.Version, files []string) error { s3client, err := newS3ClientFromBucketConfig(ctx, prodBucket) if err != nil { return trace.Wrap(err) @@ -89,7 +89,7 @@ func updateRegistry(ctx context.Context, prodBucket *bucketConfig, workspace *wo // Try downloading the versions file. This may not exist in an empty/new // registry, so if the download fails with a NotFound error we ignore it // and use an empty index. - versions := registry.NewVersions() + versions := registry.Versions{} err = download(ctx, s3client, versionsFilePath, prodBucket.bucketName, versionsFileKey) switch { case err == nil: @@ -107,7 +107,7 @@ func updateRegistry(ctx context.Context, prodBucket *bucketConfig, workspace *wo // Index the available version by their semver version, so that we can find the // appropriate release if we're overwriting an existing version - versionIndex := map[semver.Version]*registry.Version{} + versionIndex := map[semver.Version]registry.Version{} for _, v := range versions.Versions { versionIndex[v.Version] = v } @@ -150,7 +150,7 @@ func uploadRegistry(ctx aws.Context, s3Client *s3.Client, bucketName string, pro doUpload := func() error { f, err := os.Open(f) if err != nil { - return err + return trace.Wrap(err) } defer f.Close() @@ -160,7 +160,7 @@ func uploadRegistry(ctx aws.Context, s3Client *s3.Client, bucketName string, pro Body: f, }) - return err + return trace.Wrap(err) } if err = doUpload(); err != nil { @@ -171,7 +171,7 @@ func uploadRegistry(ctx aws.Context, s3Client *s3.Client, bucketName string, pro return nil } -func flattenVersionIndex(versionIndex map[semver.Version]*registry.Version) []*registry.Version { +func flattenVersionIndex(versionIndex map[semver.Version]registry.Version) []registry.Version { // We want to output a list of versions with semver ordering, so first we // generate a sorted list of keys @@ -183,7 +183,7 @@ func flattenVersionIndex(versionIndex map[semver.Version]*registry.Version) []*r // Now we can simply walk the index using the sorted key list and we have // our sorted output list - result := make([]*registry.Version, 0, len(keys)) + result := make([]registry.Version, 0, len(keys)) for _, k := range keys { result = append(result, versionIndex[k]) } diff --git a/tooling/internal/filename/parse.go b/tooling/internal/filename/parse.go index 0cfff52b1..98c32a84a 100644 --- a/tooling/internal/filename/parse.go +++ b/tooling/internal/filename/parse.go @@ -13,6 +13,8 @@ var ( filenamePattern *regexp.Regexp = regexp.MustCompile(`^(?P.*)-teleport-v(?P.*)-(?Plinux|darwin|windows)-(?Pamd64|arm|aarch)-bin.tar.gz$`) ) +// Info holds information about a plugin, deduced from from its Houston-compatible +// filename. type Info struct { Type string Version semver.Version @@ -20,6 +22,9 @@ type Info struct { Arch string } +// Parse attempts to deduce information about a staged plugin from its (assumed +// Houston-compatible) filename, returning an error if the filename can't be +// parsed. func Parse(filename string) (Info, error) { filename = filepath.Base(filename) @@ -41,6 +46,9 @@ func Parse(filename string) (Info, error) { }, nil } +// Filename generates a Houston-compatible filename for the Info block, with a +// given file extension (NB: the extension is expected to include the leading +// dot). func (info *Info) Filename(extension string) string { return fmt.Sprintf("%s-teleport-v%s-%s-%s-bin%s", info.Type, info.Version, info.OS, info.Arch, extension) } diff --git a/tooling/internal/terraform/registry/archives.go b/tooling/internal/terraform/registry/archives.go index 00b672621..837ea31e1 100644 --- a/tooling/internal/terraform/registry/archives.go +++ b/tooling/internal/terraform/registry/archives.go @@ -9,11 +9,15 @@ import ( "github.com/gravitational/trace" ) +// repack copies the contents of a compressed tar archive into a +// zip archive. Only regular files are copied into the Zip +// archive - symlinks, etc are discarded. func repack(dst io.Writer, src io.Reader) error { uncompressedReader, err := gzip.NewReader(src) if err != nil { return trace.Wrap(err) } + defer uncompressedReader.Close() tarReader := tar.NewReader(uncompressedReader) zipWriter := zip.NewWriter(dst) @@ -28,6 +32,7 @@ func repack(dst io.Writer, src io.Reader) error { return trace.Wrap(err) } + // if the header represents a "regular file"... if header.Typeflag == tar.TypeReg { if err = copyfile(zipWriter, header, tarReader); err != nil { return trace.Wrap(err, "failed repacking tar file item") @@ -43,7 +48,7 @@ func repack(dst io.Writer, src io.Reader) error { func copyfile(zipfile *zip.Writer, header *tar.Header, src io.Reader) error { zipHeader, err := zip.FileInfoHeader(header.FileInfo()) if err != nil { - return trace.Wrap(err, "failed initialising zipfile header") + return trace.Wrap(err, "failed initializing zipfile header") } zipHeader.Name = header.Name zipHeader.Method = zip.Deflate diff --git a/tooling/internal/terraform/registry/protocol.go b/tooling/internal/terraform/registry/protocol.go index ae9393b99..c00e14bd3 100644 --- a/tooling/internal/terraform/registry/protocol.go +++ b/tooling/internal/terraform/registry/protocol.go @@ -24,13 +24,11 @@ func VersionsFilePath(prefix, namespace, name string) string { // Versions is a Go representation of the Terraform Registry Protocol // `versions` file type Versions struct { - Versions []*Version `json:"versions"` -} - -func NewVersions() Versions { - return Versions{Versions: []*Version{}} + Versions []Version `json:"versions"` } +// Save formats and writes the version list to the supplied filesystem +// location. func (idx *Versions) Save(filename string) error { indexFile, err := os.Create(filename) if err != nil { @@ -44,6 +42,8 @@ func (idx *Versions) Save(filename string) error { return encoder.Encode(idx) } +// LoadVersionsFile reads and parses a versions structure from the file +// at the supplied filesystem location func LoadVersionsFile(filename string) (Versions, error) { f, err := os.Open(filename) if err != nil { @@ -60,17 +60,22 @@ func LoadVersionsFile(filename string) (Versions, error) { return idx, nil } +// Platform encodes an OS/Arch pair type Platform struct { OS string `json:"os"` Arch string `json:"arch"` } +// Version describes tha compatability and available platforms for a given +// provider version type Version struct { Version semver.Version `json:"version"` Protocols []string `json:"protocols"` Platforms []Platform `json:"platforms"` } +// GpgPublicKey describes a (the public half) of a GPG key used to sign a +// provider zipfile. type GpgPublicKey struct { KeyID string `json:"key_id"` ASCIIArmor string `json:"ascii_armor"` @@ -79,10 +84,16 @@ type GpgPublicKey struct { SourceURL string `json:"source_url,omitempty"` } +// SigningKeys describes the key (or keys) that signed a given download. As +// per the registry protocol, only GPG keys are supported for now. type SigningKeys struct { GpgPublicKeys []GpgPublicKey `json:"gpg_public_keys"` } +// Download describes the file specific download package for a given platform & +// provider version. If a provider supports multiple OSs & architectures, the +// registry will contain multiple Download records, one for each unique +// architecture/OS pair. type Download struct { Protocols []string `json:"protocols"` OS string `json:"os"` @@ -95,6 +106,9 @@ type Download struct { SigningKeys SigningKeys `json:"signing_keys"` } +// Save serializes a Download record to JSON and writes it to the appropriate +// Provider Registry Protocol-defined location in a registry directory tree, i.e. +// indexDir/:namespace/:name/:version/download/:os/:arch func (dl *Download) Save(indexDir, namespace, name string, version semver.Version) (string, error) { filename := filepath.Join(indexDir, namespace, name, version.String(), "download", dl.OS, dl.Arch) @@ -114,6 +128,10 @@ func (dl *Download) Save(indexDir, namespace, name string, version semver.Versio return filename, encoder.Encode(dl) } +// NewDownloadFromRepackResult creates a new, fully-populated Download object +// from the result of repacking a Houston-compatible tarball into a Terraform +// registry-compatible zip +// archive func NewDownloadFromRepackResult(info *RepackResult, protocols []string, objectStoreURL string) (*Download, error) { keyText, err := formatPublicKey(info.SigningEntity) diff --git a/tooling/internal/terraform/registry/protocol_test.go b/tooling/internal/terraform/registry/protocol_test.go index ed5422c23..e01183d6c 100644 --- a/tooling/internal/terraform/registry/protocol_test.go +++ b/tooling/internal/terraform/registry/protocol_test.go @@ -37,7 +37,7 @@ const validIndex = ` func TestIndexJson(t *testing.T) { uut := Versions{ - Versions: []*Version{ + Versions: []Version{ { Version: semver.Version{Major: 2, Minor: 0, Patch: 0}, Protocols: []string{"4.0", "5.1"}, diff --git a/tooling/internal/terraform/registry/sha.go b/tooling/internal/terraform/registry/sha.go index 8a7d233eb..e22473f0d 100644 --- a/tooling/internal/terraform/registry/sha.go +++ b/tooling/internal/terraform/registry/sha.go @@ -12,8 +12,15 @@ import ( // sha256Sum generates the equivalent contents of running `sha256sum` on some // data and writes it to the supplied stream, returning the sha bytes. -func sha256Sum(dst io.Writer, filename string, data []byte) ([]byte, error) { - sha := sha256.Sum256(data) - _, err := fmt.Fprintf(dst, "%s %s\n", hex.EncodeToString(sha[:]), filepath.Base(filename)) - return sha[:], trace.Wrap(err, "failed generating sum file") +func sha256Sum(dst io.Writer, filename string, data io.Reader) ([]byte, error) { + hasher := sha256.New() + _, err := io.Copy(hasher, data) + if err != nil { + return nil, trace.Wrap(err) + } + + sha := hasher.Sum(make([]byte, 0, sha256.Size)) + + _, err = fmt.Fprintf(dst, "%s %s\n", hex.EncodeToString(sha[:]), filepath.Base(filename)) + return sha, trace.Wrap(err, "failed generating sum file") } diff --git a/tooling/internal/terraform/registry/sign.go b/tooling/internal/terraform/registry/sign.go index 53b032b46..026cc89c4 100644 --- a/tooling/internal/terraform/registry/sign.go +++ b/tooling/internal/terraform/registry/sign.go @@ -3,6 +3,8 @@ package registry import ( "bytes" "encoding/hex" + "io" + "io/ioutil" "os" "path/filepath" @@ -20,6 +22,8 @@ type FileNames struct { Sig string } +// IsProviderTarball tests if a given string is a Hudson-compatible filename +// indicating a terraform-provider plugin type func IsProviderTarball(fn string) bool { info, err := filename.Parse(fn) if err != nil { @@ -46,6 +50,7 @@ type RepackResult struct { SigningEntity *openpgp.Entity } +// Sha256String formats the binary SHA256 as a hex string func (r *RepackResult) Sha256String() string { return hex.EncodeToString(r.Sha256) } @@ -54,13 +59,17 @@ func (r *RepackResult) Sha256String() string { // with a terraform provider registry, generating all the required sidecar files // as well. Returns a `RepackResult` instance containing the location of the // generated files and information about the packed plugin +// +// For more information on the output files, see the Terraform Provider Registry +// Protocol documentation: +// https://www.terraform.io/internals/provider-registry-protocol func RepackProvider(dstDir string, srcFileName string, signingEntity *openpgp.Entity) (*RepackResult, error) { info, err := filename.Parse(srcFileName) if err != nil { - return nil, trace.Wrap(err, "Bad filename") + return nil, trace.Wrap(err, "bad filename %q", srcFileName) } - log.Infof("Provider platform: %s/%s/%s", info.Version, info.OS, info.Arch) + log.Debugf("Provider platform: %s/%s/%s", info.Version, info.OS, info.Arch) src, err := os.Open(srcFileName) if err != nil { @@ -68,11 +77,22 @@ func RepackProvider(dstDir string, srcFileName string, signingEntity *openpgp.En } defer src.Close() - // Create the zip archive in memory in order to make it easier to - // hash and sign - var zipArchive bytes.Buffer - - err = repack(&zipArchive, src) + tmpZipFile, err := os.CreateTemp("", "") + if err != nil { + return nil, trace.Wrap(err, "failed creating tempfile for zip archive") + } + defer func() { + // we will only want to clean up the tmp file in the failure case, + // because if RepackProvider has succeeded then the temp file has + // already been closed and moved into place in the output directory. + if err != nil { + tmpZipFile.Close() + os.Remove(tmpZipFile.Name()) + } + }() + + log.Debugf("Repacking into zipfile: %s", tmpZipFile.Name()) + err = repack(tmpZipFile, src) if err != nil { return nil, trace.Wrap(err, "failed repacking provider") } @@ -83,14 +103,27 @@ func RepackProvider(dstDir string, srcFileName string, signingEntity *openpgp.En SigningEntity: signingEntity, } - // compute sha256 and format the sha file as per sha256sum + // compute sha256 and format the SHA file as per sha256sum + _, err = tmpZipFile.Seek(0, io.SeekStart) + if err != nil { + return nil, trace.Wrap(err, "failed rewinding temp zipfile for summing") + } + var sums bytes.Buffer - result.Sha256, err = sha256Sum(&sums, result.Zip, zipArchive.Bytes()) + result.Sha256, err = sha256Sum(&sums, result.Zip, tmpZipFile) if err != nil { return nil, trace.Wrap(err) } - // sign the sums with our private key and generate a signature file + // we're done with the temp archive for now, and we'll need the file to be + // closed to move it into place anyway... + tmpZipFileName := tmpZipFile.Name() + err = tmpZipFile.Close() + if err != nil { + return nil, trace.Wrap(err, "failed closing temp zipfile") + } + + // sign the sums with our private key and generate a signature var sig bytes.Buffer err = openpgp.DetachSign(&sig, signingEntity, bytes.NewReader(sums.Bytes()), nil) if err != nil { @@ -98,7 +131,7 @@ func RepackProvider(dstDir string, srcFileName string, signingEntity *openpgp.En } // Write everything out to the dstdir - err = writeOutput(result, zipArchive.Bytes(), sums.Bytes(), sig.Bytes()) + err = writeOutput(result, tmpZipFileName, sums.Bytes(), sig.Bytes()) if err != nil { return nil, trace.Wrap(err) } @@ -106,38 +139,26 @@ func RepackProvider(dstDir string, srcFileName string, signingEntity *openpgp.En return result, nil } -func writeOutput(entry *RepackResult, zip, sums, sig []byte) error { - zipFile, err := os.Create(entry.Zip) - if err != nil { - return trace.Wrap(err, "opening zipfile failed") - } - defer zipFile.Close() - - _, err = zipFile.Write(zip) - if err != nil { - return trace.Wrap(err, "writing zipfile failed") - } - - sumFile, err := os.Create(entry.Sum) - if err != nil { - return trace.Wrap(err, "opening sum file failed") - } - defer sumFile.Close() - - _, err = sumFile.Write(sums) +// writeOutput writes the in-memory signature data to file, and moves the temporary +// zip file into place +func writeOutput(entry *RepackResult, zipFilePath string, sums, sig []byte) error { + log.Debugf("Writing sum file to %s", entry.Sum) + err := ioutil.WriteFile(entry.Sum, sums, 0644) if err != nil { return trace.Wrap(err, "writing sumfile failed") } - sigFile, err := os.Create(entry.Sig) + log.Debugf("Writing signature file to %s", entry.Sig) + err = ioutil.WriteFile(entry.Sig, sig, 0644) if err != nil { - return trace.Wrap(err, "opening sig file failed") + return trace.Wrap(err, "writing sumfile failed") } - defer sigFile.Close() - _, err = sigFile.Write(sig) + // Do this _last_, as we want the temp file cleaned up if any of the above fails. + log.Debugf("Moving tmp zipfile %s into place at %s", zipFilePath, entry.Zip) + err = os.Rename(zipFilePath, entry.Zip) if err != nil { - return trace.Wrap(err, "writing sigFile failed") + return trace.Wrap(err, "moving zipfile into place") } return nil diff --git a/tooling/internal/terraform/registry/archives_test.go b/tooling/internal/terraform/registry/sign_test.go similarity index 98% rename from tooling/internal/terraform/registry/archives_test.go rename to tooling/internal/terraform/registry/sign_test.go index 845323227..839830f62 100644 --- a/tooling/internal/terraform/registry/archives_test.go +++ b/tooling/internal/terraform/registry/sign_test.go @@ -64,7 +64,7 @@ func newKey(t *testing.T) *openpgp.Entity { return entity } -func TestRepack(t *testing.T) { +func TestRepackProvider(t *testing.T) { signer := newKey(t) timestamp := time.Now() srcPkg := newPackage(t, timestamp, "1.2.3", "linux", "arm") From bef690a1d2606b349b349460200bcb0020ab43df Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Thu, 31 Mar 2022 18:50:47 +1100 Subject: [PATCH 17/20] Added concurrenvy warning to updateRegistry() --- tooling/cmd/promote-terraform/main.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tooling/cmd/promote-terraform/main.go b/tooling/cmd/promote-terraform/main.go index bb5ac261c..a6e810b12 100644 --- a/tooling/cmd/promote-terraform/main.go +++ b/tooling/cmd/promote-terraform/main.go @@ -77,6 +77,14 @@ func logLevel(args *args) log.Level { } } +// updateRegistry fetches the live registry and adds our new providers to it. +// It's possible for another process to update the `versions` index in the +// bucket while we are modifying it here, and unfortunately AWS doesn't give +// us a nice way to prevent this simply with S3. +// +// We could layer a locking mechanism on top of another AWS service, but for +// now we are relying on drone to honour its concurrency limits (i.e. 1) to +// serialise access to the live `versions` file. func updateRegistry(ctx context.Context, prodBucket *bucketConfig, workspace *workspacePaths, namespace, provider string, newVersion registry.Version, files []string) error { s3client, err := newS3ClientFromBucketConfig(ctx, prodBucket) if err != nil { From 0a5d5e21d32c5233602add354bb195d4cacc3cf9 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Thu, 31 Mar 2022 18:59:07 +1100 Subject: [PATCH 18/20] Drone updates --- .drone.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.drone.yml b/.drone.yml index db0a35fb2..cfe4c9b2b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -651,7 +651,7 @@ steps: - | go run ./cmd/promote-terraform \ --tag ${DRONE_TAG} \ - -p 4.0 -p 5.1 \ + -p 6 \ --namespace gravitational \ --name teleport @@ -664,13 +664,14 @@ steps: STAGING_SECRET_ACCESS_KEY: from_secret: AWS_SECRET_ACCESS_KEY - PRODUCTION_REGION: us-east-2 + PRODUCTION_REGION: + from_secret: STAGING_TERRAFORM_AWS_REGION PROD_BUCKET: - from_secret: STAGING_AWS_TERRAFORM_BUCKET + from_secret: STAGING_TERRAFORM_AWS_BUCKET PROD_ACCESS_KEY_ID: - from_secret: STAGING_AWS_TERRAFORM_ACCESS_KEY_ID + from_secret: STAGING_TERRAFORM_AWS_ACCESS_KEY_ID PROD_SECRET_ACCESS_KEY: - from_secret: STAGING_AWS_TERRAFORM_SECRET_ACCESS_KEY + from_secret: STAGING_TERRAFORM_AWS_SECRET_ACCESS_KEY SIGNING_KEY: from_secret: STAGING_TERRAFORM_REGISTRY_SIGNING_KEY @@ -696,7 +697,7 @@ steps: - | go run ./cmd/promote-terraform \ --tag ${DRONE_TAG} \ - -p 4.0 -p 5.1 \ + -p 6 \ --namespace gravitational \ --name teleport @@ -709,18 +710,19 @@ steps: STAGING_SECRET_ACCESS_KEY: from_secret: AWS_SECRET_ACCESS_KEY - PRODUCTION_REGION: ap-southeast-2 + PRODUCTION_REGION: + from_secret: PRODUCTION_TERRAFORM_AWS_REGION PROD_BUCKET: - from_secret: PRODUCTION_AWS_TERRAFORM_BUCKET + from_secret: PRODUCTION_TERRAFORM_AWS_BUCKET PROD_ACCESS_KEY_ID: - from_secret: PRODUCTION_AWS_TERRAFORM_ACCESS_KEY_ID + from_secret: PRODUCTION_TERRAFORM_AWS_ACCESS_KEY_ID PROD_SECRET_ACCESS_KEY: - from_secret: PRODUCTION_AWS_TERRAFORM_SECRET_ACCESS_KEY + from_secret: PRODUCTION_TERRAFORM_AWS_SECRET_ACCESS_KEY SIGNING_KEY: from_secret: PRODUCTION_TERRAFORM_REGISTRY_SIGNING_KEY --- kind: signature -hmac: 7c3217716b77812d57ebf5f3d6965dfef25aedcc6593bd50365903d65bd2b79f +hmac: d453538bb4b8281960cff6363267944ebd5fd943bc95f9b095a5dddff33be7f6 ... From 868639455f5785f49541b1e6aaea948c452f6b90 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Thu, 31 Mar 2022 22:11:51 +1100 Subject: [PATCH 19/20] Update parse.go --- tooling/internal/filename/parse.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tooling/internal/filename/parse.go b/tooling/internal/filename/parse.go index 98c32a84a..103d07ac5 100644 --- a/tooling/internal/filename/parse.go +++ b/tooling/internal/filename/parse.go @@ -16,9 +16,13 @@ var ( // Info holds information about a plugin, deduced from from its Houston-compatible // filename. type Info struct { + // Type represents the plugin type, e.g. "terraform-provider" Type string + // Version holds the parsed plugin version number Version semver.Version + // OS is the operating system the plugin was built for OS string + // Arch is the CPU architecture the plugin was built for Arch string } From d57fac2596105e7e0e4abbbc86067ab2384a2d13 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Thu, 21 Apr 2022 12:59:34 +1000 Subject: [PATCH 20/20] Adds role awareness to upload script --- .drone.yml | 4 +-- tooling/cmd/promote-terraform/args.go | 10 +++++-- tooling/cmd/promote-terraform/main.go | 28 ++++++++++++++++--- .../internal/terraform/registry/package.go | 3 ++ 4 files changed, 37 insertions(+), 8 deletions(-) create mode 100644 tooling/internal/terraform/registry/package.go diff --git a/.drone.yml b/.drone.yml index cfe4c9b2b..89b845c3a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -710,7 +710,7 @@ steps: STAGING_SECRET_ACCESS_KEY: from_secret: AWS_SECRET_ACCESS_KEY - PRODUCTION_REGION: + PROD_REGION: from_secret: PRODUCTION_TERRAFORM_AWS_REGION PROD_BUCKET: from_secret: PRODUCTION_TERRAFORM_AWS_BUCKET @@ -723,6 +723,6 @@ steps: from_secret: PRODUCTION_TERRAFORM_REGISTRY_SIGNING_KEY --- kind: signature -hmac: d453538bb4b8281960cff6363267944ebd5fd943bc95f9b095a5dddff33be7f6 +hmac: f6ad25a38eda9195f3af3a9e3e10f8c17f879aff7a3d0362f49acc9887924e63 ... diff --git a/tooling/cmd/promote-terraform/args.go b/tooling/cmd/promote-terraform/args.go index 2405b59ef..672496163 100644 --- a/tooling/cmd/promote-terraform/args.go +++ b/tooling/cmd/promote-terraform/args.go @@ -12,6 +12,7 @@ type bucketConfig struct { bucketName string accessKeyID string secretAccessKey string + roleARN string } type args struct { @@ -60,10 +61,15 @@ func parseCommandLine() *args { StringVar(&result.production.bucketName) app.Flag("prod-region", "AWS region the production bucket is in"). - Envar("PRODUCTION_REGION"). - Default("us-east-1"). + Envar("PROD_REGION"). + Default("us-west-2"). StringVar(&result.production.region) + app.Flag("prod-role", "AWS role to use when interacting with the deployment bucket."). + Required(). + PlaceHolder("ARN"). + StringVar(&result.production.roleARN) + app.Flag("prod-access-key-id", "AWS access key id for production bucket"). Envar("PROD_ACCESS_KEY_ID"). Required(). diff --git a/tooling/cmd/promote-terraform/main.go b/tooling/cmd/promote-terraform/main.go index a6e810b12..16a349fd2 100644 --- a/tooling/cmd/promote-terraform/main.go +++ b/tooling/cmd/promote-terraform/main.go @@ -12,12 +12,14 @@ import ( "github.com/ProtonMail/go-crypto/openpgp" "github.com/ProtonMail/go-crypto/openpgp/armor" "github.com/ProtonMail/go-crypto/openpgp/packet" + "github.com/aws/aws-sdk-go-v2/aws" awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/coreos/go-semver/semver" "github.com/gravitational/teleport-plugins/tooling/internal/staging" "github.com/gravitational/teleport-plugins/tooling/internal/terraform/registry" @@ -138,7 +140,7 @@ func updateRegistry(ctx context.Context, prodBucket *bucketConfig, workspace *wo return nil } -func uploadRegistry(ctx aws.Context, s3Client *s3.Client, bucketName string, productionDir string, files []string) error { +func uploadRegistry(ctx context.Context, s3Client *s3.Client, bucketName string, productionDir string, files []string) error { uploader := manager.NewUploader(s3Client) log.Infof("Production dir: %s", productionDir) for _, f := range files { @@ -325,7 +327,8 @@ func ensureWorkspaceExists(workspaceDir string) (*workspacePaths, error) { } func downloadStagedArtifacts(ctx context.Context, tag string, dstDir string, stagingBucket *bucketConfig) ([]string, error) { - + log.Debugf("listing plugins in %s %s", stagingBucket.region, stagingBucket.bucketName) + log.Debugf("listing plugins as %s", stagingBucket.accessKeyID) client, err := newS3ClientFromBucketConfig(ctx, stagingBucket) if err != nil { return nil, trace.Wrap(err) @@ -335,6 +338,7 @@ func downloadStagedArtifacts(ctx context.Context, tag string, dstDir string, sta } func newS3ClientFromBucketConfig(ctx context.Context, bucket *bucketConfig) (*s3.Client, error) { + creds := credentials.NewStaticCredentialsProvider( bucket.accessKeyID, bucket.secretAccessKey, "") @@ -345,7 +349,23 @@ func newS3ClientFromBucketConfig(ctx context.Context, bucket *bucketConfig) (*s3 return nil, trace.Wrap(err) } - return s3.NewFromConfig(cfg), nil + if bucket.roleARN == "" { + return s3.NewFromConfig(cfg), nil + } + + log.Debugf("Configuring deployment role %q", bucket.roleARN) + + stsClient := sts.NewFromConfig(cfg) + stsCreds := stscreds.NewAssumeRoleProvider(stsClient, bucket.roleARN) + stsAwareCfg, err := config.LoadDefaultConfig(ctx, + config.WithRegion(bucket.region), + config.WithCredentialsProvider(stsCreds)) + + if err != nil { + return nil, trace.Wrap(err) + } + + return s3.NewFromConfig(stsAwareCfg), nil } func loadSigningEntity(keyText string) (*openpgp.Entity, error) { diff --git a/tooling/internal/terraform/registry/package.go b/tooling/internal/terraform/registry/package.go new file mode 100644 index 000000000..f921a7cd9 --- /dev/null +++ b/tooling/internal/terraform/registry/package.go @@ -0,0 +1,3 @@ +// Package registry provides tools for interacting with a Terraform custom +// registry +package registry