From 3a29c831bce21f6e88165753e98f4d5129d2c344 Mon Sep 17 00:00:00 2001 From: JP Robinson Date: Tue, 19 Feb 2019 12:46:58 -0500 Subject: [PATCH 1/9] [wip] initial max_versions work --- main.go | 60 +++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/main.go b/main.go index a5332ef..8b149ed 100644 --- a/main.go +++ b/main.go @@ -52,6 +52,11 @@ type GAE struct { // and autoscaling configurations. AppFile string `json:"app_file"` + // MaxVersions is an optional value that can be used along with the "deploy" action. + // If set to a non-zero value, the plugin will look up the versions of the deployed + // service and delete any older versions beyond the "max" value provided. + MaxVersions int `json:"max_versions"` + // CronFile is the name of the cron.yaml file to use for this deployment. This field // is only required if your cron.yaml file is not named 'cron.yaml' CronFile string `json:"cron_file"` @@ -153,14 +158,25 @@ func wrapMain() error { return fmt.Errorf("error: %s\n", err) } + var serviceName string // if gcloud app cmd, run it - if found := gcloudCmds[vargs.Action]; found { - return runGcloud(runner, workspace, vargs) + if gcloudCmds[vargs.Action] { + serviceName, err = runGcloud(runner, workspace, vargs) + } else { + // otherwise, do appcfg.py command + err = runAppCfg(runner, workspace, vargs) + } + + if err != nil { + return err } - // otherwise, do appcfg.py command - return runAppCfg(runner, workspace, vargs) + // check if MaxVersions is supplied + deploy action + if vargs.MaxVersions > 0 && serviceName != "" && vargs.Action == "deploy" { + return removeOldVersions(runner, workspace, serviceName, vargs) + } + return nil } func configFromStdin(vargs *GAE, workspace *string) error { @@ -287,7 +303,7 @@ var gcloudCmds = map[string]bool{ "instances": true, } -func runGcloud(runner *Environ, workspace string, vargs GAE) error { +func runGcloud(runner *Environ, workspace string, vargs GAE) (string, error) { // add the action first (gcloud app X) args := []string{ "app", @@ -338,27 +354,47 @@ func runGcloud(runner *Environ, workspace string, vargs GAE) error { } if err := setupAppFile(workspace, vargs); err != nil { - return err + return "", err } if err := setupCronFile(workspace, vargs); err != nil { - return err + return "", err } if err := setupDispatchFile(workspace, vargs); err != nil { - return err + return "", err } if err := setupQueueFile(workspace, vargs); err != nil { - return err + return "", err } - + var output bytes.Buffer + runner.stdout = &output err := runner.Run(vargs.GCloudCmd, args...) if err != nil { - return fmt.Errorf("error: %s\n", err) + return "", fmt.Errorf("error: %s\n", err) } - return nil + // if this was a deployment, grab the name of the service we just deployed + if vargs.Action == "deploy" { + var versions struct { + Versions []struct { + Service string `json:"service"` + } + } + err = json.NewDecoder(&output).Decode(versions) + if err != nil { + return "", err + } + + if len(versions.Versions) == 0 { + return "", nil + } + + return versions.Versions[0].Service, nil + } + + return "", nil } func runAppCfg(runner *Environ, workspace string, vargs GAE) error { From 1bbb573da737464278d5aef4dfb76769d77b378e Mon Sep 17 00:00:00 2001 From: JP Robinson Date: Tue, 19 Feb 2019 12:47:04 -0500 Subject: [PATCH 2/9] [wip] initial max_versions work --- remove.go | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 remove.go diff --git a/remove.go b/remove.go new file mode 100644 index 0000000..62252a1 --- /dev/null +++ b/remove.go @@ -0,0 +1,49 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "strings" +) + +func removeOldVersions(runner *Environ, workspace, service string, vargs GAE) error { + var versionJSON bytes.Buffer + sout := runner.stdout + runner.stdout = &versionJSON + // look up existing versions for given service ordered by create time desc + err := runner.Run(vargs.GCloudCmd, "app", "versions", "list", "--service", service, + "--format", "json", "--sort-by", "~version.createTime", "--quiet") + if err != nil { + return fmt.Errorf("error: %s\n", err) + } + + var results []struct { + ID string `json:"id"` + TrafficSplit float64 `json:"traffic_split"` + } + err = json.NewDecoder(&versionJSON).Decode(&results) + if err != nil { + return err + } + + var toDelete []string + for i, res := range results { + // keep newer versions, the newly deployed version or anything that has traffic + if (i+1) < vargs.MaxVersions || res.ID == vargs.Version || res.TrafficSplit > 0 { + continue + } + toDelete = append(toDelete, res.ID) + } + log.Printf("deleting %d versions: %s", len(toDelete), toDelete) + + runner.stdout = sout + err = runner.Run(vargs.GCloudCmd, "app", "versions", "delete", "--service", service, + "--quiet", strings.Join(toDelete, " ")) + if err != nil { + return fmt.Errorf("error: %s\n", err) + } + + return nil +} From 80979b7f8a4ee3a5b70369cb6b88eb220423db3a Mon Sep 17 00:00:00 2001 From: JP Robinson Date: Tue, 19 Feb 2019 12:48:19 -0500 Subject: [PATCH 3/9] making sure JSON is outputted --- main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 8b149ed..d2e4560 100644 --- a/main.go +++ b/main.go @@ -334,8 +334,8 @@ func runGcloud(runner *Environ, workspace string, vargs GAE) (string, error) { args = append(args, "--project", vargs.Project) } - // add flag to prevent interactive - args = append(args, "--quiet") + // add flag to prevent interactive + get JSON output + args = append(args, "--format", "json", "--quiet") // add the remaining arguments if len(vargs.AddlArgs) > 0 { From b03b95b3be043c9d67f801a04a7bebd862334243 Mon Sep 17 00:00:00 2001 From: JP Robinson Date: Tue, 19 Feb 2019 17:54:07 -0500 Subject: [PATCH 4/9] hooking in env config, fixing delete ID arguments --- main.go | 4 +++- remove.go | 10 ++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index d2e4560..0228056 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "text/template" @@ -218,6 +219,7 @@ func configFromEnv(vargs *GAE, workspace *string) error { vargs.Dir = os.Getenv("PLUGIN_DIR") vargs.AppCfgCmd = os.Getenv("PLUGIN_APPCFG_CMD") vargs.GCloudCmd = os.Getenv("PLUGIN_GCLOUD_CMD") + vargs.MaxVersions, _ = strconv.Atoi(os.Getenv("PLUGIN_MAX_VERSIONS")) // Maps dummyVargs := dummyGAE{} @@ -382,7 +384,7 @@ func runGcloud(runner *Environ, workspace string, vargs GAE) (string, error) { Service string `json:"service"` } } - err = json.NewDecoder(&output).Decode(versions) + err = json.NewDecoder(&output).Decode(&versions) if err != nil { return "", err } diff --git a/remove.go b/remove.go index 62252a1..a53a122 100644 --- a/remove.go +++ b/remove.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "log" - "strings" ) func removeOldVersions(runner *Environ, workspace, service string, vargs GAE) error { @@ -13,7 +12,8 @@ func removeOldVersions(runner *Environ, workspace, service string, vargs GAE) er sout := runner.stdout runner.stdout = &versionJSON // look up existing versions for given service ordered by create time desc - err := runner.Run(vargs.GCloudCmd, "app", "versions", "list", "--service", service, + err := runner.Run(vargs.GCloudCmd, "app", "versions", "list", + "--service", service, "--project", vargs.Project, "--format", "json", "--sort-by", "~version.createTime", "--quiet") if err != nil { return fmt.Errorf("error: %s\n", err) @@ -39,8 +39,10 @@ func removeOldVersions(runner *Environ, workspace, service string, vargs GAE) er log.Printf("deleting %d versions: %s", len(toDelete), toDelete) runner.stdout = sout - err = runner.Run(vargs.GCloudCmd, "app", "versions", "delete", "--service", service, - "--quiet", strings.Join(toDelete, " ")) + args := []string{"app", "versions", "delete", + "--service", service, "--project", vargs.Project, "--quiet"} + args = append(args, toDelete...) + err = runner.Run(vargs.GCloudCmd, args...) if err != nil { return fmt.Errorf("error: %s\n", err) } From 731d713b844fc6201f360d554f8069ba3e889b4e Mon Sep 17 00:00:00 2001 From: JP Robinson Date: Tue, 19 Feb 2019 17:55:53 -0500 Subject: [PATCH 5/9] properly using max-version --- remove.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/remove.go b/remove.go index a53a122..1dfeff0 100644 --- a/remove.go +++ b/remove.go @@ -31,7 +31,7 @@ func removeOldVersions(runner *Environ, workspace, service string, vargs GAE) er var toDelete []string for i, res := range results { // keep newer versions, the newly deployed version or anything that has traffic - if (i+1) < vargs.MaxVersions || res.ID == vargs.Version || res.TrafficSplit > 0 { + if i < vargs.MaxVersions || res.ID == vargs.Version || res.TrafficSplit > 0 { continue } toDelete = append(toDelete, res.ID) From 5f76852fa80bac7465b10b062b6b6219b90da0b2 Mon Sep 17 00:00:00 2001 From: JP Robinson Date: Tue, 19 Feb 2019 18:12:25 -0500 Subject: [PATCH 6/9] using app.yaml to get service name --- main.go | 48 +++++++++++++----------------------------------- remove.go | 31 ++++++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 38 deletions(-) diff --git a/main.go b/main.go index 0228056..7927fa3 100644 --- a/main.go +++ b/main.go @@ -159,10 +159,9 @@ func wrapMain() error { return fmt.Errorf("error: %s\n", err) } - var serviceName string // if gcloud app cmd, run it if gcloudCmds[vargs.Action] { - serviceName, err = runGcloud(runner, workspace, vargs) + err = runGcloud(runner, workspace, vargs) } else { // otherwise, do appcfg.py command err = runAppCfg(runner, workspace, vargs) @@ -173,8 +172,8 @@ func wrapMain() error { } // check if MaxVersions is supplied + deploy action - if vargs.MaxVersions > 0 && serviceName != "" && vargs.Action == "deploy" { - return removeOldVersions(runner, workspace, serviceName, vargs) + if vargs.MaxVersions > 0 && (vargs.Action == "deploy" || vargs.Action == "update") { + return removeOldVersions(runner, workspace, vargs) } return nil @@ -305,7 +304,7 @@ var gcloudCmds = map[string]bool{ "instances": true, } -func runGcloud(runner *Environ, workspace string, vargs GAE) (string, error) { +func runGcloud(runner *Environ, workspace string, vargs GAE) error { // add the action first (gcloud app X) args := []string{ "app", @@ -336,8 +335,8 @@ func runGcloud(runner *Environ, workspace string, vargs GAE) (string, error) { args = append(args, "--project", vargs.Project) } - // add flag to prevent interactive + get JSON output - args = append(args, "--format", "json", "--quiet") + // add flag to prevent interactive + args = append(args, "--quiet") // add the remaining arguments if len(vargs.AddlArgs) > 0 { @@ -356,47 +355,26 @@ func runGcloud(runner *Environ, workspace string, vargs GAE) (string, error) { } if err := setupAppFile(workspace, vargs); err != nil { - return "", err + return err } if err := setupCronFile(workspace, vargs); err != nil { - return "", err + return err } if err := setupDispatchFile(workspace, vargs); err != nil { - return "", err + return err } if err := setupQueueFile(workspace, vargs); err != nil { - return "", err + return err } - var output bytes.Buffer - runner.stdout = &output + err := runner.Run(vargs.GCloudCmd, args...) if err != nil { - return "", fmt.Errorf("error: %s\n", err) - } - - // if this was a deployment, grab the name of the service we just deployed - if vargs.Action == "deploy" { - var versions struct { - Versions []struct { - Service string `json:"service"` - } - } - err = json.NewDecoder(&output).Decode(&versions) - if err != nil { - return "", err - } - - if len(versions.Versions) == 0 { - return "", nil - } - - return versions.Versions[0].Service, nil + return fmt.Errorf("error: %s\n", err) } - - return "", nil + return nil } func runAppCfg(runner *Environ, workspace string, vargs GAE) error { diff --git a/remove.go b/remove.go index 1dfeff0..1b56ecc 100644 --- a/remove.go +++ b/remove.go @@ -5,14 +5,39 @@ import ( "encoding/json" "fmt" "log" + "os" + "path/filepath" + + "gopkg.in/yaml.v2" ) -func removeOldVersions(runner *Environ, workspace, service string, vargs GAE) error { +func removeOldVersions(runner *Environ, workspace string, vargs GAE) error { + // read in the app.yaml file to grab the module/service mame we just deployed + appLoc := filepath.Join(workspace, vargs.Dir, "app.yaml") + appFile, err := os.Open(appLoc) + if err != nil { + return fmt.Errorf("error: %s\n", err) + } + defer appFile.Close() + var appStruct struct { + Service string `yaml:"service"` + Module string `yaml:"module"` + } + err = yaml.NewDecoder(appFile).Decode(&appStruct) + if err != nil { + return fmt.Errorf("error: %s\n", err) + } + + service := appStruct.Service + if service == "" { + service = appStruct.Module + } + + // look up existing versions for given service ordered by create time desc var versionJSON bytes.Buffer sout := runner.stdout runner.stdout = &versionJSON - // look up existing versions for given service ordered by create time desc - err := runner.Run(vargs.GCloudCmd, "app", "versions", "list", + err = runner.Run(vargs.GCloudCmd, "app", "versions", "list", "--service", service, "--project", vargs.Project, "--format", "json", "--sort-by", "~version.createTime", "--quiet") if err != nil { From 6ba1e3389b12216d496c234950b5029a3bec62de Mon Sep 17 00:00:00 2001 From: JP Robinson Date: Tue, 19 Feb 2019 18:15:30 -0500 Subject: [PATCH 7/9] docfix --- main.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index 7927fa3..9e80b69 100644 --- a/main.go +++ b/main.go @@ -53,9 +53,10 @@ type GAE struct { // and autoscaling configurations. AppFile string `json:"app_file"` - // MaxVersions is an optional value that can be used along with the "deploy" action. - // If set to a non-zero value, the plugin will look up the versions of the deployed - // service and delete any older versions beyond the "max" value provided. + // MaxVersions is an optional value that can be used along with the "deploy" or + // "update" actions. If set to a non-zero value, the plugin will look up the versions + // of the deployed service and delete any older versions beyond the "max" value + // provided. MaxVersions int `json:"max_versions"` // CronFile is the name of the cron.yaml file to use for this deployment. This field From 5ca2d9e06681dcc2cab5c712e68753ac732e7838 Mon Sep 17 00:00:00 2001 From: JP Robinson Date: Wed, 20 Feb 2019 10:55:45 -0500 Subject: [PATCH 8/9] pr feedback --- main.go | 4 +++- remove.go | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 9e80b69..4d64504 100644 --- a/main.go +++ b/main.go @@ -56,7 +56,9 @@ type GAE struct { // MaxVersions is an optional value that can be used along with the "deploy" or // "update" actions. If set to a non-zero value, the plugin will look up the versions // of the deployed service and delete any older versions beyond the "max" value - // provided. + // provided. If any of the "older" versions that should be deleted is actually + // serving traffic, they will not be deleted. This may result in the actual version + // count being higher than the max listed here. MaxVersions int `json:"max_versions"` // CronFile is the name of the cron.yaml file to use for this deployment. This field diff --git a/remove.go b/remove.go index 1b56ecc..5a786fe 100644 --- a/remove.go +++ b/remove.go @@ -33,7 +33,7 @@ func removeOldVersions(runner *Environ, workspace string, vargs GAE) error { service = appStruct.Module } - // look up existing versions for given service ordered by create time desc + // look up existing versions for given service ordered by create time desc var versionJSON bytes.Buffer sout := runner.stdout runner.stdout = &versionJSON @@ -61,6 +61,11 @@ func removeOldVersions(runner *Environ, workspace string, vargs GAE) error { } toDelete = append(toDelete, res.ID) } + + if len(toDelete) == 0 { + return nil + } + log.Printf("deleting %d versions: %s", len(toDelete), toDelete) runner.stdout = sout From 89903dc1ceede4fb62f52f19afc9186980fd24e0 Mon Sep 17 00:00:00 2001 From: JP Robinson Date: Wed, 20 Feb 2019 10:58:16 -0500 Subject: [PATCH 9/9] grammarz --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 4d64504..7a2ba41 100644 --- a/main.go +++ b/main.go @@ -56,7 +56,7 @@ type GAE struct { // MaxVersions is an optional value that can be used along with the "deploy" or // "update" actions. If set to a non-zero value, the plugin will look up the versions // of the deployed service and delete any older versions beyond the "max" value - // provided. If any of the "older" versions that should be deleted is actually + // provided. If any of the "older" versions that should be deleted are actually // serving traffic, they will not be deleted. This may result in the actual version // count being higher than the max listed here. MaxVersions int `json:"max_versions"`