From 4c82a6d5d48ae2562a86a8e3afd851f5ea22caa9 Mon Sep 17 00:00:00 2001 From: chrislovecnm Date: Sat, 15 Jul 2017 13:52:55 -0600 Subject: [PATCH] Fixing YAML and JSON output across multiple cli functions. Also Added --dry-run for create_ig and create_cluster --- cmd/kops/create.go | 1 + cmd/kops/create_cluster.go | 49 +++++++++++++++++++++++- cmd/kops/create_ig.go | 37 ++++++++++++++++++ cmd/kops/get.go | 41 ++++++++------------ cmd/kops/get_cluster.go | 54 +++++++++++++++++++++------ cmd/kops/get_federation.go | 30 ++++++--------- cmd/kops/get_instancegroups.go | 36 +++++------------- docs/cli/kops_create_cluster.md | 8 +++- docs/cli/kops_create_instancegroup.md | 6 +++ pkg/kopscodecs/codecs.go | 25 ++++++++++--- 10 files changed, 198 insertions(+), 89 deletions(-) diff --git a/cmd/kops/create.go b/cmd/kops/create.go index ef4a9e8a99a91..9db4254a3f225 100644 --- a/cmd/kops/create.go +++ b/cmd/kops/create.go @@ -130,6 +130,7 @@ func RunCreate(f *util.Factory, out io.Writer, c *CreateOptions) error { return fmt.Errorf("error reading file %q: %v", f, err) } + // TODO: this does not support a JSON array sections := bytes.Split(contents, []byte("\n---\n")) for _, section := range sections { defaults := &schema.GroupVersionKind{ diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index e752c7d6a5fb5..32930a64c072e 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -28,6 +28,7 @@ import ( "github.com/golang/glog" "github.com/spf13/cobra" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kops" "k8s.io/kops/cmd/kops/util" @@ -124,6 +125,11 @@ type CreateClusterOptions struct { // ConfigBase is the location where we will store the configuration, it defaults to the state store ConfigBase string + + // DryRun mode output a cluster manifest of Output type. + DryRun bool + // Output type during a DryRun + Output string } func (o *CreateClusterOptions) InitDefaults() { @@ -190,6 +196,11 @@ var ( --project my-gce-project \ --image "ubuntu-os-cloud/ubuntu-1604-xenial-v20170202" \ --yes + # Create manifest for a cluster in AWS + kops create cluster --name=kubernetes-cluster.example.com \ + --state=s3://kops-state-1234 --zones=eu-west-1a \ + --node-count=2 --dry-run -oyaml + `)) create_cluster_short = i18n.T("Create a Kubernetes cluster.") @@ -227,7 +238,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { } cmd.Flags().BoolVar(&options.Yes, "yes", options.Yes, "Specify --yes to immediately create the cluster") - cmd.Flags().StringVar(&options.Target, "target", options.Target, "Target - direct, terraform, cloudformation") + cmd.Flags().StringVar(&options.Target, "target", options.Target, fmt.Sprintf("Valid targets: %s, %s, %s. Set this flag to %s if you want kops to generate terraform", cloudup.TargetDirect, cloudup.TargetTerraform, cloudup.TargetDirect, cloudup.TargetTerraform)) cmd.Flags().StringVar(&options.Models, "model", options.Models, "Models to apply (separate multiple models with commas)") // Configuration / state location @@ -297,6 +308,10 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().StringVar(&options.APILoadBalancerType, "api-loadbalancer-type", options.APILoadBalancerType, "Sets the API loadbalancer type to either 'public' or 'internal'") + // DryRun mode that will print YAML or JSON + cmd.Flags().BoolVar(&options.DryRun, "dry-run", options.DryRun, "If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest.") + cmd.Flags().StringVarP(&options.Output, "output", "o", options.Output, "Ouput format. One of json|yaml. Used with the --dry-run flag.") + if featureflag.SpecOverrideFlag.Enabled() { cmd.Flags().StringSliceVar(&options.Overrides, "override", options.Overrides, "Directly configure values in the spec") } @@ -326,6 +341,11 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e isDryrun = true targetName = cloudup.TargetDryRun } + + if c.DryRun && c.Output == "" { + return fmt.Errorf("unable to execute --dry-run without setting --output") + } + clusterName := c.ClusterName if clusterName == "" { return fmt.Errorf("--name is required") @@ -987,6 +1007,32 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e return err } + if c.DryRun { + var obj []runtime.Object + obj = append(obj, cluster) + + for _, group := range fullInstanceGroups { + // Cluster name is not populated, and we need it + group.ObjectMeta.Labels = make(map[string]string) + group.ObjectMeta.Labels[api.LabelClusterName] = cluster.ObjectMeta.Name + obj = append(obj, group) + } + switch c.Output { + case OutputYaml: + if err := fullOutputYAML(out, obj...); err != nil { + return fmt.Errorf("error writing cluster yaml to stdout: %v", err) + } + return nil + case OutputJSON: + if err := fullOutputJSON(out, obj...); err != nil { + return fmt.Errorf("error writing cluster json to stdout: %v", err) + } + return nil + default: + return fmt.Errorf("unsupported output type %q", c.Output) + } + } + // Note we perform as much validation as we can, before writing a bad config err = registry.CreateClusterConfig(clientset, cluster, fullInstanceGroups) if err != nil { @@ -1010,6 +1056,7 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e } } + // Can we acutally get to this if?? if targetName != "" { if isDryrun { fmt.Fprintf(out, "Previewing changes that will be made:\n\n") diff --git a/cmd/kops/create_ig.go b/cmd/kops/create_ig.go index abf8c040d7b0e..dbff7591a1bc2 100644 --- a/cmd/kops/create_ig.go +++ b/cmd/kops/create_ig.go @@ -39,6 +39,10 @@ import ( type CreateInstanceGroupOptions struct { Role string Subnets []string + // DryRun mode output an ig manifest of Output type. + DryRun bool + // Output type during a DryRun + Output string } var ( @@ -52,6 +56,10 @@ var ( # Create an instancegroup for the k8s-cluster.example.com cluster. kops create ig --name=k8s-cluster.example.com node-example \ --role node --subnet my-subnet-name + + # Create a YAML manifest for an instancegroup for the k8s-cluster.example.com cluster. + kops create ig --name=k8s-cluster.example.com node-example \ + --role node --subnet my-subnet-name --dry-run -oyaml `)) create_ig_short = i18n.T(`Create an instancegroup.`) @@ -85,6 +93,9 @@ func NewCmdCreateInstanceGroup(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().StringVar(&options.Role, "role", options.Role, "Type of instance group to create ("+strings.Join(allRoles, ",")+")") cmd.Flags().StringSliceVar(&options.Subnets, "subnet", options.Subnets, "Subnets in which to create instance group") + // DryRun mode that will print YAML or JSON + cmd.Flags().BoolVar(&options.DryRun, "dry-run", options.DryRun, "If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest.") + cmd.Flags().StringVarP(&options.Output, "output", "o", options.Output, "Ouput format. One of json|yaml") return cmd } @@ -142,6 +153,32 @@ func RunCreateInstanceGroup(f *util.Factory, cmd *cobra.Command, args []string, return err } + if options.DryRun { + + if options.Output == "" { + return fmt.Errorf("must set output flag; yaml or json") + } + + // Cluster name is not populated, and we need it + ig.ObjectMeta.Labels = make(map[string]string) + ig.ObjectMeta.Labels[api.LabelClusterName] = cluster.ObjectMeta.Name + + switch options.Output { + case OutputYaml: + if err := fullOutputYAML(out, ig); err != nil { + return fmt.Errorf("error writing cluster yaml to stdout: %v", err) + } + return nil + case OutputJSON: + if err := fullOutputJSON(out, ig); err != nil { + return fmt.Errorf("error writing cluster json to stdout: %v", err) + } + return nil + default: + return fmt.Errorf("unsupported output type %q", options.Output) + } + } + var ( edit = editor.NewDefaultEditor(editorEnvs) ) diff --git a/cmd/kops/get.go b/cmd/kops/get.go index 19371291e2841..cb345a17a5e87 100644 --- a/cmd/kops/get.go +++ b/cmd/kops/get.go @@ -17,7 +17,6 @@ limitations under the License. package main import ( - "encoding/json" "fmt" "io" @@ -153,35 +152,27 @@ func RunGet(context Factory, out io.Writer, options *GetOptions) error { return err } - switch options.output { - case OutputYaml: - - err = clusterOutputYAML(clusters, out) - if err != nil { - return err + var obj []runtime.Object + if options.output != OutputTable { + obj = append(obj, cluster) + for _, group := range instancegroups { + obj = append(obj, group) } + } - if err := writeYAMLSep(out); err != nil { - return err + switch options.output { + case OutputYaml: + if err := fullOutputYAML(out, obj...); err != nil { + return fmt.Errorf("error writing cluster yaml to stdout: %v", err) } - err = igOutputYAML(instancegroups, out) - if err != nil { - return err - } + return nil case OutputJSON: - return fmt.Errorf("not implemented") - // TODO this is not outputing valid json. Not sure what cluster and instance groups should look like - /* - err = clusterOutputJson(clusters,out) - if err != nil { - return err - } - err = igOutputJson(instancegroups,out) - if err != nil { - return err - }*/ + if err := fullOutputJSON(out, obj...); err != nil { + return fmt.Errorf("error writing cluster json to stdout: %v", err) + } + return nil case OutputTable: fmt.Fprintf(os.Stdout, "Cluster\n") @@ -235,7 +226,7 @@ func marshalYaml(obj runtime.Object) ([]byte, error) { // obj must be a pointer to a marshalable object func marshalJSON(obj runtime.Object) ([]byte, error) { - j, err := json.MarshalIndent(obj, "", " ") + j, err := kopscodecs.ToVersionedJSON(obj) if err != nil { return nil, fmt.Errorf("error marshaling json: %v", err) } diff --git a/cmd/kops/get_cluster.go b/cmd/kops/get_cluster.go index 4c28a2e5a7daa..3e2345742263b 100644 --- a/cmd/kops/get_cluster.go +++ b/cmd/kops/get_cluster.go @@ -18,13 +18,13 @@ package main import ( "fmt" + "io" "os" "strings" - "io" - "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kops/cmd/kops/util" api "k8s.io/kops/pkg/apis/kops" @@ -133,7 +133,7 @@ func RunGetClusters(context Factory, out io.Writer, options *GetClusterOptions) } if len(clusters) == 0 { - return fmt.Errorf("No clusters found") + return fmt.Errorf("no clusters found") } if options.FullSpec { @@ -146,14 +146,20 @@ func RunGetClusters(context Factory, out io.Writer, options *GetClusterOptions) fmt.Fprint(out, get_cluster_full_warning) } + var obj []runtime.Object + if options.output != OutputTable { + for _, c := range clusters { + obj = append(obj, c) + } + } + switch options.output { case OutputTable: return clusterOutputTable(clusters, out) case OutputYaml: - return clusterOutputYAML(clusters, out) + return fullOutputYAML(out, obj...) case OutputJSON: - return clusterOutputJson(clusters, out) - + return fullOutputJSON(out, obj...) default: return fmt.Errorf("Unknown output format: %q", options.output) } @@ -206,23 +212,47 @@ func clusterOutputTable(clusters []*api.Cluster, out io.Writer) error { return t.Render(clusters, out, "NAME", "CLOUD", "ZONES") } -func clusterOutputJson(clusters []*api.Cluster, out io.Writer) error { - for _, cluster := range clusters { - if err := marshalToWriter(cluster, marshalJSON, out); err != nil { +// fullOutputJson outputs the marshalled JSON of a list of clusters and instance groups. It will handle +// nils for clusters and instanceGroups slices. +func fullOutputJSON(out io.Writer, args ...runtime.Object) error { + argsLen := len(args) + + if argsLen > 1 { + if _, err := fmt.Fprint(out, "["); err != nil { return err } } + + for i, arg := range args { + if i != 0 { + if _, err := fmt.Fprint(out, ","); err != nil { + return err + } + } + if err := marshalToWriter(arg, marshalJSON, out); err != nil { + return err + } + } + + if argsLen > 1 { + if _, err := fmt.Fprint(out, "]"); err != nil { + return err + } + } + return nil } -func clusterOutputYAML(clusters []*api.Cluster, out io.Writer) error { - for i, cluster := range clusters { +// fullOutputJson outputs the marshalled JSON of a list of clusters and instance groups. It will handle +// nils for clusters and instanceGroups slices. +func fullOutputYAML(out io.Writer, args ...runtime.Object) error { + for i, obj := range args { if i != 0 { if err := writeYAMLSep(out); err != nil { return fmt.Errorf("error writing to stdout: %v", err) } } - if err := marshalToWriter(cluster, marshalYaml, out); err != nil { + if err := marshalToWriter(obj, marshalYaml, out); err != nil { return err } } diff --git a/cmd/kops/get_federation.go b/cmd/kops/get_federation.go index e1c2c0b3b1951..e8ae576064f79 100644 --- a/cmd/kops/get_federation.go +++ b/cmd/kops/get_federation.go @@ -24,6 +24,7 @@ import ( "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/kops/cmd/kops/util" api "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/util/pkg/tables" @@ -85,10 +86,16 @@ func RunGetFederations(context Factory, out io.Writer, options *GetFederationOpt if len(federations) == 0 { return fmt.Errorf("No federations found") } - switch options.output { - case OutputTable: + var obj []runtime.Object + if options.output != OutputTable { + for _, c := range federations { + obj = append(obj, c) + } + } + switch options.output { + case OutputTable: t := &tables.Table{} t.AddColumn("NAME", func(f *api.Federation) string { return f.ObjectMeta.Name @@ -102,25 +109,10 @@ func RunGetFederations(context Factory, out io.Writer, options *GetFederationOpt return t.Render(federations, out, "NAME", "CONTROLLERS", "MEMBERS") case OutputYaml: - for i, f := range federations { - if i != 0 { - _, err = out.Write([]byte("\n\n---\n\n")) - if err != nil { - return fmt.Errorf("error writing to stdout: %v", err) - } - } - if err := marshalToWriter(f, marshalYaml, os.Stdout); err != nil { - return err - } - } + return fullOutputYAML(out, obj...) case OutputJSON: - for _, f := range federations { - if err := marshalToWriter(f, marshalJSON, os.Stdout); err != nil { - return err - } - } + return fullOutputJSON(out, obj...) default: return fmt.Errorf("Unknown output format: %q", options.output) } - return nil } diff --git a/cmd/kops/get_instancegroups.go b/cmd/kops/get_instancegroups.go index 50e7f2760ffff..6dc6b90c3b979 100644 --- a/cmd/kops/get_instancegroups.go +++ b/cmd/kops/get_instancegroups.go @@ -24,6 +24,7 @@ import ( "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/kops/cmd/kops/util" api "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/formatter" @@ -112,14 +113,20 @@ func RunGetInstanceGroups(options *GetInstanceGroupsOptions, args []string) erro return fmt.Errorf("No InstanceGroup objects found") } - switch options.output { + var obj []runtime.Object + if options.output != OutputTable { + for _, c := range instancegroups { + obj = append(obj, c) + } + } + switch options.output { case OutputTable: return igOutputTable(cluster, instancegroups, out) case OutputYaml: - return igOutputYAML(instancegroups, out) + return fullOutputYAML(out, obj...) case OutputJSON: - return igOutputJson(instancegroups, out) + return fullOutputJSON(out, obj...) default: return fmt.Errorf("Unknown output format: %q", options.output) } @@ -176,29 +183,6 @@ func igOutputTable(cluster *api.Cluster, instancegroups []*api.InstanceGroup, ou return t.Render(instancegroups, os.Stdout, "NAME", "ROLE", "MACHINETYPE", "MIN", "MAX", "ZONES") } -func igOutputJson(instanceGroups []*api.InstanceGroup, out io.Writer) error { - for _, ig := range instanceGroups { - if err := marshalToWriter(ig, marshalJSON, out); err != nil { - return err - } - } - return nil -} - -func igOutputYAML(instanceGroups []*api.InstanceGroup, out io.Writer) error { - for i, ig := range instanceGroups { - if i != 0 { - if err := writeYAMLSep(out); err != nil { - return fmt.Errorf("error writing to stdout: %v", err) - } - } - if err := marshalToWriter(ig, marshalYaml, out); err != nil { - return err - } - } - return nil -} - func int32PointerToString(v *int32) string { if v == nil { return "-" diff --git a/docs/cli/kops_create_cluster.md b/docs/cli/kops_create_cluster.md index 7fdfbb704e2d0..f4bb358b7e57c 100644 --- a/docs/cli/kops_create_cluster.md +++ b/docs/cli/kops_create_cluster.md @@ -56,6 +56,10 @@ kops create cluster --project my-gce-project \ --image "ubuntu-os-cloud/ubuntu-1604-xenial-v20170202" \ --yes + # Create manifest for a cluster in AWS + kops create cluster --name=kubernetes-cluster.example.com \ + --state=s3://kops-state-1234 --zones=eu-west-1a \ + --node-count=2 --dry-run -oyaml ``` ### Options @@ -71,6 +75,7 @@ kops create cluster --cloud-labels string A list of KV pairs used to tag all instance groups in AWS (eg "Owner=John Doe,Team=Some Team"). --dns string DNS hosted zone to use: public|private. Default is 'public'. (default "Public") --dns-zone string DNS hosted zone to use (defaults to longest matching zone) + --dry-run If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest. --encrypt-etcd-storage Generate key in aws kms and use it for encrypt etcd volumes --image string Image to use for all instances. --kubernetes-version string Version of kubernetes to run (defaults to version in channel) @@ -89,10 +94,11 @@ kops create cluster --node-tenancy string The tenancy of the node group on AWS. Can be either default or dedicated. --node-volume-size int32 Set instance volume size (in GB) for nodes --out string Path to write any local output + -o, --output string Ouput format. One of json|yaml. Used with the --dry-run flag. --project string Project to use (must be set on GCE) --ssh-access stringSlice Restrict SSH access to this CIDR. If not set, access will not be restricted by IP. (default [0.0.0.0/0]) --ssh-public-key string SSH public key to use (default "~/.ssh/id_rsa.pub") - --target string Target - direct, terraform, cloudformation (default "direct") + --target string Valid targets: direct, terraform, direct. Set this flag to terraform if you want kops to generate terraform (default "direct") -t, --topology string Controls network topology for the cluster. public|private. Default is 'public'. (default "public") --vpc string Set to use a shared VPC --yes Specify --yes to immediately create the cluster diff --git a/docs/cli/kops_create_instancegroup.md b/docs/cli/kops_create_instancegroup.md index e6d087b899022..e37d9ce2737a7 100644 --- a/docs/cli/kops_create_instancegroup.md +++ b/docs/cli/kops_create_instancegroup.md @@ -20,11 +20,17 @@ kops create instancegroup # Create an instancegroup for the k8s-cluster.example.com cluster. kops create ig --name=k8s-cluster.example.com node-example \ --role node --subnet my-subnet-name + + # Create a YAML manifest for an instancegroup for the k8s-cluster.example.com cluster. + kops create ig --name=k8s-cluster.example.com node-example \ + --role node --subnet my-subnet-name --dry-run -oyaml ``` ### Options ``` + --dry-run If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest. + -o, --output string Ouput format. One of json|yaml --role string Type of instance group to create (Node,Master,Bastion) (default "Node") --subnet stringSlice Subnets in which to create instance group ``` diff --git a/pkg/kopscodecs/codecs.go b/pkg/kopscodecs/codecs.go index 13545df2f98c0..80274c53ee846 100644 --- a/pkg/kopscodecs/codecs.go +++ b/pkg/kopscodecs/codecs.go @@ -45,12 +45,12 @@ func init() { install.Install(GroupFactoryRegistry, Registry, Scheme) } -func encoder(gv runtime.GroupVersioner) runtime.Encoder { - yaml, ok := runtime.SerializerInfoForMediaType(Codecs.SupportedMediaTypes(), "application/yaml") +func encoder(gv runtime.GroupVersioner, mediaType string) runtime.Encoder { + e, ok := runtime.SerializerInfoForMediaType(Codecs.SupportedMediaTypes(), mediaType) if !ok { - glog.Fatalf("no YAML serializer registered") + glog.Fatalf("no %s serializer registered", mediaType) } - return Codecs.EncoderForVersion(yaml.Serializer, gv) + return Codecs.EncoderForVersion(e.Serializer, gv) } func decoder() runtime.Decoder { @@ -68,7 +68,22 @@ func ToVersionedYaml(obj runtime.Object) ([]byte, error) { // ToVersionedYamlWithVersion encodes the object to YAML, in a specified API version func ToVersionedYamlWithVersion(obj runtime.Object, version runtime.GroupVersioner) ([]byte, error) { var w bytes.Buffer - err := encoder(version).Encode(obj, &w) + err := encoder(version, "application/yaml").Encode(obj, &w) + if err != nil { + return nil, fmt.Errorf("error encoding %T: %v", obj, err) + } + return w.Bytes(), nil +} + +// ToVersionedJSON encodes the object to JSON +func ToVersionedJSON(obj runtime.Object) ([]byte, error) { + return ToVersionedJSONWithVersion(obj, v1alpha2.SchemeGroupVersion) +} + +// ToVersionedJSONWithVersion encodes the object to JSON, in a specified API version +func ToVersionedJSONWithVersion(obj runtime.Object, version runtime.GroupVersioner) ([]byte, error) { + var w bytes.Buffer + err := encoder(version, "application/json").Encode(obj, &w) if err != nil { return nil, fmt.Errorf("error encoding %T: %v", obj, err) }