diff --git a/calicoctl/calicoctl.go b/calicoctl/calicoctl.go index e906359db..16c91789d 100644 --- a/calicoctl/calicoctl.go +++ b/calicoctl/calicoctl.go @@ -36,6 +36,7 @@ func main() { name. get Get a resource identified by file, stdin or resource type and name. + convert Convert config files between different API versions. ipam IP address management. node Calico node management. version Display the version of calicoctl. @@ -81,6 +82,8 @@ Description: commands.Delete(args) case "get": commands.Get(args) + case "convert": + commands.Convert(args) case "version": commands.Version(args) case "node": diff --git a/calicoctl/commands/convert.go b/calicoctl/commands/convert.go new file mode 100644 index 000000000..60e43bc72 --- /dev/null +++ b/calicoctl/commands/convert.go @@ -0,0 +1,176 @@ +// Copyright (c) 2017 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package commands + +import ( + "fmt" + "os" + "strings" + + "github.com/docopt/docopt-go" + log "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/projectcalico/calicoctl/calicoctl/commands/argutils" + "github.com/projectcalico/calicoctl/calicoctl/commands/constants" + "github.com/projectcalico/calicoctl/calicoctl/commands/v1resourceloader" + "github.com/projectcalico/libcalico-go/lib/apis/v1/unversioned" + conversion "github.com/projectcalico/libcalico-go/lib/upgrade/etcd/conversionv1v3" + validator "github.com/projectcalico/libcalico-go/lib/validator/v3" +) + +func Convert(args []string) { + doc := constants.DatastoreIntro + `Usage: + calicoctl convert --filename= + [--output=] [--ignore-validation] + +Examples: + # Convert the contents of policy.yaml to v3 policy. + calicoctl convert -f ./policy.yaml -o yaml + + # Convert a policy based on the JSON passed into stdin. + cat policy.json | calicoctl convert -f - + +Options: + -h --help Show this screen. + -f --filename= Filename to use to create the resource. If set to + "-" loads from stdin. + -o --output= Output format. One of: yaml or json. + [Default: yaml] + --ignore-validation Skip validation on the converted manifest. + + +Description: + Convert config files from Calico v1 to v3 API versions. Both YAML and JSON formats are accepted. + + The default output will be printed to stdout in YAML format. +` + parsedArgs, err := docopt.Parse(doc, args, true, "", false, false) + if err != nil { + fmt.Printf("Invalid option: 'calicoctl %s'. Use flag '--help' to read about a specific subcommand.\n", strings.Join(args, " ")) + os.Exit(1) + } + if len(parsedArgs) == 0 { + return + } + + var rp resourcePrinter + output := parsedArgs["--output"].(string) + // Only supported output formats are yaml (default) and json. + switch output { + case "yaml", "yml": + rp = resourcePrinterYAML{} + case "json": + rp = resourcePrinterJSON{} + default: + fmt.Printf("unrecognized output format '%s'\n", output) + os.Exit(1) + } + + filename := argutils.ArgStringOrBlank(parsedArgs, "--filename") + + // Load the V1 resource from file and convert to a slice + // of resources for easier handling. + resV1, err := v1resourceloader.CreateResourcesFromFile(filename) + if err != nil { + fmt.Printf("Failed to execute command: %v\n", err) + os.Exit(1) + } + + var results []runtime.Object + for _, v1Resource := range resV1 { + v3Resource, err := convertResource(v1Resource) + if err != nil { + fmt.Printf("Failed to execute command: %v\n", err) + os.Exit(1) + } + + // Remove any extra metadata the object might have. + rom := v3Resource.(v1.ObjectMetaAccessor).GetObjectMeta() + rom.SetNamespace("") + rom.SetUID("") + rom.SetResourceVersion("") + rom.SetCreationTimestamp(v1.Time{}) + rom.SetDeletionTimestamp(nil) + rom.SetDeletionGracePeriodSeconds(nil) + rom.SetClusterName("") + + ignoreValidation := argutils.ArgBoolOrFalse(parsedArgs, "--ignore-validation") + if !ignoreValidation { + if err := validator.Validate(v3Resource); err != nil { + fmt.Printf("Converted manifest resource(s) failed validation: %s\n", err) + fmt.Printf("Re-run the command with '--ignore-validation' flag to see the converted output.\n") + os.Exit(1) + } + } + + results = append(results, v3Resource) + } + + log.Infof("results: %+v", results) + + err = rp.print(nil, results) + if err != nil { + fmt.Printf("Failed to execute command: %v\n", err) + os.Exit(1) + } +} + +// convertResource converts v1 resource into a v3 resource. +func convertResource(v1resource unversioned.Resource) (conversion.Resource, error) { + // Get the type converter for the v1 resource. + convRes, err := getTypeConverter(v1resource.GetTypeMetadata().Kind) + if err != nil { + return nil, err + } + + // Convert v1 API resource to v1 backend KVPair. + kvp, err := convRes.APIV1ToBackendV1(v1resource) + if err != nil { + return nil, err + } + + // Convert v1 backend KVPair to v3 API resource. + res, err := convRes.BackendV1ToAPIV3(kvp) + if err != nil { + return nil, err + } + + return res, nil +} + +// getTypeConverter returns a type specific converter for a given v1 resource. +func getTypeConverter(resKind string) (conversion.Converter, error) { + switch strings.ToLower(resKind) { + case "node": + return conversion.Node{}, nil + case "hostendpoint": + return conversion.HostEndpoint{}, nil + case "workloadendpoint": + return conversion.WorkloadEndpoint{}, nil + case "profile": + return conversion.Profile{}, nil + case "policy": + return conversion.Policy{}, nil + case "ippool": + return conversion.IPPool{}, nil + case "bgppeer": + return conversion.BGPPeer{}, nil + + default: + return nil, fmt.Errorf("conversion for the resource type '%s' is not supported", resKind) + } +} diff --git a/calicoctl/commands/v1resourceloader/v1resourceloader.go b/calicoctl/commands/v1resourceloader/v1resourceloader.go new file mode 100644 index 000000000..f0df9eab6 --- /dev/null +++ b/calicoctl/commands/v1resourceloader/v1resourceloader.go @@ -0,0 +1,202 @@ +// Copyright (c) 2017 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1resourceloader + +import ( + "errors" + "fmt" + "io" + "os" + "reflect" + + log "github.com/sirupsen/logrus" + + yamlsep "github.com/projectcalico/calicoctl/calicoctl/util/yaml" + "github.com/projectcalico/go-yaml-wrapper" + apiv1 "github.com/projectcalico/libcalico-go/lib/apis/v1" + "github.com/projectcalico/libcalico-go/lib/apis/v1/unversioned" + v1validator "github.com/projectcalico/libcalico-go/lib/validator/v1" +) + +// Store a resourceHelper for each resource unversioned.TypeMetadata. +var resourceToType map[unversioned.TypeMetadata]reflect.Type + +func init() { + resourceToType = make(map[unversioned.TypeMetadata]reflect.Type) + populateResourceTypes() +} + +// populateResourceTypes register all the V1 resource types in the resourceToType map. +func populateResourceTypes() { + resTypes := []unversioned.Resource{ + apiv1.NewBGPPeer(), + apiv1.NewIPPool(), + apiv1.NewHostEndpoint(), + apiv1.NewNode(), + apiv1.NewPolicy(), + apiv1.NewProfile(), + apiv1.NewWorkloadEndpoint(), + } + + for _, rt := range resTypes { + resourceToType[rt.GetTypeMetadata()] = reflect.ValueOf(rt).Elem().Type() + } +} + +// Create a new concrete resource structure based on the type. If the type is +// a list, this creates a concrete Resource-List of the required type. +func newResource(tm unversioned.TypeMetadata) (unversioned.Resource, error) { + rh, ok := resourceToType[tm] + if !ok { + return nil, errors.New(fmt.Sprintf("Unknown resource type (%s) and/or version (%s)", tm.Kind, tm.APIVersion)) + } + log.Debugf("Found resource helper: %s", rh) + + // Create new resource and fill in the type metadata. + new := reflect.New(rh) + elem := new.Elem() + elem.FieldByName("Kind").SetString(tm.GetTypeMetadata().Kind) + elem.FieldByName("APIVersion").SetString(tm.GetTypeMetadata().APIVersion) + + return new.Interface().(unversioned.Resource), nil +} + +// Create the resource from the specified byte array encapsulating the resource. +// - The byte array may be JSON or YAML encoding of either a single resource or list of +// resources as defined by the API objects in /api. +// +// The returned Resource will either be a single resource document or a List of documents. +// If the file does not contain any valid Resources this function returns an error. +func createResourcesFromBytes(b []byte) ([]unversioned.Resource, error) { + // Start by unmarshalling the bytes into a TypeMetadata structure - this will ignore + // other fields. + var err error + tm := unversioned.TypeMetadata{} + tms := []unversioned.TypeMetadata{} + if err = yaml.Unmarshal(b, &tm); err == nil { + // We processed a metadata, so create a concrete resource struct to unpack + // into. + return unmarshalResource(tm, b) + } else if err = yaml.Unmarshal(b, &tms); err == nil { + // We processed a slice of metadata's, create a list of concrete resource + // structs to unpack into. + return unmarshalSliceOfResources(tms, b) + } else { + // Failed to parse a single resource or list of resources. + return nil, err + } +} + +// Unmarshal a bytearray containing a single resource of the specified type into +// a concrete structure for that resource type. +// +// Return as a slice of Resource interfaces, containing a single element that is +// the unmarshalled resource. +func unmarshalResource(tm unversioned.TypeMetadata, b []byte) ([]unversioned.Resource, error) { + log.Infof("Processing type %s", tm.Kind) + unpacked, err := newResource(tm) + if err != nil { + return nil, err + } + + if err = yaml.UnmarshalStrict(b, unpacked); err != nil { + return nil, err + } + + log.Infof("Type of unpacked data: %v", reflect.TypeOf(unpacked)) + if err = v1validator.Validate(unpacked); err != nil { + return nil, err + } + + log.Infof("Unpacked: %+v", unpacked) + + return []unversioned.Resource{unpacked}, nil +} + +// Unmarshal a bytearray containing a list of resources of the specified types into +// a slice of concrete structures for those resource types. +// +// Return as a slice of Resource interfaces, containing an element that is each of +// the unmarshalled resources. +func unmarshalSliceOfResources(tml []unversioned.TypeMetadata, b []byte) ([]unversioned.Resource, error) { + log.Infof("Processing list of resources") + unpacked := make([]unversioned.Resource, len(tml)) + for i, tm := range tml { + log.Infof(" - processing type %s", tm.Kind) + r, err := newResource(tm) + if err != nil { + return nil, err + } + unpacked[i] = r + } + + if err := yaml.UnmarshalStrict(b, &unpacked); err != nil { + return nil, err + } + + // Validate the data in the structures. The v1validator does not handle slices, so + // validate each resource separately. + for _, r := range unpacked { + if err := v1validator.Validate(r); err != nil { + return nil, err + } + } + + log.Infof("Unpacked: %+v", unpacked) + + return unpacked, nil +} + +// Create the Resource from the specified file f. +// - The file format may be JSON or YAML encoding of either a single resource or list of +// resources as defined by the API objects in /api. +// - A filename of "-" means "Read from stdin". +// +// The returned Resource will either be a single Resource or a List containing zero or more +// Resources. If the file does not contain any valid Resources this function returns an error. +func CreateResourcesFromFile(f string) ([]unversioned.Resource, error) { + // Load the bytes from file or from stdin. + var reader io.Reader + var err error + if f == "-" { + reader = os.Stdin + } else { + reader, err = os.Open(f) + } + if err != nil { + return nil, err + } + + var resources []unversioned.Resource + separator := yamlsep.NewYAMLDocumentSeparator(reader) + for { + b, err := separator.Next() + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + + r, err := createResourcesFromBytes(b) + if err != nil { + return nil, err + } + + resources = append(resources, r...) + } + + return resources, nil +} diff --git a/glide.lock b/glide.lock index ec9b847c9..e801a8550 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: ad5b4d15b730b4b0f4fcb3be9ad71f89216b56c2e1ee27ecb60001a9a5b84ce3 -updated: 2017-12-03T20:35:27.841168863Z +hash: bf679ef506400603ff7f79cd5fa2d450a52459a9bc12987a7fe3859caf9e4b73 +updated: 2017-12-08T11:31:56.38292149-08:00 imports: - name: cloud.google.com/go version: 3b1ae45394a234c385be014e9a488f2bb6eef821 @@ -48,7 +48,7 @@ imports: - name: github.com/emicklei/go-restful-swagger12 version: dcef7f55730566d41eae5db10e7d6981829720f6 - name: github.com/fsnotify/fsnotify - version: 4da3e2cfbabc9f751898f250b49f2439785783a1 + version: 9297c46340639aebbb6820f260c699465528fb60 - name: github.com/ghodss/yaml version: 73d445a93680fa1a78ae23a5839bad48f32ba1ee - name: github.com/go-openapi/jsonpointer @@ -102,7 +102,7 @@ imports: subpackages: - simplelru - name: github.com/hashicorp/hcl - version: 23c074d0eceb2b8a5bfdbb271ab780cde70f05a8 + version: 1c284ec98f4b398443cbabb0d9197f7f4cc0077c subpackages: - hcl/ast - hcl/parser @@ -147,9 +147,9 @@ imports: - name: github.com/mitchellh/go-ps version: 4fdf99ab29366514c69ccccddab5dc58b8d84062 - name: github.com/mitchellh/mapstructure - version: 06020f85339e21b2478f756a78e295255ffa4d6a + version: d2dd0262208475919e1a362f675cfc0e7c10e905 - name: github.com/olekukonko/tablewriter - version: 65fec0d89a572b4367094e2058d3ebe667de3b60 + version: a7a4c189eb47ed33ce7b35f2880070a0c82a67d4 - name: github.com/onsi/ginkgo version: 9eda700730cba42af70d53180f9dcce9266bc2bc subpackages: @@ -213,7 +213,7 @@ imports: - name: github.com/projectcalico/go-yaml-wrapper version: 598e54215bee41a19677faa4f0c32acd2a87eb56 - name: github.com/projectcalico/libcalico-go - version: c07bf34a9a362e92e5e51853fd5fe03c27ecf333 + version: 7fdfaa3a86b1f6de47000ae4772b87e8979cde6f subpackages: - lib/apiconfig - lib/apis/v1 @@ -247,7 +247,9 @@ imports: - lib/selector/tokenizer - lib/set - lib/testutils + - lib/upgrade/etcd/conversionv1v3 - lib/validator + - lib/validator/v1 - lib/validator/v3 - lib/watch - name: github.com/prometheus/client_golang @@ -353,7 +355,7 @@ imports: - unicode/norm - width - name: google.golang.org/appengine - version: 9d8544a6b2c7df9cff240fcf92d7b2f59bc13416 + version: 4f7eeb5305a4ba1966344836ba4af9996b7b4e05 subpackages: - internal - internal/app_identity @@ -387,7 +389,7 @@ imports: - name: gopkg.in/yaml.v2 version: 53feefa2559fb8dfa8d81baad31be332c97d6c77 - name: k8s.io/api - version: 4df58c811fe2e65feb879227b2b245e4dc26e7ad + version: 9b9dca205a15b6ce9ef10091f05d60a13fdcf418 subpackages: - admissionregistration/v1alpha1 - apps/v1beta1 @@ -414,7 +416,7 @@ imports: - storage/v1 - storage/v1beta1 - name: k8s.io/apimachinery - version: 019ae5ada31de202164b118aee88ee2d14075c31 + version: 5134afd2c0c91158afac0d8a28bd2177185a3bcc subpackages: - pkg/api/equality - pkg/api/errors diff --git a/glide.yaml b/glide.yaml index d928aa221..068c26946 100644 --- a/glide.yaml +++ b/glide.yaml @@ -28,7 +28,7 @@ import: - ssh/terminal - package: github.com/projectcalico/go-yaml-wrapper - package: github.com/projectcalico/libcalico-go - version: c07bf34a9a362e92e5e51853fd5fe03c27ecf333 + version: 7fdfaa3a86b1f6de47000ae4772b87e8979cde6f subpackages: - lib/apis/v3 - lib/clientv3 diff --git a/test-data/v1/bgppeer-node2.yaml b/test-data/v1/bgppeer-node2.yaml new file mode 100644 index 000000000..910f428e5 --- /dev/null +++ b/test-data/v1/bgppeer-node2.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: bgpPeer +metadata: + scope: node + node: Node2 + peerIP: 1.2.3.4 +spec: + asNumber: 6455 + diff --git a/test-data/v1/multi-resource.yaml b/test-data/v1/multi-resource.yaml new file mode 100644 index 000000000..014c52902 --- /dev/null +++ b/test-data/v1/multi-resource.yaml @@ -0,0 +1,39 @@ +- apiVersion: v1 + kind: bgpPeer + metadata: + node: Node1 + peerIP: aa:bb::ff + scope: node + spec: + asNumber: 64514 +- apiVersion: v1 + kind: bgpPeer + metadata: + node: Node2 + peerIP: 1.2.3.4 + scope: node + spec: + asNumber: 6455 +--- +apiVersion: v1 +kind: bgpPeer +metadata: + scope: global + peerIP: 192.20.30.40 +spec: + asNumber: 64567 + +--- +- apiVersion: v1 + kind: ipPool + metadata: + cidr: 192.168.0.0/16 + spec: + ipip: + enabled: true + +- apiVersion: v1 + kind: ipPool + metadata: + cidr: 2001::00/120 + spec: