From e1339dd98be3f2fac596a910fb6a9840dbda75af Mon Sep 17 00:00:00 2001 From: Yuwen Ma Date: Fri, 18 Sep 2020 17:20:07 -0700 Subject: [PATCH] [kpt deployer] Add "local-config" annotation to kpt fn configs. (#4803) * [kpt deployer] Add "local-config" annotation to kpt fn resource. The `config.kubernetes.io/local-config:true` annotation will skip the kpt fn resource from deploying to the cluster in `kpt live apply`. * fix linter --- go.mod | 3 +- pkg/skaffold/deploy/kpt.go | 52 ++++++++++++++++- pkg/skaffold/deploy/kpt_test.go | 99 +++++++++++++++++++++++++++++++++ vendor/modules.txt | 3 +- 4 files changed, 154 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 97f3526c8ee..99e75e6fb52 100644 --- a/go.mod +++ b/go.mod @@ -80,9 +80,10 @@ require ( gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c k8s.io/api v0.18.1 k8s.io/apiextensions-apiserver v0.18.1 // indirect - k8s.io/apimachinery v0.18.1 + k8s.io/apimachinery v0.19.2 k8s.io/client-go v0.18.1 k8s.io/kubectl v0.0.0-20190831163037-3b58a944563f k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 knative.dev/pkg v0.0.0-20200416021448-f68639f04b39 // indirect + sigs.k8s.io/yaml v1.2.0 ) diff --git a/pkg/skaffold/deploy/kpt.go b/pkg/skaffold/deploy/kpt.go index 7256c59bd3a..978371d9628 100644 --- a/pkg/skaffold/deploy/kpt.go +++ b/pkg/skaffold/deploy/kpt.go @@ -28,6 +28,9 @@ import ( "regexp" "strings" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + k8syaml "sigs.k8s.io/yaml" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/build" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/config" deploy "github.com/GoogleContainerTools/skaffold/pkg/skaffold/deploy/kubectl" @@ -36,10 +39,12 @@ import ( "github.com/GoogleContainerTools/skaffold/pkg/skaffold/util" ) -var ( +const ( inventoryTemplate = "inventory-template.yaml" kptHydrated = ".kpt-hydrated" pipeline = ".pipeline" + kptFnAnnotation = "config.kubernetes.io/function" + kptFnLocalConfig = "config.kubernetes.io/local-config" ) // KptDeployer deploys workflows with kpt CLI @@ -186,6 +191,12 @@ func (k *KptDeployer) renderManifests(ctx context.Context, _ io.Writer, builds [ return nil, nil } + // exclude the kpt function from the manipulated resources. + manifests, err = k.excludeKptFn(manifests) + if err != nil { + return nil, fmt.Errorf("exclude kpt fn from manipulated resources: %w", err) + } + manifests, err = manifests.ReplaceImages(builds) if err != nil { return nil, fmt.Errorf("replacing images in manifests: %w", err) @@ -270,6 +281,45 @@ func (k *KptDeployer) kptFnRun(ctx context.Context) (deploy.ManifestList, error) return manifests, nil } +// excludeKptFn adds an annotation "config.kubernetes.io/local-config: 'true'" to kpt function. +// This will exclude kpt functions from deployed to the cluster in kpt live apply. +func (k *KptDeployer) excludeKptFn(manifest deploy.ManifestList) (deploy.ManifestList, error) { + var newManifest deploy.ManifestList + for _, yByte := range manifest { + // Convert yaml byte config to unstructured.Unstructured + jByte, _ := k8syaml.YAMLToJSON(yByte) + var obj unstructured.Unstructured + if err := obj.UnmarshalJSON(jByte); err != nil { + return nil, fmt.Errorf("unmarshaling config: %w", err) + } + // skip if the resource is not kpt fn config. + if _, ok := obj.GetAnnotations()[kptFnAnnotation]; !ok { + newManifest = append(newManifest, yByte) + continue + } + // skip if the kpt fn has local-config annotation specified. + if _, ok := obj.GetAnnotations()[kptFnLocalConfig]; ok { + newManifest = append(newManifest, yByte) + continue + } + + // Add "local-config" annotation to kpt fn config. + anns := obj.GetAnnotations() + anns[kptFnLocalConfig] = "true" + obj.SetAnnotations(anns) + jByte, err := obj.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("marshaling to json: %w", err) + } + newYByte, err := k8syaml.JSONToYAML(jByte) + if err != nil { + return nil, fmt.Errorf("converting json to yaml: %w", err) + } + newManifest.Append(newYByte) + } + return newManifest, nil +} + // getApplyDir returns the path to applyDir if specified by the user. Otherwise, getApplyDir // creates a hidden directory named .kpt-hydrated in place of applyDir. func (k *KptDeployer) getApplyDir(ctx context.Context) (string, error) { diff --git a/pkg/skaffold/deploy/kpt_test.go b/pkg/skaffold/deploy/kpt_test.go index 97c839b16a4..127d89a1ddf 100644 --- a/pkg/skaffold/deploy/kpt_test.go +++ b/pkg/skaffold/deploy/kpt_test.go @@ -28,6 +28,7 @@ import ( "testing" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/build" + deploy "github.com/GoogleContainerTools/skaffold/pkg/skaffold/deploy/kubectl" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/runner/runcontext" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/util" @@ -889,6 +890,104 @@ func TestKpt_KptCommandArgs(t *testing.T) { } } +// TestKpt_ExcludeKptFn checks the declarative kpt fn has expected annotations added. +func TestKpt_ExcludeKptFn(t *testing.T) { + // A declarative fn. + testFn1 := []byte(`apiVersion: v1 +data: + annotation_name: k1 + annotation_value: v1 +kind: ConfigMap +metadata: + annotations: + config.kubernetes.io/function: fake`) + // A declarative fn which has `local-config` annotation specified. + testFn2 := []byte(`apiVersion: v1 +kind: ConfigMap +metadata: + annotations: + config.kubernetes.io/function: fake + config.kubernetes.io/local-config: "false" +data: + annotation_name: k2 + annotation_value: v2`) + testPod := []byte(`apiVersion: v1 +kind: Pod +metadata: + namespace: default +spec: + containers: + - image: gcr.io/project/image1 + name: image1`) + tests := []struct { + description string + manifests deploy.ManifestList + expected deploy.ManifestList + }{ + { + description: "Add `local-config` annotation to kpt fn", + manifests: deploy.ManifestList{testFn1}, + expected: deploy.ManifestList{[]byte(`apiVersion: v1 +data: + annotation_name: k1 + annotation_value: v1 +kind: ConfigMap +metadata: + annotations: + config.kubernetes.io/function: fake + config.kubernetes.io/local-config: "true"`)}, + }, + { + description: "Skip preset `local-config` annotation", + manifests: deploy.ManifestList{testFn2}, + expected: deploy.ManifestList{[]byte(`apiVersion: v1 +kind: ConfigMap +metadata: + annotations: + config.kubernetes.io/function: fake + config.kubernetes.io/local-config: "false" +data: + annotation_name: k2 + annotation_value: v2`)}, + }, + { + description: "Valid in kpt fn pipeline.", + manifests: deploy.ManifestList{testFn1, testFn2, testPod}, + expected: deploy.ManifestList{[]byte(`apiVersion: v1 +data: + annotation_name: k1 + annotation_value: v1 +kind: ConfigMap +metadata: + annotations: + config.kubernetes.io/function: fake + config.kubernetes.io/local-config: "true"`), []byte(`apiVersion: v1 +kind: ConfigMap +metadata: + annotations: + config.kubernetes.io/function: fake + config.kubernetes.io/local-config: "false" +data: + annotation_name: k2 + annotation_value: v2`), []byte(`apiVersion: v1 +kind: Pod +metadata: + namespace: default +spec: + containers: + - image: gcr.io/project/image1 + name: image1`)}, + }, + } + for _, test := range tests { + testutil.Run(t, test.description, func(t *testutil.T) { + k := NewKptDeployer(&kptConfig{}, nil) + actualManifest, err := k.excludeKptFn(test.manifests) + t.CheckErrorAndDeepEqual(false, err, test.expected.String(), actualManifest.String()) + }) + } +} + type kptConfig struct { runcontext.RunContext // Embedded to provide the default values. workingDir string diff --git a/vendor/modules.txt b/vendor/modules.txt index 691b8ab4cac..03572d3915c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -837,7 +837,7 @@ k8s.io/api/storage/v1alpha1 k8s.io/api/storage/v1beta1 # k8s.io/apiextensions-apiserver v0.18.1 ## explicit -# k8s.io/apimachinery v0.18.1 => k8s.io/apimachinery v0.17.4 +# k8s.io/apimachinery v0.19.2 => k8s.io/apimachinery v0.17.4 ## explicit k8s.io/apimachinery/pkg/api/equality k8s.io/apimachinery/pkg/api/errors @@ -1131,6 +1131,7 @@ knative.dev/pkg/kmeta knative.dev/pkg/kmp knative.dev/pkg/tracker # sigs.k8s.io/yaml v1.2.0 +## explicit sigs.k8s.io/yaml # github.com/Azure/go-autorest => github.com/Azure/go-autorest v13.0.1+incompatible # github.com/containerd/containerd => github.com/containerd/containerd v1.3.4