From 5f89535c202561e39e7caf82d67015db302f923d Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 11:58:30 -0400 Subject: [PATCH 01/31] update policy/v1beta to policy/v1 for kubernetes v1.25 support Signed-off-by: Jack Carrig --- .../etcdlockserver/etcdlockserver_controller.go | 4 ++-- pkg/controller/etcdlockserver/reconcile_pdb.go | 6 +++--- pkg/operator/etcd/pdb.go | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/controller/etcdlockserver/etcdlockserver_controller.go b/pkg/controller/etcdlockserver/etcdlockserver_controller.go index 1b6afd71..1073f445 100644 --- a/pkg/controller/etcdlockserver/etcdlockserver_controller.go +++ b/pkg/controller/etcdlockserver/etcdlockserver_controller.go @@ -23,7 +23,7 @@ import ( "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" - policyv1beta1 "k8s.io/api/policy/v1beta1" + policyv1 "k8s.io/api/policy/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -58,7 +58,7 @@ var watchResources = []client.Object{ &corev1.Pod{}, &corev1.Service{}, &corev1.PersistentVolumeClaim{}, - &policyv1beta1.PodDisruptionBudget{}, + &policyv1.PodDisruptionBudget{}, } // Add creates a new EtcdLockserver Controller and adds it to the Manager. The Manager will set fields on the Controller diff --git a/pkg/controller/etcdlockserver/reconcile_pdb.go b/pkg/controller/etcdlockserver/reconcile_pdb.go index c8260649..be201a85 100644 --- a/pkg/controller/etcdlockserver/reconcile_pdb.go +++ b/pkg/controller/etcdlockserver/reconcile_pdb.go @@ -19,7 +19,7 @@ package etcdlockserver import ( "context" - policyv1beta1 "k8s.io/api/policy/v1beta1" + policyv1 "k8s.io/api/policy/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -50,13 +50,13 @@ func (r *ReconcileEtcdLockserver) reconcilePodDisruptionBudget(ctx context.Conte Name: etcd.PDBName(lockserverName), } err := r.reconciler.ReconcileObject(ctx, ls, key, labels, true, reconciler.Strategy{ - Kind: &policyv1beta1.PodDisruptionBudget{}, + Kind: &policyv1.PodDisruptionBudget{}, New: func(key client.ObjectKey) runtime.Object { return etcd.NewPDB(key, labels) }, UpdateInPlace: func(key client.ObjectKey, obj runtime.Object) { - curObj := obj.(*policyv1beta1.PodDisruptionBudget) + curObj := obj.(*policyv1.PodDisruptionBudget) etcd.UpdatePDBInPlace(curObj, labels) }, }) diff --git a/pkg/operator/etcd/pdb.go b/pkg/operator/etcd/pdb.go index 7773ceec..4aea2f5b 100644 --- a/pkg/operator/etcd/pdb.go +++ b/pkg/operator/etcd/pdb.go @@ -17,7 +17,7 @@ limitations under the License. package etcd import ( - policyv1beta1 "k8s.io/api/policy/v1beta1" + policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -37,18 +37,18 @@ func PDBName(lockserverName string) string { } // NewPDB creates a new PDB. -func NewPDB(key client.ObjectKey, labels map[string]string) *policyv1beta1.PodDisruptionBudget { +func NewPDB(key client.ObjectKey, labels map[string]string) *policyv1.PodDisruptionBudget { // This tells `kubectl drain` not to delete one of the members unless the // number of remaining members will still be at least QuorumSize. minAvailable := intstr.FromInt(QuorumSize) - return &policyv1beta1.PodDisruptionBudget{ + return &policyv1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Namespace: key.Namespace, Name: key.Name, Labels: labels, }, - Spec: policyv1beta1.PodDisruptionBudgetSpec{ + Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: labels, }, @@ -58,7 +58,7 @@ func NewPDB(key client.ObjectKey, labels map[string]string) *policyv1beta1.PodDi } // UpdatePDBInPlace updates an existing PDB in-place. -func UpdatePDBInPlace(obj *policyv1beta1.PodDisruptionBudget, labels map[string]string) { +func UpdatePDBInPlace(obj *policyv1.PodDisruptionBudget, labels map[string]string) { // Update labels, but ignore existing ones we don't set. update.Labels(&obj.Labels, labels) } From e56a74797bbcec58939ea18f3c2e2b1e499b73cd Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 12:02:19 -0400 Subject: [PATCH 02/31] trigger workflows Signed-off-by: Jack Carrig From bc5758df30769aac8a085cef3941905ad808fbf1 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 12:10:09 -0400 Subject: [PATCH 03/31] Revert "update policy/v1beta to policy/v1 for kubernetes v1.25 support" This reverts commit faeb0f3617ee0df094079c0bea5cb5f48fb4a11a. Signed-off-by: Jack Carrig --- .../etcdlockserver/etcdlockserver_controller.go | 4 ++-- pkg/controller/etcdlockserver/reconcile_pdb.go | 6 +++--- pkg/operator/etcd/pdb.go | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/controller/etcdlockserver/etcdlockserver_controller.go b/pkg/controller/etcdlockserver/etcdlockserver_controller.go index 1073f445..1b6afd71 100644 --- a/pkg/controller/etcdlockserver/etcdlockserver_controller.go +++ b/pkg/controller/etcdlockserver/etcdlockserver_controller.go @@ -23,7 +23,7 @@ import ( "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" - policyv1 "k8s.io/api/policy/v1" + policyv1beta1 "k8s.io/api/policy/v1beta1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -58,7 +58,7 @@ var watchResources = []client.Object{ &corev1.Pod{}, &corev1.Service{}, &corev1.PersistentVolumeClaim{}, - &policyv1.PodDisruptionBudget{}, + &policyv1beta1.PodDisruptionBudget{}, } // Add creates a new EtcdLockserver Controller and adds it to the Manager. The Manager will set fields on the Controller diff --git a/pkg/controller/etcdlockserver/reconcile_pdb.go b/pkg/controller/etcdlockserver/reconcile_pdb.go index be201a85..c8260649 100644 --- a/pkg/controller/etcdlockserver/reconcile_pdb.go +++ b/pkg/controller/etcdlockserver/reconcile_pdb.go @@ -19,7 +19,7 @@ package etcdlockserver import ( "context" - policyv1 "k8s.io/api/policy/v1" + policyv1beta1 "k8s.io/api/policy/v1beta1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -50,13 +50,13 @@ func (r *ReconcileEtcdLockserver) reconcilePodDisruptionBudget(ctx context.Conte Name: etcd.PDBName(lockserverName), } err := r.reconciler.ReconcileObject(ctx, ls, key, labels, true, reconciler.Strategy{ - Kind: &policyv1.PodDisruptionBudget{}, + Kind: &policyv1beta1.PodDisruptionBudget{}, New: func(key client.ObjectKey) runtime.Object { return etcd.NewPDB(key, labels) }, UpdateInPlace: func(key client.ObjectKey, obj runtime.Object) { - curObj := obj.(*policyv1.PodDisruptionBudget) + curObj := obj.(*policyv1beta1.PodDisruptionBudget) etcd.UpdatePDBInPlace(curObj, labels) }, }) diff --git a/pkg/operator/etcd/pdb.go b/pkg/operator/etcd/pdb.go index 4aea2f5b..7773ceec 100644 --- a/pkg/operator/etcd/pdb.go +++ b/pkg/operator/etcd/pdb.go @@ -17,7 +17,7 @@ limitations under the License. package etcd import ( - policyv1 "k8s.io/api/policy/v1" + policyv1beta1 "k8s.io/api/policy/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -37,18 +37,18 @@ func PDBName(lockserverName string) string { } // NewPDB creates a new PDB. -func NewPDB(key client.ObjectKey, labels map[string]string) *policyv1.PodDisruptionBudget { +func NewPDB(key client.ObjectKey, labels map[string]string) *policyv1beta1.PodDisruptionBudget { // This tells `kubectl drain` not to delete one of the members unless the // number of remaining members will still be at least QuorumSize. minAvailable := intstr.FromInt(QuorumSize) - return &policyv1.PodDisruptionBudget{ + return &policyv1beta1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Namespace: key.Namespace, Name: key.Name, Labels: labels, }, - Spec: policyv1.PodDisruptionBudgetSpec{ + Spec: policyv1beta1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: labels, }, @@ -58,7 +58,7 @@ func NewPDB(key client.ObjectKey, labels map[string]string) *policyv1.PodDisrupt } // UpdatePDBInPlace updates an existing PDB in-place. -func UpdatePDBInPlace(obj *policyv1.PodDisruptionBudget, labels map[string]string) { +func UpdatePDBInPlace(obj *policyv1beta1.PodDisruptionBudget, labels map[string]string) { // Update labels, but ignore existing ones we don't set. update.Labels(&obj.Labels, labels) } From 604214c6951c920aa4c9480fb55fc2b84b2d4d74 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 12:14:15 -0400 Subject: [PATCH 04/31] Revert "Revert "update policy/v1beta to policy/v1 for kubernetes v1.25 support"" This reverts commit 65639b318aa916f985cbfd89e6320c1183d90847. Signed-off-by: Jack Carrig --- .../etcdlockserver/etcdlockserver_controller.go | 4 ++-- pkg/controller/etcdlockserver/reconcile_pdb.go | 6 +++--- pkg/operator/etcd/pdb.go | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/controller/etcdlockserver/etcdlockserver_controller.go b/pkg/controller/etcdlockserver/etcdlockserver_controller.go index 1b6afd71..1073f445 100644 --- a/pkg/controller/etcdlockserver/etcdlockserver_controller.go +++ b/pkg/controller/etcdlockserver/etcdlockserver_controller.go @@ -23,7 +23,7 @@ import ( "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" - policyv1beta1 "k8s.io/api/policy/v1beta1" + policyv1 "k8s.io/api/policy/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -58,7 +58,7 @@ var watchResources = []client.Object{ &corev1.Pod{}, &corev1.Service{}, &corev1.PersistentVolumeClaim{}, - &policyv1beta1.PodDisruptionBudget{}, + &policyv1.PodDisruptionBudget{}, } // Add creates a new EtcdLockserver Controller and adds it to the Manager. The Manager will set fields on the Controller diff --git a/pkg/controller/etcdlockserver/reconcile_pdb.go b/pkg/controller/etcdlockserver/reconcile_pdb.go index c8260649..be201a85 100644 --- a/pkg/controller/etcdlockserver/reconcile_pdb.go +++ b/pkg/controller/etcdlockserver/reconcile_pdb.go @@ -19,7 +19,7 @@ package etcdlockserver import ( "context" - policyv1beta1 "k8s.io/api/policy/v1beta1" + policyv1 "k8s.io/api/policy/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -50,13 +50,13 @@ func (r *ReconcileEtcdLockserver) reconcilePodDisruptionBudget(ctx context.Conte Name: etcd.PDBName(lockserverName), } err := r.reconciler.ReconcileObject(ctx, ls, key, labels, true, reconciler.Strategy{ - Kind: &policyv1beta1.PodDisruptionBudget{}, + Kind: &policyv1.PodDisruptionBudget{}, New: func(key client.ObjectKey) runtime.Object { return etcd.NewPDB(key, labels) }, UpdateInPlace: func(key client.ObjectKey, obj runtime.Object) { - curObj := obj.(*policyv1beta1.PodDisruptionBudget) + curObj := obj.(*policyv1.PodDisruptionBudget) etcd.UpdatePDBInPlace(curObj, labels) }, }) diff --git a/pkg/operator/etcd/pdb.go b/pkg/operator/etcd/pdb.go index 7773ceec..4aea2f5b 100644 --- a/pkg/operator/etcd/pdb.go +++ b/pkg/operator/etcd/pdb.go @@ -17,7 +17,7 @@ limitations under the License. package etcd import ( - policyv1beta1 "k8s.io/api/policy/v1beta1" + policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -37,18 +37,18 @@ func PDBName(lockserverName string) string { } // NewPDB creates a new PDB. -func NewPDB(key client.ObjectKey, labels map[string]string) *policyv1beta1.PodDisruptionBudget { +func NewPDB(key client.ObjectKey, labels map[string]string) *policyv1.PodDisruptionBudget { // This tells `kubectl drain` not to delete one of the members unless the // number of remaining members will still be at least QuorumSize. minAvailable := intstr.FromInt(QuorumSize) - return &policyv1beta1.PodDisruptionBudget{ + return &policyv1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Namespace: key.Namespace, Name: key.Name, Labels: labels, }, - Spec: policyv1beta1.PodDisruptionBudgetSpec{ + Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: labels, }, @@ -58,7 +58,7 @@ func NewPDB(key client.ObjectKey, labels map[string]string) *policyv1beta1.PodDi } // UpdatePDBInPlace updates an existing PDB in-place. -func UpdatePDBInPlace(obj *policyv1beta1.PodDisruptionBudget, labels map[string]string) { +func UpdatePDBInPlace(obj *policyv1.PodDisruptionBudget, labels map[string]string) { // Update labels, but ignore existing ones we don't set. update.Labels(&obj.Labels, labels) } From c8bf1e848903b92f0ca0980113443f7a9c0c3d18 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 12:19:48 -0400 Subject: [PATCH 05/31] change kube version to v1.24.11 Signed-off-by: Jack Carrig --- tools/get-kube-binaries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/get-kube-binaries.sh b/tools/get-kube-binaries.sh index 1e11e297..20c9b596 100755 --- a/tools/get-kube-binaries.sh +++ b/tools/get-kube-binaries.sh @@ -9,7 +9,7 @@ set -euo pipefail # The integration test framework expects these binaries to be found in the PATH. # This is the kube-apiserver version to test against. -KUBE_VERSION="${KUBE_VERSION:-v1.19.1}" +KUBE_VERSION="${KUBE_VERSION:-v1.24.11}" KUBERNETES_RELEASE_URL="${KUBERNETES_RELEASE_URL:-https://dl.k8s.io}" # This should be the etcd version downloaded by kubernetes/hack/lib/etcd.sh From 0e2bd4adabfe3be213aede9733f2c0b40d0ea29d Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 12:26:52 -0400 Subject: [PATCH 06/31] more logs in apiserver Signed-off-by: Jack Carrig --- test/integration/framework/apiserver.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/framework/apiserver.go b/test/integration/framework/apiserver.go index 112ac1da..a8710248 100644 --- a/test/integration/framework/apiserver.go +++ b/test/integration/framework/apiserver.go @@ -80,8 +80,8 @@ func startApiserver() (func(), error) { // Uncomment these to see kube-apiserver output in test logs. // For operator tests, we generally don't expect problems at this level. - //cmd.Stdout = os.Stdout - //cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr stop := func() { cancel() From 4cf5b3c46782bbec22a771f036c28feeca954cc2 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 12:52:25 -0400 Subject: [PATCH 07/31] remove --insecure-port Signed-off-by: Jack Carrig --- test/integration/framework/apiserver.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/integration/framework/apiserver.go b/test/integration/framework/apiserver.go index a8710248..250c0dfb 100644 --- a/test/integration/framework/apiserver.go +++ b/test/integration/framework/apiserver.go @@ -57,7 +57,7 @@ func startApiserver() (func(), error) { if err != nil { return nil, fmt.Errorf("could not get a port: %v", err) } - apiserverURL = fmt.Sprintf("http://127.0.0.1:%d", apiserverPort) + apiserverURL = fmt.Sprintf("https://127.0.0.1:%d", apiserverPort) klog.Infof("starting kube-apiserver on %s", apiserverURL) apiserverDataDir, err := ioutil.TempDir(os.TempDir(), "integration_test_apiserver_data") @@ -71,10 +71,9 @@ func startApiserver() (func(), error) { ctx, apiserverPath, "--cert-dir", apiserverDataDir, - "--insecure-port", strconv.Itoa(apiserverPort), // We don't use the secure port, but we need to pick something that // doesn't conflict with other test apiservers. - "--secure-port", strconv.Itoa(apiserverPort+1), + "--secure-port", strconv.Itoa(apiserverPort), "--etcd-servers", etcdURL, ) From 5c08a1e7a001a37f95512ba3b8b047005bdd81e2 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 13:24:59 -0400 Subject: [PATCH 08/31] add required flags Signed-off-by: Jack Carrig --- test/integration/framework/apiserver.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/integration/framework/apiserver.go b/test/integration/framework/apiserver.go index 250c0dfb..ef4f1077 100644 --- a/test/integration/framework/apiserver.go +++ b/test/integration/framework/apiserver.go @@ -75,6 +75,9 @@ func startApiserver() (func(), error) { // doesn't conflict with other test apiservers. "--secure-port", strconv.Itoa(apiserverPort), "--etcd-servers", etcdURL, + "--service-account-issuer","https://kubernetes.default.svc.cluster.local", + "--service-account-key-file","/etc/kubernetes/pki/sa.pub", + "--service-account-signing-key-file","/etc/kubernetes/pki/sa.key" ) // Uncomment these to see kube-apiserver output in test logs. From 6ca61ab6b1a6393436b1e1579e9b56607d34a6cd Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 13:30:34 -0400 Subject: [PATCH 09/31] syntax Signed-off-by: Jack Carrig --- test/integration/framework/apiserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/framework/apiserver.go b/test/integration/framework/apiserver.go index ef4f1077..98f80878 100644 --- a/test/integration/framework/apiserver.go +++ b/test/integration/framework/apiserver.go @@ -77,7 +77,7 @@ func startApiserver() (func(), error) { "--etcd-servers", etcdURL, "--service-account-issuer","https://kubernetes.default.svc.cluster.local", "--service-account-key-file","/etc/kubernetes/pki/sa.pub", - "--service-account-signing-key-file","/etc/kubernetes/pki/sa.key" + "--service-account-signing-key-file","/etc/kubernetes/pki/sa.key", ) // Uncomment these to see kube-apiserver output in test logs. From 47a79ed597c33079543517154822f0e5e4c1e15f Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 13:48:42 -0400 Subject: [PATCH 10/31] use data dir Signed-off-by: Jack Carrig --- test/integration/framework/apiserver.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/framework/apiserver.go b/test/integration/framework/apiserver.go index 98f80878..4e46f393 100644 --- a/test/integration/framework/apiserver.go +++ b/test/integration/framework/apiserver.go @@ -75,9 +75,9 @@ func startApiserver() (func(), error) { // doesn't conflict with other test apiservers. "--secure-port", strconv.Itoa(apiserverPort), "--etcd-servers", etcdURL, - "--service-account-issuer","https://kubernetes.default.svc.cluster.local", - "--service-account-key-file","/etc/kubernetes/pki/sa.pub", - "--service-account-signing-key-file","/etc/kubernetes/pki/sa.key", + "--service-account-issuer", "https://kubernetes.default.svc.cluster.local", + "--service-account-key-file", fmt.Sprintf("%s/apiserver.crt", apiserverDataDir), + "--service-account-signing-key-file", fmt.Sprintf("%s/apiserver.key", apiserverDataDir), ) // Uncomment these to see kube-apiserver output in test logs. From aa62f61a06bfd80ff19780a19f75f628a66603e8 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 14:27:54 -0400 Subject: [PATCH 11/31] short circuit test Signed-off-by: Jack Carrig --- test/integration/framework/main.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/integration/framework/main.go b/test/integration/framework/main.go index 2d8aac42..06b52321 100644 --- a/test/integration/framework/main.go +++ b/test/integration/framework/main.go @@ -110,6 +110,12 @@ func TestMain(tests func() int) { func testMain(tests func() int) error { controllermanager.InitFlags() + klog.Info("hello world -- klog.Info") + klog.Infof("hello world -- klog.Infof") + klog.Warningf("hello world -- klog.Warningf") + + return nil + if _, err := getKubectlPath(); err != nil { return errors.New(installKubectl) } From 75c64ff4aa8a16faa5fa5c20f743d11df0e761c3 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 14:42:04 -0400 Subject: [PATCH 12/31] only try to start api server Signed-off-by: Jack Carrig --- test/integration/framework/main.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/test/integration/framework/main.go b/test/integration/framework/main.go index 06b52321..08fc1ef4 100644 --- a/test/integration/framework/main.go +++ b/test/integration/framework/main.go @@ -110,21 +110,15 @@ func TestMain(tests func() int) { func testMain(tests func() int) error { controllermanager.InitFlags() - klog.Info("hello world -- klog.Info") - klog.Infof("hello world -- klog.Infof") - klog.Warningf("hello world -- klog.Warningf") - - return nil - if _, err := getKubectlPath(); err != nil { return errors.New(installKubectl) } - stopEtcd, err := startEtcd() - if err != nil { - return fmt.Errorf("cannot run integration tests: unable to start etcd: %v", err) - } - defer stopEtcd() + // stopEtcd, err := startEtcd() + // if err != nil { + // return fmt.Errorf("cannot run integration tests: unable to start etcd: %v", err) + // } + // defer stopEtcd() stopApiserver, err := startApiserver() if err != nil { @@ -135,6 +129,7 @@ func testMain(tests func() int) error { klog.Info("Waiting for kube-apiserver to be ready...") start := time.Now() for { + klog.Info("checking kubectl version") out, kubectlErr := execKubectl("version") if kubectlErr == nil { break @@ -145,6 +140,8 @@ func testMain(tests func() int) error { time.Sleep(time.Second) } + return nil + if out, err := execKubectlStdin(strings.NewReader(defaultNamespace), "apply", "-f", "-"); err != nil { return fmt.Errorf("cannot create default Namespace: %v\n%s", err, out) } From f0ae685873742f5d3a28ba153396f97ec9d3733a Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 15:14:13 -0400 Subject: [PATCH 13/31] use abac auth Signed-off-by: Jack Carrig --- test/integration/framework/apiserver.go | 15 +++++++++++++++ test/integration/framework/main.go | 3 ++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/test/integration/framework/apiserver.go b/test/integration/framework/apiserver.go index 4e46f393..65be53f1 100644 --- a/test/integration/framework/apiserver.go +++ b/test/integration/framework/apiserver.go @@ -66,6 +66,19 @@ func startApiserver() (func(), error) { } klog.Infof("storing kube-apiserver data in: %v", apiserverDataDir) + authPolicy := `{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"admin", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} + {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"kubecfg", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} + {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"kubelet", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} + {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"kube-proxy", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} + {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"kube", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} + {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"system:controller_manager", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} + {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"system:dns", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} + {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"system:logging", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} + {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"system:monitoring", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} + {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"system:scheduler", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} + {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"system:serviceaccount:kube-system:default", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}}` + os.WriteFile(fmt.Sprintf("%s/auth-policy.json",apiserverDataDir), []byte(authPolicy), 0644) + ctx, cancel := context.WithCancel(context.Background()) cmd := exec.CommandContext( ctx, @@ -78,6 +91,8 @@ func startApiserver() (func(), error) { "--service-account-issuer", "https://kubernetes.default.svc.cluster.local", "--service-account-key-file", fmt.Sprintf("%s/apiserver.crt", apiserverDataDir), "--service-account-signing-key-file", fmt.Sprintf("%s/apiserver.key", apiserverDataDir), + "--authorization-policy-file", fmt.Sprintf("%s/auth-policy.json",apiserverDataDir), + "--authorization-mode", "ABAC", ) // Uncomment these to see kube-apiserver output in test logs. diff --git a/test/integration/framework/main.go b/test/integration/framework/main.go index 08fc1ef4..2fe7fadf 100644 --- a/test/integration/framework/main.go +++ b/test/integration/framework/main.go @@ -129,7 +129,6 @@ func testMain(tests func() int) error { klog.Info("Waiting for kube-apiserver to be ready...") start := time.Now() for { - klog.Info("checking kubectl version") out, kubectlErr := execKubectl("version") if kubectlErr == nil { break @@ -140,6 +139,8 @@ func testMain(tests func() int) error { time.Sleep(time.Second) } + klog.Info("kube-apiserver is ready!") + return nil if out, err := execKubectlStdin(strings.NewReader(defaultNamespace), "apply", "-f", "-"); err != nil { From 10714bd8cd719b2696b3231b3d374bafe4344f7f Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 15:33:14 -0400 Subject: [PATCH 14/31] config set-context Signed-off-by: Jack Carrig --- test/integration/framework/apiserver.go | 12 +----------- test/integration/framework/main.go | 4 ++++ 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/test/integration/framework/apiserver.go b/test/integration/framework/apiserver.go index 65be53f1..4b5486f5 100644 --- a/test/integration/framework/apiserver.go +++ b/test/integration/framework/apiserver.go @@ -66,17 +66,7 @@ func startApiserver() (func(), error) { } klog.Infof("storing kube-apiserver data in: %v", apiserverDataDir) - authPolicy := `{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"admin", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} - {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"kubecfg", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} - {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"kubelet", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} - {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"kube-proxy", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} - {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"kube", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} - {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"system:controller_manager", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} - {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"system:dns", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} - {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"system:logging", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} - {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"system:monitoring", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} - {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"system:scheduler", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}} - {"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"system:serviceaccount:kube-system:default", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}}` + authPolicy := "{\"apiVersion\": \"abac.authorization.kubernetes.io/v1beta1\", \"kind\": \"Policy\", \"spec\": {\"user\":\"testrunner\", \"namespace\": \"*\", \"resource\": \"*\", \"apiGroup\": \"*\"}}" os.WriteFile(fmt.Sprintf("%s/auth-policy.json",apiserverDataDir), []byte(authPolicy), 0644) ctx, cancel := context.WithCancel(context.Background()) diff --git a/test/integration/framework/main.go b/test/integration/framework/main.go index 2fe7fadf..ab89430d 100644 --- a/test/integration/framework/main.go +++ b/test/integration/framework/main.go @@ -126,6 +126,10 @@ func testMain(tests func() int) error { } defer stopApiserver() + klog.Info("set kubectl context") + + execKubectl("config", "set-context", "--user=testrunner") + klog.Info("Waiting for kube-apiserver to be ready...") start := time.Now() for { From bfda9c294e30677107dba0b9525698fbf1a464f4 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 16:30:12 -0400 Subject: [PATCH 15/31] try with anonymous user Signed-off-by: Jack Carrig --- test/integration/framework/apiserver.go | 2 +- test/integration/framework/main.go | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/test/integration/framework/apiserver.go b/test/integration/framework/apiserver.go index 4b5486f5..3ffe15d0 100644 --- a/test/integration/framework/apiserver.go +++ b/test/integration/framework/apiserver.go @@ -66,7 +66,7 @@ func startApiserver() (func(), error) { } klog.Infof("storing kube-apiserver data in: %v", apiserverDataDir) - authPolicy := "{\"apiVersion\": \"abac.authorization.kubernetes.io/v1beta1\", \"kind\": \"Policy\", \"spec\": {\"user\":\"testrunner\", \"namespace\": \"*\", \"resource\": \"*\", \"apiGroup\": \"*\"}}" + authPolicy := "{\"apiVersion\": \"abac.authorization.kubernetes.io/v1beta1\", \"kind\": \"Policy\", \"spec\": {\"user\":\"system:anonymous\", \"namespace\": \"*\", \"resource\": \"*\", \"apiGroup\": \"*\"}}" os.WriteFile(fmt.Sprintf("%s/auth-policy.json",apiserverDataDir), []byte(authPolicy), 0644) ctx, cancel := context.WithCancel(context.Background()) diff --git a/test/integration/framework/main.go b/test/integration/framework/main.go index ab89430d..dee883cc 100644 --- a/test/integration/framework/main.go +++ b/test/integration/framework/main.go @@ -250,7 +250,13 @@ func execKubectlStdin(stdin io.Reader, args ...string) ([]byte, error) { return nil, fmt.Errorf("cannot exec kubectl: %v", err) } cmdline := append([]string{"--server", ApiserverURL()}, args...) - cmd := exec.Command(execPath, cmdline...) + cmdline2 := append([]string{"--username", "foo"}, cmdline...) + cmdline3 := append([]string{"--password", "bar"}, cmdline2...) + cmdline4 := append([]string{"--insecure-skip-tls-verify", "true"}, cmdline3...) + + klog.Info(fmt.Sprintf("kubectl %q", cmdline4)) + + cmd := exec.Command(execPath, cmdline4...) cmd.Stdin = stdin return cmd.CombinedOutput() } From 56f8e7d1792f54b3c81081785b1b3f2f601b0657 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 17:04:19 -0400 Subject: [PATCH 16/31] try with equals Signed-off-by: Jack Carrig --- test/integration/framework/main.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/test/integration/framework/main.go b/test/integration/framework/main.go index dee883cc..9c79e225 100644 --- a/test/integration/framework/main.go +++ b/test/integration/framework/main.go @@ -249,14 +249,10 @@ func execKubectlStdin(stdin io.Reader, args ...string) ([]byte, error) { if err != nil { return nil, fmt.Errorf("cannot exec kubectl: %v", err) } - cmdline := append([]string{"--server", ApiserverURL()}, args...) - cmdline2 := append([]string{"--username", "foo"}, cmdline...) - cmdline3 := append([]string{"--password", "bar"}, cmdline2...) - cmdline4 := append([]string{"--insecure-skip-tls-verify", "true"}, cmdline3...) + // cmdline := append([]string{"--server", ApiserverURL()}, args...) + cmdline := append([]string{"--insecure-skip-tls-verify=true", "--username=foo", "--password=bar", fmt.Sprintf("--server=%s",ApiserverURL())}, cmdline3...) + cmd := exec.Command(execPath, cmdline...) - klog.Info(fmt.Sprintf("kubectl %q", cmdline4)) - - cmd := exec.Command(execPath, cmdline4...) cmd.Stdin = stdin return cmd.CombinedOutput() } From aa1e5cb4d67f704bd2be5415c20f0936497c1539 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 17:09:01 -0400 Subject: [PATCH 17/31] fix args Signed-off-by: Jack Carrig --- test/integration/framework/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/framework/main.go b/test/integration/framework/main.go index 9c79e225..747de61f 100644 --- a/test/integration/framework/main.go +++ b/test/integration/framework/main.go @@ -250,7 +250,7 @@ func execKubectlStdin(stdin io.Reader, args ...string) ([]byte, error) { return nil, fmt.Errorf("cannot exec kubectl: %v", err) } // cmdline := append([]string{"--server", ApiserverURL()}, args...) - cmdline := append([]string{"--insecure-skip-tls-verify=true", "--username=foo", "--password=bar", fmt.Sprintf("--server=%s",ApiserverURL())}, cmdline3...) + cmdline := append([]string{"--insecure-skip-tls-verify=true", "--username=foo", "--password=bar", fmt.Sprintf("--server=%s",ApiserverURL())}, args...) cmd := exec.Command(execPath, cmdline...) cmd.Stdin = stdin From d405bd4b15334590da2b992576a4203a840f28ef Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 18:04:49 -0400 Subject: [PATCH 18/31] start etcd Signed-off-by: Jack Carrig --- test/integration/framework/apiserver.go | 3 ++- test/integration/framework/main.go | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/test/integration/framework/apiserver.go b/test/integration/framework/apiserver.go index 3ffe15d0..073d3721 100644 --- a/test/integration/framework/apiserver.go +++ b/test/integration/framework/apiserver.go @@ -78,11 +78,12 @@ func startApiserver() (func(), error) { // doesn't conflict with other test apiservers. "--secure-port", strconv.Itoa(apiserverPort), "--etcd-servers", etcdURL, - "--service-account-issuer", "https://kubernetes.default.svc.cluster.local", + "--service-account-issuer", "api", "--service-account-key-file", fmt.Sprintf("%s/apiserver.crt", apiserverDataDir), "--service-account-signing-key-file", fmt.Sprintf("%s/apiserver.key", apiserverDataDir), "--authorization-policy-file", fmt.Sprintf("%s/auth-policy.json",apiserverDataDir), "--authorization-mode", "ABAC", + // "--client-ca-file", fmt.Sprintf("%s/apiserver.crt", apiserverDataDir), ) // Uncomment these to see kube-apiserver output in test logs. diff --git a/test/integration/framework/main.go b/test/integration/framework/main.go index 747de61f..a69bcc0e 100644 --- a/test/integration/framework/main.go +++ b/test/integration/framework/main.go @@ -114,11 +114,11 @@ func testMain(tests func() int) error { return errors.New(installKubectl) } - // stopEtcd, err := startEtcd() - // if err != nil { - // return fmt.Errorf("cannot run integration tests: unable to start etcd: %v", err) - // } - // defer stopEtcd() + stopEtcd, err := startEtcd() + if err != nil { + return fmt.Errorf("cannot run integration tests: unable to start etcd: %v", err) + } + defer stopEtcd() stopApiserver, err := startApiserver() if err != nil { From 0b3636558efa8443adb7ada5840e64c6c5caf171 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Thu, 16 Mar 2023 18:11:13 -0400 Subject: [PATCH 19/31] revert issuer Signed-off-by: Jack Carrig --- test/integration/framework/apiserver.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/integration/framework/apiserver.go b/test/integration/framework/apiserver.go index 073d3721..3ffe15d0 100644 --- a/test/integration/framework/apiserver.go +++ b/test/integration/framework/apiserver.go @@ -78,12 +78,11 @@ func startApiserver() (func(), error) { // doesn't conflict with other test apiservers. "--secure-port", strconv.Itoa(apiserverPort), "--etcd-servers", etcdURL, - "--service-account-issuer", "api", + "--service-account-issuer", "https://kubernetes.default.svc.cluster.local", "--service-account-key-file", fmt.Sprintf("%s/apiserver.crt", apiserverDataDir), "--service-account-signing-key-file", fmt.Sprintf("%s/apiserver.key", apiserverDataDir), "--authorization-policy-file", fmt.Sprintf("%s/auth-policy.json",apiserverDataDir), "--authorization-mode", "ABAC", - // "--client-ca-file", fmt.Sprintf("%s/apiserver.crt", apiserverDataDir), ) // Uncomment these to see kube-apiserver output in test logs. From a1373960fdedbec968660334c3df3391a2d27636 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Fri, 17 Mar 2023 07:12:07 -0400 Subject: [PATCH 20/31] add docker images Signed-off-by: Jack Carrig --- Dockerfile.integrationgo | 16 ++++++++++++++++ Dockerfile.integrationubuntu | 10 ++++++++++ 2 files changed, 26 insertions(+) create mode 100644 Dockerfile.integrationgo create mode 100644 Dockerfile.integrationubuntu diff --git a/Dockerfile.integrationgo b/Dockerfile.integrationgo new file mode 100644 index 00000000..64457386 --- /dev/null +++ b/Dockerfile.integrationgo @@ -0,0 +1,16 @@ +FROM golang:1.19.4-alpine + + +WORKDIR /vtop + +COPY ./ ./ + +RUN apk update && apk upgrade +RUN apk add bash curl + +RUN tools/get-kube-binaries.sh +RUN ln -s `pwd`/tools/_bin/kube-apiserver /usr/local/bin && ln -s `pwd`/tools/_bin/kubectl /usr/local/bin && ln -s `pwd`/tools/_bin/etcd /usr/local/bin +RUN go test -i ./test/integration/... + + +CMD tail -f /dev/null diff --git a/Dockerfile.integrationubuntu b/Dockerfile.integrationubuntu new file mode 100644 index 00000000..83f9abc4 --- /dev/null +++ b/Dockerfile.integrationubuntu @@ -0,0 +1,10 @@ +FROM ubuntu:22.04 + +USER root +# RUN apt install sudo + +RUN apt update && apt upgrade +RUN apt install -y curl coreutils +# RUN curl -L https://github.com/actions/go-versions/releases/download/1.19.4-3636993500/go-1.19.4-linux-x64.tar.gz | tar -xz && mv `pwd`/go-1.18.4-linux-x64 `pwd`/go/ && ln -s `pwd`/go/go /usr/local/bin + +CMD tail -f /dev/null From 1c9323b2ff3d0709fba5a4d689280fea84354175 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Fri, 17 Mar 2023 11:24:54 -0400 Subject: [PATCH 21/31] add script to start etcd Signed-off-by: Jack Carrig --- Dockerfile.integrationgo | 6 ++++-- Dockerfile.integrationubuntu | 1 + tools/etcd-up.sh | 26 ++++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100755 tools/etcd-up.sh diff --git a/Dockerfile.integrationgo b/Dockerfile.integrationgo index 64457386..6bcf908a 100644 --- a/Dockerfile.integrationgo +++ b/Dockerfile.integrationgo @@ -5,12 +5,14 @@ WORKDIR /vtop COPY ./ ./ +RUN mkdir tmp +RUN mkdir etcd + RUN apk update && apk upgrade RUN apk add bash curl RUN tools/get-kube-binaries.sh RUN ln -s `pwd`/tools/_bin/kube-apiserver /usr/local/bin && ln -s `pwd`/tools/_bin/kubectl /usr/local/bin && ln -s `pwd`/tools/_bin/etcd /usr/local/bin -RUN go test -i ./test/integration/... - +# RUN go test -i ./test/integration/... CMD tail -f /dev/null diff --git a/Dockerfile.integrationubuntu b/Dockerfile.integrationubuntu index 83f9abc4..bc68585f 100644 --- a/Dockerfile.integrationubuntu +++ b/Dockerfile.integrationubuntu @@ -7,4 +7,5 @@ RUN apt update && apt upgrade RUN apt install -y curl coreutils # RUN curl -L https://github.com/actions/go-versions/releases/download/1.19.4-3636993500/go-1.19.4-linux-x64.tar.gz | tar -xz && mv `pwd`/go-1.18.4-linux-x64 `pwd`/go/ && ln -s `pwd`/go/go /usr/local/bin + CMD tail -f /dev/null diff --git a/tools/etcd-up.sh b/tools/etcd-up.sh new file mode 100755 index 00000000..9b5be20f --- /dev/null +++ b/tools/etcd-up.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +ETCD_PORT=2379 +ETCD_URL="http://127.0.0.1:${ETCD_PORT}" + + +function fail() { + echo "ERROR: $1" + exit 1 +} + +curl "${ETCD_URL}" > /dev/null 2>&1 && fail "etcd is already running. Exiting." + +echo "starting etcd on ${ETCD_URL}" + +etcd \ + --data-dir /vtop/etcd/ \ + --listen-client-urls "${ETCD_URL}" \ + --advertise-client-urls "${ETCD_URL}" \ + --listen-peer-urls http://127.0.0.1:0 \ + > /vtop/tmp/etcd.out 2>&1 & +PID=$! +echo $PID > /vtop/tmp/etcd.pid +sleep 5 + +echo "etcd is up" From 186645b8f9a144abf271f7a56d79e97e9fab8875 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Fri, 17 Mar 2023 11:58:32 -0400 Subject: [PATCH 22/31] add kube-apiserver-up Signed-off-by: Jack Carrig --- Dockerfile.integrationgo | 1 + tools/env.sh | 8 ++++++++ tools/etcd-up.sh | 3 +-- tools/kube-apiserver-up.sh | 23 +++++++++++++++++++++++ 4 files changed, 33 insertions(+), 2 deletions(-) create mode 100755 tools/env.sh create mode 100755 tools/kube-apiserver-up.sh diff --git a/Dockerfile.integrationgo b/Dockerfile.integrationgo index 6bcf908a..947a6e7e 100644 --- a/Dockerfile.integrationgo +++ b/Dockerfile.integrationgo @@ -7,6 +7,7 @@ COPY ./ ./ RUN mkdir tmp RUN mkdir etcd +RUN mkdir kube_apiserver RUN apk update && apk upgrade RUN apk add bash curl diff --git a/tools/env.sh b/tools/env.sh new file mode 100755 index 00000000..2bb42b66 --- /dev/null +++ b/tools/env.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +export ETCD_PORT=2379 +export ETCD_URL="http://127.0.0.1:${ETCD_PORT}" + +export KUBE_APISERVER_PORT=5000 +export KUBE_APISERVER_URL="https://127.0.0.1:${KUBE_APISERVER_PORT}" +export KUBE_APISERVER_DATADIR=/vtop/kube_apiserver diff --git a/tools/etcd-up.sh b/tools/etcd-up.sh index 9b5be20f..2e841646 100755 --- a/tools/etcd-up.sh +++ b/tools/etcd-up.sh @@ -1,7 +1,6 @@ #!/bin/bash -ETCD_PORT=2379 -ETCD_URL="http://127.0.0.1:${ETCD_PORT}" +source /vtop/tools/env.sh function fail() { diff --git a/tools/kube-apiserver-up.sh b/tools/kube-apiserver-up.sh new file mode 100755 index 00000000..db9271af --- /dev/null +++ b/tools/kube-apiserver-up.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +source /vtop/tools/env.sh + +echo "starting kube-apiserver on ${KUBE_APISERVER_URL}" + +echo "{\"apiVersion\": \"abac.authorization.kubernetes.io/v1beta1\", \"kind\": \"Policy\", \"spec\": {\"user\":\"system:anonymous\", \"namespace\": \"*\", \"resource\": \"*\", \"apiGroup\": \"*\"}}" >> "${KUBE_APISERVER_DATADIR}/auth-policy.json" + +kube-apiserver \ + --cert-dir "${KUBE_APISERVER_DATADIR}" \ + --secure-port "${KUBE_APISERVER_PORT}" \ + --etcd-servers "${ETCD_URL}" \ + --service-account-issuer https://kubernetes.default.svc.cluster.local \ + --service-account-key-file "${KUBE_APISERVER_DATADIR}/apiserver.crt" \ + --service-account-signing-key-file "${KUBE_APISERVER_DATADIR}/apiserver.key" \ + --authorization-policy-file "${KUBE_APISERVER_DATADIR}/auth-policy.json" \ + --authorization-mode ABAC \ + > /vtop/tmp/kube-apiserver.out 2>&1 & +PID=$! +echo $PID > /vtop/tmp/kube-apiserver.pid +sleep 5 + +echo "kube-apiserver is up" From 030800d2e5079e829eeec3ff9dd740f67e45b0c5 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Fri, 17 Mar 2023 12:13:19 -0400 Subject: [PATCH 23/31] move into folder and add readme Signed-off-by: Jack Carrig --- .../Dockerfile.integrationgo | 0 .../Dockerfile.integrationubuntu | 0 docker/README.md | 21 +++++++++++ docker/docker.sh | 36 +++++++++++++++++++ {tools => docker/scripts}/env.sh | 0 {tools => docker/scripts}/etcd-up.sh | 2 +- .../scripts}/kube-apiserver-up.sh | 2 +- 7 files changed, 59 insertions(+), 2 deletions(-) rename Dockerfile.integrationgo => docker/Dockerfile.integrationgo (100%) rename Dockerfile.integrationubuntu => docker/Dockerfile.integrationubuntu (100%) create mode 100644 docker/README.md create mode 100755 docker/docker.sh rename {tools => docker/scripts}/env.sh (100%) rename {tools => docker/scripts}/etcd-up.sh (92%) rename {tools => docker/scripts}/kube-apiserver-up.sh (96%) diff --git a/Dockerfile.integrationgo b/docker/Dockerfile.integrationgo similarity index 100% rename from Dockerfile.integrationgo rename to docker/Dockerfile.integrationgo diff --git a/Dockerfile.integrationubuntu b/docker/Dockerfile.integrationubuntu similarity index 100% rename from Dockerfile.integrationubuntu rename to docker/Dockerfile.integrationubuntu diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..c9a5d608 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,21 @@ +#local docker testing + +build the image +``` +./docker/docker.sh build +``` + +run the image +``` +./docker/docker.sh run +``` + +once inside use scripts to start etcd and kube-apiserver +``` +3ad722450f17:/vtop# docker/scripts/etcd-up.sh +starting etcd on http://127.0.0.1:2379 +etcd is up +3ad722450f17:/vtop# docker/scripts/kube-apiserver-up.sh +starting kube-apiserver on https://127.0.0.1:5000 +kube-apiserver is up +``` diff --git a/docker/docker.sh b/docker/docker.sh new file mode 100755 index 00000000..feacac9d --- /dev/null +++ b/docker/docker.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +cd "$(dirname "$0")" + +function usage() { + echo "Usage: docker.sh " + echo "command = build | run" +} + +function run_build() { + # clean up image of old builds + docker rmi "vtop:go" + + # build image + docker build --no-cache -t "vtop:go" -f Dockerfile.integrationgo ../ +} + +function run_run() { + docker run -it --rm vtop:go /bin/bash +} + +if [ "$1" != "" ]; then + case $1 in + build ) + run_build + ;; + run ) + run_run + ;; + * ) usage + exit 1 + esac +else + usage + exit 1 +fi diff --git a/tools/env.sh b/docker/scripts/env.sh similarity index 100% rename from tools/env.sh rename to docker/scripts/env.sh diff --git a/tools/etcd-up.sh b/docker/scripts/etcd-up.sh similarity index 92% rename from tools/etcd-up.sh rename to docker/scripts/etcd-up.sh index 2e841646..d0d36df2 100755 --- a/tools/etcd-up.sh +++ b/docker/scripts/etcd-up.sh @@ -1,6 +1,6 @@ #!/bin/bash -source /vtop/tools/env.sh +source /vtop/docker/scripts/env.sh function fail() { diff --git a/tools/kube-apiserver-up.sh b/docker/scripts/kube-apiserver-up.sh similarity index 96% rename from tools/kube-apiserver-up.sh rename to docker/scripts/kube-apiserver-up.sh index db9271af..7c52743c 100755 --- a/tools/kube-apiserver-up.sh +++ b/docker/scripts/kube-apiserver-up.sh @@ -1,6 +1,6 @@ #!/bin/bash -source /vtop/tools/env.sh +source /vtop/docker/scripts/env.sh echo "starting kube-apiserver on ${KUBE_APISERVER_URL}" From a028ac7eb84a0c1b07efa4773aa8a82413b379ee Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Fri, 17 Mar 2023 13:43:59 -0400 Subject: [PATCH 24/31] see if test passes Signed-off-by: Jack Carrig --- docker/scripts/kube-apiserver-up.sh | 9 ++++++--- test/integration/framework/apiserver.go | 26 ++++++++++++++++++++----- test/integration/framework/main.go | 10 +++++++--- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/docker/scripts/kube-apiserver-up.sh b/docker/scripts/kube-apiserver-up.sh index 7c52743c..b26c32a2 100755 --- a/docker/scripts/kube-apiserver-up.sh +++ b/docker/scripts/kube-apiserver-up.sh @@ -4,17 +4,20 @@ source /vtop/docker/scripts/env.sh echo "starting kube-apiserver on ${KUBE_APISERVER_URL}" -echo "{\"apiVersion\": \"abac.authorization.kubernetes.io/v1beta1\", \"kind\": \"Policy\", \"spec\": {\"user\":\"system:anonymous\", \"namespace\": \"*\", \"resource\": \"*\", \"apiGroup\": \"*\"}}" >> "${KUBE_APISERVER_DATADIR}/auth-policy.json" +echo "31ada4fd-adec-460c-809a-9e56ceb75269,testrunner,1" >> "${KUBE_APISERVER_DATADIR}/token.csv" +echo "{\"apiVersion\": \"abac.authorization.kubernetes.io/v1beta1\", \"kind\": \"Policy\", \"spec\": {\"user\":\"testrunner\", \"namespace\": \"*\", \"resource\": \"*\", \"apiGroup\": \"*\"}}" >> "${KUBE_APISERVER_DATADIR}/auth-policy.json" +echo "{\"apiVersion\": \"abac.authorization.kubernetes.io/v1beta1\", \"kind\": \"Policy\", \"spec\": {\"group\": \"system:authenticated\", \"readonly\": true, \"nonResourcePath\": \"*\"}}" >> "${KUBE_APISERVER_DATADIR}/auth-policy.json" kube-apiserver \ --cert-dir "${KUBE_APISERVER_DATADIR}" \ --secure-port "${KUBE_APISERVER_PORT}" \ --etcd-servers "${ETCD_URL}" \ - --service-account-issuer https://kubernetes.default.svc.cluster.local \ - --service-account-key-file "${KUBE_APISERVER_DATADIR}/apiserver.crt" \ + --service-account-issuer api \ + --service-account-key-file "${KUBE_APISERVER_DATADIR}/apiserver.key" \ --service-account-signing-key-file "${KUBE_APISERVER_DATADIR}/apiserver.key" \ --authorization-policy-file "${KUBE_APISERVER_DATADIR}/auth-policy.json" \ --authorization-mode ABAC \ + --token-auth-file "${KUBE_APISERVER_DATADIR}/token.csv" \ > /vtop/tmp/kube-apiserver.out 2>&1 & PID=$! echo $PID > /vtop/tmp/kube-apiserver.pid diff --git a/test/integration/framework/apiserver.go b/test/integration/framework/apiserver.go index 3ffe15d0..9bd72642 100644 --- a/test/integration/framework/apiserver.go +++ b/test/integration/framework/apiserver.go @@ -31,6 +31,8 @@ import ( ) var apiserverURL = "" +var apiserverToken = "31ada4fd-adec-460c-809a-9e56ceb75269" +var apiserverDataDir = "" const installApiserver = ` Cannot find kube-apiserver, cannot run integration tests @@ -66,8 +68,13 @@ func startApiserver() (func(), error) { } klog.Infof("storing kube-apiserver data in: %v", apiserverDataDir) - authPolicy := "{\"apiVersion\": \"abac.authorization.kubernetes.io/v1beta1\", \"kind\": \"Policy\", \"spec\": {\"user\":\"system:anonymous\", \"namespace\": \"*\", \"resource\": \"*\", \"apiGroup\": \"*\"}}" - os.WriteFile(fmt.Sprintf("%s/auth-policy.json",apiserverDataDir), []byte(authPolicy), 0644) + os.WriteFile(fmt.Sprintf("%s/token.csv", apiserverDataDir), []byte(fmt.Sprintf("%s,aclient,1", apiserverToken)), 0644) + + abac1 := "{\"apiVersion\": \"abac.authorization.kubernetes.io/v1beta1\", \"kind\": \"Policy\", \"spec\": {\"user\": \"testrunner\", \"namespace\": \"*\", \"resource\": \"*\", \"apiGroup\": \"*\"}}" + abac2 := "{\"apiVersion\": \"abac.authorization.kubernetes.io/v1beta1\", \"kind\": \"Policy\", \"spec\": {\"group\": \"system:authenticated\", \"readonly\": true, \"nonResourcePath\": \"*\"}}" + + os.WriteFile(fmt.Sprintf("%s/auth-policy.json", apiserverDataDir), []byte(fmt.Sprintf("%s\n%s", abac1, abac2)), 0644) + ctx, cancel := context.WithCancel(context.Background()) cmd := exec.CommandContext( @@ -78,11 +85,12 @@ func startApiserver() (func(), error) { // doesn't conflict with other test apiservers. "--secure-port", strconv.Itoa(apiserverPort), "--etcd-servers", etcdURL, - "--service-account-issuer", "https://kubernetes.default.svc.cluster.local", - "--service-account-key-file", fmt.Sprintf("%s/apiserver.crt", apiserverDataDir), + "--service-account-issuer", "api", + "--service-account-key-file", fmt.Sprintf("%s/apiserver.key", apiserverDataDir), "--service-account-signing-key-file", fmt.Sprintf("%s/apiserver.key", apiserverDataDir), - "--authorization-policy-file", fmt.Sprintf("%s/auth-policy.json",apiserverDataDir), + "--authorization-policy-file", fmt.Sprintf("%s/auth-policy.json", apiserverDataDir), "--authorization-mode", "ABAC", + "--token-auth-file", fmt.Sprintf("%s/token.csv", apiserverDataDir), ) // Uncomment these to see kube-apiserver output in test logs. @@ -117,3 +125,11 @@ func ApiserverConfig() *rest.Config { Host: ApiserverURL(), } } + +func ApiserverToken() string { + return apiserverToken +} + +func ApiserverDataDir() string { + return apiserverDataDir +} diff --git a/test/integration/framework/main.go b/test/integration/framework/main.go index a69bcc0e..62557207 100644 --- a/test/integration/framework/main.go +++ b/test/integration/framework/main.go @@ -249,9 +249,13 @@ func execKubectlStdin(stdin io.Reader, args ...string) ([]byte, error) { if err != nil { return nil, fmt.Errorf("cannot exec kubectl: %v", err) } - // cmdline := append([]string{"--server", ApiserverURL()}, args...) - cmdline := append([]string{"--insecure-skip-tls-verify=true", "--username=foo", "--password=bar", fmt.Sprintf("--server=%s",ApiserverURL())}, args...) - cmd := exec.Command(execPath, cmdline...) + + cmdline := append([]string{"--server", ApiserverURL()}, args...) + cmdline2 := append([]string{"--tls-server-name", "10.0.0.1"}, cmdline...) + cmdline3 := append([]string{"--certificate-authority", fmt.Sprintf("%s/apiserver.crt", ApiserverDataDir())}, cmdline2...) + cmdline4 := append([]string{"--token", ApiserverToken()}, cmdline3...) + + cmd := exec.Command(execPath, cmdline4...) cmd.Stdin = stdin return cmd.CombinedOutput() From 4743242a913cc52562420f941bf5954783f8e681 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Fri, 17 Mar 2023 15:58:31 -0400 Subject: [PATCH 25/31] test again Signed-off-by: Jack Carrig --- test/integration/framework/apiserver.go | 49 ++++++++++++++----------- test/integration/framework/main.go | 2 +- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/test/integration/framework/apiserver.go b/test/integration/framework/apiserver.go index 9bd72642..effccf1c 100644 --- a/test/integration/framework/apiserver.go +++ b/test/integration/framework/apiserver.go @@ -32,7 +32,7 @@ import ( var apiserverURL = "" var apiserverToken = "31ada4fd-adec-460c-809a-9e56ceb75269" -var apiserverDataDir = "" +var apiserverDatadir = "" const installApiserver = ` Cannot find kube-apiserver, cannot run integration tests @@ -66,31 +66,32 @@ func startApiserver() (func(), error) { if err != nil { return nil, fmt.Errorf("unable to make temp kube-apiserver data dir: %v", err) } - klog.Infof("storing kube-apiserver data in: %v", apiserverDataDir) - os.WriteFile(fmt.Sprintf("%s/token.csv", apiserverDataDir), []byte(fmt.Sprintf("%s,aclient,1", apiserverToken)), 0644) + apiserverDatadir = apiserverDataDir + + klog.Infof("storing kube-apiserver data in: %v", apiserverDatadir) + + os.WriteFile(fmt.Sprintf("%s/token.csv", apiserverDatadir), []byte(fmt.Sprintf("%s,testrunner,1", apiserverToken)), 0644) abac1 := "{\"apiVersion\": \"abac.authorization.kubernetes.io/v1beta1\", \"kind\": \"Policy\", \"spec\": {\"user\": \"testrunner\", \"namespace\": \"*\", \"resource\": \"*\", \"apiGroup\": \"*\"}}" abac2 := "{\"apiVersion\": \"abac.authorization.kubernetes.io/v1beta1\", \"kind\": \"Policy\", \"spec\": {\"group\": \"system:authenticated\", \"readonly\": true, \"nonResourcePath\": \"*\"}}" - os.WriteFile(fmt.Sprintf("%s/auth-policy.json", apiserverDataDir), []byte(fmt.Sprintf("%s\n%s", abac1, abac2)), 0644) + os.WriteFile(fmt.Sprintf("%s/auth-policy.json", apiserverDatadir), []byte(fmt.Sprintf("%s\n%s", abac1, abac2)), 0644) ctx, cancel := context.WithCancel(context.Background()) cmd := exec.CommandContext( ctx, apiserverPath, - "--cert-dir", apiserverDataDir, - // We don't use the secure port, but we need to pick something that - // doesn't conflict with other test apiservers. - "--secure-port", strconv.Itoa(apiserverPort), + "--authorization-policy-file", fmt.Sprintf("%s/auth-policy.json", apiserverDatadir), + "--authorization-mode", "ABAC", + "--cert-dir", apiserverDatadir, "--etcd-servers", etcdURL, + "--secure-port", strconv.Itoa(apiserverPort), "--service-account-issuer", "api", - "--service-account-key-file", fmt.Sprintf("%s/apiserver.key", apiserverDataDir), - "--service-account-signing-key-file", fmt.Sprintf("%s/apiserver.key", apiserverDataDir), - "--authorization-policy-file", fmt.Sprintf("%s/auth-policy.json", apiserverDataDir), - "--authorization-mode", "ABAC", - "--token-auth-file", fmt.Sprintf("%s/token.csv", apiserverDataDir), + "--service-account-key-file", fmt.Sprintf("%s/apiserver.key", apiserverDatadir), + "--service-account-signing-key-file", fmt.Sprintf("%s/apiserver.key", apiserverDatadir), + "--token-auth-file", fmt.Sprintf("%s/token.csv", apiserverDatadir), ) // Uncomment these to see kube-apiserver output in test logs. @@ -102,7 +103,7 @@ func startApiserver() (func(), error) { cancel() err := cmd.Wait() klog.Infof("kube-apiserver exit status: %v", err) - err = os.RemoveAll(apiserverDataDir) + err = os.RemoveAll(apiserverDatadir) if err != nil { klog.Warningf("error during kube-apiserver cleanup: %v", err) } @@ -114,22 +115,28 @@ func startApiserver() (func(), error) { return stop, nil } -// ApiserverURL returns the URL of the kube-apiserver instance started by TestMain. -func ApiserverURL() string { - return apiserverURL -} - // ApiserverConfig returns a rest.Config to connect to the test instance. func ApiserverConfig() *rest.Config { return &rest.Config{ Host: ApiserverURL(), + BearerToken: apiserverToken, + TLSClientConfig: rest.TLSClientConfig{ + Insecure: true, + }, } } +// ApiserverCert returns the generated kube-apiserver certificate authority +func ApiserverCert() string { + return fmt.Sprintf("%s/apiserver.crt", apiserverDatadir) +} + +// ApiserverToken returns the token used for authentication func ApiserverToken() string { return apiserverToken } -func ApiserverDataDir() string { - return apiserverDataDir +// ApiserverURL returns the URL of the kube-apiserver instance started by TestMain. +func ApiserverURL() string { + return apiserverURL } diff --git a/test/integration/framework/main.go b/test/integration/framework/main.go index 62557207..999b2cc9 100644 --- a/test/integration/framework/main.go +++ b/test/integration/framework/main.go @@ -252,7 +252,7 @@ func execKubectlStdin(stdin io.Reader, args ...string) ([]byte, error) { cmdline := append([]string{"--server", ApiserverURL()}, args...) cmdline2 := append([]string{"--tls-server-name", "10.0.0.1"}, cmdline...) - cmdline3 := append([]string{"--certificate-authority", fmt.Sprintf("%s/apiserver.crt", ApiserverDataDir())}, cmdline2...) + cmdline3 := append([]string{"--certificate-authority", ApiserverCert()}, cmdline2...) cmdline4 := append([]string{"--token", ApiserverToken()}, cmdline3...) cmd := exec.Command(execPath, cmdline4...) From d6fdb77062ca27dd47ef1aa0f43a19a23e0036aa Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Fri, 17 Mar 2023 16:07:06 -0400 Subject: [PATCH 26/31] run tests Signed-off-by: Jack Carrig --- test/integration/framework/main.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test/integration/framework/main.go b/test/integration/framework/main.go index 999b2cc9..455aa25f 100644 --- a/test/integration/framework/main.go +++ b/test/integration/framework/main.go @@ -125,10 +125,6 @@ func testMain(tests func() int) error { return fmt.Errorf("cannot run integration tests: unable to start kube-apiserver: %v", err) } defer stopApiserver() - - klog.Info("set kubectl context") - - execKubectl("config", "set-context", "--user=testrunner") klog.Info("Waiting for kube-apiserver to be ready...") start := time.Now() @@ -143,10 +139,6 @@ func testMain(tests func() int) error { time.Sleep(time.Second) } - klog.Info("kube-apiserver is ready!") - - return nil - if out, err := execKubectlStdin(strings.NewReader(defaultNamespace), "apply", "-f", "-"); err != nil { return fmt.Errorf("cannot create default Namespace: %v\n%s", err, out) } From ba0576862b0c4a3c881f8f9703594961d3be8d36 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Fri, 17 Mar 2023 16:12:32 -0400 Subject: [PATCH 27/31] cleanup cmdline kubectl Signed-off-by: Jack Carrig --- test/integration/framework/main.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/integration/framework/main.go b/test/integration/framework/main.go index 455aa25f..e1f0d208 100644 --- a/test/integration/framework/main.go +++ b/test/integration/framework/main.go @@ -242,12 +242,14 @@ func execKubectlStdin(stdin io.Reader, args ...string) ([]byte, error) { return nil, fmt.Errorf("cannot exec kubectl: %v", err) } - cmdline := append([]string{"--server", ApiserverURL()}, args...) - cmdline2 := append([]string{"--tls-server-name", "10.0.0.1"}, cmdline...) - cmdline3 := append([]string{"--certificate-authority", ApiserverCert()}, cmdline2...) - cmdline4 := append([]string{"--token", ApiserverToken()}, cmdline3...) - - cmd := exec.Command(execPath, cmdline4...) + cmdline := append([]string{ + "--server", ApiserverURL(), + "--tls-server-name", "10.0.0.1", + "--certificate-authority", ApiserverCert(), + "--token", ApiserverToken(), + }, args...) + + cmd := exec.Command(execPath, cmdline...) cmd.Stdin = stdin return cmd.CombinedOutput() From bdb47e5ab1c6087dd60e34202948f80a88b276b8 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Fri, 17 Mar 2023 16:18:28 -0400 Subject: [PATCH 28/31] cleanup Signed-off-by: Jack Carrig --- docker/Dockerfile.integrationgo | 19 ------------- docker/Dockerfile.integrationubuntu | 11 -------- docker/README.md | 21 --------------- docker/docker.sh | 36 ------------------------- docker/scripts/env.sh | 8 ------ docker/scripts/etcd-up.sh | 25 ----------------- docker/scripts/kube-apiserver-up.sh | 26 ------------------ test/integration/framework/apiserver.go | 9 +++---- test/integration/framework/main.go | 3 +-- 9 files changed, 5 insertions(+), 153 deletions(-) delete mode 100644 docker/Dockerfile.integrationgo delete mode 100644 docker/Dockerfile.integrationubuntu delete mode 100644 docker/README.md delete mode 100755 docker/docker.sh delete mode 100755 docker/scripts/env.sh delete mode 100755 docker/scripts/etcd-up.sh delete mode 100755 docker/scripts/kube-apiserver-up.sh diff --git a/docker/Dockerfile.integrationgo b/docker/Dockerfile.integrationgo deleted file mode 100644 index 947a6e7e..00000000 --- a/docker/Dockerfile.integrationgo +++ /dev/null @@ -1,19 +0,0 @@ -FROM golang:1.19.4-alpine - - -WORKDIR /vtop - -COPY ./ ./ - -RUN mkdir tmp -RUN mkdir etcd -RUN mkdir kube_apiserver - -RUN apk update && apk upgrade -RUN apk add bash curl - -RUN tools/get-kube-binaries.sh -RUN ln -s `pwd`/tools/_bin/kube-apiserver /usr/local/bin && ln -s `pwd`/tools/_bin/kubectl /usr/local/bin && ln -s `pwd`/tools/_bin/etcd /usr/local/bin -# RUN go test -i ./test/integration/... - -CMD tail -f /dev/null diff --git a/docker/Dockerfile.integrationubuntu b/docker/Dockerfile.integrationubuntu deleted file mode 100644 index bc68585f..00000000 --- a/docker/Dockerfile.integrationubuntu +++ /dev/null @@ -1,11 +0,0 @@ -FROM ubuntu:22.04 - -USER root -# RUN apt install sudo - -RUN apt update && apt upgrade -RUN apt install -y curl coreutils -# RUN curl -L https://github.com/actions/go-versions/releases/download/1.19.4-3636993500/go-1.19.4-linux-x64.tar.gz | tar -xz && mv `pwd`/go-1.18.4-linux-x64 `pwd`/go/ && ln -s `pwd`/go/go /usr/local/bin - - -CMD tail -f /dev/null diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index c9a5d608..00000000 --- a/docker/README.md +++ /dev/null @@ -1,21 +0,0 @@ -#local docker testing - -build the image -``` -./docker/docker.sh build -``` - -run the image -``` -./docker/docker.sh run -``` - -once inside use scripts to start etcd and kube-apiserver -``` -3ad722450f17:/vtop# docker/scripts/etcd-up.sh -starting etcd on http://127.0.0.1:2379 -etcd is up -3ad722450f17:/vtop# docker/scripts/kube-apiserver-up.sh -starting kube-apiserver on https://127.0.0.1:5000 -kube-apiserver is up -``` diff --git a/docker/docker.sh b/docker/docker.sh deleted file mode 100755 index feacac9d..00000000 --- a/docker/docker.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -cd "$(dirname "$0")" - -function usage() { - echo "Usage: docker.sh " - echo "command = build | run" -} - -function run_build() { - # clean up image of old builds - docker rmi "vtop:go" - - # build image - docker build --no-cache -t "vtop:go" -f Dockerfile.integrationgo ../ -} - -function run_run() { - docker run -it --rm vtop:go /bin/bash -} - -if [ "$1" != "" ]; then - case $1 in - build ) - run_build - ;; - run ) - run_run - ;; - * ) usage - exit 1 - esac -else - usage - exit 1 -fi diff --git a/docker/scripts/env.sh b/docker/scripts/env.sh deleted file mode 100755 index 2bb42b66..00000000 --- a/docker/scripts/env.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -export ETCD_PORT=2379 -export ETCD_URL="http://127.0.0.1:${ETCD_PORT}" - -export KUBE_APISERVER_PORT=5000 -export KUBE_APISERVER_URL="https://127.0.0.1:${KUBE_APISERVER_PORT}" -export KUBE_APISERVER_DATADIR=/vtop/kube_apiserver diff --git a/docker/scripts/etcd-up.sh b/docker/scripts/etcd-up.sh deleted file mode 100755 index d0d36df2..00000000 --- a/docker/scripts/etcd-up.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -source /vtop/docker/scripts/env.sh - - -function fail() { - echo "ERROR: $1" - exit 1 -} - -curl "${ETCD_URL}" > /dev/null 2>&1 && fail "etcd is already running. Exiting." - -echo "starting etcd on ${ETCD_URL}" - -etcd \ - --data-dir /vtop/etcd/ \ - --listen-client-urls "${ETCD_URL}" \ - --advertise-client-urls "${ETCD_URL}" \ - --listen-peer-urls http://127.0.0.1:0 \ - > /vtop/tmp/etcd.out 2>&1 & -PID=$! -echo $PID > /vtop/tmp/etcd.pid -sleep 5 - -echo "etcd is up" diff --git a/docker/scripts/kube-apiserver-up.sh b/docker/scripts/kube-apiserver-up.sh deleted file mode 100755 index b26c32a2..00000000 --- a/docker/scripts/kube-apiserver-up.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -source /vtop/docker/scripts/env.sh - -echo "starting kube-apiserver on ${KUBE_APISERVER_URL}" - -echo "31ada4fd-adec-460c-809a-9e56ceb75269,testrunner,1" >> "${KUBE_APISERVER_DATADIR}/token.csv" -echo "{\"apiVersion\": \"abac.authorization.kubernetes.io/v1beta1\", \"kind\": \"Policy\", \"spec\": {\"user\":\"testrunner\", \"namespace\": \"*\", \"resource\": \"*\", \"apiGroup\": \"*\"}}" >> "${KUBE_APISERVER_DATADIR}/auth-policy.json" -echo "{\"apiVersion\": \"abac.authorization.kubernetes.io/v1beta1\", \"kind\": \"Policy\", \"spec\": {\"group\": \"system:authenticated\", \"readonly\": true, \"nonResourcePath\": \"*\"}}" >> "${KUBE_APISERVER_DATADIR}/auth-policy.json" - -kube-apiserver \ - --cert-dir "${KUBE_APISERVER_DATADIR}" \ - --secure-port "${KUBE_APISERVER_PORT}" \ - --etcd-servers "${ETCD_URL}" \ - --service-account-issuer api \ - --service-account-key-file "${KUBE_APISERVER_DATADIR}/apiserver.key" \ - --service-account-signing-key-file "${KUBE_APISERVER_DATADIR}/apiserver.key" \ - --authorization-policy-file "${KUBE_APISERVER_DATADIR}/auth-policy.json" \ - --authorization-mode ABAC \ - --token-auth-file "${KUBE_APISERVER_DATADIR}/token.csv" \ - > /vtop/tmp/kube-apiserver.out 2>&1 & -PID=$! -echo $PID > /vtop/tmp/kube-apiserver.pid -sleep 5 - -echo "kube-apiserver is up" diff --git a/test/integration/framework/apiserver.go b/test/integration/framework/apiserver.go index effccf1c..2f40493e 100644 --- a/test/integration/framework/apiserver.go +++ b/test/integration/framework/apiserver.go @@ -66,16 +66,15 @@ func startApiserver() (func(), error) { if err != nil { return nil, fmt.Errorf("unable to make temp kube-apiserver data dir: %v", err) } - apiserverDatadir = apiserverDataDir - klog.Infof("storing kube-apiserver data in: %v", apiserverDatadir) + // create token auth file os.WriteFile(fmt.Sprintf("%s/token.csv", apiserverDatadir), []byte(fmt.Sprintf("%s,testrunner,1", apiserverToken)), 0644) + // create authorization policy file abac1 := "{\"apiVersion\": \"abac.authorization.kubernetes.io/v1beta1\", \"kind\": \"Policy\", \"spec\": {\"user\": \"testrunner\", \"namespace\": \"*\", \"resource\": \"*\", \"apiGroup\": \"*\"}}" abac2 := "{\"apiVersion\": \"abac.authorization.kubernetes.io/v1beta1\", \"kind\": \"Policy\", \"spec\": {\"group\": \"system:authenticated\", \"readonly\": true, \"nonResourcePath\": \"*\"}}" - os.WriteFile(fmt.Sprintf("%s/auth-policy.json", apiserverDatadir), []byte(fmt.Sprintf("%s\n%s", abac1, abac2)), 0644) @@ -96,8 +95,8 @@ func startApiserver() (func(), error) { // Uncomment these to see kube-apiserver output in test logs. // For operator tests, we generally don't expect problems at this level. - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr + // cmd.Stdout = os.Stdout + // cmd.Stderr = os.Stderr stop := func() { cancel() diff --git a/test/integration/framework/main.go b/test/integration/framework/main.go index e1f0d208..ae6f05e5 100644 --- a/test/integration/framework/main.go +++ b/test/integration/framework/main.go @@ -125,7 +125,7 @@ func testMain(tests func() int) error { return fmt.Errorf("cannot run integration tests: unable to start kube-apiserver: %v", err) } defer stopApiserver() - + klog.Info("Waiting for kube-apiserver to be ready...") start := time.Now() for { @@ -241,7 +241,6 @@ func execKubectlStdin(stdin io.Reader, args ...string) ([]byte, error) { if err != nil { return nil, fmt.Errorf("cannot exec kubectl: %v", err) } - cmdline := append([]string{ "--server", ApiserverURL(), "--tls-server-name", "10.0.0.1", From 5b89a2577798f2ecbe1318a948ed287e2837ecc8 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Fri, 17 Mar 2023 16:22:43 -0400 Subject: [PATCH 29/31] more cleanup Signed-off-by: Jack Carrig --- test/integration/framework/apiserver.go | 4 ++-- test/integration/framework/main.go | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/integration/framework/apiserver.go b/test/integration/framework/apiserver.go index 2f40493e..024338df 100644 --- a/test/integration/framework/apiserver.go +++ b/test/integration/framework/apiserver.go @@ -95,8 +95,8 @@ func startApiserver() (func(), error) { // Uncomment these to see kube-apiserver output in test logs. // For operator tests, we generally don't expect problems at this level. - // cmd.Stdout = os.Stdout - // cmd.Stderr = os.Stderr + //cmd.Stdout = os.Stdout + //cmd.Stderr = os.Stderr stop := func() { cancel() diff --git a/test/integration/framework/main.go b/test/integration/framework/main.go index ae6f05e5..0e305246 100644 --- a/test/integration/framework/main.go +++ b/test/integration/framework/main.go @@ -249,7 +249,6 @@ func execKubectlStdin(stdin io.Reader, args ...string) ([]byte, error) { }, args...) cmd := exec.Command(execPath, cmdline...) - cmd.Stdin = stdin return cmd.CombinedOutput() } From e4f183da7eb63e0d0b0d9939d7457de737b2a4d5 Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Fri, 17 Mar 2023 16:30:48 -0400 Subject: [PATCH 30/31] generate random token Signed-off-by: Jack Carrig --- test/integration/framework/apiserver.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/framework/apiserver.go b/test/integration/framework/apiserver.go index 024338df..8a5bdb23 100644 --- a/test/integration/framework/apiserver.go +++ b/test/integration/framework/apiserver.go @@ -26,12 +26,13 @@ import ( "os/exec" "strconv" + "github.com/google/uuid" "k8s.io/client-go/rest" "k8s.io/klog" ) var apiserverURL = "" -var apiserverToken = "31ada4fd-adec-460c-809a-9e56ceb75269" +var apiserverToken = uuid.New().String() var apiserverDatadir = "" const installApiserver = ` From e89227939edea272f6359908a72a5e0069379e1c Mon Sep 17 00:00:00 2001 From: Jack Carrig Date: Mon, 20 Mar 2023 11:00:41 -0400 Subject: [PATCH 31/31] update readme and k8s version to v1.25.8 Signed-off-by: Jack Carrig --- README.md | 2 +- tools/get-kube-binaries.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5940c171..4d6f3ea4 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Vitess Operator Version | Recommended Vitess Versions | Recommended Kubernetes V `v2.7.*` | `v14.0.*` | `v1.20.*`, `v1.21.*`, or `v1.22.*` `v2.8.*` | `v15.0.*` | `v1.22.*`, `v1.23.*`, or `v1.24.*` `v2.9.*` | `v16.0.*` | `v1.22.*`, `v1.23.*`, or `v1.24.*` -`latest` | `latest` | `v1.22.*`, `v1.23.*`, or `v1.24.*` +`latest` | `latest` | `v1.22.*`, `v1.23.*`, `v1.24.*`, or `v1.25.*` If for some reason you must attempt to use versions outside the recommend window, we still welcome bug reports since a workaround might be possible. diff --git a/tools/get-kube-binaries.sh b/tools/get-kube-binaries.sh index 20c9b596..3742a041 100755 --- a/tools/get-kube-binaries.sh +++ b/tools/get-kube-binaries.sh @@ -9,7 +9,7 @@ set -euo pipefail # The integration test framework expects these binaries to be found in the PATH. # This is the kube-apiserver version to test against. -KUBE_VERSION="${KUBE_VERSION:-v1.24.11}" +KUBE_VERSION="${KUBE_VERSION:-v1.25.8}" KUBERNETES_RELEASE_URL="${KUBERNETES_RELEASE_URL:-https://dl.k8s.io}" # This should be the etcd version downloaded by kubernetes/hack/lib/etcd.sh