diff --git a/Dockerfile b/Dockerfile index 84dfe697..808980e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,4 +7,5 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -a -ldflags '-extldflags " FROM scratch WORKDIR / COPY --from=builder /go/src/github.com/Azure/aks-app-routing-operator/aks-app-routing-operator . +COPY --from=builder /go/src/github.com/Azure/aks-app-routing-operator/config/crd/bases ./crd ENTRYPOINT ["/aks-app-routing-operator"] diff --git a/pkg/config/config.go b/pkg/config/config.go index f85b1860..09edf893 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -7,6 +7,7 @@ import ( "errors" "flag" "fmt" + "os" "strings" "time" @@ -42,6 +43,7 @@ func init() { flag.StringVar(&Flags.OperatorDeployment, "operator-deployment", "app-routing-operator", "name of the operator's k8s deployment") flag.StringVar(&Flags.ClusterUid, "cluster-uid", "", "unique identifier of the cluster the add-on belongs to") flag.DurationVar(&Flags.DnsSyncInterval, "dns-sync-interval", defaultDnsSyncInterval, "interval at which to sync DNS records") + flag.StringVar(&Flags.CrdPath, "crd", "/crd", "location of the CRD manifests. manifests should be directly in this directory, not in a subdirectory") } type DnsZoneConfig struct { @@ -64,6 +66,7 @@ type Config struct { OperatorDeployment string ClusterUid string DnsSyncInterval time.Duration + CrdPath string } func (c *Config) Validate() error { @@ -106,6 +109,17 @@ func (c *Config) Validate() error { c.DnsSyncInterval = defaultDnsSyncInterval } + crdPathStat, err := os.Stat(c.CrdPath) + if os.IsNotExist(err) { + return fmt.Errorf("crd path %s does not exist", c.CrdPath) + } + if err != nil { + return fmt.Errorf("checking crd path %s: %s", c.CrdPath, err) + } + if !crdPathStat.IsDir() { + return fmt.Errorf("crd path %s is not a directory", c.CrdPath) + } + return nil } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 42f9c59a..41869082 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -5,12 +5,19 @@ package config import ( "errors" + "fmt" "strings" "testing" "github.com/stretchr/testify/require" ) +const ( + validCrdPath = "../../config/crd/bases/" + notADirectoryPath = "./config.go" + notAValidPath = "./does/not/exist" +) + var validateTestCases = []struct { Name string Conf *Config @@ -29,6 +36,7 @@ var validateTestCases = []struct { ConcurrencyWatchdogThres: 101, ConcurrencyWatchdogVotes: 2, ClusterUid: "cluster-uid", + CrdPath: validCrdPath, }, }, { @@ -43,7 +51,40 @@ var validateTestCases = []struct { ConcurrencyWatchdogThres: 101, ConcurrencyWatchdogVotes: 2, ClusterUid: "test-cluster-uid", + CrdPath: validCrdPath, + }, + }, + { + Name: "nonexistent crd path", + Conf: &Config{ + NS: "test-namespace", + Registry: "test-registry", + MSIClientID: "test-msi-client-id", + TenantID: "test-tenant-id", + Cloud: "test-cloud", + Location: "test-location", + ConcurrencyWatchdogThres: 101, + ConcurrencyWatchdogVotes: 2, + ClusterUid: "test-cluster-uid", + CrdPath: notAValidPath, + }, + Error: fmt.Sprintf("crd path %s does not exist", notAValidPath), + }, + { + Name: "non-directory crd path", + Conf: &Config{ + NS: "test-namespace", + Registry: "test-registry", + MSIClientID: "test-msi-client-id", + TenantID: "test-tenant-id", + Cloud: "test-cloud", + Location: "test-location", + ConcurrencyWatchdogThres: 101, + ConcurrencyWatchdogVotes: 2, + ClusterUid: "test-cluster-uid", + CrdPath: notADirectoryPath, }, + Error: fmt.Sprintf("crd path %s is not a directory", notADirectoryPath), }, { Name: "missing-namespace", diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 35bcfbea..c4845ad3 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -7,11 +7,13 @@ import ( "context" "net/http" + approutingv1alpha1 "github.com/Azure/aks-app-routing-operator/api/v1alpha1" "github.com/go-logr/logr" cfgv1alpha2 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha2" policyv1alpha1 "github.com/openservicemesh/osm/pkg/apis/policy/v1alpha1" ubzap "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -21,6 +23,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" secv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" @@ -33,7 +36,9 @@ import ( "github.com/Azure/aks-app-routing-operator/pkg/controller/osm" ) -var scheme = runtime.NewScheme() +var ( + scheme = runtime.NewScheme() +) func init() { registerSchemes(scheme) @@ -57,6 +62,8 @@ func registerSchemes(s *runtime.Scheme) { utilruntime.Must(secv1.Install(s)) utilruntime.Must(cfgv1alpha2.AddToScheme(s)) utilruntime.Must(policyv1alpha1.AddToScheme(s)) + utilruntime.Must(approutingv1alpha1.AddToScheme(s)) + utilruntime.Must(apiextensionsv1.AddToScheme(s)) } func NewManager(conf *config.Config) (ctrl.Manager, error) { @@ -84,17 +91,26 @@ func NewManagerForRestConfig(conf *config.Config, rc *rest.Config) (ctrl.Manager m.AddHealthzCheck("liveness", func(req *http.Request) error { return nil }) - kcs, err := kubernetes.NewForConfig(rc) // need non-caching client since manager hasn't started yet + // create non-caching clients, non-caching for use before manager has started + kcs, err := kubernetes.NewForConfig(rc) + if err != nil { + return nil, err + } + cl, err := client.New(rc, client.Options{Scheme: scheme}) if err != nil { return nil, err } - log := m.GetLogger() - deploy, err := getSelfDeploy(kcs, conf, log) + setupLog := m.GetLogger().WithName("setup") + deploy, err := getSelfDeploy(kcs, conf, setupLog) if err != nil { return nil, err } - log.V(2).Info("using namespace: " + conf.NS) + setupLog.V(2).Info("using namespace: " + conf.NS) + + if err := loadCRDs(cl, conf, setupLog); err != nil { + return nil, err + } if err := dns.NewExternalDns(m, conf, deploy); err != nil { return nil, err diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index c73b429c..1fce1c82 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -8,12 +8,10 @@ import ( "context" "encoding/json" "errors" - "os" "strings" "testing" "github.com/Azure/aks-app-routing-operator/pkg/config" - "github.com/Azure/aks-app-routing-operator/pkg/controller/testutils" "github.com/go-logr/logr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -21,28 +19,9 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) -var ( - restConfig *rest.Config - err error - env *envtest.Environment -) - -func TestMain(m *testing.M) { - restConfig, env, err = testutils.StartTestingEnv() - if err != nil { - panic(err) - } - - code := m.Run() - testutils.CleanupTestingEnv(env) - os.Exit(code) -} - func TestLogger(t *testing.T) { t.Run("logs are json structured", func(t *testing.T) { logOut := new(bytes.Buffer) @@ -96,9 +75,3 @@ func TestGetSelfDeploy(t *testing.T) { require.Nil(t, self) }) } - -func TestNewManagerForRestConfig(t *testing.T) { - conf := &config.Config{NS: "app-routing-system", OperatorDeployment: "operator-test", MetricsAddr: "0"} - _, err := NewManagerForRestConfig(conf, restConfig) - require.NoError(t, err) -} diff --git a/pkg/controller/crd.go b/pkg/controller/crd.go new file mode 100644 index 00000000..aacec71b --- /dev/null +++ b/pkg/controller/crd.go @@ -0,0 +1,60 @@ +package controller + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/Azure/aks-app-routing-operator/pkg/config" + "github.com/Azure/aks-app-routing-operator/pkg/util" + "github.com/go-logr/logr" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" +) + +// loadCRDs loads the CRDs from the specified path into the cluster +func loadCRDs(c client.Client, cfg *config.Config, log logr.Logger) error { + if cfg == nil { + return errors.New("config cannot be nil") + } + + log = log.WithValues("crdPath", cfg.CrdPath) + log.Info("reading crd directory") + files, err := os.ReadDir(cfg.CrdPath) + if err != nil { + return fmt.Errorf("reading crd directory %s: %w", cfg.CrdPath, err) + } + + log.Info("applying crds") + for _, file := range files { + if file.IsDir() { + continue + } + + path := filepath.Join(cfg.CrdPath, file.Name()) + log := log.WithValues("path", path) + log.Info("reading crd file") + var content []byte + content, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading crd file %s: %w", path, err) + } + + log.Info("unmarshalling crd file") + crd := &apiextensionsv1.CustomResourceDefinition{} + if err := yaml.Unmarshal(content, crd); err != nil { + return fmt.Errorf("unmarshalling crd file %s: %w", path, err) + } + + log.Info("upserting crd") + if err := util.Upsert(context.Background(), c, crd); err != nil { + return fmt.Errorf("upserting crd %s: %w", crd.Name, err) + } + } + + log.Info("crds loaded") + return nil +} diff --git a/pkg/controller/crd_test.go b/pkg/controller/crd_test.go new file mode 100644 index 00000000..dc59f31e --- /dev/null +++ b/pkg/controller/crd_test.go @@ -0,0 +1,61 @@ +package controller + +import ( + "context" + "strings" + "testing" + + "github.com/Azure/aks-app-routing-operator/pkg/config" + "github.com/go-logr/logr" + "github.com/stretchr/testify/require" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +const ( + validCrdPath = "../../config/crd/bases/" + validCrdName = "nginxingresscontrollers.approuting.kubernetes.azure.com" + validCrdPathWithDir = "../../config/crd/" + + nonCrdManifestsPath = "../manifests/fixtures/nginx" + nonExistentFilePath = "./this/does/not/exist" +) + +func TestLoadCRDs(t *testing.T) { + t.Run("valid crds", func(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(scheme).Build() + require.NoError(t, loadCRDs(cl, &config.Config{CrdPath: validCrdPath}, logr.Discard()), "expected no error loading valid crds") + + crd := &apiextensionsv1.CustomResourceDefinition{} + crd.Name = validCrdName + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(crd), crd, nil), "getting loaded valid crd") + }) + + t.Run("valid crds with directory", func(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(scheme).Build() + require.NoError(t, loadCRDs(cl, &config.Config{CrdPath: validCrdPath}, logr.Discard()), "expected no error loading valid crds") + }) + + t.Run("invalid crds", func(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(scheme).Build() + err := loadCRDs(cl, &config.Config{CrdPath: nonCrdManifestsPath}, logr.Discard()) + require.Error(t, err, "expected error loading invalid crds") + require.True(t, strings.Contains(err.Error(), "unmarshalling crd file"), "expected error to be about umarshalling crd") + }) + + t.Run("non-existent crd path", func(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(scheme).Build() + err := loadCRDs(cl, &config.Config{CrdPath: nonExistentFilePath}, logr.Discard()) + require.Error(t, err, "expected error loading non-existent crd path") + require.True(t, strings.Contains(err.Error(), "reading crd directory"), "expected error to be about reading crd directory") + }) + + t.Run("nil config", func(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(scheme).Build() + err := loadCRDs(cl, nil, logr.Discard()) + require.Error(t, err, "expected error loading nil config") + require.True(t, strings.Contains(err.Error(), "config cannot be nil"), "expected error to be about nil config") + }) + +}