From 577d46afc0ec165a58a445fbc0f405a51ea3e48e Mon Sep 17 00:00:00 2001 From: Manabu McCloskey Date: Fri, 7 Jun 2024 20:54:18 +0000 Subject: [PATCH] parallel install core Signed-off-by: Manabu McCloskey --- pkg/controllers/localbuild/controller.go | 69 +++++++++++++++++++----- pkg/controllers/localbuild/gitea.go | 40 ++++++++++++-- pkg/controllers/localbuild/installer.go | 24 ++------- 3 files changed, 95 insertions(+), 38 deletions(-) diff --git a/pkg/controllers/localbuild/controller.go b/pkg/controllers/localbuild/controller.go index aaeb22f8..fd1c3999 100644 --- a/pkg/controllers/localbuild/controller.go +++ b/pkg/controllers/localbuild/controller.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strings" + "sync" "time" argocdapp "github.com/cnoe-io/argocd-api/api/argo/application" @@ -33,7 +34,8 @@ const ( ) var ( - defaultRequeueTime = time.Second * 30 + defaultRequeueTime = time.Second * 15 + errRequeueTime = time.Second * 5 ) type LocalbuildReconciler struct { @@ -64,23 +66,62 @@ func (r *LocalbuildReconciler) Reconcile(ctx context.Context, req ctrl.Request) // Make sure we post process defer r.postProcessReconcile(ctx, req, &localBuild) - // respecting order of installation matters as there are hard dependencies - subReconcilers := []subReconciler{ - r.ReconcileProjectNamespace, - r.ReconcileNginx, - r.ReconcileArgo, - r.ReconcileGitea, - r.ReconcileArgoAppsWithGitea, + _, err := r.ReconcileProjectNamespace(ctx, req, &localBuild) + if err != nil { + return ctrl.Result{}, err } - for _, sub := range subReconcilers { - result, err := sub(ctx, req, &localBuild) - if err != nil || result.Requeue || result.RequeueAfter != 0 { - return result, err + instCtx, cancel := context.WithCancel(ctx) + defer cancel() + errChan := make(chan error, 3) + + go r.installCorePackages(instCtx, req, &localBuild, errChan) + + select { + case <-ctx.Done(): + return ctrl.Result{}, nil + case instErr := <-errChan: + if instErr != nil { + // likely due to ingress-nginx admission hook not ready. debug log and try again. + logger.V(1).Info("failed installing core package. likely not fatal. will try again", "error", instErr) + return ctrl.Result{RequeueAfter: errRequeueTime}, nil } } - return ctrl.Result{}, nil + logger.V(1).Info("done installing core packages. passing control to argocd") + _, err = r.ReconcileArgoAppsWithGitea(ctx, req, &localBuild) + if err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{RequeueAfter: defaultRequeueTime}, nil +} + +func (r *LocalbuildReconciler) installCorePackages(ctx context.Context, req ctrl.Request, resource *v1alpha1.Localbuild, errChan chan error) { + logger := log.FromContext(ctx) + defer close(errChan) + var wg sync.WaitGroup + + installers := map[string]subReconciler{ + "nginx": r.ReconcileNginx, + "argocd": r.ReconcileArgo, + "gitea": r.ReconcileGitea, + } + logger.V(1).Info("installing core packages") + for k, v := range installers { + wg.Add(1) + name := k + inst := v + go func() { + defer wg.Done() + _, iErr := inst(ctx, req, resource) + if iErr != nil { + logger.V(1).Info("failed installing %s: %s", name, iErr) + errChan <- fmt.Errorf("failed installing %s: %w", name, iErr) + } + }() + } + wg.Wait() } // Responsible to updating ObservedGeneration in status @@ -164,7 +205,7 @@ func (r *LocalbuildReconciler) ReconcileArgoAppsWithGitea(ctx context.Context, r } r.shouldShutdown = shutdown - return ctrl.Result{RequeueAfter: defaultRequeueTime}, nil + return ctrl.Result{}, nil } func (r *LocalbuildReconciler) reconcileEmbeddedApp(ctx context.Context, appName string, resource *v1alpha1.Localbuild) (ctrl.Result, error) { diff --git a/pkg/controllers/localbuild/gitea.go b/pkg/controllers/localbuild/gitea.go index 3905812a..1ecc6008 100644 --- a/pkg/controllers/localbuild/gitea.go +++ b/pkg/controllers/localbuild/gitea.go @@ -2,8 +2,11 @@ package localbuild import ( "context" + "crypto/tls" "embed" "fmt" + "net/http" + "time" "github.com/cnoe-io/idpbuilder/api/v1alpha1" "github.com/cnoe-io/idpbuilder/pkg/k8s" @@ -14,6 +17,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" ) const ( @@ -34,7 +38,7 @@ func RawGiteaInstallResources(templateData any, config v1alpha1.PackageCustomiza return k8s.BuildCustomizedManifests(config.FilePath, "resources/gitea/k8s", installGiteaFS, scheme, templateData) } -func newGiteAdminSecret() (corev1.Secret, error) { +func newGiteaAdminSecret() (corev1.Secret, error) { pass, err := util.GeneratePassword() if err != nil { return corev1.Secret{}, err @@ -56,6 +60,7 @@ func newGiteAdminSecret() (corev1.Secret, error) { } func (r *LocalbuildReconciler) ReconcileGitea(ctx context.Context, req ctrl.Request, resource *v1alpha1.Localbuild) (ctrl.Result, error) { + logger := log.FromContext(ctx, "installer", "gitea") gitea := EmbeddedInstallation{ name: "Gitea", resourcePath: "resources/gitea/k8s", @@ -70,7 +75,7 @@ func (r *LocalbuildReconciler) ReconcileGitea(ctx context.Context, req ctrl.Requ }, } - giteCreds, err := newGiteAdminSecret() + giteCreds, err := newGiteaAdminSecret() if err != nil { return ctrl.Result{}, fmt.Errorf("generating gitea admin secret: %w", err) } @@ -80,10 +85,39 @@ func (r *LocalbuildReconciler) ReconcileGitea(ctx context.Context, req ctrl.Requ if result, err := gitea.Install(ctx, req, resource, r.Client, r.Scheme, r.Config); err != nil { return result, err } - resource.Status.Gitea.ExternalURL = fmt.Sprintf(giteaIngressURL, r.Config.Protocol, r.Config.Port) + + baseUrl := giteaBaseUrl(r.Config) + // need this to ensure gitrepository controller can reach the api endpoint. + logger.V(1).Info("checking gitea api endpoint", "url", baseUrl) + c := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + Timeout: 2 * time.Second, + } + resp, err := c.Get(baseUrl) + + if err != nil { + return ctrl.Result{}, err + } + if resp != nil { + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + logger.V(1).Info("gitea manifests installed successfully. endpoint not ready", "statusCode", resp.StatusCode) + return ctrl.Result{RequeueAfter: errRequeueTime}, nil + } + } + + resource.Status.Gitea.ExternalURL = baseUrl resource.Status.Gitea.InternalURL = giteaSvcURL resource.Status.Gitea.AdminUserSecretName = giteaAdminSecret resource.Status.Gitea.AdminUserSecretNamespace = giteaNamespace resource.Status.Gitea.Available = true return ctrl.Result{}, nil } + +func giteaBaseUrl(config util.CorePackageTemplateConfig) string { + return fmt.Sprintf(giteaIngressURL, config.Protocol, config.Port) +} diff --git a/pkg/controllers/localbuild/installer.go b/pkg/controllers/localbuild/installer.go index c22d1333..95cbd06b 100644 --- a/pkg/controllers/localbuild/installer.go +++ b/pkg/controllers/localbuild/installer.go @@ -19,7 +19,6 @@ import ( "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -79,25 +78,10 @@ func (e *EmbeddedInstallation) Install(ctx context.Context, req ctrl.Request, re } } - logger.V(1).Info(fmt.Sprintf("Installing/Reconciling %s resources", e.name)) - for _, obj := range installObjs { - if gvk, ok := e.monitoredResources[obj.GetName()]; ok { - if obj.GetObjectKind().GroupVersionKind() == gvk { - sch := runtime.NewScheme() - _ = appsv1.AddToScheme(sch) - if gvkObj, err := sch.New(gvk); err == nil { - if gotObj, ok := gvkObj.(client.Object); ok { - if err := cli.Get(ctx, types.NamespacedName{Namespace: e.namespace, Name: obj.GetName()}, gotObj); err != nil { - if err = controllerutil.SetControllerReference(resource, obj, sc); err != nil { - logger.Error(err, "Setting controller reference for deployment", obj.GetName(), obj) - return ctrl.Result{}, err - } - } - } - } - } - } + sch := runtime.NewScheme() + appsv1.AddToScheme(sch) + for _, obj := range installObjs { // Create object if err = k8s.EnsureObject(ctx, nsClient, obj, e.namespace); err != nil { return ctrl.Result{}, err @@ -123,8 +107,6 @@ func (e *EmbeddedInstallation) Install(ctx context.Context, req ctrl.Request, re go func(obj client.Object, gvk schema.GroupVersionKind) { defer wg.Done() - sch := runtime.NewScheme() - _ = appsv1.AddToScheme(sch) gvkObj, err := sch.New(gvk) if err != nil { errCh <- err