Skip to content

Commit

Permalink
parallel install core
Browse files Browse the repository at this point in the history 
Signed-off-by: Manabu McCloskey <[email protected]>
  • Loading branch information
@nabuskey
nabuskey committed Jun 7, 2024
1 parent 4a972e3 commit 577d46a
Show file tree
Hide file tree
Showing 3 changed files with 95 additions and 38 deletions.
69 changes: 55 additions & 14 deletions pkg/controllers/localbuild/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"time"

argocdapp "github.com/cnoe-io/argocd-api/api/argo/application"
Expand Down Expand Up @@ -33,7 +34,8 @@ const (
)

var (
defaultRequeueTime = time.Second * 30
defaultRequeueTime = time.Second * 15
errRequeueTime = time.Second * 5
)

type LocalbuildReconciler struct {
Expand Down Expand Up @@ -64,23 +66,62 @@ func (r *LocalbuildReconciler) Reconcile(ctx context.Context, req ctrl.Request)
// Make sure we post process
defer r.postProcessReconcile(ctx, req, &localBuild)

// respecting order of installation matters as there are hard dependencies
subReconcilers := []subReconciler{
r.ReconcileProjectNamespace,
r.ReconcileNginx,
r.ReconcileArgo,
r.ReconcileGitea,
r.ReconcileArgoAppsWithGitea,
_, err := r.ReconcileProjectNamespace(ctx, req, &localBuild)
if err != nil {
return ctrl.Result{}, err
}

for _, sub := range subReconcilers {
result, err := sub(ctx, req, &localBuild)
if err != nil || result.Requeue || result.RequeueAfter != 0 {
return result, err
instCtx, cancel := context.WithCancel(ctx)
defer cancel()
errChan := make(chan error, 3)

go r.installCorePackages(instCtx, req, &localBuild, errChan)

select {
case <-ctx.Done():
return ctrl.Result{}, nil
case instErr := <-errChan:
if instErr != nil {
// likely due to ingress-nginx admission hook not ready. debug log and try again.
logger.V(1).Info("failed installing core package. likely not fatal. will try again", "error", instErr)
return ctrl.Result{RequeueAfter: errRequeueTime}, nil
}
}

return ctrl.Result{}, nil
logger.V(1).Info("done installing core packages. passing control to argocd")
_, err = r.ReconcileArgoAppsWithGitea(ctx, req, &localBuild)
if err != nil {
return ctrl.Result{}, err
}

return ctrl.Result{RequeueAfter: defaultRequeueTime}, nil
}

func (r *LocalbuildReconciler) installCorePackages(ctx context.Context, req ctrl.Request, resource *v1alpha1.Localbuild, errChan chan error) {
logger := log.FromContext(ctx)
defer close(errChan)
var wg sync.WaitGroup

installers := map[string]subReconciler{
"nginx": r.ReconcileNginx,
"argocd": r.ReconcileArgo,
"gitea": r.ReconcileGitea,
}
logger.V(1).Info("installing core packages")
for k, v := range installers {
wg.Add(1)
name := k
inst := v
go func() {
defer wg.Done()
_, iErr := inst(ctx, req, resource)
if iErr != nil {
logger.V(1).Info("failed installing %s: %s", name, iErr)
errChan <- fmt.Errorf("failed installing %s: %w", name, iErr)
}
}()
}
wg.Wait()
}

// Responsible to updating ObservedGeneration in status
Expand Down Expand Up @@ -164,7 +205,7 @@ func (r *LocalbuildReconciler) ReconcileArgoAppsWithGitea(ctx context.Context, r
}
r.shouldShutdown = shutdown

return ctrl.Result{RequeueAfter: defaultRequeueTime}, nil
return ctrl.Result{}, nil
}

func (r *LocalbuildReconciler) reconcileEmbeddedApp(ctx context.Context, appName string, resource *v1alpha1.Localbuild) (ctrl.Result, error) {
Expand Down
40 changes: 37 additions & 3 deletions pkg/controllers/localbuild/gitea.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,11 @@ package localbuild

import (
"context"
"crypto/tls"
"embed"
"fmt"
"net/http"
"time"

"github.com/cnoe-io/idpbuilder/api/v1alpha1"
"github.com/cnoe-io/idpbuilder/pkg/k8s"
Expand All @@ -14,6 +17,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
)

const (
Expand All @@ -34,7 +38,7 @@ func RawGiteaInstallResources(templateData any, config v1alpha1.PackageCustomiza
return k8s.BuildCustomizedManifests(config.FilePath, "resources/gitea/k8s", installGiteaFS, scheme, templateData)
}

func newGiteAdminSecret() (corev1.Secret, error) {
func newGiteaAdminSecret() (corev1.Secret, error) {
pass, err := util.GeneratePassword()
if err != nil {
return corev1.Secret{}, err
Expand All @@ -56,6 +60,7 @@ func newGiteAdminSecret() (corev1.Secret, error) {
}

func (r *LocalbuildReconciler) ReconcileGitea(ctx context.Context, req ctrl.Request, resource *v1alpha1.Localbuild) (ctrl.Result, error) {
logger := log.FromContext(ctx, "installer", "gitea")
gitea := EmbeddedInstallation{
name: "Gitea",
resourcePath: "resources/gitea/k8s",
Expand All @@ -70,7 +75,7 @@ func (r *LocalbuildReconciler) ReconcileGitea(ctx context.Context, req ctrl.Requ
},
}

giteCreds, err := newGiteAdminSecret()
giteCreds, err := newGiteaAdminSecret()
if err != nil {
return ctrl.Result{}, fmt.Errorf("generating gitea admin secret: %w", err)
}
Expand All @@ -80,10 +85,39 @@ func (r *LocalbuildReconciler) ReconcileGitea(ctx context.Context, req ctrl.Requ
if result, err := gitea.Install(ctx, req, resource, r.Client, r.Scheme, r.Config); err != nil {
return result, err
}
resource.Status.Gitea.ExternalURL = fmt.Sprintf(giteaIngressURL, r.Config.Protocol, r.Config.Port)

baseUrl := giteaBaseUrl(r.Config)
// need this to ensure gitrepository controller can reach the api endpoint.
logger.V(1).Info("checking gitea api endpoint", "url", baseUrl)
c := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
},
Timeout: 2 * time.Second,
}
resp, err := c.Get(baseUrl)

if err != nil {
return ctrl.Result{}, err
}
if resp != nil {
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
logger.V(1).Info("gitea manifests installed successfully. endpoint not ready", "statusCode", resp.StatusCode)
return ctrl.Result{RequeueAfter: errRequeueTime}, nil
}
}

resource.Status.Gitea.ExternalURL = baseUrl
resource.Status.Gitea.InternalURL = giteaSvcURL
resource.Status.Gitea.AdminUserSecretName = giteaAdminSecret
resource.Status.Gitea.AdminUserSecretNamespace = giteaNamespace
resource.Status.Gitea.Available = true
return ctrl.Result{}, nil
}

func giteaBaseUrl(config util.CorePackageTemplateConfig) string {
return fmt.Sprintf(giteaIngressURL, config.Protocol, config.Port)
}
24 changes: 3 additions & 21 deletions pkg/controllers/localbuild/installer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/log"
)

Expand Down Expand Up @@ -79,25 +78,10 @@ func (e *EmbeddedInstallation) Install(ctx context.Context, req ctrl.Request, re
}
}

logger.V(1).Info(fmt.Sprintf("Installing/Reconciling %s resources", e.name))
for _, obj := range installObjs {
if gvk, ok := e.monitoredResources[obj.GetName()]; ok {
if obj.GetObjectKind().GroupVersionKind() == gvk {
sch := runtime.NewScheme()
_ = appsv1.AddToScheme(sch)
if gvkObj, err := sch.New(gvk); err == nil {
if gotObj, ok := gvkObj.(client.Object); ok {
if err := cli.Get(ctx, types.NamespacedName{Namespace: e.namespace, Name: obj.GetName()}, gotObj); err != nil {
if err = controllerutil.SetControllerReference(resource, obj, sc); err != nil {
logger.Error(err, "Setting controller reference for deployment", obj.GetName(), obj)
return ctrl.Result{}, err
}
}
}
}
}
}
sch := runtime.NewScheme()
appsv1.AddToScheme(sch)

for _, obj := range installObjs {
// Create object
if err = k8s.EnsureObject(ctx, nsClient, obj, e.namespace); err != nil {
return ctrl.Result{}, err
Expand All @@ -123,8 +107,6 @@ func (e *EmbeddedInstallation) Install(ctx context.Context, req ctrl.Request, re
go func(obj client.Object, gvk schema.GroupVersionKind) {
defer wg.Done()

sch := runtime.NewScheme()
_ = appsv1.AddToScheme(sch)
gvkObj, err := sch.New(gvk)
if err != nil {
errCh <- err
Expand Down

0 comments on commit 577d46a

Please sign in to comment.