From 27948e913ffcdf19c05b0a6075d687b91b8898d8 Mon Sep 17 00:00:00 2001 From: share2kanna Date: Thu, 30 May 2024 22:29:07 +0530 Subject: [PATCH] Incorporated review comments --- capten/agent/gin-api-server/server.go | 9 +- capten/agent/internal/app/app.go | 4 +- .../internal/clusterissuer/ca_cert_issuer.go | 153 ------------------ capten/agent/internal/config/config.go | 2 + capten/common-pkg/cert/generate_certs.go | 2 +- capten/common-pkg/k8s/cert_issuer.go | 33 ++++ charts/kad/templates/agent-certificate.yaml | 18 +++ charts/kad/templates/agent-deployment.yaml | 13 +- charts/kad/values.yaml | 3 + dockerfiles/agent/Dockerfile | 7 - 10 files changed, 71 insertions(+), 173 deletions(-) delete mode 100644 capten/agent/internal/clusterissuer/ca_cert_issuer.go create mode 100644 charts/kad/templates/agent-certificate.yaml diff --git a/capten/agent/gin-api-server/server.go b/capten/agent/gin-api-server/server.go index c5f04400..46002d85 100644 --- a/capten/agent/gin-api-server/server.go +++ b/capten/agent/gin-api-server/server.go @@ -6,19 +6,12 @@ import ( "github.com/gin-gonic/gin" "github.com/intelops/go-common/logging" "github.com/kube-tarian/kad/capten/agent/gin-api-server/api" - "github.com/kube-tarian/kad/capten/agent/internal/clusterissuer" "github.com/kube-tarian/kad/capten/agent/internal/config" ) func StartRestServer(rpcapi api.ServerInterface, cfg *config.SericeConfig, log logging.Logger) error { - err := clusterissuer.GenerateServerCertificates(cfg.ClusterCAIssuerName, log) - if err != nil { - log.Errorf("Failed to generate Server certificate, %v", err) - return err - } - r := gin.Default() api.RegisterHandlers(r, rpcapi) - return r.RunTLS(fmt.Sprintf("%s:%d", cfg.Host, cfg.RestPort), clusterissuer.CertFileName, clusterissuer.KeyFileName) + return r.RunTLS(fmt.Sprintf("%s:%d", cfg.Host, cfg.RestPort), cfg.CertFileName, cfg.KeyFileName) } diff --git a/capten/agent/internal/app/app.go b/capten/agent/internal/app/app.go index c16cc488..c242d98b 100644 --- a/capten/agent/internal/app/app.go +++ b/capten/agent/internal/app/app.go @@ -10,11 +10,11 @@ import ( "github.com/intelops/go-common/logging" ginapiserver "github.com/kube-tarian/kad/capten/agent/gin-api-server" agentapi "github.com/kube-tarian/kad/capten/agent/internal/api" - "github.com/kube-tarian/kad/capten/agent/internal/clusterissuer" "github.com/kube-tarian/kad/capten/agent/internal/config" "github.com/kube-tarian/kad/capten/agent/internal/job" captenstore "github.com/kube-tarian/kad/capten/common-pkg/capten-store" "github.com/kube-tarian/kad/capten/common-pkg/crossplane" + "github.com/kube-tarian/kad/capten/common-pkg/k8s" "github.com/kube-tarian/kad/capten/common-pkg/pb/agentpb" "github.com/kube-tarian/kad/capten/common-pkg/pb/captenpluginspb" "github.com/kube-tarian/kad/capten/common-pkg/pb/clusterpluginspb" @@ -82,7 +82,7 @@ func Start() { } }() - err = clusterissuer.SetupCACertIssuser(cfg.ClusterCAIssuerName, log) + err = k8s.SetupCACertIssuser(cfg.ClusterCAIssuerName, log) if err != nil { log.Fatalf("Failed to setupt CA Cert Issuer in cert-manager %v", err) } diff --git a/capten/agent/internal/clusterissuer/ca_cert_issuer.go b/capten/agent/internal/clusterissuer/ca_cert_issuer.go deleted file mode 100644 index db5bfa03..00000000 --- a/capten/agent/internal/clusterissuer/ca_cert_issuer.go +++ /dev/null @@ -1,153 +0,0 @@ -package clusterissuer - -import ( - "context" - "fmt" - "os" - "time" - - v1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" - cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" - cmclient "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned" - "github.com/intelops/go-common/logging" - "github.com/kube-tarian/kad/capten/common-pkg/cert" - "github.com/kube-tarian/kad/capten/common-pkg/credential" - "github.com/kube-tarian/kad/capten/common-pkg/k8s" - "github.com/pkg/errors" - k8serror "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/rest" -) - -const ( - CertFileName = "server.cert" - KeyFileName = "server.key" - namespace = "capten" - serverCertSecretName = "agent-server-mtls" -) - -func SetupCACertIssuser(clusterIssuerName string, log logging.Logger) error { - k8sclient, err := k8s.NewK8SClient(log) - if err != nil { - log.Errorf("failed to initalize k8s client, %v", err) - return err - } - - err = setupCertificateIssuer(k8sclient, clusterIssuerName, log) - if err != nil { - log.Errorf("Setup Certificates Issuer failed, %v", err) - return err - } - return nil -} - -// Setup agent certificate issuer -func setupCertificateIssuer(k8sClient *k8s.K8SClient, clusterIssuerName string, log logging.Logger) error { - // Create Agent Cluster Issuer - certsData, err := k8s.CreateOrUpdateClusterIssuer(clusterIssuerName, k8sClient, false) - if err != nil { - return fmt.Errorf("failed to create/update CA Issuer %s in cert-manager: %v", clusterIssuerName, err) - } - - // Update Vault - err = credential.PutClusterCerts(context.TODO(), "kad-agent", "kad-agent", string(certsData.CaChainCertData), string(certsData.RootKey.KeyData), string(certsData.RootCert.CertData)) - if err != nil { - log.Errorf("Failed to write to vault, %v", err) - log.Infof("Continued to start the agent as these certs from vault are not used...") - } - return nil -} - -func GenerateServerCertificates(clusterIssuerName string, log logging.Logger) error { - k8sClient, err := k8s.NewK8SClient(log) - if err != nil { - log.Errorf("failed to initalize k8s client, %v", err) - return err - } - - config, err := rest.InClusterConfig() - if err != nil { - return errors.WithMessage(err, "error while building kubeconfig") - } - - cmClient, err := cmclient.NewForConfig(config) - if err != nil { - return err - } - - err = k8sClient.CreateNamespace(context.Background(), namespace) - if err != nil { - return fmt.Errorf("failed to create namespace %v, reason: %v", namespace, err) - } - log.Infof("Created namesapce: %v", namespace) - - err = generateCertManagerServerCertificate(cmClient, namespace, serverCertSecretName, clusterIssuerName, log) - if err != nil { - return fmt.Errorf("failed to genereate server certificate: %v", err) - } - - // TODO: it may take some time for certificate to get create - // So have to keep wait and retry - time.Sleep(10 * time.Second) - secretData, err := k8sClient.GetSecretData(namespace, serverCertSecretName) - if err != nil { - return fmt.Errorf("failed to fetch certificates from secret, %v", err) - } - - // Write certificates to files - err = os.WriteFile(CertFileName, []byte(secretData.Data["tls.crt"]), cert.FilePermission) - if err != nil { - return fmt.Errorf("failed to create cert file, %v", err) - } - err = os.WriteFile(KeyFileName, []byte(secretData.Data["tls.key"]), cert.FilePermission) - if err != nil { - return fmt.Errorf("failed to create key file, %v", err) - } - - return nil -} - -func generateCertManagerServerCertificate( - cmClient *cmclient.Clientset, - namespace string, - certName string, - issuerRefName string, - log logging.Logger, -) error { - usages := []v1.KeyUsage{v1.UsageDigitalSignature, v1.UsageKeyEncipherment, v1.UsageServerAuth} - - _, err := cmClient.CertmanagerV1().Certificates(namespace).Create( - context.TODO(), - &v1.Certificate{ - ObjectMeta: metav1.ObjectMeta{ - Name: certName, - }, - Spec: v1.CertificateSpec{ - IssuerRef: cmmeta.ObjectReference{ - Name: issuerRefName, // "capten-ca-issuer" - Kind: v1.ClusterIssuerKind, - }, - SecretName: certName, - CommonName: certName, - Usages: usages, - PrivateKey: &v1.CertificatePrivateKey{ - Algorithm: v1.RSAKeyAlgorithm, - Size: 2048, - Encoding: v1.PKCS1, - }, - }, - }, - metav1.CreateOptions{}, - ) - if k8serror.IsAlreadyExists(err) { - log.Infof("%v Certificate already exists", certName) - return nil - } - if err != nil { - log.Infof("%v Certificate generation failed, reason: %v", certName, err) - } else { - log.Infof("%v Certificate generation successful", certName) - } - - return err -} diff --git a/capten/agent/internal/config/config.go b/capten/agent/internal/config/config.go index 9aa9e899..682be652 100644 --- a/capten/agent/internal/config/config.go +++ b/capten/agent/internal/config/config.go @@ -16,6 +16,8 @@ type SericeConfig struct { TektonSyncJobInterval string `envconfig:"TEKTON_SYNC_JOB_INTERVAL" default:"@every 1h"` DomainName string `envconfig:"DOMAIN_NAME" default:"example.com"` ClusterCAIssuerName string `envconfig:"AGENT_CLUSTER_CA_ISSUER_NAME" default:"agent-ca-issuer"` + CertFileName string `envconfig:"CERT_FILE_NAME" default:"/tmp/certs/tls.crt"` + KeyFileName string `envconfig:"KEY_FILE_NAME" default:"/tmp/certs/tls.key"` } func GetServiceConfig() (*SericeConfig, error) { diff --git a/capten/common-pkg/cert/generate_certs.go b/capten/common-pkg/cert/generate_certs.go index 59b10893..35eaa1c3 100644 --- a/capten/common-pkg/cert/generate_certs.go +++ b/capten/common-pkg/cert/generate_certs.go @@ -51,7 +51,7 @@ func GenerateRootCerts() (*CertificatesData, error) { }, nil } -func generateCACert() (*Key, *Cert, error) { //(rootKey *rsa.PrivateKey, rootCertTemplate *x509.Certificate, err error) { +func generateCACert() (*Key, *Cert, error) { rootKey, err := rsa.GenerateKey(rand.Reader, caBitSize) if err != nil { err = errors.WithMessage(err, "failed to generate RSA key for root certificate") diff --git a/capten/common-pkg/k8s/cert_issuer.go b/capten/common-pkg/k8s/cert_issuer.go index 5e5a6a6a..9d7d1a62 100644 --- a/capten/common-pkg/k8s/cert_issuer.go +++ b/capten/common-pkg/k8s/cert_issuer.go @@ -6,6 +6,7 @@ import ( "github.com/intelops/go-common/logging" "github.com/kube-tarian/kad/capten/common-pkg/cert" + "github.com/kube-tarian/kad/capten/common-pkg/credential" "github.com/pkg/errors" certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" @@ -18,6 +19,38 @@ import ( var log = logging.NewLogger() +func SetupCACertIssuser(clusterIssuerName string, log logging.Logger) error { + k8sclient, err := NewK8SClient(log) + if err != nil { + log.Errorf("failed to initalize k8s client, %v", err) + return err + } + + err = setupCertificateIssuer(k8sclient, clusterIssuerName, log) + if err != nil { + log.Errorf("Setup Certificates Issuer failed, %v", err) + return err + } + return nil +} + +// Setup agent certificate issuer +func setupCertificateIssuer(k8sClient *K8SClient, clusterIssuerName string, log logging.Logger) error { + // Create Agent Cluster Issuer + certsData, err := CreateOrUpdateClusterIssuer(clusterIssuerName, k8sClient, false) + if err != nil { + return fmt.Errorf("failed to create/update CA Issuer %s in cert-manager: %v", clusterIssuerName, err) + } + + // Update Vault + err = credential.PutClusterCerts(context.TODO(), "kad-agent", "kad-agent", string(certsData.CaChainCertData), string(certsData.RootKey.KeyData), string(certsData.RootCert.CertData)) + if err != nil { + log.Errorf("Failed to write to vault, %v", err) + log.Infof("Continued to start the agent as these certs from vault are not used...") + } + return nil +} + func CreateOrUpdateClusterIssuer(clusterCAIssuer string, k8sclient *K8SClient, forceUpdate bool) (*cert.CertificatesData, error) { config, err := rest.InClusterConfig() if err != nil { diff --git a/charts/kad/templates/agent-certificate.yaml b/charts/kad/templates/agent-certificate.yaml new file mode 100644 index 00000000..a973aada --- /dev/null +++ b/charts/kad/templates/agent-certificate.yaml @@ -0,0 +1,18 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "kad.fullname" . }}-agent-server-mtls +spec: + commonName: {{ include "kad.fullname" . }}-agent-server-mtls + issuerRef: + kind: ClusterIssuer + name: {{ .Values.agent.clusterIssuerReference }} + privateKey: + algorithm: RSA + encoding: PKCS1 + size: 2048 + secretName: {{ include "kad.fullname" . }}-agent-server-mtls + usages: + - digital signature + - key encipherment + - server auth diff --git a/charts/kad/templates/agent-deployment.yaml b/charts/kad/templates/agent-deployment.yaml index 2ccfd70a..8319e333 100644 --- a/charts/kad/templates/agent-deployment.yaml +++ b/charts/kad/templates/agent-deployment.yaml @@ -96,8 +96,17 @@ spec: resources: {{- toYaml .Values.resources | nindent 12 }} volumeMounts: - - name: plugin-store-clone-dir - mountPath: {{ .Values.env.pluginsStoreProjectMount }} + - name: plugin-store-clone-dir + mountPath: {{ .Values.env.pluginsStoreProjectMount }} + - name: server-certificate + mountPath: "/tmp/certs" + readOnly: true + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumes: + - name: server-certificate + secret: + secretName: {{ include "kad.fullname" . }}-agent-server-mtls {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/kad/values.yaml b/charts/kad/values.yaml index f0b56c6d..28ff4083 100644 --- a/charts/kad/values.yaml +++ b/charts/kad/values.yaml @@ -134,3 +134,6 @@ temporal: argocd: serviceURL: argocd-server.default.svc.cluster.local + +agent: + clusterIssuerReference: "capten-issuer" \ No newline at end of file diff --git a/dockerfiles/agent/Dockerfile b/dockerfiles/agent/Dockerfile index f08cb765..db173b94 100644 --- a/dockerfiles/agent/Dockerfile +++ b/dockerfiles/agent/Dockerfile @@ -6,15 +6,8 @@ RUN rm -rf vendor RUN go mod download RUN CGO_ENABLED=0 go build -o ./build/agent agent/cmd/main.go -RUN touch ./server.cert -RUN touch ./server.key -RUN chown 65532 ./server.cert -RUN chown 65532 ./server.key - FROM scratch COPY --from=builder ./build/agent agent -COPY --from=builder ./server.cert server.cert -COPY --from=builder ./server.key server.key COPY ./capten/database/postgres/ postgres/ USER 65532:65532