diff --git a/.github/workflows/dso-automatic-sync.yml b/.github/workflows/dso-automatic-sync.yml new file mode 100644 index 00000000..11a8e109 --- /dev/null +++ b/.github/workflows/dso-automatic-sync.yml @@ -0,0 +1,20 @@ +name: Sync DSO gitlab repository + +on: + workflow_dispatch: + push: + branches: + - feature/adapt-infra-for-dso-env + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - run: | + curl "https://gitlab.apps.c6.numerique-interieur.com/api/v4/projects/62/trigger/pipeline" \ + -X POST \ + --fail \ + -F token=${{ secrets.DSO_TOKEN }} \ + -F ref=main \ + -F variables[GIT_BRANCH_DEPLOY]=${{ github.ref_name }} \ + -F variables[PROJECT_NAME]=infra diff --git a/.gitignore b/.gitignore index d91a1d2d..79b63117 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,7 @@ logs/ temp/ *openrc.sh .terraform* -.secrets +*.secrets* version.txt *.crt *.pem \ No newline at end of file diff --git a/.gitlab-ci-dso.yml b/.gitlab-ci-dso.yml new file mode 100644 index 00000000..bea57bb5 --- /dev/null +++ b/.gitlab-ci-dso.yml @@ -0,0 +1,43 @@ +include: + - project: $CATALOG_PATH + file: vault-ci.yml + ref: main + - project: $CATALOG_PATH + file: kaniko-ci.yml + ref: main + +default: + image: alpine:latest + +variables: + TAG: "${CI_COMMIT_REF_SLUG}" + REGISTRY_URL: "${REGISTRY_HOST}/${PROJECT_PATH}" + +stages: + - read-secret + - docker-build + +read_secret: + stage: read-secret + extends: + - .vault:read_secret + +docker-build-frontend: + variables: + WORKING_DIR: "frontend" + IMAGE_NAMES: basegun-frontend:${CI_COMMIT_REF_SLUG} basegun-frontend:${CI_COMMIT_SHORT_SHA} + DOCKERFILE: "frontend/Dockerfile" + BUILD_ARGS: --build-arg=VERSION=1.1.1 + stage: docker-build + extends: + - .kaniko:build-push + +docker-build-backend: + variables: + WORKING_DIR: "backend" + IMAGE_NAMES: basegun-backend:${CI_COMMIT_REF_SLUG} basegun-backend:${CI_COMMIT_SHORT_SHA} + DOCKERFILE: "backend/Dockerfile" + BUILD_ARGS: --build-arg=VERSION=1.1.1 + stage: docker-build + extends: + - .kaniko:build-push diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index e215d3a7..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,82 +0,0 @@ -include: - - local: '/templates/docker.yml' - - local: '/templates/vault.yml' - -default: - image: python:3.9-slim-buster - - -variables: - http_proxy: $http_proxy - https_proxy: $http_proxy - no_proxy: $no_proxy - HTTP_PROXY: $http_proxy - HTTPS_PROXY: $http_proxy - NO_PROXY: $no_proxy - PROJECT_NAME: "basegun" - PROJECT_REPOSITORY: "basegun" - PROJECT_ORGANISATION: "ministere-interieur" - BUILD_CONFIG_FILE: $BUILD_CONFIG - REGISTRY_URL: "${QUAY_ROOT_URL}/${PROJECT_ORGANISATION}-${PROJECT_NAME}" - TAG: "1.5" - #TAG: "${CI_COMMIT_REF_SLUG}" - DOCKERFILE: 'Dockerfile' - -# GIT_CURL_VERBOSE: "1" -# GIT_DEBUG_LOOKUP: "1" -# GIT_TRANSLOOP_DEBUG: "1" -# GIT_TRANSPORT_HELPER_DEBUG: "1" - -stages: - - read-secret - - test-app - - build-docker - -read_secret: - stage: read-secret - extends: - - .vault:read_secret - -test: - image: python:3.9-slim-buster - stage: test-app - variables: - OS_USERNAME: data - OS_PASSWORD: data - OS_PROJECT_NAME: data - script: - - pip install --upgrade pip && pip install --no-cache-dir -f https://download.pytorch.org/whl/cpu/torch_stable.html -r backend/requirements.txt - - python -m unittest discover -v -s ./backend - allow_failure: true - -build_docker_front: - variables: - WORKING_DIR: 'frontend' - IMAGE_NAME: 'frontend' - DOCKERFILE: 'Dockerfile-dso' - stage: build-docker - extends: - - .kaniko:build - -build_docker_back: - variables: - WORKING_DIR: 'backend' - IMAGE_NAME: 'backend' - DOCKERFILE: 'Dockerfile-dso' - stage: build-docker - extends: - - .kaniko:build - -build_docker_logs: - variables: - WORKING_DIR: 'logs' - IMAGE_NAME: 'logs' - DOCKERFILE: 'Dockerfile-dso' - NO_PROXY: "*,gitlab-op.apps.ocp4-8.infocepo.com,dindservice,quay.apps.ocp4-8.infocepo.com" - no_proxy: "*,gitlab-op.apps.ocp4-8.infocepo.com,dindservice,quay.apps.ocp4-8.infocepo.com" - stage: build-docker - extends: - - .docker:build - tags: - - docker - - vms diff --git a/infra/kube/helm/templates/configmap.yaml b/infra/kube/helm/templates/configmap.yaml index 8daa0ba6..9c6e72bd 100644 --- a/infra/kube/helm/templates/configmap.yaml +++ b/infra/kube/helm/templates/configmap.yaml @@ -4,7 +4,6 @@ metadata: name: {{ include "basegun.name" . }}-config data: WORKSPACE: {{ .Values.backend.config.workspace }} - PATH_IMGS: {{ .Values.backend.config.path_imgs }} PATH_LOGS: /tmp/log/basegun/ --- apiVersion: v1 diff --git a/infra/kube/helm/templates/deployment-backend.yaml b/infra/kube/helm/templates/deployment-backend.yaml index 7b213736..9d0b4bf9 100644 --- a/infra/kube/helm/templates/deployment-backend.yaml +++ b/infra/kube/helm/templates/deployment-backend.yaml @@ -33,6 +33,7 @@ spec: {{- toYaml .Values.backend.securityContext | nindent 12 }} image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.backend.image.pullPolicy }} + resources: {{- toYaml .Values.backend.resources | nindent 12 }} envFrom: - configMapRef: name: {{ include "basegun.name" . }}-config @@ -60,17 +61,17 @@ spec: image: "{{ .Values.backend.logs.repository }}:{{ .Values.backend.logs.tag }}" command: ["/bin/bash"] args: ["-c", "filebeat -e -c /tmp/filebeat/filebeat.yml --strict.perms=false"] - resources: {} + resources: {{- toYaml .Values.backend.logs.resources | nindent 12 }} env: - name: X_OVH_TOKEN valueFrom: secretKeyRef: - name: {{ include "basegun.name" . }}-secret + name: {{ .Values.backend.secret.secretName }} key: X_OVH_TOKEN - name: API_OVH_TOKEN valueFrom: secretKeyRef: - name: {{ include "basegun.name" . }}-secret + name: {{ .Values.backend.secret.secretName }} key: API_OVH_TOKEN volumeMounts: - name: {{ include "basegun.name" . }}-filebeat-conf diff --git a/infra/kube/helm/templates/deployment-frontend.yaml b/infra/kube/helm/templates/deployment-frontend.yaml index d6f0ce31..4c94652a 100644 --- a/infra/kube/helm/templates/deployment-frontend.yaml +++ b/infra/kube/helm/templates/deployment-frontend.yaml @@ -31,6 +31,7 @@ spec: {{- toYaml .Values.frontend.securityContext | nindent 12 }} image: "{{ .Values.frontend.image.repository }}:{{ .Values.frontend.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.frontend.image.pullPolicy }} + resources: {{- toYaml .Values.frontend.resources | nindent 12 }} ports: - name: http containerPort: {{ .Values.frontend.service.containerPort }} diff --git a/infra/kube/helm/templates/sops.enc.yaml b/infra/kube/helm/templates/sops.enc.yaml new file mode 100644 index 00000000..29b2ea56 --- /dev/null +++ b/infra/kube/helm/templates/sops.enc.yaml @@ -0,0 +1,42 @@ +{{- if .Values.backend.secret.sops -}} +apiVersion: isindir.github.com/v1alpha3 +kind: SopsSecret +metadata: + name: sops-secret +spec: + secretTemplates: + - name: ENC[AES256_GCM,data:C7eCSGXEdWtuVa+WYplXamERpCuF2A==,iv:oDDv384GFK0ynFybO0GJXKPjXPUe/++jxUh6oPgSROI=,tag:7auyaf2Pk1gUNCKI+lQjxA==,type:str] + stringData: + API_OVH_TOKEN: ENC[AES256_GCM,data:9fDrMsKCWW4qU5EFsaWhQdA6TIWNueA5sSknmUydichzF1zczSj3nrPtfF7O+dwuWqXUxg==,iv:E8vw8EdDzAigbonjNa57RfTfVpGG9K/Xil+yIAAxPSE=,tag:4qqNEc5RJb1w/WL2dIvt+w==,type:str] + OS_PASSWORD: ENC[AES256_GCM,data:vGHEXzNVjviNsyOam48tvdLbvM+XGBwo204jiH6AruY=,iv:4QXRGhyRjQYyovR68tJzbhzzBiOPHsyNBvruCtk8pl0=,tag:snlHGsj+i5nEIV4aeFz2nQ==,type:str] + OS_PROJECT_NAME: ENC[AES256_GCM,data:Oe4oIqDnNMxjBA1xAHDuSQ==,iv:3pfX8fZ/3hy5LAP0Z0C+joleY33WnXAHUKa377rObto=,tag:CGqewlAyweMW5BSB80qVrg==,type:str] + OS_USERNAME: ENC[AES256_GCM,data:T1BTSS3/nRKMTS7Nk5ZCYi4=,iv:BtWpyd/zxiQPogucbpSzrR6Nn6oIHdbCCpkNhXYzxo8=,tag:mnd+6VxolKMO0vUR/acy4Q==,type:str] + X_OVH_TOKEN: ENC[AES256_GCM,data:Qz1uggOKElNvNBS9qxDfybUMBYEIOfuppySaoXEBx00jWv0u,iv:+cklaR+WWjjJLnD1gmZ38atrqCPNrje0BWofWJstIWA=,tag:czA3G3fU4VC7njajF7xaRw==,type:str] + - name: ENC[AES256_GCM,data:N8b/GxqS/MdpK/ZH1cFzYyppfw==,iv:HIKkI1y6FIVP323NhZMjrf1Ulp7N29jQ0zlMIv3Y7gg=,tag:ZVG+z7ncvoNsi47lofXjZw==,type:str] + stringData: + API_OVH_TOKEN: ENC[AES256_GCM,data:T9TY8BUSKH2fJfhcSX71mD+kpB7Ac9WVNyYOIV1FQpumc5XNsVFad015f3MizRn+rJiHkQ==,iv:bZ74ywut3HGCMbb+9US8n9VWQt5YJmPY1hN1+PefoJY=,tag:0cRa6vMOyWOx5Dd1sqigtg==,type:str] + OS_PASSWORD: ENC[AES256_GCM,data:uT2J6nJyIZEpXwN9L4lvpoMDv/hZXkIbfyZQK5qVRaM=,iv:ZwgDZOaS7Pt4+/1XBZ4sOshuyuSMIkvSPeadZMk2OSQ=,tag:n3djDDeafNArb+p+nF1pGg==,type:str] + OS_PROJECT_NAME: ENC[AES256_GCM,data:TXud2R//KeDgYY1NUH8NnQ==,iv:MydfYwEV58wNKpSn9Mj7tP40RDdOhini4zbByNdvf00=,tag:wI0c8ZCjTQiqmvDRj3p9/A==,type:str] + OS_USERNAME: ENC[AES256_GCM,data:Qule8RjaVy5+zfAtdhxYEQM=,iv:8qPLyyjn1Vr+TgM5Vp9lXLsI8MGExXXecScWaRXeSE0=,tag:j2dcIrxE8fecfr31Rtq3SQ==,type:str] + X_OVH_TOKEN: ENC[AES256_GCM,data:5Zsze+3JSqxle08ePuvHyHDfTelvnrQ2/INbbbwcvOHvPu/9,iv:xlcyVOkwGl0QGAFlWUT+/2LR4lGLcAGDzswkuq6cDUU=,tag:UviczjiBLQ61jEbpbE9YXA==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1g867s7tcftkgkdraz3ezs8xk5c39x6l4thhekhp9s63qxz0m7cgs5kan9a + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoRG5XVTJTR3NPOVp6dWtX + d2NrNXZybGc2akUzQ1NpbEh1OWlWK2VZd3ljCko0SStxRWZpdVNjNXFmdDdDRlRX + N0ZBckFUNlRjRTNqdU1sVGgza2J5WTgKLS0tIGdycXh2QWZIWFJXVjZBY29xM2xk + eU4vU29uaFdjdk5xQklJanBuUWFkbFUKaZYD36McjUvedtf6vsjDJPlseiYmcPhu + 4sQPd4kORdtquDoDFD76y/aY2Rna2XlVd8jMUDyFYssudKjik3y4AQ== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2023-09-22T15:49:44Z" + mac: ENC[AES256_GCM,data:m7h+73fmAbnb8R2xyytB7kA1gdVmoxOg2rTPSDPbsX0lL5dLay4Jljbz7VvrAnq0DoxJj0AOX/XOopkTnBDGaVUxiPTrzwrZUQQCO/IEB4Tor46EKSKDiglNqPziFuvwBW0Y7UFkBXj2dv8E9YJRk8fFJLHRhgsXiYEeKR+AzNQ=,iv:kx1yeNgJyycWVl+pb1bU+P9dNc1mxs3mRq/E6f2BWUw=,tag:VyD4mDjY24AKEbHZNHoE8w==,type:str] + pgp: [] + encrypted_suffix: Templates + version: 3.7.1 +{{- end }} \ No newline at end of file diff --git a/infra/kube/helm/values-dso.yaml b/infra/kube/helm/values-dso.yaml new file mode 100644 index 00000000..2db767a5 --- /dev/null +++ b/infra/kube/helm/values-dso.yaml @@ -0,0 +1,122 @@ +# Default values for basegun. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +nameOverride: "" +fullnameOverride: "" +ingress: + enabled: true + className: "" + #annotations: + #traefik.ingress.kubernetes.io/router.tls: "true" + #traefik.ingress.kubernetes.io/router.tls.certresolver: letsencrypt + hosts: + - host: basegun.dev.numerique-interieur.com + paths: + - path: / + pathType: Prefix + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" +# Config for backend +backend: + replicaCount: 1 + image: + repository: harbor.apps.c6.numerique-interieur.com/mi-basegun-test/basegun-backend + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: feature-adapt-infra-for-dso-env + imagePullSecrets: [] + podAnnotations: {} + podSecurityContext: {} + # fsGroup: 2000 + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + logs: + repository: ghcr.io/datalab-mi/basegun/filebeat-rootless + tag: 6.5.4 + pullPolicy: IfNotPresent + service: + type: ClusterIP + port: 5000 + containerPort: 5000 + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 10 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + config: + # Used for showing current workspace in app + workspace: preprod + secret: + create: false + sops: true + secretName: basegun-preprod-secret + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + nodeSelector: {} + tolerations: [] + affinity: {} +# Config for frontend +frontend: + replicaCount: 1 + image: + repository: harbor.apps.c6.numerique-interieur.com/mi-basegun-test/basegun-frontend + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: feature-adapt-infra-for-dso-env + imagePullSecrets: [] + podAnnotations: {} + podSecurityContext: {} + # fsGroup: 2000 + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 8080 + containerPort: 8080 + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + nodeSelector: {} + tolerations: [] + affinity: {} diff --git a/infra/kube/helm/values.yaml b/infra/kube/helm/values.yaml index 9e2eeb46..c7d5a92d 100644 --- a/infra/kube/helm/values.yaml +++ b/infra/kube/helm/values.yaml @@ -50,6 +50,13 @@ backend: repository: elastic/filebeat tag: 6.5.4 pullPolicy: IfNotPresent + resources: + limits: + cpu: 250m + memory: 512Mi + requests: + cpu: 100m + memory: 256Mi service: type: ClusterIP port: 5000 @@ -61,31 +68,23 @@ backend: targetCPUUtilizationPercentage: 80 # targetMemoryUtilizationPercentage: 80 config: - # Where images are stored locally - path_imgs: /app/images # Used for showing current workspace in app workspace: preprod secret: create: false - # If create is true, you can provide values else it use existing secret - #values: - # OS_PASSWORD: "" - # OS_PROJECT_NAME: "" - # OS_USERNAME: "" - # X_OVH_TOKEN: "test" - # API_OVH_TOKEN: "test" - secretName: "basegun-secret" - resources: {} + sops: false + secretName: basegun-secret + resources: # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi + limits: + cpu: 250m + memory: 1Gi + requests: + cpu: 100m + memory: 512Mi nodeSelector: {} tolerations: [] affinity: {} @@ -114,17 +113,17 @@ frontend: type: ClusterIP port: 8080 containerPort: 8080 - resources: {} + resources: # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 50m + memory: 64Mi nodeSelector: {} tolerations: [] affinity: {}