From 349f3f92f636fda97f6a6cb0d2f5834002c9c13f Mon Sep 17 00:00:00 2001 From: janosch Date: Wed, 24 Apr 2024 20:32:36 +0000 Subject: [PATCH] Update values.yaml and adding more importers --- charts/hashr/Chart.lock | 6 ++ charts/hashr/README.md | 79 +++---------------- charts/hashr/templates/hashr-deb-cronjob.yaml | 3 +- charts/hashr/templates/hashr-gcp-cronjob.yaml | 62 +++++++++++++++ .../templates/hashr-iso9660-cronjob.yaml | 54 +++++++++++++ charts/hashr/templates/hashr-rpm-cronjob.yaml | 54 +++++++++++++ .../hashr/templates/hashr-targz-cronjob.yaml | 54 +++++++++++++ charts/hashr/templates/hashr-zip-cronjob.yaml | 54 +++++++++++++ charts/hashr/values.yaml | 35 +++++++- 9 files changed, 329 insertions(+), 72 deletions(-) create mode 100644 charts/hashr/Chart.lock create mode 100644 charts/hashr/templates/hashr-gcp-cronjob.yaml create mode 100644 charts/hashr/templates/hashr-iso9660-cronjob.yaml create mode 100644 charts/hashr/templates/hashr-rpm-cronjob.yaml create mode 100644 charts/hashr/templates/hashr-targz-cronjob.yaml create mode 100644 charts/hashr/templates/hashr-zip-cronjob.yaml diff --git a/charts/hashr/Chart.lock b/charts/hashr/Chart.lock new file mode 100644 index 00000000..2b96170e --- /dev/null +++ b/charts/hashr/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 14.3.3 +digest: sha256:803fc388f1186ca5e0bf7af8597a7f94714bfe2fb4536d3ab2136e0b5ce1e59c +generated: "2024-04-24T19:58:19.38937249Z" diff --git a/charts/hashr/README.md b/charts/hashr/README.md index 1c81bdae..cfe8ce9f 100644 --- a/charts/hashr/README.md +++ b/charts/hashr/README.md @@ -62,7 +62,17 @@ for a list of values that will be used for production. helm pull osdfir-charts/hashr --untar ``` -Enable the HashR importers you want to use and define a schedule. +### Configure the HashR importers + +HashR provides different importers. Each importer has its own CronJob and can be +configured separately. Enable and configure all importers you want to use in the +`hashr.importers` section of the `values.yaml` file. + +Ensure that you have setup all requirements for the importers defined in the +HashR project. See [HashR importers](https://github.com/google/hashr?tab=readme-ov-file#setting-up-importers) +for more details. + +### Install chart Install the chart with the values in `values.yaml`, then using a release name such as `my-release`, run: @@ -113,73 +123,6 @@ Please be cautious before doing it. ## Parameters -### Global parameters - -| Name | Description | Value | -| ------------------------------- | -------------------------------------------------------------------------------------------- | ------- | -| `global.timesketch.enabled` | Enables the Timesketch deployment (only used in the main OSDFIR Infrastructure Helm chart) | `false` | -| `global.timesketch.servicePort` | Timesketch service port (overrides `timesketch.service.port`) | `nil` | -| `global.turbinia.enabled` | Enables the Turbinia deployment (only used within the main OSDFIR Infrastructure Helm chart) | `false` | -| `global.turbinia.servicePort` | Turbinia API service port (overrides `turbinia.service.port`) | `nil` | -| `global.yeti.enabled` | Enables the Yeti deployment (only used in the main OSDFIR Infrastructure Helm chart) | `false` | -| `global.yeti.servicePort` | Yeti API service port (overrides `yeti.api.service.port`) | `nil` | -| `global.existingPVC` | Existing claim for HashR persistent volume (overrides `persistent.name`) | `""` | -| `global.storageClass` | StorageClass for the HashR persistent volume (overrides `persistent.storageClass`) | `""` | - -### HashR image configuration - -| Name | Description | Value | -| ------------------------ | ------------------------------------------------------------- | ------------------------------------------------------- | -| `image.repository` | HashR image repository | `us-docker.pkg.dev/osdfir-registry/hashr/release/hashr` | -| `image.pullPolicy` | HashR image pull policy | `IfNotPresent` | -| `image.tag` | Overrides the image tag whose default is the chart appVersion | `latest` | -| `image.imagePullSecrets` | Specify secrets if pulling from a private repository | `[]` | - -### HashR Configuration Paramters - - -### Enable/Disable HashR importers - -| Name | Description | Value | -| ---------------------------------- | ---------------------------------- | ------------------- | -| `hashr.importers.gcp.enabled` | Enables the GCP importer | `false` | -| `hashr.importers.gcp.schedule` | sets the CronJob schedule times | `0 3 * * 1` | -| `hashr.importers.targz.enabled` | Enables the tar.gz importer | `false` | -| `hashr.importers.targz.schedule` | sets the CronJob schedule times | `0 3 * * 2` | -| `hashr.importers.windows.enabled` | Enables the Windows importer | `false` | -| `hashr.importers.windows.schedule` | sets the CronJob schedule times | `0 3 * * 3` | -| `hashr.importers.wsus.enabled` | Enables the WSUS importer | `false` | -| `hashr.importers.wsus.schedule` | sets the CronJob schedule times | `0 3 * * 4` | -| `hashr.importers.rpm.enabled` | Enables the RPM importer | `false` | -| `hashr.importers.rpm.schedule` | sets the CronJob schedule times | `0 3 * * 5` | -| `hashr.importers.zip.enabled` | Enables the ZIP importer | `false` | -| `hashr.importers.zip.schedule` | sets the CronJob schedule times | `0 3 * * 6` | -| `hashr.importers.gcr.enabled` | Enables the GCR importer | `false` | -| `hashr.importers.gcr.schedule` | sets the CronJob schedule times | `0 3 * * 7` | -| `hashr.importers.iso9660.enabled` | Enables the iso9660 importer | `false` | -| `hashr.importers.iso9660.schedule` | sets the CronJob schedule times | `0 15 * * 1` | -| `hashr.importers.deb.enabled` | Enables the DEB importer | `false` | -| `hashr.importers.deb.schedule` | sets the CronJob schedule times | `0 15 * * 2` | -| `persistence.name` | HashR persistent volume name | `hashrvolume` | -| `persistence.size` | HashR persistent volume size | `50Gi` | -| `persistence.storageClass` | PVC Storage Class for HashR volume | `""` | -| `persistence.accessModes` | PVC Access Mode for HashR volume | `["ReadWriteOnce"]` | - -### Postgresql Configuration Parameters - -| Name | Description | Value | -| ---------------------------------------------- | --------------------------------------------------------------------------- | ------------ | -| `postgresql.enabled` | Enables the Postgresql deployment | `true` | -| `postgresql.architecture` | PostgreSQL architecture (`standalone` or `replication`) | `standalone` | -| `postgresql.auth.username` | Name for a custom PostgreSQL user to create | `postgres` | -| `postgresql.auth.database` | Name for a custom PostgreSQL database to create (overrides `auth.database`) | `hashr` | -| `postgresql.primary.service.type` | PostgreSQL primary service type | `ClusterIP` | -| `postgresql.primary.service.ports.postgresql` | PostgreSQL primary service port | `5432` | -| `postgresql.primary.persistence.size` | PostgreSQL Persistent Volume size | `10Gi` | -| `postgresql.primary.resources.limits` | The resources limits for the PostgreSQL primary containers | `{}` | -| `postgresql.primary.resources.requests.cpu` | The requested cpu for the PostgreSQL primary containers | `250m` | -| `postgresql.primary.resources.requests.memory` | The requested memory for the PostgreSQL primary containers | `256Mi` | - ## Persistence diff --git a/charts/hashr/templates/hashr-deb-cronjob.yaml b/charts/hashr/templates/hashr-deb-cronjob.yaml index 8a038ca7..cf03e1d2 100644 --- a/charts/hashr/templates/hashr-deb-cronjob.yaml +++ b/charts/hashr/templates/hashr-deb-cronjob.yaml @@ -6,7 +6,7 @@ metadata: spec: schedule: {{ .Values.hashr.importers.deb.schedule | quote }} concurrencyPolicy: Forbid - successfulJobsHistoryLimit: 3 + successfulJobsHistoryLimit: 2 failedJobsHistoryLimit: 1 jobTemplate: spec: @@ -17,6 +17,7 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} args: + - --logtostderr=1 - -storage - postgres - -exporters diff --git a/charts/hashr/templates/hashr-gcp-cronjob.yaml b/charts/hashr/templates/hashr-gcp-cronjob.yaml new file mode 100644 index 00000000..ed80a852 --- /dev/null +++ b/charts/hashr/templates/hashr-gcp-cronjob.yaml @@ -0,0 +1,62 @@ +{{- if .Values.hashr.importers.gcp.enabled -}} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ .Release.Name }}-hashr-gcp +spec: + schedule: {{ .Values.hashr.importers.gcp.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + containers: + - name: hashr-gcp + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --logtostderr=1 + - -storage + - postgres + - -exporters + - postgres + - -postgres_host + - {{ include "common.names.fullname" (dict "Chart" (dict "Name" "postgresql") "Release" .Release "Values" .Values.postgresql) }} + - -postgres_port + - {{ .Values.postgresql.primary.service.ports.postgresql | quote }} + - -postgres_user + - {{ .Values.postgresql.auth.username | quote }} + - -postgres_password + - "$(POSTGRES_PASSWORD)" + - -postgres_db + - {{ .Values.postgresql.auth.database | quote }} + - -importers + - GCP + - -gcp_projects + - {{ .Values.hashr.importers.gcp.gcp_projects | quote }} + - -hashr_gcp_project + - {{ .Values.hashr.importers.gcp.hashr_gcp_project | quote }} + - -hashr_gcs_bucket + - {{ .Values.hashr.importers.gcp.hashr_gcs_bucket | quote }} + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + # Store your SA key in the hashrvolume/creds/ folder via "kubectl cp"! + # chown 999:1000 hashr-sa-private-key.json to prevent permission issues + value: {{ (include "hashr.dataPath" .) }}/creds/hashr-sa-private-key.json + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "postgresql.v1.secretName" .Subcharts.postgresql }} + key: {{ include "postgresql.v1.adminPasswordKey" .Subcharts.postgresql }} + volumeMounts: + - name: hashrvolume + mountPath: {{ (include "hashr.dataPath" .) | quote }} + restartPolicy: Never + volumes: + - name: hashrvolume + persistentVolumeClaim: + claimName: {{ include "hashr.pvc.name" . }} + readOnly: false +{{- end }} diff --git a/charts/hashr/templates/hashr-iso9660-cronjob.yaml b/charts/hashr/templates/hashr-iso9660-cronjob.yaml new file mode 100644 index 00000000..9423ec71 --- /dev/null +++ b/charts/hashr/templates/hashr-iso9660-cronjob.yaml @@ -0,0 +1,54 @@ +{{- if .Values.hashr.importers.iso9660.enabled -}} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ .Release.Name }}-hashr-iso9660 +spec: + schedule: {{ .Values.hashr.importers.iso9660.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + containers: + - name: hashr-iso9660 + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --logtostderr=1 + - -storage + - postgres + - -exporters + - postgres + - -postgres_host + - {{ include "common.names.fullname" (dict "Chart" (dict "Name" "postgresql") "Release" .Release "Values" .Values.postgresql) }} + - -postgres_port + - {{ .Values.postgresql.primary.service.ports.postgresql | quote }} + - -postgres_user + - {{ .Values.postgresql.auth.username | quote }} + - -postgres_password + - "$(POSTGRES_PASSWORD)" + - -postgres_db + - {{ .Values.postgresql.auth.database | quote }} + - -importers + - iso9660 + - -iso_repo_path + - {{ (include "hashr.dataPath" .) }}/iso9660/ + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "postgresql.v1.secretName" .Subcharts.postgresql }} + key: {{ include "postgresql.v1.adminPasswordKey" .Subcharts.postgresql }} + volumeMounts: + - name: hashrvolume + mountPath: {{ (include "hashr.dataPath" .) | quote }} + restartPolicy: Never + volumes: + - name: hashrvolume + persistentVolumeClaim: + claimName: {{ include "hashr.pvc.name" . }} + readOnly: false +{{- end }} diff --git a/charts/hashr/templates/hashr-rpm-cronjob.yaml b/charts/hashr/templates/hashr-rpm-cronjob.yaml new file mode 100644 index 00000000..4bdfc1fb --- /dev/null +++ b/charts/hashr/templates/hashr-rpm-cronjob.yaml @@ -0,0 +1,54 @@ +{{- if .Values.hashr.importers.rpm.enabled -}} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ .Release.Name }}-hashr-rpm +spec: + schedule: {{ .Values.hashr.importers.rpm.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + containers: + - name: hashr-rpm + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --logtostderr=1 + - -storage + - postgres + - -exporters + - postgres + - -postgres_host + - {{ include "common.names.fullname" (dict "Chart" (dict "Name" "postgresql") "Release" .Release "Values" .Values.postgresql) }} + - -postgres_port + - {{ .Values.postgresql.primary.service.ports.postgresql | quote }} + - -postgres_user + - {{ .Values.postgresql.auth.username | quote }} + - -postgres_password + - "$(POSTGRES_PASSWORD)" + - -postgres_db + - {{ .Values.postgresql.auth.database | quote }} + - -importers + - rpm + - -rpm_repo_path + - {{ (include "hashr.dataPath" .) }}/rpm/ + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "postgresql.v1.secretName" .Subcharts.postgresql }} + key: {{ include "postgresql.v1.adminPasswordKey" .Subcharts.postgresql }} + volumeMounts: + - name: hashrvolume + mountPath: {{ (include "hashr.dataPath" .) | quote }} + restartPolicy: Never + volumes: + - name: hashrvolume + persistentVolumeClaim: + claimName: {{ include "hashr.pvc.name" . }} + readOnly: false +{{- end }} diff --git a/charts/hashr/templates/hashr-targz-cronjob.yaml b/charts/hashr/templates/hashr-targz-cronjob.yaml new file mode 100644 index 00000000..a519e88b --- /dev/null +++ b/charts/hashr/templates/hashr-targz-cronjob.yaml @@ -0,0 +1,54 @@ +{{- if .Values.hashr.importers.targz.enabled -}} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ .Release.Name }}-hashr-targz +spec: + schedule: {{ .Values.hashr.importers.targz.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + containers: + - name: hashr-targz + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --logtostderr=1 + - -storage + - postgres + - -exporters + - postgres + - -postgres_host + - {{ include "common.names.fullname" (dict "Chart" (dict "Name" "postgresql") "Release" .Release "Values" .Values.postgresql) }} + - -postgres_port + - {{ .Values.postgresql.primary.service.ports.postgresql | quote }} + - -postgres_user + - {{ .Values.postgresql.auth.username | quote }} + - -postgres_password + - "$(POSTGRES_PASSWORD)" + - -postgres_db + - {{ .Values.postgresql.auth.database | quote }} + - -importers + - targz + - -targz_repo_path + - {{ (include "hashr.dataPath" .) }}/targz/ + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "postgresql.v1.secretName" .Subcharts.postgresql }} + key: {{ include "postgresql.v1.adminPasswordKey" .Subcharts.postgresql }} + volumeMounts: + - name: hashrvolume + mountPath: {{ (include "hashr.dataPath" .) | quote }} + restartPolicy: Never + volumes: + - name: hashrvolume + persistentVolumeClaim: + claimName: {{ include "hashr.pvc.name" . }} + readOnly: false +{{- end }} diff --git a/charts/hashr/templates/hashr-zip-cronjob.yaml b/charts/hashr/templates/hashr-zip-cronjob.yaml new file mode 100644 index 00000000..efc23466 --- /dev/null +++ b/charts/hashr/templates/hashr-zip-cronjob.yaml @@ -0,0 +1,54 @@ +{{- if .Values.hashr.importers.zip.enabled -}} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ .Release.Name }}-hashr-zip +spec: + schedule: {{ .Values.hashr.importers.zip.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + containers: + - name: hashr-zip + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --logtostderr=1 + - -storage + - postgres + - -exporters + - postgres + - -postgres_host + - {{ include "common.names.fullname" (dict "Chart" (dict "Name" "postgresql") "Release" .Release "Values" .Values.postgresql) }} + - -postgres_port + - {{ .Values.postgresql.primary.service.ports.postgresql | quote }} + - -postgres_user + - {{ .Values.postgresql.auth.username | quote }} + - -postgres_password + - "$(POSTGRES_PASSWORD)" + - -postgres_db + - {{ .Values.postgresql.auth.database | quote }} + - -importers + - zip + - -zip_repo_path + - {{ (include "hashr.dataPath" .) }}/zip/ + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "postgresql.v1.secretName" .Subcharts.postgresql }} + key: {{ include "postgresql.v1.adminPasswordKey" .Subcharts.postgresql }} + volumeMounts: + - name: hashrvolume + mountPath: {{ (include "hashr.dataPath" .) | quote }} + restartPolicy: Never + volumes: + - name: hashrvolume + persistentVolumeClaim: + claimName: {{ include "hashr.pvc.name" . }} + readOnly: false +{{- end }} diff --git a/charts/hashr/values.yaml b/charts/hashr/values.yaml index 2ce6c2e2..332167fc 100644 --- a/charts/hashr/values.yaml +++ b/charts/hashr/values.yaml @@ -62,28 +62,52 @@ hashr: importers: ## List of HashR importers and their settings ## + aws: # TODO: Add cronjob file! + # https://github.com/google/hashr?tab=readme-ov-file#aws + ## @param hashr.importers.aws.enabled Enables the AWS importer + ## + enabled: false + ## @param hashr.importers.aws.schedule sets the CronJob schedule times + ## + schedule: "0 3 * * 1" # At 03:00 on Monday gcp: + # Ensure you have the correct setup before enabling this importer: + # https://github.com/google/hashr?tab=readme-ov-file#gcp-google-cloud-platform + # IMPORTANT: Store your SA key in the hashrvolume via kubectl cp! + # e.g. kubectl cp ~/hashr-sa-private-key.json hashr-data-manager:/mnt/hashrvolume/data/creds/hashr-sa-private-key.json ## @param hashr.importers.gcp.enabled Enables the GCP importer ## enabled: false ## @param hashr.importers.gcp.schedule sets the CronJob schedule times ## schedule: "0 3 * * 1" # At 03:00 on Monday + ## @param hashr.importers.gcp.gcpProjects sets a comma separated list of cloud projects containing disk images + ## + gcp_projects: "" + ## @param hashr.importers.gcp.hashrGCPProject sets GCP project that will be used to store copy of disk images for processing and also run Cloud Build + ## + hashr_gcp_project: "" + ## @param hashr.importers.gcp.hashrGCSBucket sets GCS bucket that will be used to store output of Cloud Build (disk images in .tar.gz format) + ## + hashr_gcs_bucket: "" targz: + # https://github.com/google/hashr?tab=readme-ov-file#targz ## @param hashr.importers.targz.enabled Enables the tar.gz importer ## enabled: false ## @param hashr.importers.targz.schedule sets the CronJob schedule times ## schedule: "0 3 * * 2" # At 03:00 on Tuesday - windows: + windows: # TODO: Add cronjob file! + # https://github.com/google/hashr?tab=readme-ov-file#windows ## @param hashr.importers.windows.enabled Enables the Windows importer ## enabled: false ## @param hashr.importers.windows.schedule sets the CronJob schedule times ## schedule: "0 3 * * 3" # At 03:00 on Wednesday - wsus: + wsus: # TODO: Add cronjob file! + # https://github.com/google/hashr?tab=readme-ov-file#wsus ## @param hashr.importers.wsus.enabled Enables the WSUS importer ## enabled: false @@ -91,6 +115,7 @@ hashr: ## schedule: "0 3 * * 4" # At 03:00 on Thursday rpm: + # https://github.com/google/hashr?tab=readme-ov-file#rpm ## @param hashr.importers.rpm.enabled Enables the RPM importer ## enabled: false @@ -98,13 +123,15 @@ hashr: ## schedule: "0 3 * * 5" # At 03:00 on Friday zip: + # https://github.com/google/hashr?tab=readme-ov-file#zip-and-other-zip-like-formats ## @param hashr.importers.zip.enabled Enables the ZIP importer ## enabled: false ## @param hashr.importers.zip.schedule sets the CronJob schedule times ## schedule: "0 3 * * 6" # At 03:00 on Saturday - gcr: + gcr: # TODO: Add cronjob file! + # https://github.com/google/hashr?tab=readme-ov-file#gcr-google-container-registry ## @param hashr.importers.gcr.enabled Enables the GCR importer ## enabled: false @@ -112,6 +139,7 @@ hashr: ## schedule: "0 3 * * 7" # At 03:00 on Sunday iso9660: + # https://github.com/google/hashr?tab=readme-ov-file#iso-9660 ## @param hashr.importers.iso9660.enabled Enables the iso9660 importer ## enabled: false @@ -119,6 +147,7 @@ hashr: ## schedule: "0 15 * * 1" # At 15:00 on Monday deb: + # https://github.com/google/hashr?tab=readme-ov-file#deb ## @param hashr.importers.deb.enabled Enables the DEB importer ## enabled: false