From 706ada9854615acfc66fc0167a5a45bd8922bb25 Mon Sep 17 00:00:00 2001
From: Manjunath Davanam <manjunath@sanketika.in>
Date: Thu, 3 Nov 2022 15:05:23 +0530
Subject: [PATCH] OB-33 feat: Flink helm and ansible changes to support the GCP

---
 .../datapipeline_jobs/templates/flink_job_deployment.yaml  | 5 +++++
 kubernetes/helm_charts/datapipeline_jobs/values.j2         | 7 ++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml
index 10e6b62181..d0ea3318d5 100644
--- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml
+++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml
@@ -199,6 +199,11 @@ spec:
           "-Ds3.secret-key={{ .Values.s3_secret_key }}",
           "-Ds3.endpoint={{ .Values.s3_endpoint }}",
           "-Ds3.path.style.access={{ .Values.s3_path_style_access }}",
+{{- end }}  
+{{- if eq .Values.checkpoint_store_type "gcloud" }}
+          "-Dfs.gs.auth.client.id={{ .Values.gcloud_client_key }}",
+          "-Dfs.gs.auth.client.secret={{ .Values.gcloud_private_secret }}",
+          "-Dfs.gs.project.id={{ .Values.gcloud_project_id }}"
 {{- end }}
           "-Dweb.submit.enable=false",
           "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter",
diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2
index 22630c3015..07d432606e 100644
--- a/kubernetes/helm_charts/datapipeline_jobs/values.j2
+++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2
@@ -10,6 +10,9 @@ s3_access_key: {{ s3_storage_key }}
 s3_secret_key: {{ s3_storage_secret }}
 s3_endpoint: {{ s3_storage_endpoint }}
 s3_path_style_access: {{ s3_path_style_access }}
+gcloud_client_key: {{ gcloud_client_key }}
+gcloud_private_secret: {{ gcloud_private_secret }}
+gcloud_project_id: {{ gcloud_project_id }}
 
 serviceMonitor:
   enabled: {{ service_monitor_enabled | lower}}
@@ -150,7 +153,7 @@ base_config: |
       statebackend {
         blob {
           storage {
-            account = "{% if checkpoint_store_type == "azure" %}{{ azure_account }}.blob.core.windows.net{% elif checkpoint_store_type == "s3" %}{{ flink_dp_storage_container }}{% endif %}"
+            account = "{% if checkpoint_store_type == "azure" %}{{ azure_account }}.blob.core.windows.net{% elif checkpoint_store_type == "s3" %}{{ flink_dp_storage_container }}{% elif checkpoint_store_type == "gcloud" %}{{ flink_dp_storage_container }}{% endif %}"
             container = "{{ flink_container_name }}"
             checkpointing.dir = "checkpoint"
           }
@@ -159,6 +162,8 @@ base_config: |
         base.url = "wasbs://"${job.statebackend.blob.storage.container}"@"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.checkpointing.dir}
 {% elif checkpoint_store_type == "s3" %}
         base.url = "s3://"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.container}"/"${job.statebackend.blob.storage.checkpointing.dir}
+{% elif checkpoint_store_type == "gcloud" %}
+        base.url = "gs://"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.container}"/"${job.statebackend.blob.storage.checkpointing.dir}
 {% endif %}
       }
     }