Csp changes

ansible/artifacts-download.yml

@@ -3,8 +3,38 @@
   become: yes
   vars_files:
     - "{{inventory_dir}}/secrets.yml"
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_artifact_storage_account_name}}"
-    AZURE_STORAGE_SAS_TOKEN: "{{sunbird_artifact_storage_account_sas}}"
-  roles:
-    - artifacts-download-azure
+  tasks:
+    - name: download artifact from azure storage
+      include_role:
+        name: azure-cloud-storage
+        tasks_from: blob-download.yml
+      vars:
+        blob_container_name: "{{ cloud_storage_artifacts_bucketname }}"
+        blob_file_name: "{{ artifact }}"
+        local_file_or_folder_path: "{{ artifact_path }}"
+        storage_account_name: "{{ cloud_artifact_storage_accountname }}"
+        storage_account_key: "{{ cloud_artifact_storage_secret }}"
+      when: cloud_service_provider == "azure"
+
+    - name: download artifact from gcloud storage
+      include_role:
+        name: gcp-cloud-storage
+        tasks_from: download.yml
+      vars:
+        gcp_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
+        gcp_path: "{{ artifact }}"
+        local_file_or_folder_path: "{{ artifact_path }}"
+      when: cloud_service_provider == "gcloud"
+
+    - name: download artifact from aws s3
+      include_role:
+        name: aws-cloud-storage
+        tasks_from: download.yml
+      vars:
+        local_file_or_folder_path: "{{ artifact_path }}"
+        s3_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
+        s3_path: "{{ artifact }}"
+        aws_default_region: "{{ cloud_public_storage_region }}"
+        aws_access_key_id: "{{ cloud_artifact_storage_accountname }}"
+        aws_secret_access_key: "{{ cloud_artifact_storage_secret }}"
+      when: cloud_service_provider == "aws"

ansible/artifacts-upload.yml

@@ -3,8 +3,39 @@
   become: yes
   vars_files:
     - "{{inventory_dir}}/secrets.yml"
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_artifact_storage_account_name}}"
-    AZURE_STORAGE_SAS_TOKEN: "{{sunbird_artifact_storage_account_sas}}"
-  roles:
-    - artifacts-upload-azure
+  tasks:
+    - name: upload artifact to azure storage
+      include_role:
+        name: azure-cloud-storage
+        tasks_from: blob-upload.yml
+      vars:
+        blob_container_name: "{{ cloud_storage_artifacts_bucketname }}"
+        container_public_access: "off"
+        blob_file_name: "{{ artifact }}"
+        local_file_or_folder_path: "{{ artifact_path }}"
+        storage_account_name: "{{ cloud_artifact_storage_accountname }}"
+        storage_account_key: "{{ cloud_artifact_storage_secret }}"
+      when: cloud_service_provider == "azure"
+
+    - name: upload artifact to gcloud storage
+      include_role:
+        name: gcp-cloud-storage
+        tasks_from: upload.yml
+      vars:
+        gcp_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
+        gcp_path: "{{ artifact }}"
+        local_file_or_folder_path: "{{ artifact_path }}"
+      when: cloud_service_provider == "gcloud"
+
+    - name: upload artifact to aws s3
+      include_role:
+        name: aws-cloud-storage
+        tasks_from: upload.yml
+      vars:
+        local_file_or_folder_path: "{{ artifact_path }}"
+        s3_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
+        s3_path: "{{ artifact }}"
+        aws_default_region: "{{ cloud_public_storage_region }}"
+        aws_access_key_id: "{{ cloud_artifact_storage_accountname }}"
+        aws_secret_access_key: "{{ cloud_artifact_storage_secret }}"
+      when: cloud_service_provider == "aws"

ansible/inventory/env/group_vars/all.yml

@@ -6,12 +6,12 @@ log4j_appender_kafka_topic: "{{env}}.telemetry.backend"
 # Shall we change the value to telemetry-data-store in dev also?
 # What's the implication
 # Is it azure blob or s3 bucket??
-channel_data_exhaust_bucket: dev-data-store
-secrets_path: '{{inventory_dir}}/secrets.yml'
-artifacts_container: "{{dp_vault_artifacts_container}}"
+cloud_storage_telemetry_bucketname: dev-data-store
+secrets_path: "{{inventory_dir}}/secrets.yml"
+cloud_storage_artifacts_bucketname: "{{dp_vault_artifacts_container}}"
 
-report_azure_account_name: "{{sunbird_private_storage_account_name}}"
-report_azure_storage_secret: "{{sunbird_private_storage_account_key}}"
+report_azure_account_name: "{{cloud_private_storage_accountname}}"
+report_azure_storage_secret: "{{cloud_private_storage_secret}}"
 
 redis_host: "{{ groups['redis'][0] }}"
 metadata_redis_host: "{{ groups['redis'][0] }}"
@@ -34,7 +34,7 @@ telemetry_schema_directory: /etc/{{env}}/telemetry
 telemetry_schema_path: /etc/{{env}}/telemetry/schemas
 schema_repo_url: https://github.com/project-sunbird/sunbird-data-pipeline.git
 # Create learningall group with LP ips
-cassandra_host:  "{{ groups['cassandra'][0] }}"
+cassandra_host: "{{ groups['cassandra'][0] }}"
 core_cassandra_host: "{{ groups['core-cassandra'][0] }}"
 lp_cassandra_host: "{{ groups['lp-cassandra'][0] }}"
 report_cassandra_host: "{{ groups['report-cassandra'][0] }}"
@@ -46,36 +46,45 @@ sbin_path: "{{ analytics_user_home }}/sbin"
 
 # Secor vars
 secor:
-  properties: ['secor.azure', 'secor.common', 'secor', 'secor.partition', 'log4j']
-  artifact_dir: /mount/secor
-  artifact_ver: "0.29"
-  azure:
-    account_name: "{{sunbird_private_storage_account_name}}"
-    account_key: "{{sunbird_private_storage_account_key}}"
-    container_name: "{{channel_data_exhaust_bucket}}"
-  paths: ['/mount/secor', '/mount/secor/reports', '/mount/secor/logs', '/home/analytics/sbin', '/mount/data/analytics']
-  channel: "{{secor_alerts_slack_channel}}"
+    properties:
+        ["secor.azure", "secor.common", "secor", "secor.partition", "log4j"]
+    artifact_dir: /mount/secor
+    artifact_ver: "0.29"
+    azure:
+        account_name: "{{cloud_private_storage_accountname}}"
+        account_key: "{{cloud_private_storage_secret}}"
+        container_name: "{{cloud_storage_telemetry_bucketname}}"
+
+    paths:
+        [
+            "/mount/secor",
+            "/mount/secor/reports",
+            "/mount/secor/logs",
+            "/home/analytics/sbin",
+            "/mount/data/analytics",
+        ]
+    channel: "{{secor_alerts_slack_channel}}"
 
 # postgres
 # list of databases to be created
 # Can move this dictionary to postgres role; but incase we want to generalize roles!!
 postgresql_databases:
-  - name: analytics
-    owner: analytics
+    - name: analytics
+      owner: analytics
 
 postgresql_users:
-  - name: analytics
-    password: "{{dp_vault_pgdb_password}}"
+    - name: analytics
+      password: "{{dp_vault_pgdb_password}}"
 
 postgres:
-  db_url: "{{ groups['postgres'][0] }}"
-  db_username: analytics
-  db_name: analytics
-  db_password: "{{dp_vault_pgdb_password}}"
-  db_table_name: "{{env}}_consumer_channel_mapping"
-  db_port: 5432
-  db_admin_user: analytics
-  db_admin_password: "{{dp_vault_pgdb_admin_password}}"
+    db_url: "{{ groups['postgres'][0] }}"
+    db_username: analytics
+    db_name: analytics
+    db_password: "{{dp_vault_pgdb_password}}"
+    db_table_name: "{{env}}_consumer_channel_mapping"
+    db_port: 5432
+    db_admin_user: analytics
+    db_admin_password: "{{dp_vault_pgdb_admin_password}}"
 
 postgres_address_space: 0.0.0.0/0 # Postgres trust address space
 
@@ -84,9 +93,9 @@ lp_composite_search_host: "{{ groups['composite-search-cluster'][0] }}"
 lp_search: "http://{{private_ingressgateway_ip}}/search"
 lp_url: http://{{ groups['learning'][0] }}:8080/learning-service
 service:
-  search:
-    url: http://{{private_ingressgateway_ip}}/search
-    path: /v3/search
+    search:
+        url: http://{{private_ingressgateway_ip}}/search
+        path: /v3/search
 
 cassandra_hierarchy_store_prefix: "{{env}}_"
 data_exhaust_token: "{{dp_vault_data_exhaust_token}}"
@@ -106,46 +115,46 @@ CONTAINER_NAME_SAMZA: samza-logs
 script_path: /usr/local/hadoop
 
 job_names:
-  DeDuplication_1:
-      job_file_name: 'de-duplication'
-  DeNormalization_1:
-      job_file_name: 'de-normalization'
-  DruidEventsValidator_1:
-      job_file_name: 'druid-events-validator'
-  EventsRouter_1:
-      job_file_name: 'events-router'
-  TelemetryExtractor_1:
-      job_file_name: 'telemetry-extractor'
-  TelemetryLocationUpdater_1:
-      job_file_name: 'telemetry-location-updater'
-  TelemetryRouter_1:
-      job_file_name: 'telemetry-router'
-  TelemetryRedacter_1:
-      job_file_name: 'telemetry-redacter'
-  TelemetryValidator_1:
-      job_file_name: 'telemetry-validator'
-  DeviceProfileUpdater_1:
-      job_file_name: 'device-profile-updater'
-  AssessmentAggregator_1:
-      job_file_name: 'assessment-aggregator'
-  DerivedDeDuplication_1:
-      job_file_name: 'derived-de-duplication'
-  UserCacheUpdater_1:
-      job_file_name: 'user-cache-updater'
-  ContentCacheUpdater_1:
-      job_file_name: 'content-cache-updater'
-  ShareEventsFlattener_1:
-      job_file_name: 'share-events-flattener'
+    DeDuplication_1:
+        job_file_name: "de-duplication"
+    DeNormalization_1:
+        job_file_name: "de-normalization"
+    DruidEventsValidator_1:
+        job_file_name: "druid-events-validator"
+    EventsRouter_1:
+        job_file_name: "events-router"
+    TelemetryExtractor_1:
+        job_file_name: "telemetry-extractor"
+    TelemetryLocationUpdater_1:
+        job_file_name: "telemetry-location-updater"
+    TelemetryRouter_1:
+        job_file_name: "telemetry-router"
+    TelemetryRedacter_1:
+        job_file_name: "telemetry-redacter"
+    TelemetryValidator_1:
+        job_file_name: "telemetry-validator"
+    DeviceProfileUpdater_1:
+        job_file_name: "device-profile-updater"
+    AssessmentAggregator_1:
+        job_file_name: "assessment-aggregator"
+    DerivedDeDuplication_1:
+        job_file_name: "derived-de-duplication"
+    UserCacheUpdater_1:
+        job_file_name: "user-cache-updater"
+    ContentCacheUpdater_1:
+        job_file_name: "content-cache-updater"
+    ShareEventsFlattener_1:
+        job_file_name: "share-events-flattener"
 
 druid_ingestion_specs:
-  telemetry-events:
-      ingestion_file_name: 'telemetry_index_kafka'
-  summary-events:
-      ingestion_file_name: 'summary_index_kafka'
-  telemtry-feedback-events:
-      ingestion_file_name: 'telemetry_feedback_index_kafka'
-  pipeline-metrics:
-      ingestion_file_name: 'pipeline_metrics_index_kafka'
+    telemetry-events:
+        ingestion_file_name: "telemetry_index_kafka"
+    summary-events:
+        ingestion_file_name: "summary_index_kafka"
+    telemtry-feedback-events:
+        ingestion_file_name: "telemetry_feedback_index_kafka"
+    pipeline-metrics:
+        ingestion_file_name: "pipeline_metrics_index_kafka"
 
 #Druid Proxy APi service
 sunbird_druid_broker_host: "http://{{ groups['raw-broker'][0] }}"
@@ -154,9 +163,9 @@ sunbird_learner_service_url: "http://{{private_ingressgateway_ip}}/learner"
 location_search_url: "{{ domain_name }}/api/data/"
 location_search_token: "Bearer {{ sunbird_api_auth_token }}"
 
-druid_report_url_endpoint : "{{ proto}}://{{domain_name}}/api/data/v1/report/jobs"
-druid_report_url : "{{ proto}}://{{domain_name}}/api/data/v1/"
-druid_report_token : "Bearer {{ sunbird_api_auth_token }}"
+druid_report_url_endpoint: "{{ proto}}://{{domain_name}}/api/data/v1/report/jobs"
+druid_report_url: "{{ proto}}://{{domain_name}}/api/data/v1/"
+druid_report_token: "Bearer {{ sunbird_api_auth_token }}"
 
 #redis multiprocess config
 content_port: 6379

ansible/kubernetes/helm_charts/datapipeline/flink-jobs/templates/job-cluster-jobmanager.yaml

@@ -21,6 +21,7 @@ spec:
         workingDir: /opt/flink
         command: ["/opt/flink/bin/standalone-job.sh"]
 {{- $job-config-key := .Release.Name }}
+
         args: ["start-foreground",
                "--job-classname={{ index .Values $job-config-key.job_classname }}", 
                "-Djobmanager.rpc.address={{ .Release.Name }}-jobmanager",
@@ -29,7 +30,20 @@ spec:
                "-Dblob.server.port=6124", 
                "-Dqueryable-state.server.ports=6125",
                "-Djobmanager.heap.size={{ index .Values $job-config-key.job_manager_heap_size }}",
-               "-Dfs.azure.account.key.{{ .Values.azure_storage_account }}.blob.core.windows.net: {{ .Values.azure_storage_secret }}",
+{{- if eq .Values.checkpoint_store_type "azure" }}
+               "-Dfs.azure.account.key.{{ .Values.cloud_storage_key }}.blob.core.windows.net: {{ .Values.cloud_storage_secret }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "aws" }} 
+               "-Ds3.access-key={{ .Values.cloud_storage_key }}",
+               "-Ds3.secret-key={{ .Values.cloud_storage_secret }}",
+               "-Ds3.endpoint={{ .Values.cloud_private_endpoint }}",
+               "-Ds3.path.style.access={{ .Values.s3_path_style_access }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "gcloud" }}
+               "-Dfs.gs.auth.client.id={{ .Values.cloud_storage_key }}",
+               "-Dfs.gs.auth.client.secret={{ .Values.cloud_storage_secret }}",
+               "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}"
+{{- end }}
                "-Dconfig.file=/opt/flink/conf/{{ .Release.Name }}.conf"]
         ports:
         - containerPort: 6123

ansible/kubernetes/helm_charts/datapipeline/flink-jobs/templates/job-cluster-taskmanager.yaml

@@ -29,7 +29,20 @@ spec:
           "-Djobmanager.rpc.address={{ .Release.Name }}-jobmanager",
           "-Dtaskmanager.rpc.port=6122",
           "-Dtaskmanager.heap.size={{ index .Values $job-config-key.task_manager_heap_size }}",
-          "-Dfs.azure.account.key.{{ .Values.azure_storage_account }}.blob.core.windows.net: {{ .Values.azure_storage_secret }}",
+{{- if eq .Values.checkpoint_store_type "azure" }}
+          "-Dfs.azure.account.key.{{ .Values.cloud_storage_key }}.blob.core.windows.net: {{ .Values.cloud_storage_secret }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "aws" }} 
+          "-Ds3.access-key={{ .Values.cloud_storage_key }}",
+          "-Ds3.secret-key={{ .Values.cloud_storage_secret }}",
+          "-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}",
+          "-Ds3.path.style.access={{ .Values.cloud_storage_path_style_access }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "gcloud" }}
+          "-Dfs.gs.auth.client.id={{ .Values.cloud_storage_key }}",
+          "-Dfs.gs.auth.client.secret={{ .Values.cloud_storage_secret }}",
+          "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}",
+{{- end }}
           "-Dconfig.file=/opt/flink/conf/{{ .Release.Name }}.conf"]
         ports:
         - containerPort: 6122

ansible/kubernetes/helm_charts/datapipeline/flink-jobs/values.j2

@@ -6,8 +6,12 @@ dockerhub: {{ dockerhub }}
 repository: {{ datapipeline_repository|default('data-pipeline') }}
 image_tag: {{ image_tag }}
 
-azure_storage_account={{ sunbird_private_storage_account_name }}
-azure_storage_secret={{ sunbird_private_storage_account_key }}
+checkpoint_store_type: {{ cloud_service_provider }} 
+cloud_storage_key: {{ cloud_private_storage_accountname }}
+cloud_storage_secret: {{ cloud_private_storage_secret }}
+cloud_storage_endpoint: {{ cloud_private_storage_endpoint }}
+cloud_storage_path_style_access: {{ cloud_storage_pathstyle_access }}
+cloud_storage_project_id: {{ cloud_private_storage_project }}
 
 telemetry-extractor:
   job_name=telemetry-extractor

ansible/lpa_data-products_deploy.yml

@@ -5,7 +5,7 @@
   become: yes
   become_user: "{{ analytics_user }}"
   environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_private_storage_account_name}}"
-    AZURE_STORAGE_KEY: "{{sunbird_private_storage_account_key}}"
+    AZURE_STORAGE_ACCOUNT: "{{cloud_private_storage_accountname}}"
+    AZURE_STORAGE_KEY: "{{cloud_private_storage_secret}}"
   roles:
    - data-products-deploy

ansible/roles/analytics-bootstrap-always/tasks/main.yml

@@ -17,6 +17,10 @@
     createhome: yes
     group: "{{ analytics_group }}"
 
+- name: change permission
+  become: yes
+  command: chown -R "{{ analytics_user }}":"{{ analytics_user }}" /home/"{{ analytics_user }}"
+
 
 - name: Installing packages
   become: yes