diff --git a/ansible/roles/analytics-bootstrap-spark/defaults/main.yml b/ansible/roles/analytics-bootstrap-spark/defaults/main.yml index 06eb65f463..905102edf1 100644 --- a/ansible/roles/analytics-bootstrap-spark/defaults/main.yml +++ b/ansible/roles/analytics-bootstrap-spark/defaults/main.yml @@ -4,7 +4,7 @@ analytics_user_home: /home/{{ analytics_user }} analytics_model_version: "2.0" analytics: home: /mount/data/analytics - paths: ['/home/analytics/sbin', '/mount/data/analytics', '/mount/data/analytics/logs', '/mount/data/analytics/logs/services', '/mount/data/analytics/logs/data-products', '/mount/data/analytics/logs/api-service', '/mount/data/analytics/api', '/mount/data/analytics/tmp', '/mount/data/analytics/scripts/monitor-data', '/mount/data/analytics/models-{{ analytics_model_version }}' ] + paths: ['/home/analytics/sbin', '/mount/data/analytics', '/mount/data/analytics/logs', '/mount/data/analytics/logs/services', '/mount/data/analytics/logs/data-products', '/mount/data/analytics/logs/api-service', '/mount/data/analytics/api', '/mount/data/analytics/tmp', '/mount/data/analytics/scripts/monitor-data', '/mount/data/analytics/models-{{ analytics_model_version }}', '/mount/data/analytics/logs/lern-data-products' ] # analytics: # home: /mount/data/analytics diff --git a/ansible/roles/data-products-deploy/defaults/main.yml b/ansible/roles/data-products-deploy/defaults/main.yml index 252dcc0784..4442312508 100755 --- a/ansible/roles/data-products-deploy/defaults/main.yml +++ b/ansible/roles/data-products-deploy/defaults/main.yml @@ -81,8 +81,6 @@ start_jobmanager: minute: 30 have_weekly_jobs: false -course_batch_status_updater_job_schedule: 60 - run_wfs_job: wfs: hour: 00 @@ -92,22 +90,6 @@ run_monitor_job: hour: 03 minute: 00 -run_admin_user_reports_job: - admin-user-reports-3AMIST: - hour: 21 - minute: 30 - admin-user-reports-2PMIST: - hour: 8 - minute: 30 - -run_admin_geo_reports_job: - admin-geo-reports-4AMIST: - hour: 22 - minute: 30 - admin-geo-reports-3PMIST: - hour: 9 - minute: 30 - run_assessment_aggregator_report_job: assessment-aggregator-report: hour: 18 @@ -132,36 +114,11 @@ run_etb_metrics_weekly_job: # These are the dummy times till sept30 for exhaust reports #To-Do: Update time after 3.2.7 deployment -run_progress_exhaust: - progress-exhaust: - hour: 08 - minute: 00 - -run_response_exhaust: - response-exhaust: - hour: 09 - minute: 00 - -run_userinfo_exhaust: - userinfo-exhaust: - hour: 10 - minute: 00 - -run_collection_summary: - collection-summary: - hour: 09 - minute: 30 - run_sourcing_summary: sourcing-summary: hour: 10 minute: 30 -run_cassandra_migration: - cassandra-migration: - hour: 19 - minute: 15 - run_uci_private_exhaust_job: uci-private-exhaust: hour: 03 diff --git a/ansible/roles/data-products-deploy/tasks/main.yml b/ansible/roles/data-products-deploy/tasks/main.yml index 6947508627..b9db383f65 100644 --- a/ansible/roles/data-products-deploy/tasks/main.yml +++ b/ansible/roles/data-products-deploy/tasks/main.yml @@ -260,90 +260,14 @@ - spark-jobs - cronjobs -- name: Create course-batch-status-updater cron job - cron: name="{{env}}-course-batch-status-updater" minute=*/{{ course_batch_status_updater_job_schedule }} job="{{ analytics.home }}/scripts/run-job.sh course-batch-status-updater" - tags: - - cronjobs - - default-jobs - - spark1-jobs - -- name: Create admin-user-reports cron job - cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh admin-user-reports" - with_dict: "{{ run_admin_user_reports_job }}" - tags: - - cronjobs - - default-jobs - - spark-jobs -- name: Create admin-geo-reports cron job - cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh admin-geo-reports" - with_dict: "{{ run_admin_geo_reports_job }}" - tags: - - cronjobs - - default-jobs - - spark-jobs - - name: Create assessment-aggregator reports cron job cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="/bin/bash {{ analytics.home }}/adhoc-scripts/run_exporter.sh > /home/analytics/output.log" with_dict: "{{ run_assessment_aggregator_report_job }}" - tags: - - cronjobs - - default-jobs - - spark-jobs - -- name: Create etb metrics cron job - cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} weekday={{ item.value.weekday }} job="{{ analytics.home }}/scripts/run-job.sh etb-metrics" - with_dict: "{{ run_etb_metrics_weekly_job }}" tags: - cronjobs - default-jobs - spark-jobs -- name: Create progress-exhaust cron job - cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh progress-exhaust" - with_dict: "{{ run_progress_exhaust }}" - tags: - - cronjobs - - default-jobs - - spark1-jobs - -- name: Create response-exhaust cron job - cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh response-exhaust" - with_dict: "{{ run_response_exhaust }}" - tags: - - cronjobs - - default-jobs - - spark-jobs - -- name: Create cassandra-migration cron job - cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh cassandra-migration" - with_dict: "{{ run_cassandra_migration }}" - tags: - - cronjobs - - default-jobs - - spark-jobs - - -- name: Create userinfo-exhaust cron job - cron: name="{{ env }}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh userinfo-exhaust" - with_dict: "{{ run_userinfo_exhaust }}" - tags: - - cronjobs - - default-jobs - - spark-jobs - -- name: Create collection-summary cron job - cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh collection-summary-report" - with_dict: "{{ run_collection_summary }}" - tags: - - cronjobs - - default-jobs - - spark-jobs - -- name: Copy collection-summary ingestion spec - template: src="collection-summary-ingestion-spec.j2" dest={{ analytics.home }}/scripts/collection-summary-ingestion-spec.json mode=755 owner={{ analytics_user }} group={{ analytics_group }} - tags: - - ed-dataproducts - - name: Create sourcing-summary cron job cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-dock-job.sh sourcing-summary-report" with_dict: "{{ run_sourcing_summary }}" diff --git a/ansible/roles/data-products-deploy/templates/model-config.j2 b/ansible/roles/data-products-deploy/templates/model-config.j2 index 6fc260b343..772593b456 100644 --- a/ansible/roles/data-products-deploy/templates/model-config.j2 +++ b/ansible/roles/data-products-deploy/templates/model-config.j2 @@ -46,52 +46,25 @@ config() { "collection-reconciliation-job") echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.CollectionReconciliationJob","modelParams":{"mode":"prodrun","brokerList":"{{ingestion_kafka_broker_host}}","topic":"{{env}}.issue.certificate.request","sparkCassandraConnectionHost":"{{ core_cassandra_host }}"},"parallelization":30,"appName":"CollectionReconciliationJob"}' ;; - "collection-summary-report") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"],"contentType":"Course"},"fields":["identifier","name","organisation","channel"],"limit":10000}},"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}' - ;; "score-metric-migration-job") echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.ScoreMetricMigrationJob","modelParams":{"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Score Metric Migration Job"}' ;; "assessment-score-metric-correction") echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.AssessmentScoreCorrectionJob","modelParams":{"assessment.score.correction.batches":"","cassandraReadConsistency":"QUORUM","cassandraWriteConsistency":"QUORUM","csvPath":"/mount/data/analytics/score_correction","isDryRunMode":true,"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":30,"appName":"Assessment Score Correction Job"}' ;; - "course-batch-status-updater") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.CourseBatchStatusUpdaterJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","sparkElasticsearchConnectionHost":"http://{{ single_node_es_host }}:9200","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","kpLearningBasePath":"http://{{groups['learning'][0]}}:8080/learning-service","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Course Batch Status Updater Job"}' - ;; - "collection-summary-report-v2") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJobV2","modelParams":{"storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","batchSize":50,"generateForAllBatches":true,"contentFields":["identifier","name","organisation","channel","status","keywords","createdFor","medium","subject"],"contentStatus":["Live","Unlisted","Retired"],"store":"{{cloud_service_provider}}","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","specPath":"/mount/data/analytics/scripts/collection-summary-ingestion-spec.json","druidIngestionUrl":"'$druidIngestionURL'","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report V2"}' - ;; "uci-private-exhaust") echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.uci.UCIPrivateExhaustJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","mode":"OnDemand","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UCI Private Exhaust"}' ;; "uci-response-exhaust") echo '{"search":{"type":"{{cloud_service_provider}}","queries":[{"bucket":"'$bucket'","prefix":"unique/raw/","endDate":"'$endDate'","delta":0}]},"filters":[{"name":"eid","operator":"EQ","value":"ASSESS"}],"model":"org.sunbird.analytics.uci.UCIResponseExhaust","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","botPdataId":"{{ uci_pdata_id }}","mode":"OnDemand","fromDate":"","toDate":""},"parallelization":8,"appName":"UCI Response Exhaust"}' ;; - "userinfo-exhaust") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.UserInfoExhaustJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UserInfo Exhaust"}' - ;; - "program-collection-summary-report") + "program-collection-summary-report") echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"],"contentType":"Course","keywords":["'$keyword'"]},"fields":["identifier","name","organisation","channel"],"limit":10000}},"columns":["Published by","Batch id","Collection id","Collection name","Batch start date","Batch end date","State","Total enrolments By State","Total completion By State"],"keywords":"'$keyword'","store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}' ;; - "response-exhaust") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ResponseExhaustJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust"}' - ;; - "response-exhaust-v2") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ResponseExhaustJobV2","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust V2"}' - ;; - "progress-exhaust") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ProgressExhaustJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}", "apiVersion":"v2", "parallelization":200,"mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust"}' - ;; - "progress-exhaust-v2") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ProgressExhaustJobV2","modelParams":{"store":"{{cloud_service_provider}}","mode":"OnDemand","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust V2"}' - ;; "druid_reports") echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.DruidQueryProcessingModel","modelParams":{"mode":"batch"},"parallelization":8,"appName":"Druid Reports"}' ;; - "cassandra-migration") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.updater.CassandraMigratorJob","modelParams":{"cassandraDataHost":"{{ core_cassandra_host }}","cassandraMigrateHost":"{{ report_cassandra_host }}","keyspace":"sunbird_courses","cassandraDataTable":"user_enrolments","cassandraMigrateTable":"{{ report_user_enrolment_table }}","repartitionColumns":"batchid"},"parallelization":10,"appName":"Cassandra Migrator","deviceMapping":false}' - ;; - "monitor-job-summ") + "monitor-job-summ") echo '{"search":{"type":"local","queries":[{"file":"'$analytics_home'/scripts/logs/joblog.log"}]},"model":"org.ekstep.analytics.model.MonitorSummaryModel","modelParams":{"pushMetrics":true,"brokerList":"'$brokerList'","topic":"'$analyticsMetricsTopic'","model":[{"model":"WorkFlowSummaryModel","category":"consumption","input_dependency":"None"},{"model":"UpdateContentRating","category":"consumption","input_dependency":"None"},{"model":"DruidQueryProcessingModel","category":"consumption","input_dependency":"None"},{"model":"MetricsAuditJob","category":"consumption","input_dependency":"None"},{"model":"StateAdminReportJob","category":"consumption","input_dependency":"None"},{"model":"StateAdminGeoReportJob","category":"consumption","input_dependency":"None"},{"model":"CourseEnrollmentJob","category":"consumption","input_dependency":"None"}]},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"appName":"TestMonitorSummarizer","deviceMapping":true}' ;; "job-manager") @@ -104,12 +77,6 @@ config() { "video-streaming") echo '{"search":{"type":"{{ cloud_service_provider }}"},"model":"org.ekstep.analytics.job.VideoStreamingJob","modelParams":{"maxIterations":10},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Video Streaming Job","deviceMapping":false}' ;; - "admin-user-reports") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.report.StateAdminReportJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin User Reports","deviceMapping":false}' - ;; - "admin-geo-reports") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.report.StateAdminGeoReportJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin Geo Reports","deviceMapping":false}' - ;; "telemetry-replay") echo '{"search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"'$inputBucket'","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.job.EventsReplayJob","modelParams":{},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$sinkTopic'"}}],"parallelization":8,"appName":"TelemetryReplayJob","deviceMapping":false}' ;; @@ -122,9 +89,6 @@ config() { "experiment") echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.ExperimentDefinitionModel","modelParams":{"sparkElasticsearchConnectionHost":"{{ lp_composite_search_host }}"},"output":[{"to":"elasticsearch","params":{"index":"experiment"}}],"parallelization":8,"appName":"Experiment-Definition","deviceMapping":false}' ;; - "etb-metrics") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.ETBMetricsJob","modelParams":{"reportConfig":{"id":"etb_metrics","metrics":[],"labels":{"date":"Date","identifier":"Textbook ID","name":"Textbook Name","medium":"Medium","gradeLevel":"Grade","subject":"Subject","createdOn":"Created On","lastUpdatedOn":"Last Updated On","totalQRCodes":"Total number of QR codes","contentLinkedQR":"Number of QR codes with atleast 1 linked content","withoutContentQR":"Number of QR codes with no linked content","withoutContentT1":"Term 1 QR Codes with no linked content","withoutContentT2":"Term 2 QR Codes with no linked content","status":"Textbook Status","totalContentLinked":"Total content linked","totalQRLinked":"Total QR codes linked to content","totalQRNotLinked":"Total number of QR codes with no linked content","leafNodesCount":"Total number of leaf nodes","leafNodeUnlinked":"Number of leaf nodes with no content","l1Name":"Level 1 Name","l2Name":"Level 2 Name","l3Name":"Level 3 Name","l4Name":"Level 4 Name","l5Name":"Level 5 Name","dialcode":"QR Code","sum(scans)":"Total Scans","noOfContent":"Number of contents","nodeType":"Type of Node","term":"Term"},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"dialcode_counts.csv","postContainer":"'$reportPostContainer'"}},"dialcodeReportConfig":{"id":"etb_metrics","metrics":[],"labels":{},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":1,"reportPath":"dialcode_counts.csv","rollupAge":"ACADEMIC_YEAR","rollupCol":"Date","rollupRange":10,"postContainer":"'$reportPostContainer'"}},"etbFileConfig":{"bucket":"'$reportPostContainer'","file":"dialcode_scans/dialcode_counts.csv"},"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"channel","aliasName":"channel"},{"fieldName":"identifier","aliasName":"identifier","type":"Extraction","outputType":"STRING","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.split(\".\")[0]}"}]},{"fieldName":"name","aliasName":"name"},{"fieldName":"status","aliasName":"status"}],"filters":[{"type":"equals","dimension":"contentType","value":"TextBook"},{"type":"in","dimension":"status","values":["Live","Draft","Review"]}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"tenantConfig":{"tenantId":"","slugName":""},"store":"{{ cloud_service_provider }}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"]},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"ETB Metrics Model","deviceMapping":false}' - ;; "course-enrollment-report") echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.CourseEnrollmentJob","modelParams":{"reportConfig":{"id":"tpd_metrics","metrics":[],"labels":{"completionCount":"Completion Count","status":"Status","enrollmentCount":"Enrollment Count","courseName":"Course Name","batchName":"Batch Name"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"DAY","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"course_enrollment.csv"}},"esConfig":{"request":{"filters":{"objectType":["Content"],"contentType":["Course"],"identifier":[],"status":["Live"]},"limit":10000}},"store":"{{ cloud_service_provider }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"TPD Course Enrollment Metrics Model","deviceMapping":false}' ;; diff --git a/ansible/roles/data-products-deploy/templates/model-config.json.j2 b/ansible/roles/data-products-deploy/templates/model-config.json.j2 index aa54c3413a..0465e2b65b 100644 --- a/ansible/roles/data-products-deploy/templates/model-config.json.j2 +++ b/ansible/roles/data-products-deploy/templates/model-config.json.j2 @@ -67,58 +67,6 @@ "appName": "Video Streaming Job", "deviceMapping": false }, - "admin-user-reports": { - "search": { - "type": "none" - }, - "model": "org.ekstep.analytics.job.report.StateAdminReportJob", - "modelParams": { - "storageKeyConfig":"storage.key.config", - "storageSecretConfig":"storage.secret.config", - "storageContainer":"{{reports_container}}", - "storageEndpoint":"{{cloud_private_storage_endpoint}}", - "store":"{{ cloud_service_provider }}", - "sparkCassandraConnectionHost": "{{core_cassandra_host}}", - "sparkElasticsearchConnectionHost": "{{sunbird_es_host}}" - }, - "output": [ - { - "to": "console", - "params": { - "printEvent": false - } - } - ], - "parallelization": 8, - "appName": "Admin User Reports", - "deviceMapping": false - }, - "admin-geo-reports": { - "search": { - "type": "none" - }, - "model": "org.ekstep.analytics.job.report.StateAdminGeoReportJob", - "modelParams": { - "storageKeyConfig":"storage.key.config", - "storageSecretConfig":"storage.secret.config", - "storageContainer":"{{reports_container}}", - "storageEndpoint":"{{cloud_private_storage_endpoint}}", - "store":"{{ cloud_service_provider }}", - "sparkCassandraConnectionHost": "{{core_cassandra_host}}", - "sparkElasticsearchConnectionHost": "{{sunbird_es_host}}" - }, - "output": [ - { - "to": "console", - "params": { - "printEvent": false - } - } - ], - "parallelization": 8, - "appName": "Admin Geo Reports", - "deviceMapping": false - }, "content-rating-updater": { "search": { "type": "none" @@ -185,16 +133,6 @@ "category": "consumption", "input_dependency": "None" }, - { - "model": "StateAdminReportJob", - "category": "consumption", - "input_dependency": "None" - }, - { - "model": "StateAdminGeoReportJob", - "category": "consumption", - "input_dependency": "None" - }, { "model": "CourseEnrollmentJob", "category": "consumption", diff --git a/ansible/roles/data-products-deploy/templates/run-job.j2 b/ansible/roles/data-products-deploy/templates/run-job.j2 index 26ec84da87..f61419e4bc 100644 --- a/ansible/roles/data-products-deploy/templates/run-job.j2 +++ b/ansible/roles/data-products-deploy/templates/run-job.j2 @@ -20,34 +20,12 @@ get_report_job_model_name(){ ;; "sourcing-metrics") echo 'org.sunbird.analytics.sourcing.SourcingMetrics' ;; - "admin-geo-reports") echo 'org.sunbird.analytics.job.report.StateAdminGeoReportJob' - ;; - "etb-metrics") echo 'org.sunbird.analytics.job.report.ETBMetricsJob' - ;; - "admin-user-reports") echo 'org.sunbird.analytics.job.report.StateAdminReportJob' - ;; - "userinfo-exhaust") echo 'org.sunbird.analytics.exhaust.collection.UserInfoExhaustJob' - ;; - "response-exhaust") echo 'org.sunbird.analytics.exhaust.collection.ResponseExhaustJob' - ;; - "response-exhaust-v2") echo 'org.sunbird.analytics.exhaust.collection.ResponseExhaustJobV2' - ;; - "progress-exhaust") echo 'org.sunbird.analytics.exhaust.collection.ProgressExhaustJob' - ;; - "progress-exhaust-v2") echo 'org.sunbird.analytics.exhaust.collection.ProgressExhaustJobV2' - ;; - "cassandra-migration") echo 'org.sunbird.analytics.updater.CassandraMigratorJob' - ;; - "collection-summary-report") echo 'org.sunbird.analytics.job.report.CollectionSummaryJob' - ;; "program-collection-summary-report") echo 'org.sunbird.analytics.job.report.CollectionSummaryJob' ;; "collection-summary-report-v2") echo 'org.sunbird.analytics.job.report.CollectionSummaryJobV2' ;; "assessment-score-metric-correction") echo 'org.sunbird.analytics.audit.AssessmentScoreCorrectionJob' ;; - "course-batch-status-updater") echo 'org.sunbird.analytics.audit.CourseBatchStatusUpdaterJob' - ;; "collection-reconciliation-job") echo 'org.sunbird.analytics.audit.CollectionReconciliationJob' ;; "assessment-correction") echo 'org.sunbird.analytics.job.report.AssessmentCorrectionJob'