Merge pull request #3587 from project-sunbird/release-5.0.0

Merge release 5.0.0 to 5.0.1

ansible/roles/ml-analytics-service/defaults/main.yml

@@ -87,3 +87,15 @@ ml_analytics_survey_azure_blob_path : "survey/status/"
 ml_analytics_druid_survey_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/survey/status/sl_survey_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-survey-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":true,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["survey_submission_id", "submission_status", "user_id", "user_sub_type", "user_type", "state_externalId", "block_externalId", "district_externalId", "cluster_externalId", "school_externalId", "state_name", "block_name", "district_name", "cluster_name", "school_name", "board_name", "organisation_id", "organisation_name", "program_externalId", "program_id", "program_name", "survey_name", "survey_id", "survey_externalId", "created_date", "submission_date", "updatedAt", "parent_channel", "solution_name", "solution_id","private_program"]},"metricsSpec":[]}}}'
 ml_analytics_slack_token: "{{ ml_slack_token | default('') }}"
 ml_analytics_channel_name: "{{ ml_slack_channel | default('') }}"
+ml_analytics_public_azure_account_name: "{{ ml_public_azure_account | default('') }}"
+ml_analytics_public_azure_access_key: "{{ ml_public_azure_key | default('') }}"
+ml_analytics_public_azure_container_name: "{{ ml_public_azure_container | default('') }}"
+ml_analytics_program_dashboard_azure_blob_path: "{{ ml_program_blob_path | default('') }}"
+ml_druid_query_data: "{{ ml_druid_query | default('') }}"
+ml_program_dashboard_data: "{{ ml_program_data | default('') }}"
+ml_analytics_druid_query_url: "{{groups['druid'][0]}}:8082"
+ml_analytics_druid_observation_query_spec: '{"queryType":"scan","dataSource":"sl-observation","resultFormat":"list","columns":["completedDate","createdAt","createdBy","criteriaExternalId","criteriaId","criteriaName","entityType","entityTypeId","observationId","observationName","observationSubmissionId","questionAnswer","questionECM","questionExternalId","questionId","questionName","questionResponseLabel","questionResponseType","solutionExternalId","solutionId","solutionName","updatedAt","instanceParentId","instanceId","instanceParentResponsetype","instanceParentQuestion","questionSequenceByEcm","maxScore","minScore","percentageScore","pointsBasedScoreInParent","totalScore","scoreAchieved","totalpercentage","instanceParentExternalId","instanceParentEcmSequence","remarks","total_evidences","evidence_count","school","block","district","cluster","state","schoolName","blockName","districtName","clusterName","stateName","schoolExternalId","blockExternalId","districtExternalId","clusterExternalId","stateExternalId","schoolTypes","administrationTypes","instanceParentCriteriaId","instanceParentCriteriaExternalId","instanceParentCriteriaName","role_title","location_validated_with_geotag","distance_in_meters","entity","entityExternalId","entityName","isAPrivateProgram","programId","programName","programExternalId","questionResponseLabel_number","criteriaLevel","criteriaScore","submissionNumber","submissionTitle","channel","parent_channel","user_districtName","user_blockName","user_clusterName","appName","evidences","user_stateName","domainName","domainExternalId","childName","childType","childExternalid","level","criteriaDescription","programDescription","solutionDescription","label","imp_project_id","imp_project_title","imp_project_goal","imp_project_externalId","ancestorName","scoringSystem","domainLevel","domainScore","criteriaLevelReport","user_schoolName","user_schoolId","user_schoolUDISE_code","solution_type","organisation_name","user_boardName","district_externalId","state_externalId","block_externalId","cluster_externalId","organisation_id","user_type"],"intervals":["1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00"]}'
+ml_analytics_druid_observation_batch_ingestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/observation/batchDeletion/druidData.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"completedDate","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"completedDate"},{"type":"string","name":"createdAt"},{"type":"string","name":"createdBy"},{"type":"string","name":"criteriaExternalId"},{"type":"string","name":"criteriaId"},{"type":"string","name":"criteriaName"},{"type":"string","name":"entityType"},{"type":"string","name":"entityTypeId"},{"type":"string","name":"observationId"},{"type":"string","name":"observationName"},{"type":"string","name":"observationSubmissionId"},{"type":"string","name":"questionAnswer"},{"type":"string","name":"questionECM"},{"type":"string","name":"questionExternalId"},{"type":"string","name":"questionId"},{"type":"string","name":"questionName"},{"type":"string","name":"questionResponseLabel"},{"type":"string","name":"questionResponseType"},{"type":"string","name":"solutionExternalId"},{"type":"string","name":"solutionId"},{"type":"string","name":"solutionName"},{"type":"string","name":"updatedAt"},{"type":"string","name":"instanceParentId"},{"type":"string","name":"instanceId"},{"type":"string","name":"instanceParentResponsetype"},{"type":"string","name":"instanceParentQuestion"},{"type":"string","name":"questionSequenceByEcm"},{"type":"string","name":"maxScore"},{"type":"string","name":"minScore"},{"type":"string","name":"percentageScore"},{"type":"string","name":"pointsBasedScoreInParent"},{"type":"string","name":"totalScore"},{"type":"string","name":"scoreAchieved"},{"type":"string","name":"totalpercentage"},{"type":"string","name":"instanceParentExternalId"},{"type":"string","name":"instanceParentEcmSequence"},{"type":"string","name":"remarks"},{"type":"string","name":"total_evidences"},{"type":"string","name":"evidence_count"},{"type":"string","name":"school"},{"type":"string","name":"block"},{"type":"string","name":"district"},{"type":"string","name":"cluster"},{"type":"string","name":"state"},{"type":"string","name":"schoolName"},{"type":"string","name":"blockName"},{"type":"string","name":"districtName"},{"type":"string","name":"clusterName"},{"type":"string","name":"stateName"},{"type":"string","name":"schoolExternalId"},{"type":"string","name":"blockExternalId"},{"type":"string","name":"districtExternalId"},{"type":"string","name":"clusterExternalId"},{"type":"string","name":"stateExternalId"},{"type":"string","name":"schoolTypes"},{"type":"string","name":"administrationTypes"},{"type":"string","name":"instanceParentCriteriaId"},{"type":"string","name":"instanceParentCriteriaExternalId"},{"type":"string","name":"instanceParentCriteriaName"},{"type":"string","name":"role_title"},{"type":"string","name":"location_validated_with_geotag"},{"type":"string","name":"distance_in_meters"},{"type":"string","name":"entity"},{"type":"string","name":"entityExternalId"},{"type":"string","name":"entityName"},{"type":"string","name":"isAPrivateProgram"},{"type":"string","name":"programId"},{"type":"string","name":"programName"},{"type":"string","name":"programExternalId"},{"name":"questionResponseLabel_number","type":"float"},{"type":"string","name":"criteriaLevel"},{"type":"string","name":"criteriaScore"},{"type":"string","name":"submissionNumber"},{"type":"string","name":"submissionTitle"},{"type":"string","name":"channel"},{"type":"string","name":"parent_channel"},{"type":"string","name":"user_districtName"},{"type":"string","name":"user_blockName"},{"type":"string","name":"user_clusterName"},{"type":"string","name":"appName"},{"type":"string","name":"evidences"},{"type":"string","name":"user_stateName"},{"type":"string","name":"domainName"},{"type":"string","name":"domainExternalId"},{"type":"string","name":"childName"},{"type":"string","name":"childType"},{"type":"string","name":"childExternalid"},{"type":"string","name":"level"},{"type":"string","name":"criteriaDescription"},{"type":"string","name":"programDescription"},{"type":"string","name":"solutionDescription"},{"type":"string","name":"label"},{"type":"string","name":"imp_project_id"},{"type":"string","name":"imp_project_title"},{"type":"string","name":"imp_project_goal"},{"type":"string","name":"imp_project_externalId"},{"type":"string","name":"ancestorName"},{"type":"string","name":"scoringSystem"},{"type":"string","name":"domainLevel"},{"type":"string","name":"domainScore"},{"name":"criteriaLevelReport","type":"boolean"},{"type":"string","name":"user_schoolName"},{"type":"string","name":"user_schoolId"},{"type":"string","name":"user_schoolUDISE_code"},{"type":"string","name":"solution_type"},{"type":"string","name":"organisation_name"},{"type":"string","name":"user_boardName"},{"type":"string","name":"district_externalId"},{"type":"string","name":"state_externalId"},{"type":"string","name":"block_externalId"},{"type":"string","name":"cluster_externalId"},{"type":"string","name":"organisation_id"},{"type":"string","name":"user_type"},{"type":"string","name":"isSubmissionDeleted"}]},"metricsSpec":[{"type":"floatSum","name":"question_response_number","fieldName":"questionResponseLabel_number"}]}}}'
+ml_analytics_observation_batchupdate_azure_blob_path: "observation/batchDeletion"
+ml_analytics_observation_submission_id_filepath: "{{ WORKDIR }}/ml-analytics-service/observations/submissions.csv"
+ml_analytics_observation_batchupdate_output_dir: "{{ WORKDIR }}/source/observations/"

ansible/roles/ml-analytics-service/tasks/main.yml

@@ -105,6 +105,12 @@
     dest: "{{ config_path }}/config.ini"
     backup: yes
 
+- name: Templating the shell_script_config.j2 to shell_script_config
+  template:
+    src: "shell_script_config.j2"
+    dest: "{{ config_path }}/shell_script_config"
+    backup: yes
+    
 - name: SYSTEMD DAEMON-RELOAD
   systemd:
     daemon_reload: yes

ansible/roles/ml-analytics-service/templates/config.j2

@@ -78,6 +78,8 @@ metadata_rollup_url = http://{{ ml_analytics_druid_rollup_url }}/druid/coordinat
 
 batch_rollup_url = http://{{ ml_analytics_druid_rollup_url }}/druid/indexer/v1/task
 
+urlQuery = http://{{ ml_analytics_druid_query_url }}/druid/v2?pretty
+
 observation_status_injestion_spec = {{ ml_analytics_druid_observation_status_injestion_spec }}
 
 project_injestion_spec = {{ ml_analytics_druid_project_injestion_spec }}
@@ -100,6 +102,10 @@ ml_distinctCnt_prglevel_projects_status_spec = {{ ml_analytics_druid_distinctCnt
 
 survey_status_injestion_spec = {{ ml_analytics_druid_survey_status_injestion_spec }}
 
+observation_query_spec = {{ ml_analytics_druid_observation_query_spec }}
+
+observation_injestion_spec = {{ml_analytics_druid_observation_batch_ingestion_spec}}
+
 [KAFKA]
 
 url = {{ ml_analytics_kafka_url }}
@@ -172,6 +178,16 @@ projects_distinctCnt_prgmlevel_blob_path = {{ml_analytics_projects_distinctCnt_p
 
 survey_blob_path = {{ ml_analytics_survey_azure_blob_path }}
 
+public_account_name = {{ ml_analytics_public_azure_account_name }}
+
+public_access_key = {{ ml_analytics_public_azure_access_key }}
+
+public_container_name = {{ ml_analytics_public_azure_container_name }}
+
+projects_program_csv = {{ ml_analytics_program_dashboard_azure_blob_path }}
+
+observation_batch_ingestion_data_del = {{ ml_analytics_observation_batchupdate_azure_blob_path }}
+
 [REDIS]
 
 host = {{ ml_analytics_redis_host }}
@@ -204,6 +220,10 @@ projects_distinctCount_prgmlevel = {{ml_analytics_projects_distinctCnt_prglevel_
 
 survey_status = {{ ml_analytics_survey_status_output_dir }}
 
+observation_sub_ids = {{ ml_analytics_observation_submission_id_filepath }}
+
+observation_druid_data = {{ ml_analytics_observation_batchupdate_output_dir }}
+
 [CLOUD_STORAGE]
 
 service_name = {{ ml_analytics_AWS_service_name }}
@@ -221,3 +241,9 @@ bucket_name = {{ ml_analytics_AWS_bucket_name }}
 token = {{ ml_analytics_slack_token }}
 
 channel = {{ ml_analytics_channel_name }} 
+
+[VAM]
+
+druid_query_url = {{ ml_druid_query_data }}
+
+program_dashboard_data = {{ ml_program_dashboard_data }}

ansible/roles/ml-analytics-service/templates/shell_script_config.j2

+mongo_url={{ ml_analytics_mongodb_url }}
+mongo_db_name={{ ml_analytics_mongo_db_name }}