Merge pull request #3604 from shikshalokam/release-5.1.0

Added CSP related changes to ML Analytics

ansible/roles/ml-analytics-service/tasks/main.yml

@@ -105,6 +105,10 @@
     dest: "{{ config_path }}/config.ini"
     backup: yes
 
+- name: Copy GCP Secrets to JSON file
+  copy: dest="{{config_path}}/{{ml_GCP_secret_json_file}}" content="{{ ml_GCP_Secrets | to_nice_json}}" mode=0400 owner="{{ USER }}"
+  when: ML_Cloud_Service_Provider == 'gcloud'
+
 - name: Templating the shell_script_config.j2 to shell_script_config
   template:
     src: "shell_script_config.j2"

ansible/roles/ml-analytics-service/templates/config.j2

@@ -6,6 +6,7 @@ integrated_app = {{ ml_analytics_integrated_app_name }}
 
 integrated_portal = {{ ml_analytics_integrated_portal }}
 
+
 [API_HEADERS]
 
 content_type = application/json
@@ -14,6 +15,7 @@ authorization = {{ ml_analytics_api_authorization_key }}
 
 internal_access_token = {{ ml_analytics_api_access_token }}
 
+
 [ML_SURVEY_SERVICE_URL]
 
 url = {{ ml_analytics_survey_service }}
@@ -24,24 +26,15 @@ user_profile_end_point = assessment/api/v1/userExtension/getProfile/
 
 evidence_base_url = {{ ml_analytics_evidence_base_url }}
 
-[MONGO]
-
-# --------------
-# Mongo  url
-#---------------
 
-mongo_url = mongodb://{{ ml_analytics_mongodb_url }}
+[MONGO]
 
-# -----------------------
-# Mongo database name
-# -----------------------
+url = mongodb://{{ ml_analytics_mongodb_url }}
 
 database_name = {{ ml_analytics_mongo_db_name }}
 
-# -------------------
-# Mongo Collections
-# -------------------
 
+# ------ Mongo Collections ------- #
 observation_sub_collection = {{ ml_analytics_mongo_observation_submission_collection }}
 
 solutions_collection = {{ ml_analytics_mongo_solution_collection }}
@@ -68,6 +61,7 @@ survey_submissions_collection = {{ ml_analytics_mongo_survey_submissions_collect
 
 survey_collection = {{ ml_analytics_mongo_survey_collection }}
 
+
 [DRUID]
 
 metadata_url = http://{{ ml_analytics_druid_url }}/druid/coordinator/v1/datasources/
@@ -84,13 +78,15 @@ observation_status_injestion_spec = {{ ml_analytics_druid_observation_status_inj
 
 project_injestion_spec = {{ ml_analytics_druid_project_injestion_spec }}
 
-ml_distinctCnt_obs_status_spec = {{ ml_analytics_druid_distinctCnt_obs_injestion_spec }}	
+ml_distinctCnt_obs_status_spec = {{ ml_analytics_druid_distinctCnt_obs_injestion_spec }}
 
-ml_distinctCnt_obs_domain_spec = {{ ml_analytics_druid_distinctCnt_obs_domain_injestion_spec }}	
+ml_distinctCnt_obs_domain_spec = {{ ml_analytics_druid_distinctCnt_obs_domain_injestion_spec }}
 
-ml_distinctCnt_obs_domain_criteria_spec = {{ ml_analytics_druid_distinctCnt_obs_domain_criteria_injestion_spec }}	
+ml_distinctCnt_obs_domain_criteria_spec = {{ ml_analytics_druid_distinctCnt_obs_domain_criteria_injestion_spec }}
 
-ml_distinctCnt_projects_status_spec = {{ ml_analytics_druid_distinctCnt_projects_status_injestion_spec }}	
+ml_distinctCnt_projects_status_spec = {{ ml_analytics_druid_distinctCnt_projects_status_injestion_spec }}
+
+ml_distinctCnt_prglevel_projects_status_spec = {{ ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec }}
 
 observation_status_rollup_injestion_spec = {{ ml_analytics_druid_observation_status_rollup_injestion_spec }}
 
@@ -98,14 +94,19 @@ project_rollup_injestion_spec = {{ ml_analytics_druid_project_rollup_injestion_s
 
 ml_survey_rollup_spec = {{ml_analytics_druid_survey_rollup_injestion_spec}}
 
-ml_distinctCnt_prglevel_projects_status_spec = {{ ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec}}
-
 survey_status_injestion_spec = {{ ml_analytics_druid_survey_status_injestion_spec }}
 
 observation_query_spec = {{ ml_analytics_druid_observation_query_spec }}
 
 observation_injestion_spec = {{ml_analytics_druid_observation_batch_ingestion_spec}}
 
+survey_query_spec = {{ml_analytics_druid_survey_query_spec}}
+
+survey_injestion_spec = {{ml_analytics_druid_survey_batch_ingestion_spec}}
+
+intervals = {{ml_analytics_druid_interval_list}}
+
+
 [KAFKA]
 
 url = {{ ml_analytics_kafka_url }}
@@ -122,6 +123,7 @@ survey_raw_topic = {{ ml_analytics_kafka_survey_topic_name }}
 
 survey_druid_topic = {{ ml_analytics_kafka_survey_druid_topic_name }}
 
+
 [LOGS]
 
 observation_streaming_success = {{ ml_analytics_observation_log_folder_path }}/success.log
@@ -148,53 +150,54 @@ survey_streaming_success = {{ ml_analytics_survey_log_folder_path }}/success.log
 
 survey_streaming_error = {{ ml_analytics_survey_log_folder_path }}/error.log
 
-[AZURE]
+{% if ML_Cloud_Service_Provider is eq 'ORACLE' %}
 
-account_name = {{ ml_analytics_azure_account_name }}
+[ORACLE]
 
-sas_token = {{ ml_analytics_azure_sas_token }}
+endpoint_url = {{ ml_ORACLE_endpoint_url }}
 
-container_name = {{ ml_analytics_azure_container_name }}
+access_key = {{ ml_ORACLE_access_key }}
 
-observation_blob_path = {{ ml_analytics_observation_azure_blob_path }}
+secret_access_key = {{ ml_ORACLE_secret_access_key }}
 
-projects_blob_path = {{ ml_analytics_project_azure_blob_path }}
+region_name = {{ ml_ORACLE_region_name }}
 
-observation_distinctCount_blob_path = {{ ml_analytics_obs_distinctCnt_azure_blob_path }}	
+bucket_name = {{ ml_ORACLE_bucket_name }}
 
-observation_distinctCount_domain_blob_path = {{ ml_analytics_obs_distinctCnt_domain_azure_blob_path }}	
+{% elif ML_Cloud_Service_Provider is eq 'gcloud' %}
 
-observation_distinctCount_domain_criteria_blob_path = {{ ml_analytics_obs_distinctCnt_domain_criteria_azure_blob_path }}	
+[GCP]
 
-projects_distinctCnt_blob_path = {{ ml_analytics_projects_distinctCnt_azure_blob_path }}
+secret_data = {{ ml_GCP_secret_json_file }}
 
-projects_rollup_blob_path = {{ ml_analytics_project_rollup_azure_blob_path }}
+bucket_name = {{ ml_GCP_bucket_name }}
 
-observation_rollup_blob_path = {{ ml_analytics_observation_rollup_azure_blob_path }}
+{% elif ML_Cloud_Service_Provider is eq 'aws' %}
 
-survey_rollup_blob_path = {{ ml_analytics_survey_rollup_azure_blob_path }}
+[AWS]
 
-projects_distinctCnt_prgmlevel_blob_path = {{ml_analytics_projects_distinctCnt_prglevel_azure_blob_path}}
+service_name = {{ ml_AWS_service_name }}
 
-survey_blob_path = {{ ml_analytics_survey_azure_blob_path }}
+access_key = {{ ml_AWS_access_key }}
 
-public_account_name = {{ ml_analytics_public_azure_account_name }}
+secret_access_key = {{ ml_AWS_secret_access_key }}
 
-public_access_key = {{ ml_analytics_public_azure_access_key }}
+region_name = {{ ml_AWS_region_name }}
 
-public_container_name = {{ ml_analytics_public_azure_container_name }}
+bucket_name = {{ ml_AWS_bucket_name }}
 
-projects_program_csv = {{ ml_analytics_program_dashboard_azure_blob_path }}
+{% else %}
 
-observation_batch_ingestion_data_del = {{ ml_analytics_observation_batchupdate_azure_blob_path }}
+[AZURE]
+
+account_name = {{ ml_analytics_azure_account_name }}
 
-[REDIS]
+sas_token = {{ ml_analytics_azure_sas_token }}
 
-host = {{ ml_analytics_redis_host }}
+container_name = {{ ml_analytics_azure_container_name }}
 
-port = {{ ml_analytics_redis_port }}
+account_key = {{ ml_analytics_azure_account_key }}
 
-db_name = {{ ml_analytics_redis_db_name }}
 
 [OUTPUT_DIR]
 
@@ -202,48 +205,77 @@ project = {{ ml_analytics_project_output_dir }}
 
 observation_status = {{ ml_analytics_observation_status_output_dir }}
 
-observation_distinctCount_status = {{ ml_analytics_obs_distinctCnt_output_dir }}	
+observation_distinctCount_status = {{ ml_analytics_obs_distinctCnt_output_dir }}
 
-observation_distinctCount_domain = {{ ml_analytics_obs_distinctCnt_domain_output_dir }}	
+observation_distinctCount_domain = {{ ml_analytics_obs_distinctCnt_domain_output_dir }}
 
-observation_distinctCount_domain_criteria = {{ ml_analytics_obs_distinctCnt_domain_criteria_output_dir }}	
+observation_distinctCount_domain_criteria = {{ ml_analytics_obs_distinctCnt_domain_criteria_output_dir }}
 
 projects_distinctCount = {{ ml_analytics_projects_distinctCnt_output_dir }}
 
+projects_distinctCount_prgmlevel = {{ ml_analytics_projects_distinctCnt_prglevel_output_dir }}
+
 project_rollup = {{ ml_analytics_project_rollup_output_dir }}
 
 observation_status_rollup = {{ ml_analytics_observation_status_rollup_output_dir }}
 
 survey_rollup = {{ ml_analytics_survey_rollup_output_dir }}
 
-projects_distinctCount_prgmlevel = {{ml_analytics_projects_distinctCnt_prglevel_output_dir}}
-
 survey_status = {{ ml_analytics_survey_status_output_dir }}
 
 observation_sub_ids = {{ ml_analytics_observation_submission_id_filepath }}
 
 observation_druid_data = {{ ml_analytics_observation_batchupdate_output_dir }}
 
-[CLOUD_STORAGE]
-
-service_name = {{ ml_analytics_AWS_service_name }}
-
-access_key = {{ ml_analytics_AWS_access_key }}
-
-secret_access_key = {{ ml_analytics_AWS_secret_access_key }}
+survey_sub_ids = {{ml_analytics_survey_submission_id_filepath}}
 
-region_name = {{ ml_analytics_AWS_region_name }}
+survey_druid_data = {{ml_analytics_survey_batchupdate_output_dir}}
 
-bucket_name = {{ ml_analytics_AWS_bucket_name }}
 
 [SLACK]
 
-token = {{ ml_analytics_slack_token }}
+token = {{ml_slack_token}}
+
+channel = {{ml_slack_channel}}
 
-channel = {{ ml_analytics_channel_name }} 
 
 [VAM]
 
 druid_query_url = {{ ml_druid_query_data }}
 
 program_dashboard_data = {{ ml_program_dashboard_data }}
+
+
+[COMMON]
+
+cloud_module_path = {{ ml_analytics_cloud_package_path }}
+
+observation_blob_path = {{ ml_analytics_observation_azure_blob_path }}
+
+projects_blob_path = {{ ml_analytics_project_azure_blob_path }}
+
+observation_distinctCount_blob_path = {{ ml_analytics_obs_distinctCnt_azure_blob_path }}
+
+observation_distinctCount_domain_blob_path = {{ ml_analytics_obs_distinctCnt_domain_azure_blob_path }}
+
+observation_distinctCount_domain_criteria_blob_path = {{ ml_analytics_obs_distinctCnt_domain_criteria_azure_blob_path }}
+
+projects_distinctCnt_blob_path = {{ ml_analytics_projects_distinctCnt_azure_blob_path }}
+
+projects_distinctCnt_prgmlevel_blob_path = {{ ml_analytics_projects_distinctCnt_prglevel_azure_blob_path }}
+
+projects_rollup_blob_path = {{ ml_analytics_project_rollup_azure_blob_path }}
+
+observation_rollup_blob_path = {{ ml_analytics_observation_rollup_azure_blob_path }}
+
+survey_rollup_blob_path = {{ ml_analytics_survey_rollup_azure_blob_path }}
+
+survey_blob_path = {{ ml_analytics_survey_azure_blob_path }}
+
+projects_program_csv = {{ ml_analytics_program_dashboard_azure_blob_path }}
+
+observation_batch_ingestion_data_del = {{ ml_analytics_observation_batchupdate_azure_blob_path }}
+
+survey_batch_ingestion_data_del = {{ ml_analytics_survey_batchupdate_azure_blob_path}}
+
+cname_url = {{ ml_analytics_cname_url }}