diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index fd5e62c5f44ec6fe2fe78810a78324999da8a47a..fd73a696735b45f8e02fac15672b1af44f7d509c 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -44,8 +44,8 @@ ml_analytics_kafka_survey_druid_topic_name: "{{ env_name }}.ml.survey.druid" ml_analytics_observation_log_folder_path: "{{ WORKDIR }}/logs/observation" ml_analytics_project_log_folder_path: "{{ WORKDIR }}/logs/project" ml_analytics_survey_log_folder_path: "{{ WORKDIR }}/logs/survey" -ml_analytics_azure_account_name: "{{ sunbird_private_storage_account_name }}" -ml_analytics_azure_container_name: "telemetry-data-store" +ml_analytics_azure_account_name: "{{ cloud_private_storage_accountname }}" +ml_analytics_azure_container_name: "{{ cloud_storage_telemetry_bucketname }}" ml_analytics_observation_azure_blob_path: "observation/status/" ml_analytics_project_azure_blob_path: "projects/" ml_analytics_redis_host: "{{ml_redis_host | default(groups['dp-redis'][0])}}" @@ -57,7 +57,7 @@ ml_analytics_api_authorization_key: "{{ml_api_auth_token | default('sunbird_api_ ml_analytics_api_access_token: "{{ml_api_access_token | default('ml_core_internal_access_token')}}" ml_analytics_druid_observation_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/observation/status/sl_observation_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_code","school_externalId","state_name","state_code","state_externalId","district_name","district_code","district_externalId","block_name","block_code","block_externalId","cluster_name","cluster_code","cluster_externalId","completedDate","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name","ecm_marked_na","board_name","updatedAt","organisation_id","user_type","observed_school_name","observed_school_id","observed_school_code","observed_state_name","observed_state_id","observed_state_code","observed_district_name","observed_district_id","observed_district_code","observed_block_name","observed_block_id","observed_block_code","observed_cluster_name","observed_cluster_id","observed_cluster_code"]},"metricsSpec":[]}}}' ml_analytics_druid_project_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/projects/sl_projects.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-project","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"auto"},"dimensionsSpec":{"dimensions":[]},"metricsSpec":[]}}}' -ml_analytics_azure_sas_token: "{{ sunbird_private_storage_account_key }}" +ml_analytics_azure_sas_token: "{{ cloud_private_storage_secret }}" ml_analytics_druid_distinctCnt_obs_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount/ml_observation_distinctCount_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' ml_analytics_druid_distinctCnt_obs_domain_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount_domain/ml_observation_distinctCount_domain.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' ml_analytics_druid_distinctCnt_obs_domain_criteria_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount_domain_criteria/ml_observation_distinctCount_domain_criteria.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain-criteria","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"string","name":"criteria_name"},{"type":"string","name":"criteria_score"},{"type":"string","name":"criteria_id"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' @@ -80,11 +80,10 @@ ml_analytics_observation_status_rollup_output_dir: "/opt/sparkjobs/source/observ ml_analytics_druid_project_rollup_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/projects/rollup/projects_rollup.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"iso"},"dimensionsSpec":{"dimensions":["project_title","project_goal","area_of_improvement","status_of_project","tasks_name","tasks_status","designation","task_evidence_status","project_id","task_id","project_created_type","parent_channel","program_id","program_name","project_updated_date","createdBy","program_externalId","private_program","task_deleted_flag","project_terms_and_condition","state_externalId","block_externalId","district_externalId","cluster_externalId","school_externalId","state_name","block_name","district_name","cluster_name","school_name","board_name","organisation_name","solution_id","organisation_id",{"name":"status_code","type":"long"}]},"metricsSpec":[{"name":"count","type":"count"},{"name":"sum___v","type":"longSum","fieldName":"__v"},{"name":"sum_status_code","type":"longMax","fieldName":"status_code"},{"type":"HLLSketchBuild","name":"count_of_createBy","fieldName":"createdBy"},{"type":"HLLSketchBuild","name":"count_of_project_id","fieldName":"project_id"},{"type":"HLLSketchBuild","name":"count_of_solution_id","fieldName":"solution_id"},{"type":"HLLSketchBuild","name":"count_of_program_id","fieldName":"program_id"}]}}}' ml_analytics_druid_observation_status_rollup_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/rollup/observation_status_rollup.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-observation-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["status","user_id","solution_id","submission_id","entity_name","completedDate","program_id","private_program","solution_type","updatedAt","role_title","solution_name","program_name","channel","parent_channel","block_name","district_name","school_name","cluster_name","state_name","organisation_name","board_name","district_externalId","state_externalId","block_externalId","cluster_externalId","school_externalId","organisation_id",{"type":"long","name":"status_code"}]},"metricsSpec":[{"type":"count","name":"count"},{"type":"longSum","name":"sum___v","fieldName":"__v","expression":null},{"type":"HLLSketchBuild","name":"count_distinct_solution","fieldName":"solution_id","lgK":12,"tgtHllType":"HLL_4","round":false},{"type":"HLLSketchBuild","name":"count_distinct_submission_id","fieldName":"submission_id","lgK":12,"tgtHllType":"HLL_4","round":false},{"type":"HLLSketchBuild","name":"count_distinct_user_id","fieldName":"user_id","lgK":12,"tgtHllType":"HLL_4","round":false}]}}}' ml_analytics_druid_rollup_url: "{{groups['druid'][0]}}:8081" -ml_analytics_AWS_service_name: "{{ ml_AWS_service_name | default('') }}" -ml_analytics_AWS_access_key: "{{ ml_AWS_access_key | default('') }}" -ml_analytics_AWS_secret_access_key: "{{ ml_AWS_secret_access_key | default('') }}" -ml_analytics_AWS_region_name: "{{ ml_AWS_region_name | default('') }}" -ml_analytics_AWS_bucket_name: "{{ ml_AWS_bucket_name | default('') }}" +ml_analytics_AWS_access_key: "{{ cloud_private_storage_accountname }}" +ml_analytics_AWS_secret_access_key: "{{ cloud_private_storage_secret }}" +ml_analytics_AWS_region_name: "{{ cloud_private_storage_region }}" +ml_analytics_AWS_bucket_name: "{{ cloud_storage_telemetry_bucketname }}" ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/projects/distinctCountPrglevel/ml_projects_distinctCount_prgmlevel.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-programLevel-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' ml_analytics_projects_distinctCnt_prglevel_output_dir: "{{ WORKDIR }}/source/projects/distinctCountPrglevel/output" ml_analytics_projects_distinctCnt_prglevel_azure_blob_path: "projects/distinctCountPrglevel/" @@ -93,9 +92,6 @@ ml_analytics_survey_azure_blob_path : "survey/status/" ml_analytics_druid_survey_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/survey/status/sl_survey_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-survey-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":true,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["survey_submission_id", "submission_status", "user_id", "user_sub_type", "user_type", "state_externalId", "block_externalId", "district_externalId", "cluster_externalId", "school_externalId", "state_name", "block_name", "district_name", "cluster_name", "school_name", "board_name", "organisation_id", "organisation_name", "program_externalId", "program_id", "program_name", "survey_name", "survey_id", "survey_externalId", "created_date", "submission_date", "updatedAt", "parent_channel", "solution_name", "solution_id","private_program"]},"metricsSpec":[]}}}' ml_analytics_slack_token: "{{ ml_slack_token | default('') }}" ml_analytics_channel_name: "{{ ml_slack_channel | default('') }}" -ml_analytics_public_azure_account_name: "{{ ml_public_azure_account | default('') }}" -ml_analytics_public_azure_access_key: "{{ ml_public_azure_key | default('') }}" -ml_analytics_public_azure_container_name: "{{ ml_public_azure_container | default('') }}" ml_analytics_program_dashboard_azure_blob_path: "{{ ml_program_blob_path | default('') }}" ml_druid_query_data: "{{ ml_druid_query | default('') }}" ml_program_dashboard_data: "{{ ml_program_data | default('') }}" @@ -105,3 +101,18 @@ ml_analytics_druid_observation_batch_ingestion_spec: '{"type":"index","spec":{"i ml_analytics_observation_batchupdate_azure_blob_path: "observation/batchDeletion" ml_analytics_observation_submission_id_filepath: "{{ WORKDIR }}/ml-analytics-service/observations/submissions.csv" ml_analytics_observation_batchupdate_output_dir: "{{ WORKDIR }}/source/observations/" +ml_analytics_druid_survey_query_spec : '{"queryType":"scan","dataSource":"sl-survey","resultFormat":"list","columns":["completedDate","createdAt","createdBy","criteriaExternalId","criteriaId","criteriaName","surveyId","surveyName","surveySubmissionId","questionAnswer","questionECM","questionExternalId","questionId","questionName","questionResponseLabel","questionResponseType","solutionExternalId","solutionId","solutionName","updatedAt","instanceParentId","instanceId","instanceParentResponsetype","instanceParentQuestion","questionSequenceByEcm","maxScore","minScore","percentageScore","pointsBasedScoreInParent","totalScore","scoreAchieved","totalpercentage","instanceParentExternalId","instanceParentEcmSequence","remarks","total_evidences","evidence_count","instanceParentCriteriaId","instanceParentCriteriaExternalId","instanceParentCriteriaName","isAPrivateProgram","programId","programName","programExternalId","questionResponseLabel_number","channel","parent_channel","appName","organisation_name","user_subtype","user_type","board_name","district_code","district_name","district_externalId","block_code","block_name","block_externalId","school_code","school_name","school_externalId","cluster_code","cluster_name","cluster_externalId","state_code","state_name","state_externalId","organisation_id","evidences"],"intervals":["1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00"]}' +ml_analytics_druid_survey_batch_ingestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris": ["azure://telemetry-data-store/survey/batchDeletion/druidData.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-survey","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"completedDate","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"completedDate"},{"type":"string","name":"createdAt"},{"type":"string","name":"createdBy"},{"type":"string","name":"criteriaExternalId"},{"type":"string","name":"criteriaId"},{"type":"string","name":"criteriaName"},{"type":"string","name":"surveyId"},{"type":"string","name":"surveyName"},{"type":"string","name":"surveySubmissionId"},{"type":"string","name":"questionAnswer"},{"type":"string","name":"questionECM"},{"type":"string","name":"questionExternalId"},{"type":"string","name":"questionId"},{"type":"string","name":"questionName"},{"type":"string","name":"questionResponseLabel"},{"type":"string","name":"questionResponseType"},{"type":"string","name":"solutionExternalId"},{"type":"string","name":"solutionId"},{"type":"string","name":"solutionName"},{"type":"string","name":"updatedAt"},{"type":"string","name":"instanceParentId"},{"type":"string","name":"instanceId"},{"type":"string","name":"instanceParentResponsetype"},{"type":"string","name":"instanceParentQuestion"},{"type":"string","name":"questionSequenceByEcm"},{"type":"string","name":"maxScore"},{"type":"string","name":"minScore"},{"type":"string","name":"percentageScore"},{"type":"string","name":"pointsBasedScoreInParent"},{"type":"string","name":"totalScore"},{"type":"string","name":"scoreAchieved"},{"type":"string","name":"totalpercentage"},{"type":"string","name":"instanceParentExternalId"},{"type":"string","name":"instanceParentEcmSequence"},{"type":"string","name":"remarks"},{"type":"string","name":"total_evidences"},{"type":"string","name":"evidence_count"},{"type":"string","name":"evidences"},{"type":"string","name":"instanceParentCriteriaId"},{"type":"string","name":"instanceParentCriteriaExternalId"},{"type":"string","name":"instanceParentCriteriaName"},{"type":"string","name":"isAPrivateProgram"},{"type":"string","name":"programId"},{"type":"string","name":"programName"},{"type":"string","name":"programExternalId"},{"name":"questionResponseLabel_number","type":"float"},{"type":"string","name":"channel"},{"type":"string","name":"parent_channel"},{"type":"string","name":"appName"},{"type":"string","name":"organisation_name"},{"type":"string","name":"user_subtype"},{"type":"string","name":"user_type"},{"type":"string","name":"board_name"},{"type":"string","name":"district_code"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_code"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"school_code"},{"type":"string","name":"school_name"},{"type":"string","name":"school_externalId"},{"type":"string","name":"cluster_code"},{"type":"string","name":"cluster_name"},{"type":"string","name":"cluster_externalId"},{"type":"string","name":"state_code"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"organisation_id"},{"type":"string","name":"isSubmissionDeleted"}]},"metricsSpec":[{"type":"floatSum","name":"question_response_number","fieldName":"questionResponseLabel_number"}]}}}' +ml_analytics_survey_batchupdate_azure_blob_path : "survey/batchDeletion" +ml_analytics_survey_submission_id_filepath : "{{ WORKDIR }}/ml-analytics-service/survey/submissions.csv" +ml_analytics_survey_batchupdate_output_dir : "{{ WORKDIR }}/source/survey/" +ml_analytics_druid_interval_list: '["1901-01-01T00:00:00+00:00/2020-01-01T00:00:00+00:00","2020-01-01T00:00:00+00:00/2020-06-01T00:00:00+00:00","2020-06-01T00:00:00+00:00/2021-01-01T00:00:00+00:00","2021-01-01T00:00:00+00:00/2021-06-01T00:00:00+00:00","2021-06-01T00:00:00+00:00/2022-01-01T00:00:00+00:00","2022-01-01T00:00:00+00:00/2022-03-01T00:00:00+00:00","2022-03-01T00:00:00+00:00/2022-06-01T00:00:00+00:00","2022-06-01T00:00:00+00:00/2022-09-01T00:00:00+00:00","2022-09-01T00:00:00+00:00/2023-01-01T00:00:00+00:00"]' +ml_analytics_azure_account_key: "{{ cloud_private_storage_secret }}" +ML_Cloud_Service_Provider: "{{ cloud_service_provider | default('azure') }}" ## Valid options - ORACLE, gcloud, aws & azure +ml_analytics_cloud_package_path: "{{ config_path }}/cloud_storage" +ml_analytics_cname_url: "https://{{ cloud_private_storage_accountname }}.blob.core.windows.net/{{ cloud_storage_telemetry_bucketname }}" +ml_GCP_secret_json_file: gcp_secrets.json +ml_GCP_bucket_name: {{ cloud_storage_telemetry_bucketname }} +ml_GCP_Secrets: + account_name: {{ cloud_private_storage_accountname }} + account_key: {{ cloud_private_storage_secret }} diff --git a/ansible/roles/ml-analytics-service/tasks/main.yml b/ansible/roles/ml-analytics-service/tasks/main.yml index 0998cf8188bde33cea8c77f474b6a2bd4ed5e692..dfa015c99cb9ca8146990c61386dd63553b57815 100755 --- a/ansible/roles/ml-analytics-service/tasks/main.yml +++ b/ansible/roles/ml-analytics-service/tasks/main.yml @@ -105,6 +105,10 @@ dest: "{{ config_path }}/config.ini" backup: yes +- name: Copy GCP Secrets to JSON file + copy: dest="{{config_path}}/{{ml_GCP_secret_json_file}}" content="{{ ml_GCP_Secrets | to_nice_json}}" mode=0400 owner="{{ USER }}" + when: ML_Cloud_Service_Provider == 'gcloud' + - name: Templating the shell_script_config.j2 to shell_script_config template: src: "shell_script_config.j2" diff --git a/ansible/roles/ml-analytics-service/templates/config.j2 b/ansible/roles/ml-analytics-service/templates/config.j2 index b4945675d3c632ed4eaf1a5ba5126c591a8cb706..770de394cb87fc77b8fa940bf075fbb53f6f079b 100644 --- a/ansible/roles/ml-analytics-service/templates/config.j2 +++ b/ansible/roles/ml-analytics-service/templates/config.j2 @@ -6,6 +6,7 @@ integrated_app = {{ ml_analytics_integrated_app_name }} integrated_portal = {{ ml_analytics_integrated_portal }} + [API_HEADERS] content_type = application/json @@ -14,6 +15,7 @@ authorization = {{ ml_analytics_api_authorization_key }} internal_access_token = {{ ml_analytics_api_access_token }} + [ML_SURVEY_SERVICE_URL] url = {{ ml_analytics_survey_service }} @@ -24,24 +26,15 @@ user_profile_end_point = assessment/api/v1/userExtension/getProfile/ evidence_base_url = {{ ml_analytics_evidence_base_url }} -[MONGO] - -# -------------- -# Mongo url -#--------------- -mongo_url = mongodb://{{ ml_analytics_mongodb_url }} +[MONGO] -# ----------------------- -# Mongo database name -# ----------------------- +url = mongodb://{{ ml_analytics_mongodb_url }} database_name = {{ ml_analytics_mongo_db_name }} -# ------------------- -# Mongo Collections -# ------------------- +# ------ Mongo Collections ------- # observation_sub_collection = {{ ml_analytics_mongo_observation_submission_collection }} solutions_collection = {{ ml_analytics_mongo_solution_collection }} @@ -68,6 +61,7 @@ survey_submissions_collection = {{ ml_analytics_mongo_survey_submissions_collect survey_collection = {{ ml_analytics_mongo_survey_collection }} + [DRUID] metadata_url = http://{{ ml_analytics_druid_url }}/druid/coordinator/v1/datasources/ @@ -84,13 +78,15 @@ observation_status_injestion_spec = {{ ml_analytics_druid_observation_status_inj project_injestion_spec = {{ ml_analytics_druid_project_injestion_spec }} -ml_distinctCnt_obs_status_spec = {{ ml_analytics_druid_distinctCnt_obs_injestion_spec }} +ml_distinctCnt_obs_status_spec = {{ ml_analytics_druid_distinctCnt_obs_injestion_spec }} -ml_distinctCnt_obs_domain_spec = {{ ml_analytics_druid_distinctCnt_obs_domain_injestion_spec }} +ml_distinctCnt_obs_domain_spec = {{ ml_analytics_druid_distinctCnt_obs_domain_injestion_spec }} -ml_distinctCnt_obs_domain_criteria_spec = {{ ml_analytics_druid_distinctCnt_obs_domain_criteria_injestion_spec }} +ml_distinctCnt_obs_domain_criteria_spec = {{ ml_analytics_druid_distinctCnt_obs_domain_criteria_injestion_spec }} -ml_distinctCnt_projects_status_spec = {{ ml_analytics_druid_distinctCnt_projects_status_injestion_spec }} +ml_distinctCnt_projects_status_spec = {{ ml_analytics_druid_distinctCnt_projects_status_injestion_spec }} + +ml_distinctCnt_prglevel_projects_status_spec = {{ ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec }} observation_status_rollup_injestion_spec = {{ ml_analytics_druid_observation_status_rollup_injestion_spec }} @@ -98,14 +94,19 @@ project_rollup_injestion_spec = {{ ml_analytics_druid_project_rollup_injestion_s ml_survey_rollup_spec = {{ml_analytics_druid_survey_rollup_injestion_spec}} -ml_distinctCnt_prglevel_projects_status_spec = {{ ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec}} - survey_status_injestion_spec = {{ ml_analytics_druid_survey_status_injestion_spec }} observation_query_spec = {{ ml_analytics_druid_observation_query_spec }} observation_injestion_spec = {{ml_analytics_druid_observation_batch_ingestion_spec}} +survey_query_spec = {{ml_analytics_druid_survey_query_spec}} + +survey_injestion_spec = {{ml_analytics_druid_survey_batch_ingestion_spec}} + +intervals = {{ml_analytics_druid_interval_list}} + + [KAFKA] url = {{ ml_analytics_kafka_url }} @@ -122,6 +123,7 @@ survey_raw_topic = {{ ml_analytics_kafka_survey_topic_name }} survey_druid_topic = {{ ml_analytics_kafka_survey_druid_topic_name }} + [LOGS] observation_streaming_success = {{ ml_analytics_observation_log_folder_path }}/success.log @@ -148,53 +150,54 @@ survey_streaming_success = {{ ml_analytics_survey_log_folder_path }}/success.log survey_streaming_error = {{ ml_analytics_survey_log_folder_path }}/error.log -[AZURE] +{% if ML_Cloud_Service_Provider is eq 'ORACLE' %} -account_name = {{ ml_analytics_azure_account_name }} +[ORACLE] -sas_token = {{ ml_analytics_azure_sas_token }} +endpoint_url = {{ ml_ORACLE_endpoint_url }} -container_name = {{ ml_analytics_azure_container_name }} +access_key = {{ ml_ORACLE_access_key }} -observation_blob_path = {{ ml_analytics_observation_azure_blob_path }} +secret_access_key = {{ ml_ORACLE_secret_access_key }} -projects_blob_path = {{ ml_analytics_project_azure_blob_path }} +region_name = {{ ml_ORACLE_region_name }} -observation_distinctCount_blob_path = {{ ml_analytics_obs_distinctCnt_azure_blob_path }} +bucket_name = {{ ml_ORACLE_bucket_name }} -observation_distinctCount_domain_blob_path = {{ ml_analytics_obs_distinctCnt_domain_azure_blob_path }} +{% elif ML_Cloud_Service_Provider is eq 'gcloud' %} -observation_distinctCount_domain_criteria_blob_path = {{ ml_analytics_obs_distinctCnt_domain_criteria_azure_blob_path }} +[GCP] -projects_distinctCnt_blob_path = {{ ml_analytics_projects_distinctCnt_azure_blob_path }} +secret_data = {{ ml_GCP_secret_json_file }} -projects_rollup_blob_path = {{ ml_analytics_project_rollup_azure_blob_path }} +bucket_name = {{ ml_GCP_bucket_name }} -observation_rollup_blob_path = {{ ml_analytics_observation_rollup_azure_blob_path }} +{% elif ML_Cloud_Service_Provider is eq 'aws' %} -survey_rollup_blob_path = {{ ml_analytics_survey_rollup_azure_blob_path }} +[AWS] -projects_distinctCnt_prgmlevel_blob_path = {{ml_analytics_projects_distinctCnt_prglevel_azure_blob_path}} +service_name = {{ ml_AWS_service_name }} -survey_blob_path = {{ ml_analytics_survey_azure_blob_path }} +access_key = {{ ml_AWS_access_key }} -public_account_name = {{ ml_analytics_public_azure_account_name }} +secret_access_key = {{ ml_AWS_secret_access_key }} -public_access_key = {{ ml_analytics_public_azure_access_key }} +region_name = {{ ml_AWS_region_name }} -public_container_name = {{ ml_analytics_public_azure_container_name }} +bucket_name = {{ ml_AWS_bucket_name }} -projects_program_csv = {{ ml_analytics_program_dashboard_azure_blob_path }} +{% else %} -observation_batch_ingestion_data_del = {{ ml_analytics_observation_batchupdate_azure_blob_path }} +[AZURE] + +account_name = {{ ml_analytics_azure_account_name }} -[REDIS] +sas_token = {{ ml_analytics_azure_sas_token }} -host = {{ ml_analytics_redis_host }} +container_name = {{ ml_analytics_azure_container_name }} -port = {{ ml_analytics_redis_port }} +account_key = {{ ml_analytics_azure_account_key }} -db_name = {{ ml_analytics_redis_db_name }} [OUTPUT_DIR] @@ -202,48 +205,77 @@ project = {{ ml_analytics_project_output_dir }} observation_status = {{ ml_analytics_observation_status_output_dir }} -observation_distinctCount_status = {{ ml_analytics_obs_distinctCnt_output_dir }} +observation_distinctCount_status = {{ ml_analytics_obs_distinctCnt_output_dir }} -observation_distinctCount_domain = {{ ml_analytics_obs_distinctCnt_domain_output_dir }} +observation_distinctCount_domain = {{ ml_analytics_obs_distinctCnt_domain_output_dir }} -observation_distinctCount_domain_criteria = {{ ml_analytics_obs_distinctCnt_domain_criteria_output_dir }} +observation_distinctCount_domain_criteria = {{ ml_analytics_obs_distinctCnt_domain_criteria_output_dir }} projects_distinctCount = {{ ml_analytics_projects_distinctCnt_output_dir }} +projects_distinctCount_prgmlevel = {{ ml_analytics_projects_distinctCnt_prglevel_output_dir }} + project_rollup = {{ ml_analytics_project_rollup_output_dir }} observation_status_rollup = {{ ml_analytics_observation_status_rollup_output_dir }} survey_rollup = {{ ml_analytics_survey_rollup_output_dir }} -projects_distinctCount_prgmlevel = {{ml_analytics_projects_distinctCnt_prglevel_output_dir}} - survey_status = {{ ml_analytics_survey_status_output_dir }} observation_sub_ids = {{ ml_analytics_observation_submission_id_filepath }} observation_druid_data = {{ ml_analytics_observation_batchupdate_output_dir }} -[CLOUD_STORAGE] - -service_name = {{ ml_analytics_AWS_service_name }} - -access_key = {{ ml_analytics_AWS_access_key }} - -secret_access_key = {{ ml_analytics_AWS_secret_access_key }} +survey_sub_ids = {{ml_analytics_survey_submission_id_filepath}} -region_name = {{ ml_analytics_AWS_region_name }} +survey_druid_data = {{ml_analytics_survey_batchupdate_output_dir}} -bucket_name = {{ ml_analytics_AWS_bucket_name }} [SLACK] -token = {{ ml_analytics_slack_token }} +token = {{ml_slack_token}} + +channel = {{ml_slack_channel}} -channel = {{ ml_analytics_channel_name }} [VAM] druid_query_url = {{ ml_druid_query_data }} program_dashboard_data = {{ ml_program_dashboard_data }} + + +[COMMON] + +cloud_module_path = {{ ml_analytics_cloud_package_path }} + +observation_blob_path = {{ ml_analytics_observation_azure_blob_path }} + +projects_blob_path = {{ ml_analytics_project_azure_blob_path }} + +observation_distinctCount_blob_path = {{ ml_analytics_obs_distinctCnt_azure_blob_path }} + +observation_distinctCount_domain_blob_path = {{ ml_analytics_obs_distinctCnt_domain_azure_blob_path }} + +observation_distinctCount_domain_criteria_blob_path = {{ ml_analytics_obs_distinctCnt_domain_criteria_azure_blob_path }} + +projects_distinctCnt_blob_path = {{ ml_analytics_projects_distinctCnt_azure_blob_path }} + +projects_distinctCnt_prgmlevel_blob_path = {{ ml_analytics_projects_distinctCnt_prglevel_azure_blob_path }} + +projects_rollup_blob_path = {{ ml_analytics_project_rollup_azure_blob_path }} + +observation_rollup_blob_path = {{ ml_analytics_observation_rollup_azure_blob_path }} + +survey_rollup_blob_path = {{ ml_analytics_survey_rollup_azure_blob_path }} + +survey_blob_path = {{ ml_analytics_survey_azure_blob_path }} + +projects_program_csv = {{ ml_analytics_program_dashboard_azure_blob_path }} + +observation_batch_ingestion_data_del = {{ ml_analytics_observation_batchupdate_azure_blob_path }} + +survey_batch_ingestion_data_del = {{ ml_analytics_survey_batchupdate_azure_blob_path}} + +cname_url = {{ ml_analytics_cname_url }}