diff --git a/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js b/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js
index fa1a23a3de9bb45124c096826bc11d07d1607d26..e1f88cc741dbe280a3aeceb16ac2bc39f1bd6a45 100644
--- a/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js
+++ b/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js
@@ -2506,7 +2506,7 @@ if(client_id.toLowerCase() === 'android'){
"telemetry": {
"pdata": {
"id": pdataId,
- "ver": "4.10.3",
+ "ver": "5.0.0",
"pid": "sunbird-portal"
}
}
@@ -2684,6 +2684,13 @@ if(client_id.toLowerCase() === 'android'){
return decodeURIComponent(window.location.search.replace(new RegExp("^(?:.*[&\\?]" + encodeURIComponent(key).replace(/[\.\+\*]/g, "\\$&") + "(?:\\=([^&]*))?)?.*$", "i"), "$1"));
}
+ function stringToHTML(str) {
+ let parser = new DOMParser();
+ let doc = parser.parseFromString(str, 'text/html');
+ console.log('Doc parse => ', doc); // TODO: log!
+ return doc?.body?.innerText || document.createElement('body');
+ }
+
window.onload = function(){
var mergeaccountprocess = (new URLSearchParams(window.location.search)).get('mergeaccountprocess');
var version = getValueFromSession('version');
@@ -2708,8 +2715,16 @@ if(client_id.toLowerCase() === 'android'){
}
addVersionToURL(version);
toggleGoogleSignInBtn();
+
var error_message = (new URLSearchParams(window.location.search)).get('error_message');
+ console.log('before error_message parse => ', error_message); // TODO: log!
+ if (error_message) error_message = stringToHTML(error_message);
+ console.log('after error_message parse => ', error_message); // TODO: log!`
+
var success_message = (new URLSearchParams(window.location.search)).get('success_message');
+ console.log('before success_message parse => ', success_message); // TODO: log!
+ if (success_message) success_message = stringToHTML(success_message);
+ console.log('after success_message parse => ', success_message); // TODO: log!`
if(error_message){
var error_msg = document.getElementById('error-msg');
diff --git a/ansible/es.yml b/ansible/es.yml
index 8042e5dbfa5db1306cbd0299b8cc3cb8db292a0e..c526414f47c0d87a5efbb7956ce26d21d2d62f36 100644
--- a/ansible/es.yml
+++ b/ansible/es.yml
@@ -6,10 +6,16 @@
- es_snapshot_host: "{{app_es_snapshot_host}}"
- snapshot_base_path: "{{app_snapshot_base_path}}"
roles:
- - es-azure-snapshot
- - es5-snapshot-purge
+ - role: es-azure-snapshot
+ when: cloud_service_provider == "azure"
+ - role: es-s3-snapshot
+ when: cloud_service_provider == "aws"
+ - role: es-gcs-snapshot
+ when: cloud_service_provider == "gcloud"
+ - role: es5-snapshot-purge
tags:
- es_backup
+ run_once: true
- hosts: log-es-backup
become: yes
@@ -19,16 +25,13 @@
- es_snapshot_host: "{{log_es_snapshot_host}}"
- snapshot_base_path: "{{log_snapshot_base_path}}"
roles:
- - es-azure-snapshot
+ - role: es-azure-snapshot
+ when: cloud_service_provider == "azure"
+ - role: es-s3-snapshot
+ when: cloud_service_provider == "aws"
+ - role: es-gcs-snapshot
+ when: cloud_service_provider == "gcloud"
- es5-snapshot-purge
tags:
- log_es_backup
-
-# - hosts: log-es
-# become: yes
-# vars_files:
-# - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
-# roles:
-# - es2-data-purge
-# tags:
-# - purge_old_logs
+ run_once: true
\ No newline at end of file
diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml
index dd3e9cfcd92083b5176465c7765a7b681a204019..df20586566b684eeaf1e48dffe098e97d6718b89 100644
--- a/ansible/inventory/env/group_vars/all.yml
+++ b/ansible/inventory/env/group_vars/all.yml
@@ -132,6 +132,8 @@ cassandra_rpc_address: 0.0.0.0
cassandra_restore_dir: "/home/{{ ansible_ssh_user }}/"
cassandra_backup_azure_container_name: cassandra-backup
cassandra_backup_dir: /data/cassandra/backup
+### Release 5.0.0 ###
+cassandra_multi_dc_enabled: false
keycloak_realm: sunbird
sunbird_content_player_url: "http://kong:8000/"
@@ -363,7 +365,7 @@ content_service_whitelisted_mimetype: ''
content_service_blacklisted_mimetype: ''
sunbird_cloud_storage_urls: 'https://s3.ap-south-1.amazonaws.com/ekstep-public-{{ekstep_s3_env}}/,https://ekstep-public-{{ekstep_s3_env}}.s3-ap-south-1.amazonaws.com/,https://{{sunbird_public_storage_account_name}}.blob.core.windows.net/{{sunbird_content_azure_storage_container}}/'
sunbird_email_max_recipients_limit: 100
-sunbird_cassandra_consistency_level: one
+sunbird_cassandra_consistency_level: local_quorum
sunbird_cassandra_replication_strategy: '{"class":"SimpleStrategy","replication_factor":"1"}'
sunbird_lock_expiry_time: 3600
app_es_heap_size: 1500m
@@ -466,6 +468,10 @@ job_names:
api_report_mailing_list: "" ## This mailing list to send the daily api count report. if adaptor want to use they have to override in common.yml
kafka_assessment_topic: "{{env_name}}.telemetry.assess"
+### Release-5.0.0 kafka topic variables for merge user course and notification job ###
+kafka_topic_lms_user_account: "{{env_name}}.lms.user.account.merge"
+kafka_topic_lms_notification: "{{env_name}}.lms.notification"
+
# Portal vars for reports
#sunbird_portal_azure_storage_account: "{{sunbird_azure_storage_account}}"
#sunbird_portal_azure_storage_key: "{{sunbird_azure_storage_key}}"
@@ -501,7 +507,7 @@ sunbird_portal_updateLoginTimeEnabled: false
# Desktop app vars
#sunbird_offline_azure_storage_account: "" #added this var for adopter usecase
-offline_installer_container_name: "" #added this var for adopter usecase
+offline_installer_container_name: "{{env}}-offlineinstaller"
cloud_storage_url: "{{ sunbird_public_storage_account_name }}.blob.core.windows.net"
# Search-service
@@ -548,4 +554,4 @@ sunbird_trampoline_desktop_keycloak_client_id: trampoline-desktop
# DIAL-service schema
dial_plugin_container_name: "sunbird-dial-{{env}}"
-dial_service_schema_base_path: "https://{{sunbird_public_storage_account_name}}.blob.core.windows.net/{{dial_plugin_container_name}}/schemas/local"
+dial_service_schema_base_path: "https://{{sunbird_public_storage_account_name}}.blob.core.windows.net/{{dial_plugin_container_name}}/jsonld-schema/local"
diff --git a/ansible/provision-log-es.yml b/ansible/provision-log-es.yml
index 0959dd349c0c2aa5efe0a26be5c66323747819fc..51262b685a29586cfe8d558f51c374c9c89236ba 100644
--- a/ansible/provision-log-es.yml
+++ b/ansible/provision-log-es.yml
@@ -1,13 +1,10 @@
- hosts: log-es
become: yes
vars:
- es_version: "6.8.10"
+ es_version: "6.8.22"
es_etc_discovery_zen_ping_unicast_hosts: "{{ groups['log-es'] }}"
es_etc_cluster_name: "{{ node_name }}"
es_heap_size: "{{ log_es_heap_size | default('1g') }}"
- es_plugins_reinstall: true
- es_plugins:
- - plugin: "repository-azure"
es_config:
cluster.name: "{{ log_es_etc_cluster_name }}"
discovery.zen.ping.unicast.hosts: "{{ groups['log-es'] }}"
@@ -49,5 +46,7 @@
method: PUT
return_content: true
timeout: 30
- retries: 3
- delay: 10
+ register: _result
+ until: _result.status == 200
+ retries: 3
+ delay: 10
\ No newline at end of file
diff --git a/ansible/roles/elasticsearch_dependencies/main.yml b/ansible/roles/elasticsearch_dependencies/tasks/main.yml
similarity index 63%
rename from ansible/roles/elasticsearch_dependencies/main.yml
rename to ansible/roles/elasticsearch_dependencies/tasks/main.yml
index b4752a62af76d2f4637c5fb3147a8bc422be002e..20c7bf0c82eb625ce612953f3d9a38eaa4035efe 100644
--- a/ansible/roles/elasticsearch_dependencies/main.yml
+++ b/ansible/roles/elasticsearch_dependencies/tasks/main.yml
@@ -2,5 +2,5 @@
- debug: msg="Installing jmespath"
- name: Install jmespath
- apt: name=jmespath state=present
+ apt: name=python3-jmespath state=present
become: yes
diff --git a/ansible/roles/es-azure-restore/tasks/main.yml b/ansible/roles/es-azure-restore/tasks/main.yml
index f45a97587b2a857fa5971593a180a34791adec35..ff71c19a11c934c5e88f21a0a1825d8acdc625f1 100644
--- a/ansible/roles/es-azure-restore/tasks/main.yml
+++ b/ansible/roles/es-azure-restore/tasks/main.yml
@@ -1,7 +1,7 @@
---
- name: Set azure snapshot for the first time
uri:
- url: "http://{{ es_restore_host }}:9200/_snapshot/azurebackup"
+ url: "http://{{ es_restore_host }}:9200/_snapshot/{{ snapshot_base_path }}"
method: PUT
body: "{{ snapshot_create_request_body | to_json }}"
headers:
@@ -9,12 +9,12 @@
- name: Restore ES from Azure backup
uri:
- url: "http://{{ es_restore_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}/_restore"
+ url: "http://{{ es_restore_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}/_restore"
method: POST
- name: "Wait for restore to be completed"
uri:
- url: "http://{{ es_restore_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}/_status"
+ url: "http://{{ es_restore_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}/_status"
method: GET
return_content: yes
status_code: 200
diff --git a/ansible/roles/es-azure-snapshot/defaults/main.yml b/ansible/roles/es-azure-snapshot/defaults/main.yml
index 36295b9f9fbf3974de5b5b8ce8f9480f803ba1d1..99787b0f89be78d05d6fed7f57e31f656d1524fa 100644
--- a/ansible/roles/es-azure-snapshot/defaults/main.yml
+++ b/ansible/roles/es-azure-snapshot/defaults/main.yml
@@ -1,11 +1,13 @@
snapshot_create_request_body: {
type: azure,
settings: {
- container: "elasticsearch-snapshots",
+ container: "{{ es_azure_backup_folder_name }}",
base_path: "{{ snapshot_base_path }}_{{ base_path_date }}"
}
}
# Override these values
-es_snapshot_host: localhost
-snapshot_base_path: default
+es_snapshot_host: "localhost"
+snapshot_base_path: "default"
+
+es_azure_backup_folder_name: "elasticsearch-snapshots"
\ No newline at end of file
diff --git a/ansible/roles/es-azure-snapshot/tasks/main.yml b/ansible/roles/es-azure-snapshot/tasks/main.yml
index d9e01c55293a7debdeaf6dfa822168fff87a1671..d2880013d51239dd91ef76da027744ef0b49c71f 100644
--- a/ansible/roles/es-azure-snapshot/tasks/main.yml
+++ b/ansible/roles/es-azure-snapshot/tasks/main.yml
@@ -2,41 +2,42 @@
- set_fact: base_path_date="{{ lookup('pipe','date +%Y-%m') }}"
-- name: Create azure snapshot
+- set_fact: snapshot_number="snapshot_{{ lookup('pipe','date +%s') }}"
+
+- name: Ensure backup folder exists in azure blob
+ shell: "az storage container create --name {{ es_azure_backup_folder_name }}"
+ environment:
+ AZURE_STORAGE_ACCOUNT: "{{ azure_management_storage_account_name }}"
+ AZURE_STORAGE_KEY: "{{ azure_management_storage_account_key }}"
+
+- name: Create Azure Repository
uri:
- url: "http://{{ es_snapshot_host }}:9200/_snapshot/azurebackup"
+ url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}"
method: PUT
body: "{{ snapshot_create_request_body | to_json }}"
headers:
Content-Type: "application/json"
-- debug:
- var: "{{ snapshot_base_path }}"
-
-- set_fact: snapshot_number="snapshot_{{ansible_date_time.epoch}}"
-
- name: Take new snapshot
uri:
- url: "http://{{ es_snapshot_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}"
+ url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
method: PUT
- body: >
- {"indices":"*","include_global_state":false}
headers:
Content-Type: "application/json"
- name: Print all snapshots
uri:
- url: "http://{{ es_snapshot_host }}:9200/_snapshot/azurebackup/_all"
+ url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/_all"
method: GET
- name: Print status of current snapshot
uri:
- url: "http://{{ es_snapshot_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}"
+ url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
method: GET
- name: "Wait for backup to be completed"
uri:
- url: "http://{{ es_snapshot_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}"
+ url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
method: GET
return_content: yes
status_code: 200
diff --git a/ansible/roles/es-curator/defaults/main.yml b/ansible/roles/es-curator/defaults/main.yml
index a4728a5b39b1f650b175cb78d31f5894e568789c..9fd4efe2c8571c7ef87e290fe6e54b9313c46981 100644
--- a/ansible/roles/es-curator/defaults/main.yml
+++ b/ansible/roles/es-curator/defaults/main.yml
@@ -1,6 +1,2 @@
-# es_curator_major_version: 3
-# es_curator_version: 3.4.1-1
-
es_curator_major_version: 5
-es_curator_version: 5.7.6
-python_es_curator_version: 3.4.1-1
+es_curator_version: 5.8.4
\ No newline at end of file
diff --git a/ansible/roles/es-curator/tasks/main.yml b/ansible/roles/es-curator/tasks/main.yml
index 689b87be46d3a0985dea2ed5eb75ab12295e58f7..c4a8bacee7a3cdc0a018845e700a2f9f1178b743 100644
--- a/ansible/roles/es-curator/tasks/main.yml
+++ b/ansible/roles/es-curator/tasks/main.yml
@@ -7,11 +7,6 @@
- debug:
msg: "{{ es_curator_version }}"
-- name: Install python elasticsearch curator
- apt:
- name: python-elasticsearch-curator={{ python_es_curator_version }}
- force: yes
-
- name: Install elasticsearch curator
apt:
name: elasticsearch-curator={{ es_curator_version }}
diff --git a/ansible/roles/es-gcs-snapshot/defaults/main.yml b/ansible/roles/es-gcs-snapshot/defaults/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..3c2efa9a7f8b5496ea70b720b35ad48192e64439
--- /dev/null
+++ b/ansible/roles/es-gcs-snapshot/defaults/main.yml
@@ -0,0 +1,12 @@
+snapshot_create_request_body: {
+ type: gcs,
+ settings: {
+ bucket: "{{ gcs_management_bucket_name }}",
+ base_path: "{{ es_gcs_backup_folder_name }}/{{ snapshot_base_path }}_{{ base_path_date }}"
+ }
+}
+
+# Override these values
+es_snapshot_host: "localhost"
+snapshot_base_path: "default"
+es_gcs_backup_folder_name: "elasticsearch-snapshots"
\ No newline at end of file
diff --git a/ansible/roles/es-gcs-snapshot/tasks/main.yml b/ansible/roles/es-gcs-snapshot/tasks/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..55f50b17ad4d1f60013277d490255f2dcfee8a19
--- /dev/null
+++ b/ansible/roles/es-gcs-snapshot/tasks/main.yml
@@ -0,0 +1,42 @@
+---
+
+- set_fact: base_path_date="{{ lookup('pipe','date +%Y-%m') }}"
+
+- set_fact: snapshot_number="snapshot_{{ lookup('pipe','date +%s') }}"
+
+- name: Create GCS Repository
+ uri:
+ url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}"
+ method: PUT
+ body: "{{ snapshot_create_request_body | to_json }}"
+ headers:
+ Content-Type: "application/json"
+
+- name: Take new snapshot
+ uri:
+ url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+ method: PUT
+ headers:
+ Content-Type: "application/json"
+
+- name: Print all snapshots
+ uri:
+ url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/_all"
+ method: GET
+
+- name: Print status of current snapshot
+ uri:
+ url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+ method: GET
+
+- name: "Wait for backup to be completed"
+ uri:
+ url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+ method: GET
+ return_content: yes
+ status_code: 200
+ body_format: json
+ register: result
+ until: result.json.snapshots[0].state == 'SUCCESS'
+ retries: 120
+ delay: 10
diff --git a/ansible/roles/es-s3-snapshot/defaults/main.yml b/ansible/roles/es-s3-snapshot/defaults/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..2ca18929b470f370f13cda93381570cbbfe612e7
--- /dev/null
+++ b/ansible/roles/es-s3-snapshot/defaults/main.yml
@@ -0,0 +1,12 @@
+snapshot_create_request_body: {
+ type: s3,
+ settings: {
+ bucket: "{{ aws_management_bucket_name }}",
+ base_path: "{{ es_s3_backup_folder_name }}/{{ snapshot_base_path }}_{{ base_path_date }}"
+ }
+}
+
+# Override these values
+es_snapshot_host: "localhost"
+snapshot_base_path: "default"
+es_s3_backup_folder_name: "elasticsearch-snapshots"
\ No newline at end of file
diff --git a/ansible/roles/es-s3-snapshot/tasks/main.yml b/ansible/roles/es-s3-snapshot/tasks/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..aee768626c0b2eaa16f01d327251722fdac92ca3
--- /dev/null
+++ b/ansible/roles/es-s3-snapshot/tasks/main.yml
@@ -0,0 +1,42 @@
+---
+
+- set_fact: base_path_date="{{ lookup('pipe','date +%Y-%m') }}"
+
+- set_fact: snapshot_number="snapshot_{{ lookup('pipe','date +%s') }}"
+
+- name: Create S3 Repository
+ uri:
+ url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}"
+ method: PUT
+ body: "{{ snapshot_create_request_body | to_json }}"
+ headers:
+ Content-Type: "application/json"
+
+- name: Take new snapshot
+ uri:
+ url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+ method: PUT
+ headers:
+ Content-Type: "application/json"
+
+- name: Print all snapshots
+ uri:
+ url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/_all"
+ method: GET
+
+- name: Print status of current snapshot
+ uri:
+ url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+ method: GET
+
+- name: "Wait for backup to be completed"
+ uri:
+ url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+ method: GET
+ return_content: yes
+ status_code: 200
+ body_format: json
+ register: result
+ until: result.json.snapshots[0].state == 'SUCCESS'
+ retries: 120
+ delay: 10
diff --git a/ansible/roles/es2-snapshot-purge/defaults/main.yml b/ansible/roles/es2-snapshot-purge/defaults/main.yml
index 4f64ac4aff660a022ba7baeaab77a383a2549b11..0a619ae4b008dd40d62192e4eb50ea05ad587332 100644
--- a/ansible/roles/es2-snapshot-purge/defaults/main.yml
+++ b/ansible/roles/es2-snapshot-purge/defaults/main.yml
@@ -1,2 +1,2 @@
es_snapshot_retention_days: 30
-es_snapshot_repository: azurebackup
+es_snapshot_repository: "{{ snapshot_base_path }}"
diff --git a/ansible/roles/es5-snapshot-purge/defaults/main.yml b/ansible/roles/es5-snapshot-purge/defaults/main.yml
index 739d0bf3112e3e4c9f50f9931f19b016b8e5c3ad..cab1182a3dfa493927e420d996ed4a61408c8932 100644
--- a/ansible/roles/es5-snapshot-purge/defaults/main.yml
+++ b/ansible/roles/es5-snapshot-purge/defaults/main.yml
@@ -1,5 +1,5 @@
es_snapshot_host: localhost
-es_snapshot_repository: azurebackup
+es_snapshot_repository: "{{ snapshot_base_path }}"
es_snapshot_retention_days: 30
es_curator_config_dir: /etc/curator
es_curator_config_file: "{{ es_curator_config_dir }}/curator.yml"
diff --git a/ansible/roles/es5-snapshot-purge/meta/main.yml b/ansible/roles/es5-snapshot-purge/meta/main.yml
index e26db1f14555acf283684466064addb2f7a399f1..8b4e268b5d2de70f0f4b3838efcff2bd13442deb 100644
--- a/ansible/roles/es5-snapshot-purge/meta/main.yml
+++ b/ansible/roles/es5-snapshot-purge/meta/main.yml
@@ -1,3 +1,3 @@
---
dependencies:
- - { role: es-curator, es_curator_major_version: 5, es_curator_version: 5.8.1, python_es_curator_version: 3.4.1-1 }
+ - { role: es-curator, es_curator_major_version: 5, es_curator_version: 5.8.4 }
diff --git a/ansible/roles/es6/defaults/main.yml b/ansible/roles/es6/defaults/main.yml
index b2402c73a287b60b7cc5f66e4f52b02000b5ea43..c0063fcc909326e2bc10cfa6119bfd8d03d3893f 100644
--- a/ansible/roles/es6/defaults/main.yml
+++ b/ansible/roles/es6/defaults/main.yml
@@ -1,6 +1,6 @@
---
es_major_version: "6.x"
-es_version: "6.2.3"
+es_version: "6.8.22"
es_version_lock: false
es_use_repository: true
es_templates_fileglob: "files/templates/*.json"
@@ -11,7 +11,6 @@ es_start_service: true
es_java_install: true
update_java: false
es_restart_on_change: true
-es_plugins_reinstall: false
es_scripts: false
es_templates: true
es_user: elasticsearch
@@ -22,7 +21,7 @@ es_pid_dir: "/var/run/elasticsearch"
es_data_dirs: "/var/lib/elasticsearch"
es_log_dir: "/var/log/elasticsearch"
es_max_open_files: 65536
-es_max_threads: "{{ 2048 if ( es_version | version_compare('6.0.0', '<')) else 8192 }}"
+es_max_threads: "{{ 2048 if ( es_version is version('6.0.0', '<')) else 8192 }}"
es_max_map_count: 262144
es_allow_downgrades: false
es_enable_xpack: false
@@ -41,4 +40,9 @@ conf_dir: ''
data_dirs: ''
# JVM custom parameters
es_jvm_custom_parameters: ''
-es_heap_size: "2g"
+es_heap_size: "1g"
+es_plugins_reinstall: true
+es_plugins:
+- plugin: "repository-azure"
+- plugin: "repository-s3"
+- plugin: "repository-gcs"
\ No newline at end of file
diff --git a/ansible/roles/es6/tasks/azure-plugin.yml b/ansible/roles/es6/tasks/azure-plugin.yml
deleted file mode 100644
index 46d3079babcde748ebecc55f0d1d7fa7a4488454..0000000000000000000000000000000000000000
--- a/ansible/roles/es6/tasks/azure-plugin.yml
+++ /dev/null
@@ -1,11 +0,0 @@
-- name: Check if plugin already installed or not
- shell: "curl -s localhost:9200/_cat/plugins | awk '{print $2}' |head -1"
- register: plugin_output
-
-- name: check the output
- debug:
- var: plugin_output
-
-- name: Install plugin
- shell: ES_PATH_CONF=/etc/elasticsearch/"{{ es_instance_name }}" /usr/share/elasticsearch/bin/elasticsearch-plugin install repository-azure
- when: plugin_output.stdout != "repository-azure"
diff --git a/ansible/roles/es6/tasks/elasticsearch-parameters.yml b/ansible/roles/es6/tasks/elasticsearch-parameters.yml
index 13c6472a3d35a26d9989de12905c656dc8e23088..a9cc13830a60c1226ab0e4a6c2bbbac66d819e2f 100644
--- a/ansible/roles/es6/tasks/elasticsearch-parameters.yml
+++ b/ansible/roles/es6/tasks/elasticsearch-parameters.yml
@@ -58,7 +58,7 @@
#Relies on elasticsearch distribution installing a serviced script to determine whether one should be copied.
- name: set fact use_system_d
- set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', '>=')) or (ansible_distribution in ['RedHat','CentOS'] and ansible_distribution_version | version_compare('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15', '>=')) }}
+ set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version is version('8', '>=')) or (ansible_distribution in ['RedHat','CentOS'] and ansible_distribution_version is version('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version is version('15', '>=')) }}
- name: set fact instance_sysd_script
set_fact: instance_sysd_script={{sysd_script | dirname }}/{{es_instance_name}}_{{sysd_script | basename}}
diff --git a/ansible/roles/es6/tasks/main.yml b/ansible/roles/es6/tasks/main.yml
index 7bf1a214a9617875c6cab186dbef4d500139c3d1..1de1eeff8e222a52f5e0c9ea9fd2ce3cbdf14369 100644
--- a/ansible/roles/es6/tasks/main.yml
+++ b/ansible/roles/es6/tasks/main.yml
@@ -31,16 +31,27 @@
tags:
- plugins
-# Install Elasticsearch plugin for backups
-- name: include elasticsearch plugin install
- include: azure-plugin.yml
-
#We always execute xpack as we may need to remove features
- name: include xpack/elasticsearch-xpack.yml
include: xpack/elasticsearch-xpack.yml
tags:
- xpack
+- name: include plugins/create-keystore.yml
+ include: plugins/create-keystore.yml
+
+- name: include plugins/repository-azure.yml
+ include: plugins/repository-azure.yml
+ when: cloud_service_provider == "azure"
+
+- name: include plugins/repository-s3.yml
+ include: plugins/repository-s3.yml
+ when: cloud_service_provider == "aws"
+
+- name: include plugins/repository-gcs.yml
+ include: plugins/repository-gcs.yml
+ when: cloud_service_provider == "gcloud"
+
- name: flush handlers
meta: flush_handlers
@@ -79,15 +90,3 @@
when: es_templates
tags:
- templates
-
-# Add ES Plugin credentials for backups
-- name: Add ES Backup variables to elasticsearch to have backups enabled
- lineinfile:
- path: '{{conf_dir}}/elasticsearch.yml'
- line: 'cloud.azure.storage.default.{{item.key}}: {{item.value}}'
- regexp: "cloud.azure.storage.default.{{ item.key }}.*"
- with_items:
- - {key: 'account', value: '{{ sunbird_management_storage_account_name }}'}
- - {key: 'key', value: '{{ sunbird_management_storage_account_key }}'}
- when: sunbird_management_storage_account_name is defined and sunbird_management_storage_account_key is defined
- notify: restart elasticsearch
diff --git a/ansible/roles/es6/tasks/plugins/create-keystore.yml b/ansible/roles/es6/tasks/plugins/create-keystore.yml
new file mode 100644
index 0000000000000000000000000000000000000000..656183902d6a72594995f02c31c1be54e147e0eb
--- /dev/null
+++ b/ansible/roles/es6/tasks/plugins/create-keystore.yml
@@ -0,0 +1,13 @@
+---
+- name: Check if elasticsearch keystore exists or not
+ become: yes
+ stat:
+ path: "{{ conf_dir }}/elasticsearch.keystore"
+ register: elasticsearch_keystore_file
+
+- name: Create the elasticsearch keystore if not exists
+ become: yes
+ command: "{{es_home}}/bin/elasticsearch-keystore create"
+ environment:
+ ES_PATH_CONF: "{{ conf_dir }}"
+ when: elasticsearch_keystore_file.stat.exists == false
diff --git a/ansible/roles/es6/tasks/plugins/repository-azure.yml b/ansible/roles/es6/tasks/plugins/repository-azure.yml
new file mode 100644
index 0000000000000000000000000000000000000000..170a84000eb4d55d5090d58e2ede8a077c857684
--- /dev/null
+++ b/ansible/roles/es6/tasks/plugins/repository-azure.yml
@@ -0,0 +1,15 @@
+---
+- name: Add default azure account name for backups
+ become: yes
+ shell: echo "{{ azure_management_storage_account_name }}" | {{ es_home }}/bin/elasticsearch-keystore add -f azure.client.default.account
+ no_log: True
+ environment:
+ ES_PATH_CONF: "{{ conf_dir }}"
+
+
+- name: Add default azure account key for backups
+ become: yes
+ shell: echo "{{ azure_management_storage_account_key }}" | {{ es_home }}/bin/elasticsearch-keystore add -f azure.client.default.key
+ no_log: True
+ environment:
+ ES_PATH_CONF: "{{ conf_dir }}"
\ No newline at end of file
diff --git a/ansible/roles/es6/tasks/plugins/repository-gcs.yml b/ansible/roles/es6/tasks/plugins/repository-gcs.yml
new file mode 100644
index 0000000000000000000000000000000000000000..7d5c32e52ee6a22bb996b9c453c6046a2d10a644
--- /dev/null
+++ b/ansible/roles/es6/tasks/plugins/repository-gcs.yml
@@ -0,0 +1,18 @@
+---
+- name: Create the gcs service account file from variable
+ become: yes
+ copy:
+ dest: "{{ conf_dir }}/gcs_management_bucket_service_account.json"
+ content: "{{ gcs_management_bucket_service_account }}"
+
+- name: Add gcs service account file to keystore
+ become: yes
+ shell: "{{ es_home }}/bin/elasticsearch-keystore add-file -f gcs.client.default.credentials_file {{ conf_dir }}/gcs_management_bucket_service_account.json"
+ no_log: True
+ environment:
+ ES_PATH_CONF: "{{ conf_dir }}"
+
+- name: Remove the service account file
+ file:
+ path: "{{ conf_dir }}/gcs_management_bucket_service_account.json"
+ state: absent
\ No newline at end of file
diff --git a/ansible/roles/es6/tasks/plugins/repository-s3.yml b/ansible/roles/es6/tasks/plugins/repository-s3.yml
new file mode 100644
index 0000000000000000000000000000000000000000..b5897792abc272dd1cdeddfdcfac95479ec59cfb
--- /dev/null
+++ b/ansible/roles/es6/tasks/plugins/repository-s3.yml
@@ -0,0 +1,14 @@
+---
+- name: Add default aws account name for backups
+ become: yes
+ shell: echo "{{ aws_management_bucket_user_access_key }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.access_key
+ no_log: True
+ environment:
+ ES_PATH_CONF: "{{ conf_dir }}"
+
+- name: Add default aws account key for backups
+ become: yes
+ shell: echo "{{ aws_management_bucket_user_secret_key }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.secret_key
+ no_log: True
+ environment:
+ ES_PATH_CONF: "{{ conf_dir }}"
diff --git a/ansible/roles/es6/tasks/xpack/elasticsearch-xpack.yml b/ansible/roles/es6/tasks/xpack/elasticsearch-xpack.yml
index ec82a68a6f51114e15df5a26ac75a6841c384692..4fe1cab279dfe002af9fd0428143d490cad4630c 100644
--- a/ansible/roles/es6/tasks/xpack/elasticsearch-xpack.yml
+++ b/ansible/roles/es6/tasks/xpack/elasticsearch-xpack.yml
@@ -19,4 +19,4 @@
- name: Set elasticsearch.keystore Permissions
become: yes
file: state=file path={{ conf_dir }}/elasticsearch.keystore owner={{ es_user }} group={{ es_group }}
- when: es_enable_xpack and "security" in es_xpack_features and (es_version | version_compare('6.0.0', '>'))
+ when: es_enable_xpack and "security" in es_xpack_features and (es_version is version('6.0.0', '>'))
diff --git a/ansible/roles/es6/tasks/xpack/security/elasticsearch-security.yml b/ansible/roles/es6/tasks/xpack/security/elasticsearch-security.yml
index 14f4d203868de8117f4dcaebf4ea8f1e46dfcbec..9d5d4dc814d6866c8c53d448801a3b72733a388f 100644
--- a/ansible/roles/es6/tasks/xpack/security/elasticsearch-security.yml
+++ b/ansible/roles/es6/tasks/xpack/security/elasticsearch-security.yml
@@ -20,12 +20,12 @@
environment:
ES_PATH_CONF: "{{ conf_dir }}"
when:
- - (es_enable_xpack and "security" in es_xpack_features) and (es_version | version_compare('6.0.0', '>'))
+ - (es_enable_xpack and "security" in es_xpack_features) and (es_version is version('6.0.0', '>'))
- name: Create Bootstrap password for elastic user
shell: echo "{{es_api_basic_auth_password}}" | {{es_home}}/bin/elasticsearch-keystore add -x 'bootstrap.password'
when:
- - (es_enable_xpack and "security" in es_xpack_features) and (es_version | version_compare('6.0.0', '>')) and es_api_basic_auth_username is defined and list_keystore is defined and es_api_basic_auth_username == 'elastic' and 'bootstrap.password' not in list_keystore.stdout_lines
+ - (es_enable_xpack and "security" in es_xpack_features) and (es_version is version('6.0.0', '>')) and es_api_basic_auth_username is defined and list_keystore is defined and es_api_basic_auth_username == 'elastic' and 'bootstrap.password' not in list_keystore.stdout_lines
environment:
ES_PATH_CONF: "{{ conf_dir }}"
no_log: true
diff --git a/ansible/roles/es6/templates/elasticsearch.yml.j2 b/ansible/roles/es6/templates/elasticsearch.yml.j2
index 030cc6e1a506d59f75478c7c9a3bf8a0782c01a1..d6aaab45c9dc380590ff650dbdb71e9a47b2e7bb 100644
--- a/ansible/roles/es6/templates/elasticsearch.yml.j2
+++ b/ansible/roles/es6/templates/elasticsearch.yml.j2
@@ -21,7 +21,7 @@ node.name: {{inventory_hostname}}-{{es_instance_name}}
# Path to directory containing configuration (this file and logging.yml):
-{% if (es_version | version_compare('6.0.0', '<')) %}
+{% if (es_version is version('6.0.0', '<')) %}
path.conf: {{ conf_dir }}
{% endif %}
diff --git a/ansible/roles/es6/templates/init/debian/elasticsearch.j2 b/ansible/roles/es6/templates/init/debian/elasticsearch.j2
index efe2c37f6c40f7941475b3f66ae180a5638b6352..22af436d6b9dc32e5021a161bc2f21042f1feb9f 100755
--- a/ansible/roles/es6/templates/init/debian/elasticsearch.j2
+++ b/ansible/roles/es6/templates/init/debian/elasticsearch.j2
@@ -92,7 +92,7 @@ fi
# Define other required variables
PID_FILE="$PID_DIR/$NAME.pid"
DAEMON=$ES_HOME/bin/elasticsearch
-{% if (es_version | version_compare('6.0.0', '<')) %}
+{% if (es_version is version('6.0.0', '<')) %}
DAEMON_OPTS="-d -p $PID_FILE -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR"
{% else %}
DAEMON_OPTS="-d -p $PID_FILE"
diff --git a/ansible/roles/es6/templates/init/redhat/elasticsearch.j2 b/ansible/roles/es6/templates/init/redhat/elasticsearch.j2
index c993e14365e897ffb8dff4ffde76a64015485726..d32e393b2fdde8b363f8921c8f95ea9b8e42c85e 100755
--- a/ansible/roles/es6/templates/init/redhat/elasticsearch.j2
+++ b/ansible/roles/es6/templates/init/redhat/elasticsearch.j2
@@ -140,7 +140,7 @@ start() {
cd $ES_HOME
echo -n $"Starting $prog: "
# if not running, start it up here, usually something like "daemon $exec"
-{% if (es_version | version_compare('6.0.0', '<')) %}
+{% if (es_version is version('6.0.0', '<')) %}
daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR
{% else %}
daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d
diff --git a/ansible/roles/es6/templates/log4j2.properties.j2 b/ansible/roles/es6/templates/log4j2.properties.j2
index 269be5290bf3ed457a4b6690caa5803d5b1dcfab..330f1435b8b573ec15c5153c049ff2d1b2b13b36 100644
--- a/ansible/roles/es6/templates/log4j2.properties.j2
+++ b/ansible/roles/es6/templates/log4j2.properties.j2
@@ -11,14 +11,14 @@ appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
appender.rolling.type = RollingFile
appender.rolling.name = rolling
-{% if (es_version | version_compare('6.0.0', '<')) %}
+{% if (es_version is version('6.0.0', '<')) %}
appender.rolling.fileName = ${sys:es.logs}.log
{% else %}
appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
{% endif %}
appender.rolling.layout.type = PatternLayout
appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
-{% if (es_version | version_compare('6.0.0', '<')) %}
+{% if (es_version is version('6.0.0', '<')) %}
appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log
{% else %}
appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz
@@ -27,7 +27,7 @@ appender.rolling.policies.type = Policies
appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
appender.rolling.policies.time.interval = 1
appender.rolling.policies.time.modulate = true
-{% if (es_version | version_compare('6.0.0', '>')) %}
+{% if (es_version is version('6.0.0', '>')) %}
appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
appender.rolling.policies.size.size = 128MB
appender.rolling.strategy.type = DefaultRolloverStrategy
@@ -45,14 +45,14 @@ rootLogger.appenderRef.rolling.ref = rolling
appender.deprecation_rolling.type = RollingFile
appender.deprecation_rolling.name = deprecation_rolling
-{% if (es_version | version_compare('6.0.0', '<')) %}
+{% if (es_version is version('6.0.0', '<')) %}
appender.deprecation_rolling.fileName = ${sys:es.logs}_deprecation.log
{% else %}
appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
{% endif %}
appender.deprecation_rolling.layout.type = PatternLayout
appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
-{% if (es_version | version_compare('6.0.0', '<')) %}
+{% if (es_version is version('6.0.0', '<')) %}
appender.deprecation_rolling.filePattern = ${sys:es.logs}_deprecation-%i.log.gz
{% else %}
appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz
@@ -70,14 +70,14 @@ logger.deprecation.additivity = false
appender.index_search_slowlog_rolling.type = RollingFile
appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
-{% if (es_version | version_compare('6.0.0', '<')) %}
+{% if (es_version is version('6.0.0', '<')) %}
appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log
{% else %}
appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log
{% endif %}
appender.index_search_slowlog_rolling.layout.type = PatternLayout
appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
-{% if (es_version | version_compare('6.0.0', '<')) %}
+{% if (es_version is version('6.0.0', '<')) %}
appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log
{% else %}
appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log
@@ -94,14 +94,14 @@ logger.index_search_slowlog_rolling.additivity = false
appender.index_indexing_slowlog_rolling.type = RollingFile
appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
-{% if (es_version | version_compare('6.0.0', '<')) %}
+{% if (es_version is version('6.0.0', '<')) %}
appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log
{% else %}
appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log
{% endif %}
appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
-{% if (es_version | version_compare('6.0.0', '<')) %}
+{% if (es_version is version('6.0.0', '<')) %}
appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
{% else %}
appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
diff --git a/ansible/roles/es6/templates/systemd/elasticsearch.j2 b/ansible/roles/es6/templates/systemd/elasticsearch.j2
index 1c32b1f5a38298590a90ead3aa6e566ea0ca6919..b4ef89753899c554b997f90fb3c6bf5240e6e1a3 100644
--- a/ansible/roles/es6/templates/systemd/elasticsearch.j2
+++ b/ansible/roles/es6/templates/systemd/elasticsearch.j2
@@ -19,13 +19,13 @@ WorkingDirectory={{es_home}}
User={{es_user}}
Group={{es_group}}
-{% if (es_version | version_compare('6.0.0', '<')) %}
+{% if (es_version is version('6.0.0', '<')) %}
ExecStartPre=/usr/share/elasticsearch/bin/elasticsearch-systemd-pre-exec
{% endif %}
ExecStart={{es_home}}/bin/elasticsearch \
-p ${PID_DIR}/elasticsearch.pid \
-{% if (es_version | version_compare('6.0.0', '<')) %}
+{% if (es_version is version('6.0.0', '<')) %}
-Edefault.path.logs=${LOG_DIR} \
-Edefault.path.data=${DATA_DIR} \
-Edefault.path.conf=${CONF_DIR} \
diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml
index 3ae6e47417c17e8b3251f8660b5763bd31e0ed22..907e2504607272fa3a054aebce48b66ce6d5d396 100644
--- a/ansible/roles/kong-api/defaults/main.yml
+++ b/ansible/roles/kong-api/defaults/main.yml
@@ -14,6 +14,34 @@ statsd_pulgin:
name: statsd
config.host: "{{ statsd_host }}"
config.port: "{{ statsd_port }}"
+ config.metrics:
+ - name: request_count
+ sample_rate: 1
+ stat_type: counter
+ - name: latency
+ stat_type: timer
+ - name: request_size
+ stat_type: timer
+ - name: status_count
+ sample_rate: 1
+ stat_type: counter
+ - name: response_size
+ stat_type: timer
+ - consumer_identifier: username
+ name: unique_users
+ stat_type: set
+ - consumer_identifier: username
+ name: request_per_user
+ sample_rate: 1
+ stat_type: counter
+ - name: upstream_latency
+ stat_type: timer
+ - name: kong_latency
+ stat_type: timer
+ - consumer_identifier: username
+ name: status_count_per_user
+ sample_rate: 1
+ stat_type: counter
# Default Rate limits
small_rate_limit_per_hour: 100
@@ -3944,6 +3972,48 @@ kong_apis:
config.required: false
config.enabled: false
+- name: linkDialcodeContentV1
+ uris: "{{ content_prefix }}/v1/dialcode/link"
+ upstream_url: "{{ content_service_url }}/content/v3/dialcode/link"
+ strip_uri: true
+ plugins:
+ - name: jwt
+ - name: cors
+ - "{{ statsd_pulgin }}"
+ - name: acl
+ config.whitelist:
+ - contentUpdate
+ - name: rate-limiting
+ config.policy: local
+ config.hour: "{{ medium_rate_limit_per_hour }}"
+ config.limit_by: credential
+ - name: request-size-limiting
+ config.allowed_payload_size: "{{ small_request_size_limit }}"
+ - name: opa-checks
+ config.required: false
+ config.enabled: false
+
+- name: reserveDialcodeContent
+ uris: "{{ content_prefix }}/v1/dialcode/reserve"
+ upstream_url: "{{ content_service_url }}/content/v3/dialcode/reserve"
+ strip_uri: true
+ plugins:
+ - name: jwt
+ - name: cors
+ - "{{ statsd_pulgin }}"
+ - name: acl
+ config.whitelist:
+ - contentUpdate
+ - name: rate-limiting
+ config.policy: local
+ config.hour: "{{ medium_rate_limit_per_hour }}"
+ config.limit_by: credential
+ - name: request-size-limiting
+ config.allowed_payload_size: "{{ small_request_size_limit }}"
+ - name: opa-checks
+ config.required: false
+ config.enabled: false
+
- name: updateCourseHierarchy
uris: "{{ course_service_prefix }}/v1/hierarchy/update"
upstream_url: "{{ knowledge_mw_service_url }}/v1/course/hierarchy/update"
@@ -6019,6 +6089,27 @@ kong_apis:
config.required: false
config.enabled: false
+- name: linkDialcodeCollection
+ uris: "{{ collection_prefix }}/v1/dialcode/link"
+ upstream_url: "{{ content_service_url }}/collection/v4/dialcode/link"
+ strip_uri: true
+ plugins:
+ - name: jwt
+ - name: cors
+ - "{{ statsd_pulgin }}"
+ - name: acl
+ config.whitelist:
+ - contentUpdate
+ - name: rate-limiting
+ config.policy: local
+ config.hour: "{{ medium_rate_limit_per_hour }}"
+ config.limit_by: credential
+ - name: request-size-limiting
+ config.allowed_payload_size: "{{ small_request_size_limit }}"
+ - name: opa-checks
+ config.required: false
+ config.enabled: false
+
- name: collectionHierarchyAdd
uris: "{{ collection_prefix }}/v1/hierarchy/add"
upstream_url: "{{ content_service_url }}/collection/v4/hierarchy/add"
@@ -9293,6 +9384,48 @@ kong_apis:
config.required: true
config.enabled: true
+- name: linkDialcodeContentV2
+ uris: "{{ content_prefix }}/v2/dialcode/link"
+ upstream_url: "{{ content_service_url }}/content/v4/dialcode/link"
+ strip_uri: true
+ plugins:
+ - name: jwt
+ - name: cors
+ - "{{ statsd_pulgin }}"
+ - name: acl
+ config.whitelist:
+ - contentUpdate
+ - name: rate-limiting
+ config.policy: local
+ config.hour: "{{ medium_rate_limit_per_hour }}"
+ config.limit_by: credential
+ - name: request-size-limiting
+ config.allowed_payload_size: "{{ small_request_size_limit }}"
+ - name: opa-checks
+ config.required: false
+ config.enabled: false
+
+- name: reserveDialcodeContentV2
+ uris: "{{ content_prefix }}/v2/dialcode/reserve"
+ upstream_url: "{{ content_service_url }}/content/v4/dialcode/reserve"
+ strip_uri: true
+ plugins:
+ - name: jwt
+ - name: cors
+ - "{{ statsd_pulgin }}"
+ - name: acl
+ config.whitelist:
+ - contentUpdate
+ - name: rate-limiting
+ config.policy: local
+ config.hour: "{{ medium_rate_limit_per_hour }}"
+ config.limit_by: credential
+ - name: request-size-limiting
+ config.allowed_payload_size: "{{ small_request_size_limit }}"
+ - name: opa-checks
+ config.required: false
+ config.enabled: false
+
- name: collectionReview
uris: "{{ collection_prefix }}/v1/review"
upstream_url: "{{ content_service_url }}/collection/v4/review"
@@ -9356,6 +9489,27 @@ kong_apis:
config.required: false
config.enabled: false
+- name: reserveDialcodeCollection
+ uris: "{{ collection_prefix }}/v1/dialcode/reserve"
+ upstream_url: "{{ content_service_url }}/collection/v4/dialcode/reserve"
+ strip_uri: true
+ plugins:
+ - name: jwt
+ - name: cors
+ - "{{ statsd_pulgin }}"
+ - name: acl
+ config.whitelist:
+ - contentUpdate
+ - name: rate-limiting
+ config.policy: local
+ config.hour: "{{ medium_rate_limit_per_hour }}"
+ config.limit_by: credential
+ - name: request-size-limiting
+ config.allowed_payload_size: "{{ small_request_size_limit }}"
+ - name: opa-checks
+ config.required: false
+ config.enabled: false
+
- name: listNotificationTemplate
uris: "{{ notification_service_prefix }}/v1/template/list"
upstream_url: "{{ notification_service_url }}/v1/notification/template/list"
diff --git a/ansible/roles/log-es-azure-restore/tasks/main.yml b/ansible/roles/log-es-azure-restore/tasks/main.yml
index f45a97587b2a857fa5971593a180a34791adec35..8d06fec321c05e47ce9ebfcd2f33766c412baca6 100644
--- a/ansible/roles/log-es-azure-restore/tasks/main.yml
+++ b/ansible/roles/log-es-azure-restore/tasks/main.yml
@@ -1,7 +1,7 @@
---
- name: Set azure snapshot for the first time
uri:
- url: "http://{{ es_restore_host }}:9200/_snapshot/azurebackup"
+ url: "http://{{ es_restore_host }}:9200/_snapshot/{{ snapshot_base_path }}"
method: PUT
body: "{{ snapshot_create_request_body | to_json }}"
headers:
@@ -9,12 +9,12 @@
- name: Restore ES from Azure backup
uri:
- url: "http://{{ es_restore_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}/_restore"
+ url: "http://{{ es_restore_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{snapshot_number}}/_restore"
method: POST
- name: "Wait for restore to be completed"
uri:
- url: "http://{{ es_restore_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}/_status"
+ url: "http://{{ es_restore_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{snapshot_number}}/_status"
method: GET
return_content: yes
status_code: 200
diff --git a/ansible/roles/log-es6/defaults/main.yml b/ansible/roles/log-es6/defaults/main.yml
index bb239c3ead1d08f56b2fc1476837101bcd46125b..be7451275cf4cf27764820e682feb9684c896e31 100644
--- a/ansible/roles/log-es6/defaults/main.yml
+++ b/ansible/roles/log-es6/defaults/main.yml
@@ -15,7 +15,6 @@ es_start_service: true
es_java_install: "{{ false if (es_version is version('7.0.0', '>=')) else true }}"
update_java: false
es_restart_on_change: true
-es_plugins_reinstall: false
es_templates: false
es_user: elasticsearch
es_group: elasticsearch
@@ -69,3 +68,8 @@ es_ssl_verification_mode: "certificate"
es_validate_certs: "yes"
es_delete_unmanaged_file: true
es_delete_unmanaged_native: true
+es_plugins_reinstall: true
+es_plugins:
+ - plugin: "repository-azure"
+ - plugin: "repository-s3"
+ - plugin: "repository-gcs"
diff --git a/ansible/roles/log-es6/tasks/main.yml b/ansible/roles/log-es6/tasks/main.yml
index 185212662b0bac9cfe967db46beafe8d8cff2eb4..646947abee093053611499c54a832d9ea4f0e7ca 100644
--- a/ansible/roles/log-es6/tasks/main.yml
+++ b/ansible/roles/log-es6/tasks/main.yml
@@ -59,6 +59,21 @@
tags:
- xpack
+- name: include plugins/create-keystore.yml
+ include: plugins/create-keystore.yml
+
+- name: include plugins/repository-azure.yml
+ include: plugins/repository-azure.yml
+ when: cloud_service_provider == "azure"
+
+- name: include plugins/repository-s3.yml
+ include: plugins/repository-s3.yml
+ when: cloud_service_provider == "aws"
+
+- name: include plugins/repository-gcs.yml
+ include: plugins/repository-gcs.yml
+ when: cloud_service_provider == "gcloud"
+
- name: include elasticsearch-ssl.yml
include: elasticsearch-ssl.yml
when: es_enable_http_ssl or es_enable_transport_ssl
diff --git a/ansible/roles/log-es6/tasks/plugins/create-keystore.yml b/ansible/roles/log-es6/tasks/plugins/create-keystore.yml
new file mode 100644
index 0000000000000000000000000000000000000000..80d2ca536e7976744cbba520b37b4efbb3606de1
--- /dev/null
+++ b/ansible/roles/log-es6/tasks/plugins/create-keystore.yml
@@ -0,0 +1,13 @@
+---
+- name: Check if elasticsearch keystore exists or not
+ become: yes
+ stat:
+ path: "{{ es_conf_dir }}/elasticsearch.keystore"
+ register: elasticsearch_keystore_file
+
+- name: Create the elasticsearch keystore if not exists
+ become: yes
+ command: "{{es_home}}/bin/elasticsearch-keystore create"
+ environment:
+ ES_PATH_CONF: "{{ es_conf_dir }}"
+ when: elasticsearch_keystore_file.stat.exists == false
diff --git a/ansible/roles/log-es6/tasks/plugins/repository-azure.yml b/ansible/roles/log-es6/tasks/plugins/repository-azure.yml
new file mode 100644
index 0000000000000000000000000000000000000000..9c3b9d3774cbf917bbe32c427fe20c6a5b299637
--- /dev/null
+++ b/ansible/roles/log-es6/tasks/plugins/repository-azure.yml
@@ -0,0 +1,15 @@
+---
+- name: Add default azure account name for backups
+ become: yes
+ shell: echo "{{ azure_management_storage_account_name }}" | {{ es_home }}/bin/elasticsearch-keystore add -f azure.client.default.account
+ no_log: True
+ environment:
+ ES_PATH_CONF: "{{ es_conf_dir }}"
+
+
+- name: Add default azure account key for backups
+ become: yes
+ shell: echo "{{ azure_management_storage_account_key }}" | {{ es_home }}/bin/elasticsearch-keystore add -f azure.client.default.key
+ no_log: True
+ environment:
+ ES_PATH_CONF: "{{ es_conf_dir }}"
\ No newline at end of file
diff --git a/ansible/roles/log-es6/tasks/plugins/repository-gcs.yml b/ansible/roles/log-es6/tasks/plugins/repository-gcs.yml
new file mode 100644
index 0000000000000000000000000000000000000000..81078e173dcb59d7fa5ae1a6386f7db73c7b04e4
--- /dev/null
+++ b/ansible/roles/log-es6/tasks/plugins/repository-gcs.yml
@@ -0,0 +1,18 @@
+---
+- name: Create the gcs service account file from variable
+ become: yes
+ copy:
+ dest: "{{ es_conf_dir }}/gcs_management_bucket_service_account.json"
+ content: "{{ gcs_management_bucket_service_account }}"
+
+- name: Add gcs service account file to keystore
+ become: yes
+ shell: "{{ es_home }}/bin/elasticsearch-keystore add-file -f gcs.client.default.credentials_file {{ es_conf_dir }}/gcs_management_bucket_service_account.json"
+ no_log: True
+ environment:
+ ES_PATH_CONF: "{{ es_conf_dir }}"
+
+- name: Remove the service account file
+ file:
+ path: "{{ es_conf_dir }}/gcs_management_bucket_service_account.json"
+ state: absent
\ No newline at end of file
diff --git a/ansible/roles/log-es6/tasks/plugins/repository-s3.yml b/ansible/roles/log-es6/tasks/plugins/repository-s3.yml
new file mode 100644
index 0000000000000000000000000000000000000000..344af29e6ea9b06a70035557b7e2b20220e36640
--- /dev/null
+++ b/ansible/roles/log-es6/tasks/plugins/repository-s3.yml
@@ -0,0 +1,14 @@
+---
+- name: Add default aws account name for backups
+ become: yes
+ shell: echo "{{ aws_management_bucket_user_access_key }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.access_key
+ no_log: True
+ environment:
+ ES_PATH_CONF: "{{ es_conf_dir }}"
+
+- name: Add default aws account key for backups
+ become: yes
+ shell: echo "{{ aws_management_bucket_user_secret_key }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.secret_key
+ no_log: True
+ environment:
+ ES_PATH_CONF: "{{ es_conf_dir }}"
diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml
index 1553e1b00db634baac4c070b00ae58d14a65d54e..5c5d87dace3c3ed90854b9ca755557c30273f2f2 100755
--- a/ansible/roles/ml-analytics-service/defaults/main.yml
+++ b/ansible/roles/ml-analytics-service/defaults/main.yml
@@ -10,7 +10,8 @@ ml_analytics_survey_app_name: "{{ ml_survey_app_name | default('sunbirdsurvey')
ml_analytics_integrated_app_name: "{{ ml_integrated_app_name | default('sunbird') }}"
ml_analytics_integrated_portal: "{{ ml_integrated_portal | default('dev.sunbird.portal') }}"
ml_analytics_survey_service: "http://{{private_ingressgateway_ip}}/ml-survey"
-ml_analytics_evidence_base_url: "https://{{ sunbird_private_storage_account_name }}.blob.core.windows.net/telemetry-data-store/"
+ml_analytics_public_container: "{{ ml_analytics_container | default('samiksha') }}"
+ml_analytics_evidence_base_url: "https://{{ sunbird_public_storage_account_name }}.blob.core.windows.net/{{ ml_analytics_public_container }}/"
ml_analytics_mongodb_url: "{{ml_mongodb_host | default(groups['mongo_master'][0]+':27017')}}"
ml_analytics_mongo_db_name: "{{ml_mongodb | default('ml-survey')}}"
ml_analytics_mongo_observation_submission_collection: "observationSubmissions"
@@ -48,14 +49,13 @@ ml_analytics_project_output_dir: "{{ WORKDIR }}/source/projects/output"
ml_analytics_observation_status_output_dir: "{{ WORKDIR }}/source/observations/status/output"
ml_analytics_api_authorization_key: "{{ml_api_auth_token | default('sunbird_api_auth_token')}}"
ml_analytics_api_access_token: "{{ml_api_access_token | default('ml_core_internal_access_token')}}"
-ml_analytics_druid_observation_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/observation/status/sl_observation_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_id","school_externalId","state_name","state_id","state_externalId","district_name","district_id","district_externalId","zone_name","zone_id","zone_externalId","parent_name","parent_id","block_name","block_id","block_externalId","cluster_name","cluster_id","cluster_externalId","teacher_name","teacher_id","teacher_externalId","schoolLeader_name","schoolLeader_id","schoolLeader_externalId","taluk_id","taluk_externalId","taluk_name","hub_name","hub_id","hub_externalId","mandal_name","mandal_id","mandal_externalId","region_name","region_id","region_externalId","complex_name","complex_id","complex_externalId","completedDate","beat_name","beat_id","beat_externalId","division_name","division_id","division_externalId","Village/Ward_name","Village/Ward_id","Village/Ward_externalId","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name", "ecm_marked_na","board_name","updatedAt","organisation_id","user_type"]},"metricsSpec":[]}}}'
+ml_analytics_druid_observation_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/observation/status/sl_observation_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_code","school_externalId","state_name","state_code","state_externalId","district_name","district_code","district_externalId","block_name","block_code","block_externalId","cluster_name","cluster_code","cluster_externalId","completedDate","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name","ecm_marked_na","board_name","updatedAt","organisation_id","user_type","observed_school_name","observed_school_id","observed_school_code","observed_state_name","observed_state_id","observed_state_code","observed_district_name","observed_district_id","observed_district_code","observed_block_name","observed_block_id","observed_block_code","observed_cluster_name","observed_cluster_id","observed_cluster_code"]},"metricsSpec":[]}}}'
ml_analytics_druid_project_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/projects/sl_projects.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-project","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"auto"},"dimensionsSpec":{"dimensions":[]},"metricsSpec":[]}}}'
ml_analytics_azure_sas_token: "{{ sunbird_private_storage_account_key }}"
-ml_analytics_druid_distinctCnt_obs_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount/ml_observation_distinctCount_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
-ml_analytics_druid_distinctCnt_obs_domain_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount_domain/ml_observation_distinctCount_domain.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
-ml_analytics_druid_distinctCnt_obs_domain_criteria_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount_domain_criteria/ml_observation_distinctCount_domain_criteria.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain-criteria","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"string","name":"criteria_name"},{"type":"string","name":"criteria_score"},{"type":"string","name":"criteria_id"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
-ml_analytics_druid_distinctCnt_projects_status_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/projects/distinctCount/ml_projects_distinctCount.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"project_title"},{"type":"string","name":"solution_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
-ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/projects/distinctCountPrglevel/ml_projects_distinctCount_prgmlevel.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-programLevel-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
+ml_analytics_druid_distinctCnt_obs_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount/ml_observation_distinctCount_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
+ml_analytics_druid_distinctCnt_obs_domain_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount_domain/ml_observation_distinctCount_domain.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
+ml_analytics_druid_distinctCnt_obs_domain_criteria_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount_domain_criteria/ml_observation_distinctCount_domain_criteria.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain-criteria","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"string","name":"criteria_name"},{"type":"string","name":"criteria_score"},{"type":"string","name":"criteria_id"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
+ml_analytics_druid_distinctCnt_projects_status_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/projects/distinctCount/ml_projects_distinctCount.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"project_title"},{"type":"string","name":"solution_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
ml_analytics_obs_distinctCnt_azure_blob_path: "observation/distinctCount/"
ml_analytics_obs_distinctCnt_domain_azure_blob_path: "observation/distinctCount_domain/"
ml_analytics_obs_distinctCnt_domain_criteria_azure_blob_path: "observation/distinctCount_domain_criteria/"
@@ -64,5 +64,38 @@ ml_analytics_obs_distinctCnt_output_dir: "{{ WORKDIR }}/source/observations/dist
ml_analytics_obs_distinctCnt_domain_output_dir: "{{ WORKDIR }}/source/observations/distinctCount_domain/output"
ml_analytics_obs_distinctCnt_domain_criteria_output_dir: "{{ WORKDIR }}/source/observations/distinctCount_domain_criteria/output"
ml_analytics_projects_distinctCnt_output_dir: "{{ WORKDIR }}/source/projects/distinctCount/output"
+ml_analytics_survey_rollup_azure_blob_path: "survey/rollup/"
+ml_analytics_druid_survey_rollup_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/survey/rollup/sl_survey_rollup.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-survey-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":true,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["survey_submission_id", "submission_status", "user_id", "user_sub_type", "user_type", "state_externalId", "block_externalId", "district_externalId", "cluster_externalId", "school_externalId", "state_name", "block_name", "district_name", "cluster_name", "school_name", "board_name", "organisation_id", "organisation_name", "program_externalId", "program_id", "program_name", "survey_name", "survey_id", "survey_externalId", "created_date", "submission_date", "updatedAt", "parent_channel",{"type":"long","name":"status_code"}, "solution_name", "solution_id"]},"metricsSpec":[{"name":"count","type":"count"},{"name":"sum___v","type":"longSum","fieldName":"__v"},{"name":"sum_status_code","type":"longMax","fieldName":"status_code"},{"type":"HLLSketchBuild","name":"count_of_user_id","fieldName":"user_id"},{"type":"HLLSketchBuild","name":"count_of_survey_submission_id","fieldName":"survey_submission_id"},{"type":"HLLSketchBuild","name":"count_of_solution_id","fieldName":"solution_id"},{"type":"HLLSketchBuild","name":"count_of_program_id","fieldName":"program_id"}]}}}'
+ml_analytics_survey_rollup_output_dir: "/opt/sparkjobs/source/survey/output"
+ml_analytics_project_rollup_azure_blob_path: "projects/rollup"
+ml_analytics_observation_rollup_azure_blob_path: "observation/rollup"
+ml_analytics_project_rollup_output_dir: "/opt/sparkjobs/source/projects/output_rollup"
+ml_analytics_observation_status_rollup_output_dir: "/opt/sparkjobs/source/observations/output_rollup"
+ml_analytics_druid_project_rollup_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/projects/rollup/projects_rollup.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"iso"},"dimensionsSpec":{"dimensions":["project_title","project_goal","area_of_improvement","status_of_project","tasks_name","tasks_status","designation","task_evidence_status","project_id","task_id","project_created_type","parent_channel","program_id","program_name","project_updated_date","createdBy","program_externalId","private_program","task_deleted_flag","project_terms_and_condition","state_externalId","block_externalId","district_externalId","cluster_externalId","school_externalId","state_name","block_name","district_name","cluster_name","school_name","board_name","organisation_name","solution_id","organisation_id",{"name":"status_code","type":"long"}]},"metricsSpec":[{"name":"count","type":"count"},{"name":"sum___v","type":"longSum","fieldName":"__v"},{"name":"sum_status_code","type":"longMax","fieldName":"status_code"},{"type":"HLLSketchBuild","name":"count_of_createBy","fieldName":"createdBy"},{"type":"HLLSketchBuild","name":"count_of_project_id","fieldName":"project_id"},{"type":"HLLSketchBuild","name":"count_of_solution_id","fieldName":"solution_id"},{"type":"HLLSketchBuild","name":"count_of_program_id","fieldName":"program_id"}]}}}'
+ml_analytics_druid_observation_status_rollup_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/rollup/observation_status_rollup.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-observation-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["status","user_id","solution_id","submission_id","entity_name","completedDate","program_id","private_program","solution_type","updatedAt","role_title","solution_name","program_name","channel","parent_channel","block_name","district_name","school_name","cluster_name","state_name","organisation_name","board_name","district_externalId","state_externalId","block_externalId","cluster_externalId","school_externalId","organisation_id",{"type":"long","name":"status_code"}]},"metricsSpec":[{"type":"count","name":"count"},{"type":"longSum","name":"sum___v","fieldName":"__v","expression":null},{"type":"HLLSketchBuild","name":"count_distinct_solution","fieldName":"solution_id","lgK":12,"tgtHllType":"HLL_4","round":false},{"type":"HLLSketchBuild","name":"count_distinct_submission_id","fieldName":"submission_id","lgK":12,"tgtHllType":"HLL_4","round":false},{"type":"HLLSketchBuild","name":"count_distinct_user_id","fieldName":"user_id","lgK":12,"tgtHllType":"HLL_4","round":false}]}}}'
+ml_analytics_druid_rollup_url: "{{groups['druid'][0]}}:8081"
+ml_analytics_AWS_service_name: "{{ ml_AWS_service_name | default('') }}"
+ml_analytics_AWS_access_key: "{{ ml_AWS_access_key | default('') }}"
+ml_analytics_AWS_secret_access_key: "{{ ml_AWS_secret_access_key | default('') }}"
+ml_analytics_AWS_region_name: "{{ ml_AWS_region_name | default('') }}"
+ml_analytics_AWS_bucket_name: "{{ ml_AWS_bucket_name | default('') }}"
+ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/projects/distinctCountPrglevel/ml_projects_distinctCount_prgmlevel.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-programLevel-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
ml_analytics_projects_distinctCnt_prglevel_output_dir: "{{ WORKDIR }}/source/projects/distinctCountPrglevel/output"
ml_analytics_projects_distinctCnt_prglevel_azure_blob_path: "projects/distinctCountPrglevel/"
+ml_analytics_survey_status_output_dir : "{{ WORKDIR }}/source/survey/status/output"
+ml_analytics_survey_azure_blob_path : "survey/status/"
+ml_analytics_druid_survey_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/survey/status/sl_survey_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-survey-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":true,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["survey_submission_id", "submission_status", "user_id", "user_sub_type", "user_type", "state_externalId", "block_externalId", "district_externalId", "cluster_externalId", "school_externalId", "state_name", "block_name", "district_name", "cluster_name", "school_name", "board_name", "organisation_id", "organisation_name", "program_externalId", "program_id", "program_name", "survey_name", "survey_id", "survey_externalId", "created_date", "submission_date", "updatedAt", "parent_channel", "solution_name", "solution_id","private_program"]},"metricsSpec":[]}}}'
+ml_analytics_slack_token: "{{ ml_slack_token | default('') }}"
+ml_analytics_channel_name: "{{ ml_slack_channel | default('') }}"
+ml_analytics_public_azure_account_name: "{{ ml_public_azure_account | default('') }}"
+ml_analytics_public_azure_access_key: "{{ ml_public_azure_key | default('') }}"
+ml_analytics_public_azure_container_name: "{{ ml_public_azure_container | default('') }}"
+ml_analytics_program_dashboard_azure_blob_path: "{{ ml_program_blob_path | default('') }}"
+ml_druid_query_data: "{{ ml_druid_query | default('') }}"
+ml_program_dashboard_data: "{{ ml_program_data | default('') }}"
+ml_analytics_druid_query_url: "{{groups['druid'][0]}}:8082"
+ml_analytics_druid_observation_query_spec: '{"queryType":"scan","dataSource":"sl-observation","resultFormat":"list","columns":["completedDate","createdAt","createdBy","criteriaExternalId","criteriaId","criteriaName","entityType","entityTypeId","observationId","observationName","observationSubmissionId","questionAnswer","questionECM","questionExternalId","questionId","questionName","questionResponseLabel","questionResponseType","solutionExternalId","solutionId","solutionName","updatedAt","instanceParentId","instanceId","instanceParentResponsetype","instanceParentQuestion","questionSequenceByEcm","maxScore","minScore","percentageScore","pointsBasedScoreInParent","totalScore","scoreAchieved","totalpercentage","instanceParentExternalId","instanceParentEcmSequence","remarks","total_evidences","evidence_count","school","block","district","cluster","state","schoolName","blockName","districtName","clusterName","stateName","schoolExternalId","blockExternalId","districtExternalId","clusterExternalId","stateExternalId","schoolTypes","administrationTypes","instanceParentCriteriaId","instanceParentCriteriaExternalId","instanceParentCriteriaName","role_title","location_validated_with_geotag","distance_in_meters","entity","entityExternalId","entityName","isAPrivateProgram","programId","programName","programExternalId","questionResponseLabel_number","criteriaLevel","criteriaScore","submissionNumber","submissionTitle","channel","parent_channel","user_districtName","user_blockName","user_clusterName","appName","evidences","user_stateName","domainName","domainExternalId","childName","childType","childExternalid","level","criteriaDescription","programDescription","solutionDescription","label","imp_project_id","imp_project_title","imp_project_goal","imp_project_externalId","ancestorName","scoringSystem","domainLevel","domainScore","criteriaLevelReport","user_schoolName","user_schoolId","user_schoolUDISE_code","solution_type","organisation_name","user_boardName","district_externalId","state_externalId","block_externalId","cluster_externalId","organisation_id","user_type"],"intervals":["1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00"]}'
+ml_analytics_druid_observation_batch_ingestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/observation/batchDeletion/druidData.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"completedDate","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"completedDate"},{"type":"string","name":"createdAt"},{"type":"string","name":"createdBy"},{"type":"string","name":"criteriaExternalId"},{"type":"string","name":"criteriaId"},{"type":"string","name":"criteriaName"},{"type":"string","name":"entityType"},{"type":"string","name":"entityTypeId"},{"type":"string","name":"observationId"},{"type":"string","name":"observationName"},{"type":"string","name":"observationSubmissionId"},{"type":"string","name":"questionAnswer"},{"type":"string","name":"questionECM"},{"type":"string","name":"questionExternalId"},{"type":"string","name":"questionId"},{"type":"string","name":"questionName"},{"type":"string","name":"questionResponseLabel"},{"type":"string","name":"questionResponseType"},{"type":"string","name":"solutionExternalId"},{"type":"string","name":"solutionId"},{"type":"string","name":"solutionName"},{"type":"string","name":"updatedAt"},{"type":"string","name":"instanceParentId"},{"type":"string","name":"instanceId"},{"type":"string","name":"instanceParentResponsetype"},{"type":"string","name":"instanceParentQuestion"},{"type":"string","name":"questionSequenceByEcm"},{"type":"string","name":"maxScore"},{"type":"string","name":"minScore"},{"type":"string","name":"percentageScore"},{"type":"string","name":"pointsBasedScoreInParent"},{"type":"string","name":"totalScore"},{"type":"string","name":"scoreAchieved"},{"type":"string","name":"totalpercentage"},{"type":"string","name":"instanceParentExternalId"},{"type":"string","name":"instanceParentEcmSequence"},{"type":"string","name":"remarks"},{"type":"string","name":"total_evidences"},{"type":"string","name":"evidence_count"},{"type":"string","name":"school"},{"type":"string","name":"block"},{"type":"string","name":"district"},{"type":"string","name":"cluster"},{"type":"string","name":"state"},{"type":"string","name":"schoolName"},{"type":"string","name":"blockName"},{"type":"string","name":"districtName"},{"type":"string","name":"clusterName"},{"type":"string","name":"stateName"},{"type":"string","name":"schoolExternalId"},{"type":"string","name":"blockExternalId"},{"type":"string","name":"districtExternalId"},{"type":"string","name":"clusterExternalId"},{"type":"string","name":"stateExternalId"},{"type":"string","name":"schoolTypes"},{"type":"string","name":"administrationTypes"},{"type":"string","name":"instanceParentCriteriaId"},{"type":"string","name":"instanceParentCriteriaExternalId"},{"type":"string","name":"instanceParentCriteriaName"},{"type":"string","name":"role_title"},{"type":"string","name":"location_validated_with_geotag"},{"type":"string","name":"distance_in_meters"},{"type":"string","name":"entity"},{"type":"string","name":"entityExternalId"},{"type":"string","name":"entityName"},{"type":"string","name":"isAPrivateProgram"},{"type":"string","name":"programId"},{"type":"string","name":"programName"},{"type":"string","name":"programExternalId"},{"name":"questionResponseLabel_number","type":"float"},{"type":"string","name":"criteriaLevel"},{"type":"string","name":"criteriaScore"},{"type":"string","name":"submissionNumber"},{"type":"string","name":"submissionTitle"},{"type":"string","name":"channel"},{"type":"string","name":"parent_channel"},{"type":"string","name":"user_districtName"},{"type":"string","name":"user_blockName"},{"type":"string","name":"user_clusterName"},{"type":"string","name":"appName"},{"type":"string","name":"evidences"},{"type":"string","name":"user_stateName"},{"type":"string","name":"domainName"},{"type":"string","name":"domainExternalId"},{"type":"string","name":"childName"},{"type":"string","name":"childType"},{"type":"string","name":"childExternalid"},{"type":"string","name":"level"},{"type":"string","name":"criteriaDescription"},{"type":"string","name":"programDescription"},{"type":"string","name":"solutionDescription"},{"type":"string","name":"label"},{"type":"string","name":"imp_project_id"},{"type":"string","name":"imp_project_title"},{"type":"string","name":"imp_project_goal"},{"type":"string","name":"imp_project_externalId"},{"type":"string","name":"ancestorName"},{"type":"string","name":"scoringSystem"},{"type":"string","name":"domainLevel"},{"type":"string","name":"domainScore"},{"name":"criteriaLevelReport","type":"boolean"},{"type":"string","name":"user_schoolName"},{"type":"string","name":"user_schoolId"},{"type":"string","name":"user_schoolUDISE_code"},{"type":"string","name":"solution_type"},{"type":"string","name":"organisation_name"},{"type":"string","name":"user_boardName"},{"type":"string","name":"district_externalId"},{"type":"string","name":"state_externalId"},{"type":"string","name":"block_externalId"},{"type":"string","name":"cluster_externalId"},{"type":"string","name":"organisation_id"},{"type":"string","name":"user_type"},{"type":"string","name":"isSubmissionDeleted"}]},"metricsSpec":[{"type":"floatSum","name":"question_response_number","fieldName":"questionResponseLabel_number"}]}}}'
+ml_analytics_observation_batchupdate_azure_blob_path: "observation/batchDeletion"
+ml_analytics_observation_submission_id_filepath: "{{ WORKDIR }}/ml-analytics-service/observations/submissions.csv"
+ml_analytics_observation_batchupdate_output_dir: "{{ WORKDIR }}/source/observations/"
diff --git a/ansible/roles/ml-analytics-service/tasks/main.yml b/ansible/roles/ml-analytics-service/tasks/main.yml
index 9c9bf79a9038881b1a7947ed26ce9aa157ff2522..0998cf8188bde33cea8c77f474b6a2bd4ed5e692 100755
--- a/ansible/roles/ml-analytics-service/tasks/main.yml
+++ b/ansible/roles/ml-analytics-service/tasks/main.yml
@@ -105,6 +105,12 @@
dest: "{{ config_path }}/config.ini"
backup: yes
+- name: Templating the shell_script_config.j2 to shell_script_config
+ template:
+ src: "shell_script_config.j2"
+ dest: "{{ config_path }}/shell_script_config"
+ backup: yes
+
- name: SYSTEMD DAEMON-RELOAD
systemd:
daemon_reload: yes
@@ -120,10 +126,19 @@
- faust_survey.service
- faust_survey_evidence.service
-- name: CREATE THE CRON
+- name: CREATE THE CRON BATCH INGESTION
cron:
name: "Run Batch Ingestion Job"
user: "{{ USER }}"
minute: "30"
hour: "18"
job: "{{ BASEPATH }}/ml-analytics-service/run.sh > {{ BASEPATH }}/ml-analytics-service/crontab_job.log"
+
+- name: CREATE THE CRON NVSK Data Upload
+ cron:
+ name: "NVSK Data Upload JOB"
+ user: "{{ USER }}"
+ minute: "30"
+ hour: "7"
+ weekday: "4"
+ job: "{{ BASEPATH }}/run_weekly.sh > {{ BASEPATH }}/ml-analytics-service/nvsk_data_weekly.logs"
diff --git a/ansible/roles/ml-analytics-service/templates/config.j2 b/ansible/roles/ml-analytics-service/templates/config.j2
index 8693935e9c2543905f03d99d4fde1a1574036427..b4945675d3c632ed4eaf1a5ba5126c591a8cb706 100644
--- a/ansible/roles/ml-analytics-service/templates/config.j2
+++ b/ansible/roles/ml-analytics-service/templates/config.j2
@@ -74,6 +74,12 @@ metadata_url = http://{{ ml_analytics_druid_url }}/druid/coordinator/v1/datasour
batch_url = http://{{ ml_analytics_druid_url }}/druid/indexer/v1/task
+metadata_rollup_url = http://{{ ml_analytics_druid_rollup_url }}/druid/coordinator/v1/datasources/
+
+batch_rollup_url = http://{{ ml_analytics_druid_rollup_url }}/druid/indexer/v1/task
+
+urlQuery = http://{{ ml_analytics_druid_query_url }}/druid/v2?pretty
+
observation_status_injestion_spec = {{ ml_analytics_druid_observation_status_injestion_spec }}
project_injestion_spec = {{ ml_analytics_druid_project_injestion_spec }}
@@ -86,7 +92,19 @@ ml_distinctCnt_obs_domain_criteria_spec = {{ ml_analytics_druid_distinctCnt_obs_
ml_distinctCnt_projects_status_spec = {{ ml_analytics_druid_distinctCnt_projects_status_injestion_spec }}
-ml_distinctCnt_prglevel_projects_status_spec = {{ ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec }}
+observation_status_rollup_injestion_spec = {{ ml_analytics_druid_observation_status_rollup_injestion_spec }}
+
+project_rollup_injestion_spec = {{ ml_analytics_druid_project_rollup_injestion_spec }}
+
+ml_survey_rollup_spec = {{ml_analytics_druid_survey_rollup_injestion_spec}}
+
+ml_distinctCnt_prglevel_projects_status_spec = {{ ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec}}
+
+survey_status_injestion_spec = {{ ml_analytics_druid_survey_status_injestion_spec }}
+
+observation_query_spec = {{ ml_analytics_druid_observation_query_spec }}
+
+observation_injestion_spec = {{ml_analytics_druid_observation_batch_ingestion_spec}}
[KAFKA]
@@ -150,7 +168,33 @@ observation_distinctCount_domain_criteria_blob_path = {{ ml_analytics_obs_distin
projects_distinctCnt_blob_path = {{ ml_analytics_projects_distinctCnt_azure_blob_path }}
-projects_distinctCnt_prgmlevel_blob_path = {{ ml_analytics_projects_distinctCnt_prglevel_azure_blob_path }}
+projects_rollup_blob_path = {{ ml_analytics_project_rollup_azure_blob_path }}
+
+observation_rollup_blob_path = {{ ml_analytics_observation_rollup_azure_blob_path }}
+
+survey_rollup_blob_path = {{ ml_analytics_survey_rollup_azure_blob_path }}
+
+projects_distinctCnt_prgmlevel_blob_path = {{ml_analytics_projects_distinctCnt_prglevel_azure_blob_path}}
+
+survey_blob_path = {{ ml_analytics_survey_azure_blob_path }}
+
+public_account_name = {{ ml_analytics_public_azure_account_name }}
+
+public_access_key = {{ ml_analytics_public_azure_access_key }}
+
+public_container_name = {{ ml_analytics_public_azure_container_name }}
+
+projects_program_csv = {{ ml_analytics_program_dashboard_azure_blob_path }}
+
+observation_batch_ingestion_data_del = {{ ml_analytics_observation_batchupdate_azure_blob_path }}
+
+[REDIS]
+
+host = {{ ml_analytics_redis_host }}
+
+port = {{ ml_analytics_redis_port }}
+
+db_name = {{ ml_analytics_redis_db_name }}
[OUTPUT_DIR]
@@ -166,4 +210,40 @@ observation_distinctCount_domain_criteria = {{ ml_analytics_obs_distinctCnt_doma
projects_distinctCount = {{ ml_analytics_projects_distinctCnt_output_dir }}
-projects_distinctCount_prgmlevel = {{ ml_analytics_projects_distinctCnt_prglevel_output_dir }}
+project_rollup = {{ ml_analytics_project_rollup_output_dir }}
+
+observation_status_rollup = {{ ml_analytics_observation_status_rollup_output_dir }}
+
+survey_rollup = {{ ml_analytics_survey_rollup_output_dir }}
+
+projects_distinctCount_prgmlevel = {{ml_analytics_projects_distinctCnt_prglevel_output_dir}}
+
+survey_status = {{ ml_analytics_survey_status_output_dir }}
+
+observation_sub_ids = {{ ml_analytics_observation_submission_id_filepath }}
+
+observation_druid_data = {{ ml_analytics_observation_batchupdate_output_dir }}
+
+[CLOUD_STORAGE]
+
+service_name = {{ ml_analytics_AWS_service_name }}
+
+access_key = {{ ml_analytics_AWS_access_key }}
+
+secret_access_key = {{ ml_analytics_AWS_secret_access_key }}
+
+region_name = {{ ml_analytics_AWS_region_name }}
+
+bucket_name = {{ ml_analytics_AWS_bucket_name }}
+
+[SLACK]
+
+token = {{ ml_analytics_slack_token }}
+
+channel = {{ ml_analytics_channel_name }}
+
+[VAM]
+
+druid_query_url = {{ ml_druid_query_data }}
+
+program_dashboard_data = {{ ml_program_dashboard_data }}
diff --git a/ansible/roles/ml-analytics-service/templates/shell_script_config.j2 b/ansible/roles/ml-analytics-service/templates/shell_script_config.j2
new file mode 100644
index 0000000000000000000000000000000000000000..6ecdeba31a3cd73c4d8d6a8e561a81ccf7bd9b1b
--- /dev/null
+++ b/ansible/roles/ml-analytics-service/templates/shell_script_config.j2
@@ -0,0 +1,2 @@
+mongo_url={{ ml_analytics_mongodb_url }}
+mongo_db_name={{ ml_analytics_mongo_db_name }}
diff --git a/ansible/roles/stack-monitor-stateful/templates/elasticsearch_snapshots_exporter_config.yml b/ansible/roles/stack-monitor-stateful/templates/elasticsearch_snapshots_exporter_config.yml
index aa13c4be67b06837d20747f5d668f5e7d6d77dfc..43b12d5220115805b788421dfeef3c13bbd94530 100644
--- a/ansible/roles/stack-monitor-stateful/templates/elasticsearch_snapshots_exporter_config.yml
+++ b/ansible/roles/stack-monitor-stateful/templates/elasticsearch_snapshots_exporter_config.yml
@@ -1,6 +1,6 @@
exporter_port: 9158
log_level: info
-json_data_url: http://{{ groups['es'][0] }}:9200/_snapshot/azurebackup/_all
+json_data_url: http://{{ groups['es'][0] }}:9200/_snapshot/{{ snapshot_base_path }}/_all
metric_name_prefix: elasticsearch_snapshots
metrics:
- name: latest_successful_snapshot_timestamp
diff --git a/ansible/roles/stack-monitor/templates/elasticsearch_snapshots_exporter_config.yml b/ansible/roles/stack-monitor/templates/elasticsearch_snapshots_exporter_config.yml
index aa13c4be67b06837d20747f5d668f5e7d6d77dfc..43b12d5220115805b788421dfeef3c13bbd94530 100644
--- a/ansible/roles/stack-monitor/templates/elasticsearch_snapshots_exporter_config.yml
+++ b/ansible/roles/stack-monitor/templates/elasticsearch_snapshots_exporter_config.yml
@@ -1,6 +1,6 @@
exporter_port: 9158
log_level: info
-json_data_url: http://{{ groups['es'][0] }}:9200/_snapshot/azurebackup/_all
+json_data_url: http://{{ groups['es'][0] }}:9200/_snapshot/{{ snapshot_base_path }}/_all
metric_name_prefix: elasticsearch_snapshots
metrics:
- name: latest_successful_snapshot_timestamp
diff --git a/ansible/roles/stack-sunbird/templates/content-service_application.conf b/ansible/roles/stack-sunbird/templates/content-service_application.conf
index 9703fb47a7e02f25bc664f52be2402b2a015f6a5..d33dbecf6fe3fec8e849bff663a90537f1c35acc 100644
--- a/ansible/roles/stack-sunbird/templates/content-service_application.conf
+++ b/ansible/roles/stack-sunbird/templates/content-service_application.conf
@@ -500,8 +500,16 @@ dial_service {
api {
base_url : "http://dial-service.{{namespace}}.svc.cluster.local:9000"
auth_key : "{{ sunbird_dial_repo_api_key }}"
+ search : "/dialcode/v3/search"
+ generate : "/dialcode/v3/generate"
}
}
+
+reserve_dialcode {
+ mimeType : ["application/vnd.ekstep.content-collection"]
+ max_count : 250
+}
+
content.link_dialcode.validation=true
content.link_dialcode.max_limit=10
@@ -594,7 +602,7 @@ master.category.validation.enabled="{{ master_category_validation_enabled }}"
#Collection CSV
sunbird_dialcode_search_api="http://dial-service.{{namespace}}.svc.cluster.local:9000/dialcode/v3/list"
framework_read_api_url="{{ sunbird_content_repo_api_base_url }}/framework/v3/read"
-sunbird_link_dial_code_api="{{ sunbird_content_repo_api_base_url }}/collection/v3/dialcode/link"
+sunbird_link_dial_code_api="{{ sunbird_content_service_api_base_url }}/collection/v4/dialcode/link"
collection {
diff --git a/ansible/roles/stack-sunbird/templates/dial-service_application.conf b/ansible/roles/stack-sunbird/templates/dial-service_application.conf
index 1be178f6ff9eb87439ce0e67fb8e7404a199e2bc..745a8b9bfe5916e8ee0f044bd535f5ff970bded5 100644
--- a/ansible/roles/stack-sunbird/templates/dial-service_application.conf
+++ b/ansible/roles/stack-sunbird/templates/dial-service_application.conf
@@ -183,9 +183,11 @@ kafka {
dial_id = "https://{{domain_name}}/dial/{dialcode}"
dial_type = "https://{{domain_name}}/ns/"
-schema {
+jsonld {
basePath = "{{dial_service_schema_base_path}}"
+ type = "sbed"
localPath = "/tmp"
ttl = 300
+ sb_schema = ["http://store.knowlg.sunbird.org/dial/specs/sb/schema.jsonld"]
}
diff --git a/ansible/roles/stack-sunbird/templates/sunbird_cert-registry-service.env b/ansible/roles/stack-sunbird/templates/sunbird_cert-registry-service.env
index dcde35ae4390e75687ec5d7198cc6a4711e29892..2ea34bac0bad15ae222b19dac8b6284506d984d6 100644
--- a/ansible/roles/stack-sunbird/templates/sunbird_cert-registry-service.env
+++ b/ansible/roles/stack-sunbird/templates/sunbird_cert-registry-service.env
@@ -6,7 +6,7 @@ sunbird_es_port={{sunbird_es_port}}
sunbird_cassandra_keyspace={{sunbird_cassandra_keyspace|d('sunbird')}}
sunbird_cassandra_username={{sunbird_cassandra_username|d('cassandra')}}
sunbird_cassandra_password={{sunbird_cassandra_password|d('password')}}
-sunbird_cassandra_consistency_level=quorum
+sunbird_cassandra_consistency_level={{sunbird_cassandra_consistency_level}}
{% if (cassandra_cluster_size | int) > 1 %}
sunbird_cassandra_host={{groups['cassandra']|join(',')}}
sunbird_cassandra_port=9042
@@ -14,3 +14,5 @@ sunbird_cassandra_port=9042
sunbird_cassandra_host={{sunbird_cassandra_host}}
sunbird_cassandra_port=9042
{% endif %}
+# Release-5.0.0
+isMultiDCEnabled={{cassandra_multi_dc_enabled}}
diff --git a/ansible/roles/stack-sunbird/templates/sunbird_cert-service.env b/ansible/roles/stack-sunbird/templates/sunbird_cert-service.env
index 686df40e63afb88ad6c8b83fcb1864f50e794f88..c43c23171b687e74597513c2be139798f8fdd485 100644
--- a/ansible/roles/stack-sunbird/templates/sunbird_cert-service.env
+++ b/ansible/roles/stack-sunbird/templates/sunbird_cert-service.env
@@ -1,13 +1,13 @@
CONTAINER_NAME={{cert_service_container_name}}
-CLOUD_STORAGE_TYPE={{cert_service_cloud_storage_type}}
-AZURE_STORAGE_SECRET={{sunbird_private_storage_account_key}}
-AZURE_STORAGE_KEY={{sunbird_private_storage_account_name}}
+CLOUD_STORAGE_TYPE={{cloud_service_provider}}
+PRIVATE_CLOUD_STORAGE_SECRET={{sunbird_private_storage_account_key}}
+PRIVATE_CLOUD_STORAGE_KEY={{sunbird_private_storage_account_name}}
sunbird_cert_domain_url={{proto}}://{{proxy_server_name}}
sunbird_cert_enc_service_url=http://enc-service:8013
download_link_expiry_timeout=600
es_conn_info={{groups['es']|join(':9200,')}}:9200
ITEXT_LICENSE_ENABLED={{itext_license_enabled}}
ITEXT_LICENSE_PATH=/home/sunbird/itext_trail_license.xml
-PUBLIC_AZURE_STORAGE_KEY={{sunbird_public_storage_account_name}}
-PUBLIC_AZURE_STORAGE_SECRET={{sunbird_public_storage_account_key}}
+PUBLIC_CLOUD_STORAGE_KEY={{sunbird_public_storage_account_name}}
+PUBLIC_CLOUD_STORAGE_SECRET={{sunbird_public_storage_account_key}}
PUBLIC_CONTAINER_NAME={{sunbird_cert_qr_container_name}}
diff --git a/ansible/roles/stack-sunbird/templates/sunbird_groups-service.env b/ansible/roles/stack-sunbird/templates/sunbird_groups-service.env
index 38e1c3cf8c6ce5d325d705452f8db600b04f67fe..2c1e2fe83d8de8791b7f76da4ed3102812a3881e 100644
--- a/ansible/roles/stack-sunbird/templates/sunbird_groups-service.env
+++ b/ansible/roles/stack-sunbird/templates/sunbird_groups-service.env
@@ -1,14 +1,13 @@
{% if (cassandra_cluster_size | int) > 1 %}
sunbird_cassandra_host={{groups['cassandra']|join(',')}}
sunbird_cassandra_port=9042,9042,9042
-sunbird_cassandra_consistency_level=quorum
{% else %}
sunbird_cassandra_host={{sunbird_cassandra_host}}
sunbird_cassandra_port=9042
-sunbird_cassandra_consistency_level={{sunbird_cassandra_consistency_level}}
{% endif %}
sunbird_cassandra_password={{sunbird_cassandra_password|d('password')}}
sunbird_cassandra_username={{sunbird_cassandra_username|d('cassandra')}}
+sunbird_cassandra_consistency_level={{sunbird_cassandra_consistency_level}}
LEARNER_SERVICE_PORT=http://learner-service:9000
sunbird_redis_host={{sunbird_redis_host}}
sunbird_redis_port={{sunbird_redis_port|default(6379)}}
@@ -41,3 +40,5 @@ notification_service_api_url=/v2/notification/send
#“OFF“: To disable for all states
#To allow multiple channel . please write as comma separated : e.g : tn,cg value
enable_tenant_config={{enable_tenant_config| default('*')}}
+# Release-5.0.0
+isMultiDCEnabled={{cassandra_multi_dc_enabled}}
diff --git a/ansible/roles/stack-sunbird/templates/sunbird_learner-service.env b/ansible/roles/stack-sunbird/templates/sunbird_learner-service.env
index 27b2569ba5509bd7b6060eb6badda48ea4d59d2d..79d0bfe1e356233caff1d742df09c4355bd92b01 100644
--- a/ansible/roles/stack-sunbird/templates/sunbird_learner-service.env
+++ b/ansible/roles/stack-sunbird/templates/sunbird_learner-service.env
@@ -39,13 +39,11 @@ sunbird_installation_email={{sunbird_installation_email}}
{% if (cassandra_cluster_size | int) > 1 %}
sunbird_cassandra_host={{groups['cassandra']|join(',')}}
sunbird_cassandra_port=9042,9042,9042
-sunbird_cassandra_consistency_level=quorum
{% else %}
sunbird_cassandra_host={{sunbird_cassandra_host}}
sunbird_cassandra_port=9042
-sunbird_cassandra_consistency_level={{sunbird_cassandra_consistency_level}}
{% endif %}
-
+sunbird_cassandra_consistency_level={{sunbird_cassandra_consistency_level}}
sunbird_mw_system_host=learner-service
sunbird_mw_system_port=8088
background_actor_provider=local
@@ -98,7 +96,8 @@ sunbird_cache_enable={{sunbird_cache_enable | default(false)}}
sunbird_redis_host={{sunbird_redis_host}}
sunbird_redis_port={{sunbird_redis_port|default(6379)}}
kafka_urls={{kafka_urls}}
-sunbird_user_cert_kafka_topic={{env_name}}.lms.user.account.merge
+# Release-5.0.0
+sunbird_user_cert_kafka_topic={{kafka_topic_lms_user_account}}
# Release-2.3.0
sunbird_reset_pass_msg={{sunbird_reset_pass_msg | d('You have requested to reset password. Click on the link to set a password: {0}')}}
sunbird_fuzzy_search_threshold={{sunbird_fuzzy_search_threshold | d('0.5')}}
@@ -129,3 +128,7 @@ user_index_alias={{user_index_alias | default('user_alias')}}
org_index_alias={{org_index_alias | default('org_alias')}}
sunbird_installation_display_name_for_sms=DIKSHA
sunbird_sso_lb_ip={{keycloak_url}}
+
+# Release-5.0.0
+sunbird_cloud_service_provider={{cloud_service_provider}}
+isMultiDCEnabled={{cassandra_multi_dc_enabled}}
diff --git a/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env b/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env
index 112c702f37d587fb270d63cb5fbf7c602ed3407a..6b790eb7352142eb2aba441ffb7ecf414fd8f1da 100644
--- a/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env
+++ b/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env
@@ -39,13 +39,11 @@ sunbird_installation_email={{sunbird_installation_email}}
{% if (cassandra_cluster_size | int) > 1 %}
sunbird_cassandra_host={{groups['cassandra']|join(',')}}
sunbird_cassandra_port=9042,9042,9042
-sunbird_cassandra_consistency_level=quorum
{% else %}
sunbird_cassandra_host={{sunbird_cassandra_host}}
sunbird_cassandra_port=9042
-sunbird_cassandra_consistency_level={{sunbird_cassandra_consistency_level}}
{% endif %}
-
+sunbird_cassandra_consistency_level={{sunbird_cassandra_consistency_level}}
sunbird_mw_system_host=lms-service
sunbird_mw_system_port=8088
background_actor_provider=local
@@ -142,4 +140,8 @@ druid_proxy_api_port=8082
druid_proxy_api_endpoint=/druid/v2/
collection_summary_agg_data_source={{ summary_agg_data_source | default('audit-rollup-syncts') }}
collection_summary_agg_cache_ttl={{ summary_agg_ttl | default(21600) }}
-enrollment_list_size={{ enrollment_list_size | default(1000) }}
\ No newline at end of file
+enrollment_list_size={{ enrollment_list_size | default(1000) }}
+
+# Release-5.0.0
+sunbird_cloud_service_provider={{cloud_service_provider}}
+isMultiDCEnabled={{cassandra_multi_dc_enabled}}
\ No newline at end of file
diff --git a/ansible/roles/stack-sunbird/templates/sunbird_notification-service.env b/ansible/roles/stack-sunbird/templates/sunbird_notification-service.env
index 6189bdafa506f4845ca44050ee17180aba7437d5..19b9d2607cbc35ac3445e9fd7db9fdc55011aeea 100644
--- a/ansible/roles/stack-sunbird/templates/sunbird_notification-service.env
+++ b/ansible/roles/stack-sunbird/templates/sunbird_notification-service.env
@@ -1,17 +1,17 @@
{% if (cassandra_cluster_size | int) > 1 %}
sunbird_cassandra_host={{groups['cassandra']|join(',')}}
sunbird_cassandra_port=9042,9042,9042
-sunbird_cassandra_consistency_level=quorum
{% else %}
sunbird_cassandra_host={{ groups['cassandra'][0] }}
sunbird_cassandra_port=9042
-sunbird_cassandra_consistency_level=quorum
{% endif %}
+sunbird_cassandra_consistency_level={{sunbird_cassandra_consistency_level}}
sunbird_cassandra_notification_keyspace=sunbird_notifications
sunbird_cassandra_password={{sunbird_cassandra_password|d('password')}}
sunbird_cassandra_username={{sunbird_cassandra_username|d('cassandra')}}
sunbird_notification_kafka_servers_config={{sunbird_processing_kafka_host}}
-sunbird_notification_kafka_topic={{env_name}}.lms.notification
+# Release-5.0.0
+sunbird_notification_kafka_topic={{kafka_topic_lms_notification}}
sunbird_notification_msg_default_sender={{sunbird_msg_sender}}
sunbird_msg_91_auth={{sunbird_msg_91_auth}}
sunbird_mail_server_from_email={{sunbird_mail_server_from_email}}
@@ -35,3 +35,5 @@ LEARNER_SERVICE_PORT=http://learner-service:9000
sunbird_us_system_setting_url=/api/data/v1/system/settings/list
sunbird_us_org_read_url=/v1/org/read
notification_category_type_config=certificateUpdate
+# Release-5.0.0
+isMultiDCEnabled={{cassandra_multi_dc_enabled}}
diff --git a/ansible/roles/stack-sunbird/templates/sunbird_player.env b/ansible/roles/stack-sunbird/templates/sunbird_player.env
index 7130ccf290b7d8bd493583c0caeb104fdf073e20..0d1c34f45b3d68fc80410faf8c6de89bf21be64a 100644
--- a/ansible/roles/stack-sunbird/templates/sunbird_player.env
+++ b/ansible/roles/stack-sunbird/templates/sunbird_player.env
@@ -202,3 +202,23 @@ sunbird_portal_uci_bot_phone_number={{sunbird_portal_uci_bot_phone_number | defa
#Release-4.6.0
sunbird_collection_children_limit={{ sunbird_collection_children_limit | default(1200) }}
sunbird_questionset_children_limit={{ sunbird_questionset_children_limit | default(500) }}
+
+
+#release-5.0.0
+#AWS config for storage access
+#To be reverted post testing
+sunbird_cloud_storage_provider={{cloud_service_provider}}
+sunbird_aws_bucket_name={{aws_private_bucket_name | default("")}}
+sunbird_aws_reports={{aws_reports_folder_name | default("reports")}}
+sunbird_aws_labels={{aws_labels_folder_name | default("labels")}}
+sunbird_aws_access_key={{aws_private_bucket_access_key | default("")}}
+sunbird_aws_secret_key={{aws_private_bucket_secret_key | default("")}}
+sunbird_aws_region={{aws_private_bucket_region | default("ap-south-1")}}
+
+#GCLOUD config for storage access
+sunbird_gcloud_bucket_name={{gcloud_private_bucket_name | default("")}}
+sunbird_gcloud_reports={{gcloud_reports_folder_name | default("reports")}}
+sunbird_gcloud_labels={{gcloud_labels_folder_name | default("labels")}}
+sunbird_gcloud_client_email={{gcloud_private_bucket_client_email | default("")}}
+sunbird_gcloud_private_key={{gcloud_private_bucket_private_key | default("")}}
+sunbird_gcloud_projectId={{gcloud_private_bucket_projectId | default("")}}
diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Lern/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Lern/config.xml
new file mode 100644
index 0000000000000000000000000000000000000000..c20bd4ad2f7eb24eeae3dd66b73868d0a83dd831
--- /dev/null
+++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Lern/config.xml
@@ -0,0 +1,28 @@
+<?xml version='1.1' encoding='UTF-8'?>
+<com.cloudbees.hudson.plugins.folder.Folder plugin="cloudbees-folder@6.15">
+ <description></description>
+ <properties>
+ <org.jenkinsci.plugins.docker.workflow.declarative.FolderConfig plugin="docker-workflow@1.26">
+ <dockerLabel></dockerLabel>
+ <registry plugin="docker-commons@1.17"/>
+ </org.jenkinsci.plugins.docker.workflow.declarative.FolderConfig>
+ </properties>
+ <folderViews class="com.cloudbees.hudson.plugins.folder.views.DefaultFolderViewHolder">
+ <views>
+ <hudson.model.AllView>
+ <owner class="com.cloudbees.hudson.plugins.folder.Folder" reference="../../../.."/>
+ <name>All</name>
+ <filterExecutors>false</filterExecutors>
+ <filterQueue>false</filterQueue>
+ <properties class="hudson.model.View$PropertyList"/>
+ </hudson.model.AllView>
+ </views>
+ <tabBar class="hudson.views.DefaultViewsTabBar"/>
+ </folderViews>
+ <healthMetrics>
+ <com.cloudbees.hudson.plugins.folder.health.WorstChildHealthMetric>
+ <nonRecursive>false</nonRecursive>
+ </com.cloudbees.hudson.plugins.folder.health.WorstChildHealthMetric>
+ </healthMetrics>
+ <icon class="com.cloudbees.hudson.plugins.folder.icons.StockFolderIcon"/>
+</com.cloudbees.hudson.plugins.folder.Folder>
\ No newline at end of file
diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Lern/jobs/LernDataProducts/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Lern/jobs/LernDataProducts/config.xml
new file mode 100644
index 0000000000000000000000000000000000000000..e040465555fe390752fdb7e8372668e7dc1f397a
--- /dev/null
+++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Lern/jobs/LernDataProducts/config.xml
@@ -0,0 +1,108 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?><flow-definition plugin="workflow-job@2.40">
+ <actions>
+ <org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction plugin="workflow-multibranch@2.23">
+ <jobPropertyDescriptors>
+ <string>hudson.model.ParametersDefinitionProperty</string>
+ <string>com.sonyericsson.rebuild.RebuildSettings</string>
+ </jobPropertyDescriptors>
+ </org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction>
+ </actions>
+ <description/>
+ <keepDependencies>false</keepDependencies>
+ <properties>
+ <jenkins.model.BuildDiscarderProperty>
+ <strategy class="hudson.tasks.LogRotator">
+ <daysToKeep>-1</daysToKeep>
+ <numToKeep>10</numToKeep>
+ <artifactDaysToKeep>-1</artifactDaysToKeep>
+ <artifactNumToKeep>1</artifactNumToKeep>
+ </strategy>
+ </jenkins.model.BuildDiscarderProperty>
+ <org.jenkinsci.plugins.workflow.job.properties.DisableConcurrentBuildsJobProperty/>
+ <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.32">
+ <autoRebuild>false</autoRebuild>
+ <rebuildDisabled>false</rebuildDisabled>
+ </com.sonyericsson.rebuild.RebuildSettings>
+ <hudson.model.ParametersDefinitionProperty>
+ <parameterDefinitions>
+ <hudson.model.StringParameterDefinition>
+ <name>absolute_job_path</name>
+ <description><font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font></description>
+ <defaultValue>Build/Lern/LernDataProducts</defaultValue>
+ <trim>false</trim>
+ </hudson.model.StringParameterDefinition>
+ <hudson.model.StringParameterDefinition>
+ <name>build_number</name>
+ <description><font color=darkgreen size=2><b>OPTIONAL: Specify the build job number to upload / copy the artifact built in that job.</b></font></description>
+ <defaultValue>lastSuccessfulBuild</defaultValue>
+ <trim>false</trim>
+ </hudson.model.StringParameterDefinition>
+ <hudson.model.ChoiceParameterDefinition>
+ <name>artifact_source</name>
+ <description><font color=dimgray size=2><b>
+ArtifactRepo - Upload and keep a copy of the artifact in Jenkins, JenkinsJob - Just keep a copy of the artifact in Jenkins.</b></font></description>
+ <choices class="java.util.Arrays$ArrayList">
+ <a class="string-array">
+ <string>ArtifactRepo</string>
+ <string>JenkinsJob</string>
+ </a>
+ </choices>
+ </hudson.model.ChoiceParameterDefinition>
+ </parameterDefinitions>
+ </hudson.model.ParametersDefinitionProperty>
+ <hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.2">
+ <maxConcurrentPerNode>0</maxConcurrentPerNode>
+ <maxConcurrentTotal>0</maxConcurrentTotal>
+ <categories class="java.util.concurrent.CopyOnWriteArrayList"/>
+ <throttleEnabled>false</throttleEnabled>
+ <throttleOption>project</throttleOption>
+ <limitOneJobWithMatchingParams>false</limitOneJobWithMatchingParams>
+ <paramsToUseForLimit/>
+ </hudson.plugins.throttleconcurrents.ThrottleJobProperty>
+
+ <org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+ <triggers>
+ <jenkins.triggers.ReverseBuildTrigger>
+ <spec/>
+ <upstreamProjects>/Build/Lern/LernDataProducts</upstreamProjects>
+ <threshold>
+ <name>SUCCESS</name>
+ <ordinal>0</ordinal>
+ <color>BLUE</color>
+ <completeBuild>true</completeBuild>
+ </threshold>
+ </jenkins.triggers.ReverseBuildTrigger>
+ </triggers>
+ </org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+ </properties>
+ <definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.90">
+ <scm class="hudson.plugins.git.GitSCM" plugin="git@4.7.1">
+ <configVersion>2</configVersion>
+ <userRemoteConfigs>
+ <hudson.plugins.git.UserRemoteConfig>
+ <url>https://github.com/Sunbird-Lern/data-products</url>
+ </hudson.plugins.git.UserRemoteConfig>
+ </userRemoteConfigs>
+ <branches>
+ <hudson.plugins.git.BranchSpec>
+ <name>${public_repo_branch}</name>
+ </hudson.plugins.git.BranchSpec>
+ </branches>
+ <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+ <submoduleCfg class="empty-list"/>
+ <extensions>
+ <hudson.plugins.git.extensions.impl.CloneOption>
+ <shallow>true</shallow>
+ <noTags>false</noTags>
+ <reference/>
+ <depth>1</depth>
+ <honorRefspec>false</honorRefspec>
+ </hudson.plugins.git.extensions.impl.CloneOption>
+ </extensions>
+ </scm>
+ <scriptPath>pipelines/upload/Jenkinsfile</scriptPath>
+ <lightweight>false</lightweight>
+ </definition>
+ <triggers/>
+ <disabled>false</disabled>
+</flow-definition>
\ No newline at end of file
diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Lern/jobs/LernFlinkJobs/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Lern/jobs/LernFlinkJobs/config.xml
new file mode 100644
index 0000000000000000000000000000000000000000..0988779348d58348779ebdfdd90dcee0252a7351
--- /dev/null
+++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Lern/jobs/LernFlinkJobs/config.xml
@@ -0,0 +1,108 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?><flow-definition plugin="workflow-job@2.40">
+ <actions>
+ <org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction plugin="workflow-multibranch@2.23">
+ <jobPropertyDescriptors>
+ <string>hudson.model.ParametersDefinitionProperty</string>
+ <string>com.sonyericsson.rebuild.RebuildSettings</string>
+ </jobPropertyDescriptors>
+ </org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction>
+ </actions>
+ <description/>
+ <keepDependencies>false</keepDependencies>
+ <properties>
+ <jenkins.model.BuildDiscarderProperty>
+ <strategy class="hudson.tasks.LogRotator">
+ <daysToKeep>-1</daysToKeep>
+ <numToKeep>10</numToKeep>
+ <artifactDaysToKeep>-1</artifactDaysToKeep>
+ <artifactNumToKeep>1</artifactNumToKeep>
+ </strategy>
+ </jenkins.model.BuildDiscarderProperty>
+ <org.jenkinsci.plugins.workflow.job.properties.DisableConcurrentBuildsJobProperty/>
+ <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.32">
+ <autoRebuild>false</autoRebuild>
+ <rebuildDisabled>false</rebuildDisabled>
+ </com.sonyericsson.rebuild.RebuildSettings>
+ <hudson.model.ParametersDefinitionProperty>
+ <parameterDefinitions>
+ <hudson.model.StringParameterDefinition>
+ <name>absolute_job_path</name>
+ <description><font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font></description>
+ <defaultValue>Build/Lern/LernFlinkJobs</defaultValue>
+ <trim>false</trim>
+ </hudson.model.StringParameterDefinition>
+ <hudson.model.StringParameterDefinition>
+ <name>image_tag</name>
+ <description><font color=darkgreen size=2><b>OPTIONAL: Specify the tag to upload a specific image version to the container registry.</b></font></description>
+ <defaultValue/>
+ <trim>false</trim>
+ </hudson.model.StringParameterDefinition>
+ <hudson.model.ChoiceParameterDefinition>
+ <name>artifact_source</name>
+ <description><font color=dimgray size=2><b>
+ArtifactRepo - Push the docker image to container registry.
+</b></font></description>
+ <choices class="java.util.Arrays$ArrayList">
+ <a class="string-array">
+ <string>ArtifactRepo</string>
+ </a>
+ </choices>
+ </hudson.model.ChoiceParameterDefinition>
+ </parameterDefinitions>
+ </hudson.model.ParametersDefinitionProperty>
+ <hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.2">
+ <maxConcurrentPerNode>0</maxConcurrentPerNode>
+ <maxConcurrentTotal>0</maxConcurrentTotal>
+ <categories class="java.util.concurrent.CopyOnWriteArrayList"/>
+ <throttleEnabled>false</throttleEnabled>
+ <throttleOption>project</throttleOption>
+ <limitOneJobWithMatchingParams>false</limitOneJobWithMatchingParams>
+ <paramsToUseForLimit/>
+ </hudson.plugins.throttleconcurrents.ThrottleJobProperty>
+
+ <org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+ <triggers>
+ <jenkins.triggers.ReverseBuildTrigger>
+ <spec/>
+ <upstreamProjects>Build/Lern/LernFlinkJobs</upstreamProjects>
+ <threshold>
+ <name>SUCCESS</name>
+ <ordinal>0</ordinal>
+ <color>BLUE</color>
+ <completeBuild>true</completeBuild>
+ </threshold>
+ </jenkins.triggers.ReverseBuildTrigger>
+ </triggers>
+ </org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+ </properties>
+ <definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.90">
+ <scm class="hudson.plugins.git.GitSCM" plugin="git@4.7.1">
+ <configVersion>2</configVersion>
+ <userRemoteConfigs>
+ <hudson.plugins.git.UserRemoteConfig>
+ <url>https://github.com/Sunbird-Lern/data-pipeline.git</url>
+ </hudson.plugins.git.UserRemoteConfig>
+ </userRemoteConfigs>
+ <branches>
+ <hudson.plugins.git.BranchSpec>
+ <name>${public_repo_branch}</name>
+ </hudson.plugins.git.BranchSpec>
+ </branches>
+ <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+ <submoduleCfg class="empty-list"/>
+ <extensions>
+ <hudson.plugins.git.extensions.impl.CloneOption>
+ <shallow>true</shallow>
+ <noTags>false</noTags>
+ <reference/>
+ <depth>0</depth>
+ <honorRefspec>false</honorRefspec>
+ </hudson.plugins.git.extensions.impl.CloneOption>
+ </extensions>
+ </scm>
+ <scriptPath>kubernetes/pipelines/upload/Jenkinsfile</scriptPath>
+ <lightweight>false</lightweight>
+ </definition>
+ <triggers/>
+ <disabled>false</disabled>
+</flow-definition>
\ No newline at end of file
diff --git a/deploy/jenkins/jobs/Build/jobs/Lern/config.xml b/deploy/jenkins/jobs/Build/jobs/Lern/config.xml
new file mode 100644
index 0000000000000000000000000000000000000000..c20bd4ad2f7eb24eeae3dd66b73868d0a83dd831
--- /dev/null
+++ b/deploy/jenkins/jobs/Build/jobs/Lern/config.xml
@@ -0,0 +1,28 @@
+<?xml version='1.1' encoding='UTF-8'?>
+<com.cloudbees.hudson.plugins.folder.Folder plugin="cloudbees-folder@6.15">
+ <description></description>
+ <properties>
+ <org.jenkinsci.plugins.docker.workflow.declarative.FolderConfig plugin="docker-workflow@1.26">
+ <dockerLabel></dockerLabel>
+ <registry plugin="docker-commons@1.17"/>
+ </org.jenkinsci.plugins.docker.workflow.declarative.FolderConfig>
+ </properties>
+ <folderViews class="com.cloudbees.hudson.plugins.folder.views.DefaultFolderViewHolder">
+ <views>
+ <hudson.model.AllView>
+ <owner class="com.cloudbees.hudson.plugins.folder.Folder" reference="../../../.."/>
+ <name>All</name>
+ <filterExecutors>false</filterExecutors>
+ <filterQueue>false</filterQueue>
+ <properties class="hudson.model.View$PropertyList"/>
+ </hudson.model.AllView>
+ </views>
+ <tabBar class="hudson.views.DefaultViewsTabBar"/>
+ </folderViews>
+ <healthMetrics>
+ <com.cloudbees.hudson.plugins.folder.health.WorstChildHealthMetric>
+ <nonRecursive>false</nonRecursive>
+ </com.cloudbees.hudson.plugins.folder.health.WorstChildHealthMetric>
+ </healthMetrics>
+ <icon class="com.cloudbees.hudson.plugins.folder.icons.StockFolderIcon"/>
+</com.cloudbees.hudson.plugins.folder.Folder>
\ No newline at end of file
diff --git a/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml
new file mode 100644
index 0000000000000000000000000000000000000000..088b9fa27f40140674196e58414d2cc980a29d79
--- /dev/null
+++ b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml
@@ -0,0 +1,82 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?><flow-definition plugin="workflow-job@2.40">
+ <actions>
+ <org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction plugin="workflow-multibranch@2.23">
+ <jobPropertyDescriptors>
+ <string>hudson.model.ParametersDefinitionProperty</string>
+ <string>com.sonyericsson.rebuild.RebuildSettings</string>
+ </jobPropertyDescriptors>
+ </org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction>
+ </actions>
+ <description/>
+ <keepDependencies>false</keepDependencies>
+ <properties>
+ <jenkins.model.BuildDiscarderProperty>
+ <strategy class="hudson.tasks.LogRotator">
+ <daysToKeep>-1</daysToKeep>
+ <numToKeep>10</numToKeep>
+ <artifactDaysToKeep>-1</artifactDaysToKeep>
+ <artifactNumToKeep>1</artifactNumToKeep>
+ </strategy>
+ </jenkins.model.BuildDiscarderProperty>
+ <org.jenkinsci.plugins.workflow.job.properties.DisableConcurrentBuildsJobProperty/>
+ <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.32">
+ <autoRebuild>false</autoRebuild>
+ <rebuildDisabled>false</rebuildDisabled>
+ </com.sonyericsson.rebuild.RebuildSettings>
+ <hudson.model.ParametersDefinitionProperty>
+ <parameterDefinitions>
+ <hudson.model.StringParameterDefinition>
+ <name>github_release_tag</name>
+ <description><font style="color:dimgray;font-size:14px;"><b>
+<li>To build from a tag, use refs/tags/github_tag</li>
+<li>To build from a branch, use refs/heads/github_branch</li>
+<li>The default value of ${public_repo_branch} will be the release / tag version set in global configuration</li>
+<li>To build from a differnt branch, replace the ${public_repo_branch} with your branch</li>
+</b></font></description>
+ <defaultValue>refs/heads/${public_repo_branch}</defaultValue>
+ <trim>true</trim>
+ </hudson.model.StringParameterDefinition>
+ </parameterDefinitions>
+ </hudson.model.ParametersDefinitionProperty>
+ <hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.2">
+ <maxConcurrentPerNode>0</maxConcurrentPerNode>
+ <maxConcurrentTotal>0</maxConcurrentTotal>
+ <categories class="java.util.concurrent.CopyOnWriteArrayList"/>
+ <throttleEnabled>false</throttleEnabled>
+ <throttleOption>project</throttleOption>
+ <limitOneJobWithMatchingParams>false</limitOneJobWithMatchingParams>
+ <paramsToUseForLimit/>
+ </hudson.plugins.throttleconcurrents.ThrottleJobProperty>
+
+ <org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+ <triggers>
+ <hudson.triggers.SCMTrigger>
+ <spec>H/15 * * * *</spec>
+ <ignorePostCommitHooks>false</ignorePostCommitHooks>
+ </hudson.triggers.SCMTrigger>
+ </triggers>
+ </org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+ </properties>
+ <definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.90">
+ <scm class="hudson.plugins.git.GitSCM" plugin="git@4.7.1">
+ <configVersion>2</configVersion>
+ <userRemoteConfigs>
+ <hudson.plugins.git.UserRemoteConfig>
+ <url>https://github.com/Sunbird-Lern/data-products.git</url>
+ </hudson.plugins.git.UserRemoteConfig>
+ </userRemoteConfigs>
+ <branches>
+ <hudson.plugins.git.BranchSpec>
+ <name>${github_release_tag}</name>
+ </hudson.plugins.git.BranchSpec>
+ </branches>
+ <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+ <submoduleCfg class="empty-list"/>
+ <extensions/>
+ </scm>
+ <scriptPath>lern-data-products/build.Jenkinsfile</scriptPath>
+ <lightweight>false</lightweight>
+ </definition>
+ <triggers/>
+ <disabled>false</disabled>
+</flow-definition>
\ No newline at end of file
diff --git a/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml
new file mode 100644
index 0000000000000000000000000000000000000000..df82bece0b41e666a1580db56b62c66cf915dc00
--- /dev/null
+++ b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml
@@ -0,0 +1,82 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?><flow-definition plugin="workflow-job@2.40">
+ <actions>
+ <org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction plugin="workflow-multibranch@2.23">
+ <jobPropertyDescriptors>
+ <string>hudson.model.ParametersDefinitionProperty</string>
+ <string>com.sonyericsson.rebuild.RebuildSettings</string>
+ </jobPropertyDescriptors>
+ </org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction>
+ </actions>
+ <description/>
+ <keepDependencies>false</keepDependencies>
+ <properties>
+ <jenkins.model.BuildDiscarderProperty>
+ <strategy class="hudson.tasks.LogRotator">
+ <daysToKeep>-1</daysToKeep>
+ <numToKeep>10</numToKeep>
+ <artifactDaysToKeep>-1</artifactDaysToKeep>
+ <artifactNumToKeep>1</artifactNumToKeep>
+ </strategy>
+ </jenkins.model.BuildDiscarderProperty>
+ <org.jenkinsci.plugins.workflow.job.properties.DisableConcurrentBuildsJobProperty/>
+ <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.32">
+ <autoRebuild>false</autoRebuild>
+ <rebuildDisabled>false</rebuildDisabled>
+ </com.sonyericsson.rebuild.RebuildSettings>
+ <hudson.model.ParametersDefinitionProperty>
+ <parameterDefinitions>
+ <hudson.model.StringParameterDefinition>
+ <name>github_release_tag</name>
+ <description><font style="color:dimgray;font-size:14px;"><b>
+<li>To build from a tag, use refs/tags/github_tag</li>
+<li>To build from a branch, use refs/heads/github_branch</li>
+<li>The default value of ${public_repo_branch} will be the release / tag version set in global configuration</li>
+<li>To build from a differnt branch, replace the ${public_repo_branch} with your branch</li>
+</b></font></description>
+ <defaultValue>refs/heads/${public_repo_branch}</defaultValue>
+ <trim>true</trim>
+ </hudson.model.StringParameterDefinition>
+ </parameterDefinitions>
+ </hudson.model.ParametersDefinitionProperty>
+ <hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.2">
+ <maxConcurrentPerNode>0</maxConcurrentPerNode>
+ <maxConcurrentTotal>0</maxConcurrentTotal>
+ <categories class="java.util.concurrent.CopyOnWriteArrayList"/>
+ <throttleEnabled>false</throttleEnabled>
+ <throttleOption>project</throttleOption>
+ <limitOneJobWithMatchingParams>false</limitOneJobWithMatchingParams>
+ <paramsToUseForLimit/>
+ </hudson.plugins.throttleconcurrents.ThrottleJobProperty>
+
+ <org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+ <triggers>
+ <hudson.triggers.SCMTrigger>
+ <spec>H/15 * * * *</spec>
+ <ignorePostCommitHooks>false</ignorePostCommitHooks>
+ </hudson.triggers.SCMTrigger>
+ </triggers>
+ </org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+ </properties>
+ <definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.90">
+ <scm class="hudson.plugins.git.GitSCM" plugin="git@4.7.1">
+ <configVersion>2</configVersion>
+ <userRemoteConfigs>
+ <hudson.plugins.git.UserRemoteConfig>
+ <url>https://github.com/Sunbird-Lern/data-pipeline.git</url>
+ </hudson.plugins.git.UserRemoteConfig>
+ </userRemoteConfigs>
+ <branches>
+ <hudson.plugins.git.BranchSpec>
+ <name>${github_release_tag}</name>
+ </hudson.plugins.git.BranchSpec>
+ </branches>
+ <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+ <submoduleCfg class="empty-list"/>
+ <extensions/>
+ </scm>
+ <scriptPath>kubernetes/pipelines/build/Jenkinsfile</scriptPath>
+ <lightweight>false</lightweight>
+ </definition>
+ <triggers/>
+ <disabled>false</disabled>
+</flow-definition>
\ No newline at end of file
diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml
index 30bdcc3098a6ebd0bb5d3caff3120f7675192ebe..cb98de88c1572f863bf875283350da55b57f2bd5 100644
--- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml
+++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml
@@ -110,7 +110,8 @@ return """<b>This parameter is not used</b>"""</script>
'metrics-data-transformer',
'content-publish',
'content-auto-creator',
-'qrcode-image-generator']</script>
+'qrcode-image-generator',
+'dialcode-context-updater']</script>
<sandbox>true</sandbox>
</secureScript>
<secureFallbackScript plugin="script-security@1.76">
diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/config.xml
new file mode 100644
index 0000000000000000000000000000000000000000..c20bd4ad2f7eb24eeae3dd66b73868d0a83dd831
--- /dev/null
+++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/config.xml
@@ -0,0 +1,28 @@
+<?xml version='1.1' encoding='UTF-8'?>
+<com.cloudbees.hudson.plugins.folder.Folder plugin="cloudbees-folder@6.15">
+ <description></description>
+ <properties>
+ <org.jenkinsci.plugins.docker.workflow.declarative.FolderConfig plugin="docker-workflow@1.26">
+ <dockerLabel></dockerLabel>
+ <registry plugin="docker-commons@1.17"/>
+ </org.jenkinsci.plugins.docker.workflow.declarative.FolderConfig>
+ </properties>
+ <folderViews class="com.cloudbees.hudson.plugins.folder.views.DefaultFolderViewHolder">
+ <views>
+ <hudson.model.AllView>
+ <owner class="com.cloudbees.hudson.plugins.folder.Folder" reference="../../../.."/>
+ <name>All</name>
+ <filterExecutors>false</filterExecutors>
+ <filterQueue>false</filterQueue>
+ <properties class="hudson.model.View$PropertyList"/>
+ </hudson.model.AllView>
+ </views>
+ <tabBar class="hudson.views.DefaultViewsTabBar"/>
+ </folderViews>
+ <healthMetrics>
+ <com.cloudbees.hudson.plugins.folder.health.WorstChildHealthMetric>
+ <nonRecursive>false</nonRecursive>
+ </com.cloudbees.hudson.plugins.folder.health.WorstChildHealthMetric>
+ </healthMetrics>
+ <icon class="com.cloudbees.hudson.plugins.folder.icons.StockFolderIcon"/>
+</com.cloudbees.hudson.plugins.folder.Folder>
\ No newline at end of file
diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/jobs/LernAnalyticsFetchLogs/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/jobs/LernAnalyticsFetchLogs/config.xml
new file mode 100644
index 0000000000000000000000000000000000000000..bf6988457b0fabb35085187f11d10ba43fa8d54f
--- /dev/null
+++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/jobs/LernAnalyticsFetchLogs/config.xml
@@ -0,0 +1,146 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?><flow-definition plugin="workflow-job@2.40">
+ <actions>
+ <org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction plugin="workflow-multibranch@2.23">
+ <jobPropertyDescriptors>
+ <string>hudson.model.ParametersDefinitionProperty</string>
+ <string>com.sonyericsson.rebuild.RebuildSettings</string>
+ </jobPropertyDescriptors>
+ </org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction>
+ </actions>
+ <description/>
+ <keepDependencies>false</keepDependencies>
+ <properties>
+ <jenkins.model.BuildDiscarderProperty>
+ <strategy class="hudson.tasks.LogRotator">
+ <daysToKeep>-1</daysToKeep>
+ <numToKeep>-1</numToKeep>
+ <artifactDaysToKeep>-1</artifactDaysToKeep>
+ <artifactNumToKeep>5</artifactNumToKeep>
+ </strategy>
+ </jenkins.model.BuildDiscarderProperty>
+ <org.jenkinsci.plugins.workflow.job.properties.DisableConcurrentBuildsJobProperty/>
+ <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.32">
+ <autoRebuild>false</autoRebuild>
+ <rebuildDisabled>false</rebuildDisabled>
+ </com.sonyericsson.rebuild.RebuildSettings>
+ <hudson.model.ParametersDefinitionProperty>
+ <parameterDefinitions>
+ <hudson.model.ChoiceParameterDefinition>
+ <name>instance_name</name>
+ <description><font color=dimgray size=2><b>Choose the instance to fetch logs.</b></font></description>
+ <choices class="java.util.Arrays$ArrayList">
+ <a class="string-array">
+ <string>spark</string>
+ <string>analytics-api</string>
+ <string>secor</string>
+ <string>cassandra</string>
+ </a>
+ </choices>
+ </hudson.model.ChoiceParameterDefinition>
+ <hudson.model.ChoiceParameterDefinition>
+ <name>action</name>
+ <description><font color=dimgray size=2><b>Choose one of the option - Fetch logs or set cronjob.</b></font></description>
+ <choices class="java.util.Arrays$ArrayList">
+ <a class="string-array">
+ <string>fetch-logs</string>
+ <string>cronjobs</string>
+ </a>
+ </choices>
+ </hudson.model.ChoiceParameterDefinition>
+ <org.biouno.unochoice.DynamicReferenceParameter plugin="uno-choice@2.5.6">
+ <name>private_branch</name>
+ <description/>
+ <randomName>choice-parameter-2544395024638227</randomName>
+ <visibleItemCount>1</visibleItemCount>
+ <script class="org.biouno.unochoice.model.GroovyScript">
+ <secureScript plugin="script-security@1.76">
+ <script>if (override_private_branch.equals("true")) {
+return """<input name="value" value="${private_repo_branch}" class="setting-input" type="text"><br> <font color=dimgray size=2><b>Change this value to checkout a different branch from private repository.</b></font>"""
+}
+else
+return """<b>This parameter is not used</b>"""</script>
+ <sandbox>true</sandbox>
+ </secureScript>
+ <secureFallbackScript plugin="script-security@1.76">
+ <script>return """<b>This parameter is not used</b>"""</script>
+ <sandbox>true</sandbox>
+ </secureFallbackScript>
+ </script>
+ <projectName>LernAnalyticsFetchLogs</projectName>
+ <projectFullName>Deploy/dev/Lern/LernAnalyticsFetchLogs</projectFullName>
+ <parameters class="linked-hash-map"/>
+ <referencedParameters/>
+ <choiceType>ET_FORMATTED_HTML</choiceType>
+ <omitValueField>true</omitValueField>
+ </org.biouno.unochoice.DynamicReferenceParameter>
+ <org.biouno.unochoice.DynamicReferenceParameter plugin="uno-choice@2.5.6">
+ <name>branch_or_tag</name>
+ <description/>
+ <randomName>choice-parameter-2620434998790477</randomName>
+ <visibleItemCount>1</visibleItemCount>
+ <script class="org.biouno.unochoice.model.GroovyScript">
+ <secureScript plugin="script-security@1.76">
+ <script>if (override_public_branch.equals("true")) {
+return """<input name="value" value="" class="setting-input" type="text"><br> <font color=dimgray size=2><b>Provide the tag or branch name to checkout the Jenkinsfile and codebase.<br>Note: The tag or branch name for this job should be taken from <a href="https://github.com/Sunbird-Lern/data-products">Sunbird-Lern/data-products</a></b></font></b></font>""";
+
+}
+else
+return """<b>This parameter is not used</b>"""</script>
+ <sandbox>true</sandbox>
+ </secureScript>
+ <secureFallbackScript plugin="script-security@1.76">
+ <script>return """<b>This parameter is not used</b>"""</script>
+ <sandbox>true</sandbox>
+ </secureFallbackScript>
+ </script>
+ <projectName>LernAnalyticsFetchLogs</projectName>
+ <projectFullName>Deploy/dev/Lern/LernAnalyticsFetchLogs</projectFullName>
+ <parameters class="linked-hash-map"/>
+ <referencedParameters/>
+ <choiceType>ET_FORMATTED_HTML</choiceType>
+ <omitValueField>true</omitValueField>
+ </org.biouno.unochoice.DynamicReferenceParameter>
+ </parameterDefinitions>
+ </hudson.model.ParametersDefinitionProperty>
+ <hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.2">
+ <maxConcurrentPerNode>0</maxConcurrentPerNode>
+ <maxConcurrentTotal>0</maxConcurrentTotal>
+ <categories class="java.util.concurrent.CopyOnWriteArrayList"/>
+ <throttleEnabled>false</throttleEnabled>
+ <throttleOption>project</throttleOption>
+ <limitOneJobWithMatchingParams>false</limitOneJobWithMatchingParams>
+ <paramsToUseForLimit/>
+ </hudson.plugins.throttleconcurrents.ThrottleJobProperty>
+
+ </properties>
+ <definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.90">
+ <scm class="hudson.plugins.git.GitSCM" plugin="git@4.7.1">
+ <configVersion>2</configVersion>
+ <userRemoteConfigs>
+ <hudson.plugins.git.UserRemoteConfig>
+ <url>https://github.com/Sunbird-Lern/data-products</url>
+ </hudson.plugins.git.UserRemoteConfig>
+ </userRemoteConfigs>
+ <branches>
+ <hudson.plugins.git.BranchSpec>
+ <name>${branch_or_tag}</name>
+ </hudson.plugins.git.BranchSpec>
+ </branches>
+ <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+ <submoduleCfg class="empty-list"/>
+ <extensions>
+ <hudson.plugins.git.extensions.impl.CloneOption>
+ <shallow>true</shallow>
+ <noTags>false</noTags>
+ <reference/>
+ <depth>1</depth>
+ <honorRefspec>false</honorRefspec>
+ </hudson.plugins.git.extensions.impl.CloneOption>
+ </extensions>
+ </scm>
+ <scriptPath>pipelines/deploy/fetch-logs/Jenkinsfile</scriptPath>
+ <lightweight>false</lightweight>
+ </definition>
+ <triggers/>
+ <disabled>false</disabled>
+</flow-definition>
\ No newline at end of file
diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/jobs/LernAnalyticsReplayJobs/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/jobs/LernAnalyticsReplayJobs/config.xml
new file mode 100644
index 0000000000000000000000000000000000000000..cd2ac8b2d1602caa511b47d95c37c252c83085d0
--- /dev/null
+++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/jobs/LernAnalyticsReplayJobs/config.xml
@@ -0,0 +1,209 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?><flow-definition plugin="workflow-job@2.40">
+ <actions>
+ <org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction plugin="workflow-multibranch@2.23">
+ <jobPropertyDescriptors>
+ <string>hudson.model.ParametersDefinitionProperty</string>
+ <string>com.sonyericsson.rebuild.RebuildSettings</string>
+ </jobPropertyDescriptors>
+ </org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction>
+ </actions>
+ <description/>
+ <keepDependencies>false</keepDependencies>
+ <properties>
+ <jenkins.model.BuildDiscarderProperty>
+ <strategy class="hudson.tasks.LogRotator">
+ <daysToKeep>-1</daysToKeep>
+ <numToKeep>10</numToKeep>
+ <artifactDaysToKeep>-1</artifactDaysToKeep>
+ <artifactNumToKeep>2</artifactNumToKeep>
+ </strategy>
+ </jenkins.model.BuildDiscarderProperty>
+ <org.jenkinsci.plugins.workflow.job.properties.DisableConcurrentBuildsJobProperty/>
+ <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.32">
+ <autoRebuild>false</autoRebuild>
+ <rebuildDisabled>false</rebuildDisabled>
+ </com.sonyericsson.rebuild.RebuildSettings>
+ <hudson.model.ParametersDefinitionProperty>
+ <parameterDefinitions>
+ <hudson.model.StringParameterDefinition>
+ <name>cql_query</name>
+ <description><font color=dimgray size=2><b>Specify cql_query if any tables needs to be truncated.<b></font></description>
+ <defaultValue>0</defaultValue>
+ <trim>false</trim>
+ </hudson.model.StringParameterDefinition>
+ <hudson.model.ChoiceParameterDefinition>
+ <name>job_type</name>
+ <description><font color=dimgray size=2><b>Specify the job type - Replay or Run.</b></font></description>
+ <choices class="java.util.Arrays$ArrayList">
+ <a class="string-array">
+ <string>run-job</string>
+ <string>replay-job</string>
+ <string>generate-metrics</string>
+ <string>submit-job</string>
+ </a>
+ </choices>
+ </hudson.model.ChoiceParameterDefinition>
+ <hudson.model.ChoiceParameterDefinition>
+ <name>job_id</name>
+ <description><font color=dimgray size=2><b>Specify the job id.</b></font></description>
+ <choices class="java.util.Arrays$ArrayList">
+ <a class="string-array">
+ <string>collection-summary-report-v2</string>
+ <string>collection-summary-report</string>
+ <string>program-collection-summary-report</string>
+ <string>audit-metrics-report</string>
+ <string>admin-user-reports</string>
+ <string>admin-geo-reports</string>
+ <string>district-weekly</string>
+ <string>district-monthly</string>
+ <string>desktop-consumption-report</string>
+ <string>wfs</string>
+ <string>content-rating-updater</string>
+ <string>druid-query-processor</string>
+ <string>monitor-job-summ</string>
+ <string>cassandra-migration </string>
+ <string>course-enrollment-report</string>
+ <string>textbook-progress-report</string>
+ <string>etb-metrics</string>
+ <string>daily-metrics</string>
+ <string>progress-exhaust</string>
+ <string>userinfo-exhaust</string>
+ <string>response-exhaust</string>
+ <string>progress-exhaust-v2</string>
+ <string>response-exhaust-v2</string>
+ <string>course-batch-status-updater</string>
+ <string>druid-dataset</string>
+ <string>score-metric-migration-job</string>
+ <string>uci-response-exhaust</string>
+ <string>uci-private-exhaust</string>
+ </a>
+ </choices>
+ </hudson.model.ChoiceParameterDefinition>
+ <hudson.model.StringParameterDefinition>
+ <name>batch_identifier</name>
+ <description><font color=dimgray size=2><b>Specify The BatchId to Generate report for specific batch</b></font>
+<font color=dimgray size=2><b>NOTE: Only Applicable if you are running course-metrics and assessment metrics job</b></font></description>
+ <defaultValue/>
+ <trim>false</trim>
+ </hudson.model.StringParameterDefinition>
+ <hudson.model.StringParameterDefinition>
+ <name>start_date</name>
+ <description><font color=dimgray size=2><b>Specify the start date. The default value is just a placeholder which can be changed as per need.</b></font></description>
+ <defaultValue>2017-05-01</defaultValue>
+ <trim>false</trim>
+ </hudson.model.StringParameterDefinition>
+ <hudson.model.StringParameterDefinition>
+ <name>end_date</name>
+ <description><font color=dimgray size=2><b>Specify the end date. The default value is just a placeholder which can be changed as per need.</b></font></description>
+ <defaultValue>2017-07-11</defaultValue>
+ <trim>false</trim>
+ </hudson.model.StringParameterDefinition>
+ <hudson.model.StringParameterDefinition>
+ <name>pause_min</name>
+ <description><font color=dimgray size=2><b>Specify the # of minutes to pause. The default value is just a placeholder which can be changed as per need.</b></font></description>
+ <defaultValue>30</defaultValue>
+ <trim>false</trim>
+ </hudson.model.StringParameterDefinition>
+ <org.biouno.unochoice.DynamicReferenceParameter plugin="uno-choice@2.5.6">
+ <name>private_branch</name>
+ <description/>
+ <randomName>choice-parameter-2544395024638227</randomName>
+ <visibleItemCount>1</visibleItemCount>
+ <script class="org.biouno.unochoice.model.GroovyScript">
+ <secureScript plugin="script-security@1.76">
+ <script>if (override_private_branch.equals("true")) {
+return """<input name="value" value="${private_repo_branch}" class="setting-input" type="text"><br> <font color=dimgray size=2><b>Change this value to checkout a different branch from private repository.</b></font>"""
+}
+else
+return """<b>This parameter is not used</b>"""</script>
+ <sandbox>true</sandbox>
+ </secureScript>
+ <secureFallbackScript plugin="script-security@1.76">
+ <script>return """<b>This parameter is not used</b>"""</script>
+ <sandbox>true</sandbox>
+ </secureFallbackScript>
+ </script>
+ <projectName>LernAnalyticsReplayJobs</projectName>
+ <projectFullName>Deploy/dev/Lern/LernAnalyticsReplayJobs</projectFullName>
+ <parameters class="linked-hash-map"/>
+ <referencedParameters/>
+ <choiceType>ET_FORMATTED_HTML</choiceType>
+ <omitValueField>true</omitValueField>
+ </org.biouno.unochoice.DynamicReferenceParameter>
+ <org.biouno.unochoice.DynamicReferenceParameter plugin="uno-choice@2.5.6">
+ <name>branch_or_tag</name>
+ <description/>
+ <randomName>choice-parameter-2620434998790477</randomName>
+ <visibleItemCount>1</visibleItemCount>
+ <script class="org.biouno.unochoice.model.GroovyScript">
+ <secureScript plugin="script-security@1.76">
+ <script>if (override_public_branch.equals("true")) {
+return """<input name="value" value="" class="setting-input" type="text"><br> <font color=dimgray size=2><b>Provide the tag or branch name to checkout the Jenkinsfile and codebase.<br>Note: The tag or branch name for this job should be taken from <a href="https://github.com/Sunbird-Lern/data-products">Sunbird-Lern/data-products</a></b></font></b></font>""";
+
+}
+else
+return """<b>This parameter is not used</b>"""</script>
+ <sandbox>true</sandbox>
+ </secureScript>
+ <secureFallbackScript plugin="script-security@1.76">
+ <script>return """<b>This parameter is not used</b>"""</script>
+ <sandbox>true</sandbox>
+ </secureFallbackScript>
+ </script>
+ <projectName>LernAnalyticsReplayJobs</projectName>
+ <projectFullName>Deploy/dev/Lern/LernAnalyticsReplayJobs</projectFullName>
+ <parameters class="linked-hash-map"/>
+ <referencedParameters/>
+ <choiceType>ET_FORMATTED_HTML</choiceType>
+ <omitValueField>true</omitValueField>
+ </org.biouno.unochoice.DynamicReferenceParameter>
+ <hudson.model.StringParameterDefinition>
+ <name>keyword</name>
+ <description><font color=dimgray size=2><b>Optional - Enter a keyword to generate a report for the "program-collection-summary-report"</b></font></description>
+ <defaultValue/>
+ <trim>false</trim>
+ </hudson.model.StringParameterDefinition>
+ </parameterDefinitions>
+ </hudson.model.ParametersDefinitionProperty>
+ <hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.2">
+ <maxConcurrentPerNode>0</maxConcurrentPerNode>
+ <maxConcurrentTotal>0</maxConcurrentTotal>
+ <categories class="java.util.concurrent.CopyOnWriteArrayList"/>
+ <throttleEnabled>false</throttleEnabled>
+ <throttleOption>project</throttleOption>
+ <limitOneJobWithMatchingParams>false</limitOneJobWithMatchingParams>
+ <paramsToUseForLimit/>
+ </hudson.plugins.throttleconcurrents.ThrottleJobProperty>
+
+ </properties>
+ <definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.90">
+ <scm class="hudson.plugins.git.GitSCM" plugin="git@4.7.1">
+ <configVersion>2</configVersion>
+ <userRemoteConfigs>
+ <hudson.plugins.git.UserRemoteConfig>
+ <url>https://github.com/Sunbird-Lern/data-products</url>
+ </hudson.plugins.git.UserRemoteConfig>
+ </userRemoteConfigs>
+ <branches>
+ <hudson.plugins.git.BranchSpec>
+ <name>${branch_or_tag}</name>
+ </hudson.plugins.git.BranchSpec>
+ </branches>
+ <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+ <submoduleCfg class="empty-list"/>
+ <extensions>
+ <hudson.plugins.git.extensions.impl.CloneOption>
+ <shallow>true</shallow>
+ <noTags>false</noTags>
+ <reference/>
+ <depth>0</depth>
+ <honorRefspec>false</honorRefspec>
+ </hudson.plugins.git.extensions.impl.CloneOption>
+ </extensions>
+ </scm>
+ <scriptPath>pipelines/deploy/replay-jobs/Jenkinsfile</scriptPath>
+ <lightweight>false</lightweight>
+ </definition>
+ <triggers/>
+ <disabled>false</disabled>
+</flow-definition>
\ No newline at end of file
diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/jobs/LernDataProducts/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/jobs/LernDataProducts/config.xml
new file mode 100644
index 0000000000000000000000000000000000000000..6f8fd449c25290cc3477664edecf01202dd6569f
--- /dev/null
+++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/jobs/LernDataProducts/config.xml
@@ -0,0 +1,238 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?><flow-definition plugin="workflow-job@2.40">
+ <actions>
+ <org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction plugin="workflow-multibranch@2.23">
+ <jobPropertyDescriptors>
+ <string>hudson.model.ParametersDefinitionProperty</string>
+ <string>com.sonyericsson.rebuild.RebuildSettings</string>
+ </jobPropertyDescriptors>
+ </org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction>
+ </actions>
+ <description/>
+ <keepDependencies>false</keepDependencies>
+ <properties>
+ <jenkins.model.BuildDiscarderProperty>
+ <strategy class="hudson.tasks.LogRotator">
+ <daysToKeep>-1</daysToKeep>
+ <numToKeep>10</numToKeep>
+ <artifactDaysToKeep>-1</artifactDaysToKeep>
+ <artifactNumToKeep>2</artifactNumToKeep>
+ </strategy>
+ </jenkins.model.BuildDiscarderProperty>
+ <org.jenkinsci.plugins.workflow.job.properties.DisableConcurrentBuildsJobProperty/>
+ <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.32">
+ <autoRebuild>false</autoRebuild>
+ <rebuildDisabled>false</rebuildDisabled>
+ </com.sonyericsson.rebuild.RebuildSettings>
+ <hudson.model.ParametersDefinitionProperty>
+ <parameterDefinitions>
+ <hudson.model.StringParameterDefinition>
+ <name>absolute_job_path</name>
+ <description><font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font></description>
+ <defaultValue>ArtifactUpload/dev/Lern/LernDataProducts</defaultValue>
+ <trim>false</trim>
+ </hudson.model.StringParameterDefinition>
+ <org.biouno.unochoice.CascadeChoiceParameter plugin="uno-choice@2.5.6">
+ <name>artifact_source</name>
+ <description><font color=dimgray size=2><b>
+ArtifactRepo - Download the artifact from azure blob, JenkinsJob - Use the atrifact from Jenkins job.</b></font></description>
+ <randomName>choice-parameter-9061667106947</randomName>
+ <visibleItemCount>1</visibleItemCount>
+ <script class="org.biouno.unochoice.model.GroovyScript">
+ <secureScript plugin="script-security@1.76">
+ <script>return[
+'JenkinsJob',
+'ArtifactRepo'
+]</script>
+ <sandbox>true</sandbox>
+ </secureScript>
+ <secureFallbackScript plugin="script-security@1.76">
+ <script/>
+ <sandbox>true</sandbox>
+ </secureFallbackScript>
+ </script>
+ <projectName>LernDataProducts</projectName>
+ <projectFullName>Deploy/dev/Lern/LernDataProducts</projectFullName>
+ <parameters class="linked-hash-map"/>
+ <referencedParameters/>
+ <choiceType>PT_SINGLE_SELECT</choiceType>
+ <filterable>false</filterable>
+ <filterLength>1</filterLength>
+ </org.biouno.unochoice.CascadeChoiceParameter>
+ <org.biouno.unochoice.DynamicReferenceParameter plugin="uno-choice@2.5.6">
+ <name>build_number</name>
+ <description/>
+ <randomName>choice-parameter-9061670571657</randomName>
+ <visibleItemCount>1</visibleItemCount>
+ <script class="org.biouno.unochoice.model.GroovyScript">
+ <secureScript plugin="script-security@1.76">
+ <script>if (artifact_source.equals("JenkinsJob")) {
+return """<input name="value" value="lastSuccessfulBuild" class="setting-input" type="text"><br> <font color=darkgreen size=2><b>OPTIONAL: Specify the build number of ArtifactUpload job from which the artifact will be copied and deployed.</b></font>"""
+}
+else
+{
+return "<b>Not Applicable</b>"
+}</script>
+ <sandbox>true</sandbox>
+ </secureScript>
+ <secureFallbackScript plugin="script-security@1.76">
+ <script/>
+ <sandbox>true</sandbox>
+ </secureFallbackScript>
+ </script>
+ <projectName>LernDataProducts</projectName>
+ <projectFullName>Deploy/dev/Lern/LernDataProducts</projectFullName>
+ <parameters class="linked-hash-map"/>
+ <referencedParameters>artifact_source</referencedParameters>
+ <choiceType>ET_FORMATTED_HTML</choiceType>
+ <omitValueField>true</omitValueField>
+ </org.biouno.unochoice.DynamicReferenceParameter>
+ <org.biouno.unochoice.DynamicReferenceParameter plugin="uno-choice@2.5.6">
+ <name>artifact_version</name>
+ <description/>
+ <randomName>choice-parameter-9061673103864</randomName>
+ <visibleItemCount>1</visibleItemCount>
+ <script class="org.biouno.unochoice.model.GroovyScript">
+ <secureScript plugin="script-security@1.76">
+ <script>if (artifact_source.equals("ArtifactRepo")) {
+return """<input name="value" value="" class="setting-input" type="text"> <br> <font color=red size=2><b>CAUTION: If the value is blank, artifact version will be taken from the latest metadata.json.</b></font>"""
+}
+else {
+return "<b>Not Applicable</b>"
+}</script>
+ <sandbox>true</sandbox>
+ </secureScript>
+ <secureFallbackScript plugin="script-security@1.76">
+ <script/>
+ <sandbox>true</sandbox>
+ </secureFallbackScript>
+ </script>
+ <projectName>LernDataProducts</projectName>
+ <projectFullName>Deploy/dev/Lern/LernDataProducts</projectFullName>
+ <parameters class="linked-hash-map"/>
+ <referencedParameters>artifact_source</referencedParameters>
+ <choiceType>ET_FORMATTED_HTML</choiceType>
+ <omitValueField>true</omitValueField>
+ </org.biouno.unochoice.DynamicReferenceParameter>
+ <org.biouno.unochoice.DynamicReferenceParameter plugin="uno-choice@2.5.6">
+ <name>private_branch</name>
+ <description/>
+ <randomName>choice-parameter-2544395024638227</randomName>
+ <visibleItemCount>1</visibleItemCount>
+ <script class="org.biouno.unochoice.model.GroovyScript">
+ <secureScript plugin="script-security@1.76">
+ <script>if (override_private_branch.equals("true")) {
+return """<input name="value" value="${private_repo_branch}" class="setting-input" type="text"><br> <font color=dimgray size=2><b>Change this value to checkout a different branch from private repository.</b></font>"""
+}
+else
+return """<b>This parameter is not used</b>"""</script>
+ <sandbox>true</sandbox>
+ </secureScript>
+ <secureFallbackScript plugin="script-security@1.76">
+ <script>return """<b>This parameter is not used</b>"""</script>
+ <sandbox>true</sandbox>
+ </secureFallbackScript>
+ </script>
+ <projectName>LernDataProducts</projectName>
+ <projectFullName>Deploy/dev/Lern/LernDataProducts</projectFullName>
+ <parameters class="linked-hash-map"/>
+ <referencedParameters/>
+ <choiceType>ET_FORMATTED_HTML</choiceType>
+ <omitValueField>true</omitValueField>
+ </org.biouno.unochoice.DynamicReferenceParameter>
+ <org.biouno.unochoice.DynamicReferenceParameter plugin="uno-choice@2.5.6">
+ <name>branch_or_tag</name>
+ <description/>
+ <randomName>choice-parameter-2620434998790477</randomName>
+ <visibleItemCount>1</visibleItemCount>
+ <script class="org.biouno.unochoice.model.GroovyScript">
+ <secureScript plugin="script-security@1.76">
+ <script>if (override_public_branch.equals("true")) {
+return """<input name="value" value="" class="setting-input" type="text"><br> <font color=dimgray size=2><b>Provide the tag or branch name to checkout the Jenkinsfile and codebase.<br>Note: The tag or branch name for this job should be taken from <a href="https://github.com/Sunbird-Lern/data-products">Sunbird-Lern/data-products</a></b></font></b></font>""";
+
+}
+else
+return """<b>This parameter is not used</b>"""</script>
+ <sandbox>true</sandbox>
+ </secureScript>
+ <secureFallbackScript plugin="script-security@1.76">
+ <script>return """<b>This parameter is not used</b>"""</script>
+ <sandbox>true</sandbox>
+ </secureFallbackScript>
+ </script>
+ <projectName>LernDataProducts</projectName>
+ <projectFullName>Deploy/dev/Lern/LernDataProducts</projectFullName>
+ <parameters class="linked-hash-map"/>
+ <referencedParameters/>
+ <choiceType>ET_FORMATTED_HTML</choiceType>
+ <omitValueField>true</omitValueField>
+ </org.biouno.unochoice.DynamicReferenceParameter>
+ <hudson.model.StringParameterDefinition>
+ <name>module</name>
+ <description><font color=dimgray size=2><b>It will deploy only lpa_core_dp_artifacts(batch-models & job-manager) jar.</b></font></description>
+ <defaultValue>ed-dataproducts</defaultValue>
+ <trim>false</trim>
+ </hudson.model.StringParameterDefinition>
+ <hudson.model.StringParameterDefinition>
+ <name>remote</name>
+ <description/>
+ <defaultValue>spark</defaultValue>
+ <trim>false</trim>
+ </hudson.model.StringParameterDefinition>
+ </parameterDefinitions>
+ </hudson.model.ParametersDefinitionProperty>
+ <hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.2">
+ <maxConcurrentPerNode>0</maxConcurrentPerNode>
+ <maxConcurrentTotal>0</maxConcurrentTotal>
+ <categories class="java.util.concurrent.CopyOnWriteArrayList"/>
+ <throttleEnabled>false</throttleEnabled>
+ <throttleOption>project</throttleOption>
+ <limitOneJobWithMatchingParams>false</limitOneJobWithMatchingParams>
+ <paramsToUseForLimit/>
+ </hudson.plugins.throttleconcurrents.ThrottleJobProperty>
+
+ <org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+ <triggers>
+ <jenkins.triggers.ReverseBuildTrigger>
+ <spec/>
+ <upstreamProjects>ArtifactUpload/dev/Lern/LernDataProducts</upstreamProjects>
+ <threshold>
+ <name>SUCCESS</name>
+ <ordinal>0</ordinal>
+ <color>BLUE</color>
+ <completeBuild>true</completeBuild>
+ </threshold>
+ </jenkins.triggers.ReverseBuildTrigger>
+ </triggers>
+ </org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+ </properties>
+ <definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.90">
+ <scm class="hudson.plugins.git.GitSCM" plugin="git@4.7.1">
+ <configVersion>2</configVersion>
+ <userRemoteConfigs>
+ <hudson.plugins.git.UserRemoteConfig>
+ <url>https://github.com/Sunbird-Lern/data-products</url>
+ </hudson.plugins.git.UserRemoteConfig>
+ </userRemoteConfigs>
+ <branches>
+ <hudson.plugins.git.BranchSpec>
+ <name>${branch_or_tag}</name>
+ </hudson.plugins.git.BranchSpec>
+ </branches>
+ <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+ <submoduleCfg class="empty-list"/>
+ <extensions>
+ <hudson.plugins.git.extensions.impl.CloneOption>
+ <shallow>true</shallow>
+ <noTags>false</noTags>
+ <reference/>
+ <depth>0</depth>
+ <honorRefspec>false</honorRefspec>
+ </hudson.plugins.git.extensions.impl.CloneOption>
+ </extensions>
+ </scm>
+ <scriptPath>pipelines/deploy/ed-dataproducts/Jenkinsfile</scriptPath>
+ <lightweight>false</lightweight>
+ </definition>
+ <triggers/>
+ <disabled>false</disabled>
+</flow-definition>
\ No newline at end of file
diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/jobs/LernFlinkJobs/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/jobs/LernFlinkJobs/config.xml
new file mode 100644
index 0000000000000000000000000000000000000000..19322ba1a41a7ff2f7ddc392de3c06f4f780a50c
--- /dev/null
+++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Lern/jobs/LernFlinkJobs/config.xml
@@ -0,0 +1,170 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?><flow-definition plugin="workflow-job@2.40">
+ <actions>
+ <org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction plugin="workflow-multibranch@2.23">
+ <jobPropertyDescriptors>
+ <string>hudson.model.ParametersDefinitionProperty</string>
+ <string>com.sonyericsson.rebuild.RebuildSettings</string>
+ </jobPropertyDescriptors>
+ </org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction>
+ </actions>
+ <description/>
+ <keepDependencies>false</keepDependencies>
+ <properties>
+ <jenkins.model.BuildDiscarderProperty>
+ <strategy class="hudson.tasks.LogRotator">
+ <daysToKeep>-1</daysToKeep>
+ <numToKeep>-1</numToKeep>
+ <artifactDaysToKeep>-1</artifactDaysToKeep>
+ <artifactNumToKeep>5</artifactNumToKeep>
+ </strategy>
+ </jenkins.model.BuildDiscarderProperty>
+ <org.jenkinsci.plugins.workflow.job.properties.DisableConcurrentBuildsJobProperty/>
+ <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.32">
+ <autoRebuild>false</autoRebuild>
+ <rebuildDisabled>false</rebuildDisabled>
+ </com.sonyericsson.rebuild.RebuildSettings>
+ <hudson.model.ParametersDefinitionProperty>
+ <parameterDefinitions>
+ <hudson.model.StringParameterDefinition>
+ <name>absolute_job_path</name>
+ <description><font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font></description>
+ <defaultValue>ArtifactUpload/dev/Lern/LernFlinkJobs</defaultValue>
+ <trim>false</trim>
+ </hudson.model.StringParameterDefinition>
+ <hudson.model.StringParameterDefinition>
+ <name>image_tag</name>
+ <description><font color=red size=2><b>CAUTION: If the value is blank, image tag will be taken from the latest metadata.json.</b></font></description>
+ <defaultValue/>
+ <trim>false</trim>
+ </hudson.model.StringParameterDefinition>
+ <org.biouno.unochoice.DynamicReferenceParameter plugin="uno-choice@2.5.6">
+ <name>private_branch</name>
+ <description/>
+ <randomName>choice-parameter-2544395024638227</randomName>
+ <visibleItemCount>1</visibleItemCount>
+ <script class="org.biouno.unochoice.model.GroovyScript">
+ <secureScript plugin="script-security@1.76">
+ <script>if (override_private_branch.equals("true")) {
+return """<input name="value" value="${private_repo_branch}" class="setting-input" type="text"><br> <font color=dimgray size=2><b>Change this value to checkout a different branch from private repository.</b></font>"""
+}
+else
+return """<b>This parameter is not used</b>"""</script>
+ <sandbox>true</sandbox>
+ </secureScript>
+ <secureFallbackScript plugin="script-security@1.76">
+ <script>return """<b>This parameter is not used</b>"""</script>
+ <sandbox>true</sandbox>
+ </secureFallbackScript>
+ </script>
+ <projectName>LernFlinkJobs</projectName>
+ <projectFullName>Deploy/dev/Lern/LernFlinkJobs</projectFullName>
+ <parameters class="linked-hash-map"/>
+ <referencedParameters/>
+ <choiceType>ET_FORMATTED_HTML</choiceType>
+ <omitValueField>true</omitValueField>
+ </org.biouno.unochoice.DynamicReferenceParameter>
+ <org.biouno.unochoice.DynamicReferenceParameter plugin="uno-choice@2.5.6">
+ <name>branch_or_tag</name>
+ <description/>
+ <randomName>choice-parameter-2620434998790477</randomName>
+ <visibleItemCount>1</visibleItemCount>
+ <script class="org.biouno.unochoice.model.GroovyScript">
+ <secureScript plugin="script-security@1.76">
+ <script>if (override_public_branch.equals("true")) {
+return """<input name="value" value="" class="setting-input" type="text"><br> <font color=dimgray size=2><b>Provide the tag or branch name to checkout the Jenkinsfile and codebase.<br>Note: The tag or branch name for this job should be taken from <a href="https://github.com/project-sunbird/sunbird-devops">project-sunbird/sunbird-devops</a></b></font></b></font>""";
+
+}
+else
+return """<b>This parameter is not used</b>"""</script>
+ <sandbox>true</sandbox>
+ </secureScript>
+ <secureFallbackScript plugin="script-security@1.76">
+ <script>return """<b>This parameter is not used</b>"""</script>
+ <sandbox>true</sandbox>
+ </secureFallbackScript>
+ </script>
+ <projectName>LernFlinkJobs</projectName>
+ <projectFullName>Deploy/dev/Lern/LernFlinkJobs</projectFullName>
+ <parameters class="linked-hash-map"/>
+ <referencedParameters/>
+ <choiceType>ET_FORMATTED_HTML</choiceType>
+ <omitValueField>true</omitValueField>
+ </org.biouno.unochoice.DynamicReferenceParameter>
+ <org.biouno.unochoice.CascadeChoiceParameter plugin="uno-choice@2.5.6">
+ <name>job_names_to_deploy</name>
+ <description><font color=green size=2><b>Choose the job names to deploy. Multi-selection is available.</b></font></description>
+ <randomName>choice-parameter-6698726079983885</randomName>
+ <visibleItemCount>1</visibleItemCount>
+ <script class="org.biouno.unochoice.model.GroovyScript">
+ <secureScript plugin="script-security@1.76">
+ <script>return[
+'collection-cert-pre-processor',
+'collection-certificate-generator',
+'activity-aggregate-updater',
+'relation-cache-updater',
+'merge-user-courses',
+'assessment-aggregator',
+'enrolment-reconciliation',
+'notification-job']</script>
+ <sandbox>true</sandbox>
+ </secureScript>
+ <secureFallbackScript plugin="script-security@1.76">
+ <script/>
+ <sandbox>true</sandbox>
+ </secureFallbackScript>
+ </script>
+ <projectName>LernFlinkJobs</projectName>
+ <projectFullName>Deploy/dev/Lern/LernFlinkJobs</projectFullName>
+ <parameters class="linked-hash-map"/>
+ <referencedParameters/>
+ <choiceType>PT_MULTI_SELECT</choiceType>
+ <filterable>false</filterable>
+ <filterLength>1</filterLength>
+ </org.biouno.unochoice.CascadeChoiceParameter>
+ </parameterDefinitions>
+ </hudson.model.ParametersDefinitionProperty>
+ <hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.2">
+ <maxConcurrentPerNode>0</maxConcurrentPerNode>
+ <maxConcurrentTotal>0</maxConcurrentTotal>
+ <categories class="java.util.concurrent.CopyOnWriteArrayList"/>
+ <throttleEnabled>false</throttleEnabled>
+ <throttleOption>project</throttleOption>
+ <limitOneJobWithMatchingParams>false</limitOneJobWithMatchingParams>
+ <paramsToUseForLimit/>
+ </hudson.plugins.throttleconcurrents.ThrottleJobProperty>
+
+ <org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+ <triggers/>
+ </org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+ </properties>
+ <definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.90">
+ <scm class="hudson.plugins.git.GitSCM" plugin="git@4.7.1">
+ <configVersion>2</configVersion>
+ <userRemoteConfigs>
+ <hudson.plugins.git.UserRemoteConfig>
+ <url>https://github.com/Sunbird-Lern/data-pipeline.git</url>
+ </hudson.plugins.git.UserRemoteConfig>
+ </userRemoteConfigs>
+ <branches>
+ <hudson.plugins.git.BranchSpec>
+ <name>${branch_or_tag}</name>
+ </hudson.plugins.git.BranchSpec>
+ </branches>
+ <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+ <submoduleCfg class="empty-list"/>
+ <extensions>
+ <hudson.plugins.git.extensions.impl.CloneOption>
+ <shallow>true</shallow>
+ <noTags>false</noTags>
+ <reference/>
+ <depth>0</depth>
+ <honorRefspec>false</honorRefspec>
+ </hudson.plugins.git.extensions.impl.CloneOption>
+ </extensions>
+ </scm>
+ <scriptPath>kubernetes/pipelines/deploy/Jenkinsfile</scriptPath>
+ <lightweight>false</lightweight>
+ </definition>
+ <triggers/>
+ <disabled>false</disabled>
+</flow-definition>
\ No newline at end of file
diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ApplicationElasticsearchRestore/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ApplicationElasticsearchRestore/config.xml
index 65ec68f8a1a8edf3b8e825925fa9be887b08bc45..1bc46ecfe8d08846afb895982622ea31f63e85d0 100644
--- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ApplicationElasticsearchRestore/config.xml
+++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ApplicationElasticsearchRestore/config.xml
@@ -29,7 +29,7 @@
<name>SNAPSHOT_NUMBER</name>
<description><font color=teal size=2>Enter Snapshot number to restore.
curl command to get snapshot number:
-curl http://localhost:9200/_snapshot/azurebackup/_all, Parse the json and get the valid snapshot number to restore</font></description>
+curl http://localhost:9200/_snapshot/{{ snapshot_base_path }}/_all, Parse the json and get the valid snapshot number to restore</font></description>
<defaultValue></defaultValue>
<trim>false</trim>
</hudson.model.StringParameterDefinition>
diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/ElasticsearchRestore/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/ElasticsearchRestore/config.xml
index 2587780de44336b05a93e91bbcd5d65c6b014b52..6e186acc541d8f6014fef5a2ddec70e0a01eb996 100644
--- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/ElasticsearchRestore/config.xml
+++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/ElasticsearchRestore/config.xml
@@ -36,7 +36,7 @@
<name>SNAPSHOT_NUMBER</name>
<description><font color=teal size=2>Enter Snapshot number to restore.
curl command to get snapshot number:
-curl http://localhost:9200/_snapshot/azurebackup/_all, Parse the json and get the valid snapshot number to restore</font></description>
+curl http://localhost:9200/_snapshot/{{ snapshot_base_path }}/_all, Parse the json and get the valid snapshot number to restore</font></description>
<defaultValue></defaultValue>
<trim>false</trim>
</hudson.model.StringParameterDefinition>
diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/ElasticsearchRestore/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/ElasticsearchRestore/config.xml
index 85428f56e814f11464675461217eb606d50fcb19..6add03b312c6673a24ebd7367bd40bf85a8a5790 100644
--- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/ElasticsearchRestore/config.xml
+++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/ElasticsearchRestore/config.xml
@@ -29,7 +29,7 @@
<name>SNAPSHOT_NUMBER</name>
<description><font color=teal size=2>Enter Snapshot number to restore.
curl command to get snapshot number:
-curl http://localhost:9200/_snapshot/azurebackup/_all, Parse the json and get the valid snapshot number to restore</font></description>
+curl http://localhost:9200/_snapshot/{{ snapshot_base_path }}/_all, Parse the json and get the valid snapshot number to restore</font></description>
<defaultValue></defaultValue>
<trim>false</trim>
</hudson.model.StringParameterDefinition>
diff --git a/kubernetes/ansible/roles/kong-api/tasks/main.yml b/kubernetes/ansible/roles/kong-api/tasks/main.yml
index d85575decee8350e12fc117c4612517ca27bd69d..5b9ff4d8a2a2f4fc35592048ca96c655b4d490b4 100644
--- a/kubernetes/ansible/roles/kong-api/tasks/main.yml
+++ b/kubernetes/ansible/roles/kong-api/tasks/main.yml
@@ -23,4 +23,5 @@
shell: "python /tmp/kong-api-scripts/kong_apis.py /tmp/kong_apis.json --kong-admin-api-url=http://{{ private_ingressgateway_ip }}/admin-api"
- name: Reload kong
- shell: kubectl get pod -l app=apimanager --namespace={{namespace}} -o name | xargs -I{} kubectl exec {} -- kong reload
\ No newline at end of file
+ shell: kubectl get pod -l app=apimanager --namespace={{namespace}} -o name | xargs -I{} kubectl exec {} -- kong reload
+ ignore_errors: true
diff --git a/kubernetes/ansible/roles/kong-consumer/tasks/main.yml b/kubernetes/ansible/roles/kong-consumer/tasks/main.yml
index edb1b4c0a4feb4906600feaca07873b394931f60..f7772d26995518cc337125a23e70f08c303d2e34 100644
--- a/kubernetes/ansible/roles/kong-consumer/tasks/main.yml
+++ b/kubernetes/ansible/roles/kong-consumer/tasks/main.yml
@@ -50,4 +50,5 @@
- /tmp/jwt_token.txt
- name: Reload kong
- shell: kubectl get pod -l app=apimanager --namespace={{namespace}} -o name | xargs -I{} kubectl exec {} -- kong reload
\ No newline at end of file
+ shell: kubectl get pod -l app=apimanager --namespace={{namespace}} -o name | xargs -I{} kubectl exec {} -- kong reload
+ ignore_errors: true
diff --git a/kubernetes/ansible/roles/logging/templates/elasticsearch-curator.yaml b/kubernetes/ansible/roles/logging/templates/elasticsearch-curator.yaml
index b411d48adb047a7d65ce5e8267c4b6f028243282..6253cd44e1e151886704269cc284e7b19a2bee32 100644
--- a/kubernetes/ansible/roles/logging/templates/elasticsearch-curator.yaml
+++ b/kubernetes/ansible/roles/logging/templates/elasticsearch-curator.yaml
@@ -8,7 +8,7 @@ cronjob:
image:
repository: untergeek/curator
- tag: 5.8.1
+ tag: 5.8.4
pullPolicy: IfNotPresent
configMaps:
diff --git a/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml b/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml
index f4422e608b1c0ce6ff840292624633b1fd2cc73c..0dd8167be22df47361a286176203efdaa4c1b783 100644
--- a/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml
+++ b/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml
@@ -253,6 +253,12 @@ data:
proxy_http_version 1.1;
proxy_pass $target;
}
+ location /ml-reports/ {
+ set $target http://ml-reports-service.{{ .Values.namespace }}.svc.cluster.local:3000;
+ rewrite ^/ml-reports/(.*) /$1 break;
+ proxy_http_version 1.1;
+ proxy_pass $target;
+ }
}
}
kind: ConfigMap
diff --git a/kubernetes/helm_charts/monitoring/dashboards/dashboards/api-manager-consumerlevel-dashboard.json b/kubernetes/helm_charts/monitoring/dashboards/dashboards/api-manager-consumerlevel-dashboard.json
new file mode 100644
index 0000000000000000000000000000000000000000..2b4b93fafd1ce267f7c85d85a9f830d9e0fcb1b3
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/dashboards/dashboards/api-manager-consumerlevel-dashboard.json
@@ -0,0 +1,1050 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 0,
+ "id": 57,
+ "iteration": 1657820887217,
+ "links": [],
+ "panels": [
+ {
+ "collapsed": false,
+ "datasource": null,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 32,
+ "panels": [],
+ "repeat": null,
+ "title": "Traffic",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$cluster",
+ "decimals": 0,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 1
+ },
+ "hiddenSeries": false,
+ "id": 12,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": true,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(increase(kong_request_per_user_v2{username=~\"${username}\",api =~ \"$api\"}[$__range])) by (api)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{api}}",
+ "metric": "",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Total Requests / API",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:145",
+ "decimals": null,
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:146",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$cluster",
+ "decimals": 0,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 1
+ },
+ "hiddenSeries": false,
+ "id": 36,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": true,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(increase(kong_request_per_user_v2{username=~\"${username}\",api =~ \"${api}\"}[$__range])) by (username)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{username}}",
+ "metric": "",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Total Requests / Username",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:145",
+ "decimals": null,
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:146",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$cluster",
+ "decimals": 0,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 8
+ },
+ "hiddenSeries": false,
+ "id": 1,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "max",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(kong_status_count_per_user_v2{username=~\"${username}\",api=~\"${api}\"}[1m])) by (api)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{api}}",
+ "metric": "",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Requests per sec / API",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": 0,
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "decimals": null,
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$cluster",
+ "decimals": 0,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 8
+ },
+ "hiddenSeries": false,
+ "id": 34,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "max",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(kong_request_per_user_v2{username=~\"${username}\",api=~\"${api}\"}[1m])) by (username)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{username}}",
+ "metric": "",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Requests per sec / User",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": 0,
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "decimals": null,
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "collapsed": false,
+ "datasource": null,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 15
+ },
+ "id": 38,
+ "panels": [],
+ "title": "Success / Error %",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$cluster",
+ "decimals": 2,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 16
+ },
+ "hiddenSeries": false,
+ "id": 40,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": true,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(increase(kong_status_count_per_user_v2{username=~\"${username}\",api=~\"${api}\",status=~\"2..|3..\", status!= \"total\"}[1m])) by (api, status) / ignoring(api, status) group_left sum(increase(kong_status_count_per_user_v2{username=~\"${username}\",api=~\"${api}\", status!= \"total\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{api}}-{{status}}",
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Success %",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": 1,
+ "format": "percentunit",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "decimals": null,
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$cluster",
+ "decimals": 2,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 16
+ },
+ "hiddenSeries": false,
+ "id": 42,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "max",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(increase(kong_request_status_count{api=~\"${api}\",status_code!~\"2..|3..|total\"}[1m])) by (api, status_code) / ignoring(api, status_code) group_left sum(increase(kong_request_status_count{api=~\"${api}\", status_code!= \"total\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{api}}-{{status_code}}",
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Error %",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": 1,
+ "format": "percentunit",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "decimals": null,
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "collapsed": false,
+ "datasource": null,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 23
+ },
+ "id": 44,
+ "panels": [],
+ "title": "Success / Error Count",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$cluster",
+ "decimals": 0,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 24
+ },
+ "hiddenSeries": false,
+ "id": 46,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(increase(kong_status_count_per_user_v2{api=~\"${api}\",status=~\"2..|3..\", status!= \"total\"}[1m])) by (api, status)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{api}}-{{status}}",
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Success Count",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": null,
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "decimals": null,
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$cluster",
+ "decimals": 0,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 24
+ },
+ "hiddenSeries": false,
+ "id": 48,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(increase(kong_status_count_per_user_v2{api=~\"${api}\",status!~\"2..|3..|total\"}[1m])) by (api, status)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{api}}-{{status}}",
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Error Count",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": null,
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "decimals": null,
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "refresh": "1m",
+ "schemaVersion": 25,
+ "style": "dark",
+ "tags": [
+ "external",
+ "api"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "Prometheus",
+ "value": "Prometheus"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": null,
+ "multi": false,
+ "name": "cluster",
+ "options": [],
+ "query": "prometheus",
+ "queryValue": "",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "allValue": null,
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": [
+ "$__all"
+ ]
+ },
+ "datasource": "$cluster",
+ "definition": "label_values(kong_request_per_user_v2,api)",
+ "hide": 0,
+ "includeAll": true,
+ "label": null,
+ "multi": true,
+ "name": "api",
+ "options": [],
+ "query": "label_values(kong_request_per_user_v2,api)",
+ "refresh": 2,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "selected": true,
+ "text": "All",
+ "value": [
+ "$__all"
+ ]
+ },
+ "datasource": "$cluster",
+ "definition": "label_values(kong_request_per_user_v2,username)",
+ "hide": 0,
+ "includeAll": true,
+ "label": null,
+ "multi": true,
+ "name": "username",
+ "options": [],
+ "query": "label_values(kong_request_per_user_v2,username)",
+ "refresh": 2,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-3h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "",
+ "title": "API Manager Consumer Level",
+ "uid": "By5aKkRVk",
+ "version": 1
+}
diff --git a/kubernetes/helm_charts/monitoring/dashboards/values.yaml b/kubernetes/helm_charts/monitoring/dashboards/values.yaml
index 8365ad1dc89d1c5e982d1324b3e5ab4867f26f24..7e7f42d74e821cd02b552d911664c82d807dd567 100644
--- a/kubernetes/helm_charts/monitoring/dashboards/values.yaml
+++ b/kubernetes/helm_charts/monitoring/dashboards/values.yaml
@@ -408,6 +408,8 @@ dashboards2:
file: dashboards/api-manager-old.json
dashboardkong4:
file: dashboards/api-manager-req-resp-aggregated-beta.json
+ dashboardkong5:
+ file: dashboards/api-manager-consumerlevel-dashboard.json
fluentbit:
fluent-bit:
file: dashboards/fluent-bit.json
diff --git a/kubernetes/helm_charts/monitoring/statsd-exporter/values.yaml b/kubernetes/helm_charts/monitoring/statsd-exporter/values.yaml
index 002293c45bff1eea285c9b6ed4d4e7ed574e29f9..4a499884cb76d687f2c3f847d62383a0e577ac7b 100644
--- a/kubernetes/helm_charts/monitoring/statsd-exporter/values.yaml
+++ b/kubernetes/helm_charts/monitoring/statsd-exporter/values.yaml
@@ -125,6 +125,42 @@ configMapValues:
histogram_options:
buckets: [ 1, 5, 10, 25, 50, 100, 250, 500, 1000, 2000 ]
name: kong_response_size
+ - labels:
+ api: $1
+ match: kong.*.user.uniques
+ name: kong_unique_users_v2
+ - labels:
+ api: $1
+ username: $2
+ match: kong.*.user.*.count
+ name: kong_request_per_user
+ - labels:
+ api: $1
+ username: $2
+ match: kong.*.user.*.request.count
+ name: kong_request_per_user_v2
+ - labels:
+ api: $1
+ username: $2
+ status: $3
+ match: kong.*.user.*.status.*
+ name: kong_status_count_per_user
+ - labels:
+ api: $1
+ username: $2
+ status: $3
+ match: kong.*.user.*.request.status.*
+ name: kong_status_count_per_user_v2
+ - labels:
+ api: $1
+ username: $2
+ match: kong.*.user.*.status.total
+ name: kong_status_count_per_user_total
+ - labels:
+ api: $1
+ username: $2
+ match: kong.*.user.*.request.status.total
+ name: kong_status_count_per_user_total_v2
- match: "."
match_type: regex
action: drop
diff --git a/kubernetes/helm_charts/sunbird-RC/registry/schemas/TrainingCertificate.json b/kubernetes/helm_charts/sunbird-RC/registry/schemas/TrainingCertificate.json
index be84e538a37c99cbb20ca6dda6bd6023913fdf6a..2906929f2d186848ef7fd0e0ef2be85bd489ecb0 100644
--- a/kubernetes/helm_charts/sunbird-RC/registry/schemas/TrainingCertificate.json
+++ b/kubernetes/helm_charts/sunbird-RC/registry/schemas/TrainingCertificate.json
@@ -69,6 +69,6 @@
],
"systemFields": ["osCreatedAt", "osUpdatedAt", "osCreatedBy", "osUpdatedBy"],
"enableLogin": false,
- "credentialTemplate": "https://{{sunbird_public_storage_account_name}}.blob.core.windows.net/{{sunbird_content_azure_storage_container}}/schema/credential_template.json"
+ "credentialTemplate": "https://{{upstream_url}}/schema/credential_template.json"
}
}
diff --git a/private_repo/ansible/inventory/dev/Core/common.yml b/private_repo/ansible/inventory/dev/Core/common.yml
index 48024937f1e1889cd4cfbba6d6ef57c16d6bae38..e54937985e00686ce10494fa649471e7308aa351 100644
--- a/private_repo/ansible/inventory/dev/Core/common.yml
+++ b/private_repo/ansible/inventory/dev/Core/common.yml
@@ -23,6 +23,19 @@ sunbird_public_storage_account_name: "change.azure.storage.account.name"
sunbird_private_storage_account_name: "change.azure.storage.account.name" # Azure account name for storing private data (like reports, telemetry data)
sunbird_artifact_storage_account_name: "change.azure.storage.account.name" # Azure account name for storing artifacts data (like jenkins build zip files)
sunbird_management_storage_account_name: "change.azure.storage.account.name" # Azure account name for storing backup data (like cassandra backups)
+
+# Define the below if you are using Azure Cloud
+# Management Storage Account
+azure_management_storage_account_name: "{{ sunbird_management_storage_account_name }}"
+
+# Define the below if you are using AWS Cloud
+# Management Storage Bucket
+aws_management_bucket_name: ""
+
+# Define the below if you are using Google Cloud
+# Management Storage Bucket
+gcs_management_bucket_name: ""
+
# ------------------------------------------------------------------------------------------------------------ #
# Cloud / Infra Specific values - Check these and update accordingly
diff --git a/private_repo/ansible/inventory/dev/Core/secrets.yml b/private_repo/ansible/inventory/dev/Core/secrets.yml
index b004ab86d2639d650ef27b4f828e1686ee15bd05..798aceb7e0a4b3dcc28d16d52d85b6a3816b420a 100644
--- a/private_repo/ansible/inventory/dev/Core/secrets.yml
+++ b/private_repo/ansible/inventory/dev/Core/secrets.yml
@@ -17,6 +17,22 @@ sunbird_artifact_storage_account_sas: "change.azure.storage.account.sas"
sunbird_public_storage_account_sas: "change.azure.storage.account.sas" # SAS token value generated from above command
sunbird_public_storage_account_key: "change.azure.storage.account.key"
sunbird_private_storage_account_key: "change.azure.storage.account.key"
+sunbird_management_storage_account_key: "change.azure.storage.account.key"
+
+# Define the below if you are using Azure Cloud
+# Management Storage Account
+azure_management_storage_account_key: "{{ sunbird_management_storage_account_key }}"
+
+# Define the below if you are using AWS Cloud
+# Management Storage Bucket
+aws_management_bucket_user_access_key: ""
+aws_management_bucket_user_secret_key: ""
+
+
+# Define the below if you are using Google Cloud
+# Management Storage Bucket
+gcs_management_bucket_service_account: |
+
# The proxy key and crt values should be padded to the right by a couple of spaces
# Example:
diff --git a/utils/sunbird-RC/schema/credential_template.json b/utils/sunbird-RC/schema/credential_template.json
index 3f940546869a9bacaf771f40b5106ecfd7ecc5c9..8f467986cc4015948357a0f10b152601d9d7e406 100644
--- a/utils/sunbird-RC/schema/credential_template.json
+++ b/utils/sunbird-RC/schema/credential_template.json
@@ -1,7 +1,7 @@
{
"@context": [
- "https://{{sunbird_public_storage_account_name}}.blob.core.windows.net/{{sunbird_content_azure_storage_container}}/schema/v1_context.json",
- "https://{{sunbird_public_storage_account_name}}.blob.core.windows.net/{{sunbird_content_azure_storage_container}}/schema/sunbird_context.json"
+ "https://{{upstream_url}}/schema/v1_context.json",
+ "https://{{upstream_url}}/schema/sunbird_context.json"
],
"type": [
"VerifiableCredential"