Merge branch 'release-5.0.0' into release-5.0.0

ansible/es.yml

@@ -6,10 +6,16 @@
     - es_snapshot_host: "{{app_es_snapshot_host}}"
     - snapshot_base_path: "{{app_snapshot_base_path}}"
   roles:
-    - es-azure-snapshot
-    - es5-snapshot-purge
+    - role: es-azure-snapshot
+      when: cloud_service_provider == "azure"
+    - role: es-s3-snapshot
+      when: cloud_service_provider == "aws"
+    - role: es-gcs-snapshot
+      when: cloud_service_provider == "gcloud"
+    - role: es5-snapshot-purge
   tags:
     - es_backup
+  run_once: true
 
 - hosts: log-es-backup
   become: yes
@@ -19,16 +25,13 @@
     - es_snapshot_host: "{{log_es_snapshot_host}}"
     - snapshot_base_path: "{{log_snapshot_base_path}}"
   roles:
-    - es-azure-snapshot
+    - role: es-azure-snapshot
+      when: cloud_service_provider == "azure"
+    - role: es-s3-snapshot
+      when: cloud_service_provider == "aws"
+    - role: es-gcs-snapshot
+      when: cloud_service_provider == "gcloud"
     - es5-snapshot-purge
   tags:
     - log_es_backup
-
-# - hosts: log-es
+  run_once: true
\ No newline at end of file
-#   become: yes
-#   vars_files:
-#     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
-#   roles:
-#     - es2-data-purge
-#   tags:
-#     - purge_old_logs

ansible/inventory/env/group_vars/all.yml

@@ -132,6 +132,8 @@ cassandra_rpc_address: 0.0.0.0
 cassandra_restore_dir: "/home/{{ ansible_ssh_user }}/"
 cassandra_backup_azure_container_name: cassandra-backup
 cassandra_backup_dir: /data/cassandra/backup
+### Release 5.0.0 ###
+cassandra_isMutiDCEnabled: false
 
 keycloak_realm: sunbird
 sunbird_content_player_url: "http://kong:8000/"

ansible/provision-log-es.yml

 - hosts: log-es
   become: yes
   vars:
-    es_version: "6.8.10"
+    es_version: "6.8.22"
     es_etc_discovery_zen_ping_unicast_hosts: "{{ groups['log-es'] }}"
     es_etc_cluster_name: "{{ node_name }}"
     es_heap_size: "{{ log_es_heap_size | default('1g') }}"
-    es_plugins_reinstall: true
-    es_plugins:
-      - plugin: "repository-azure"
     es_config:
       cluster.name: "{{ log_es_etc_cluster_name }}"
       discovery.zen.ping.unicast.hosts: "{{ groups['log-es'] }}"
@@ -49,5 +46,7 @@
         method: PUT
         return_content: true
         timeout: 30
-        retries: 3
-        delay: 10
+      register: _result
+      until: _result.status == 200
+      retries: 3
+      delay: 10
\ No newline at end of file

ansible/roles/elasticsearch_dependencies/main.yml → ansible/roles/elasticsearch_dependencies/tasks/main.yml

@@ -2,5 +2,5 @@
 - debug: msg="Installing jmespath"
 
 - name: Install jmespath
-  apt: name=jmespath state=present
+  apt: name=python3-jmespath state=present
   become: yes

ansible/roles/es-azure-restore/tasks/main.yml

 ---
 - name: Set azure snapshot for the first time
   uri:
-    url: "http://{{ es_restore_host }}:9200/_snapshot/azurebackup"
+    url: "http://{{ es_restore_host }}:9200/_snapshot/{{ snapshot_base_path }}"
     method: PUT
     body: "{{ snapshot_create_request_body | to_json }}"
     headers:
@@ -9,12 +9,12 @@
 
 - name: Restore ES from Azure backup
   uri:
-    url: "http://{{ es_restore_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}/_restore"
+    url: "http://{{ es_restore_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}/_restore"
     method: POST
 
 - name: "Wait for restore to be completed"
   uri:
-    url: "http://{{ es_restore_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}/_status"
+    url: "http://{{ es_restore_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}/_status"
     method: GET
     return_content: yes
     status_code: 200

ansible/roles/es-azure-snapshot/defaults/main.yml

 snapshot_create_request_body: {
     type: azure,
     settings: {
-      container: "elasticsearch-snapshots",
+      container: "{{ es_azure_backup_folder_name }}",
       base_path: "{{ snapshot_base_path }}_{{ base_path_date }}"
     }
 }
 
 # Override these values
-es_snapshot_host: localhost
-snapshot_base_path: default
+es_snapshot_host: "localhost"
+snapshot_base_path: "default"
+
+es_azure_backup_folder_name: "elasticsearch-snapshots"
\ No newline at end of file

ansible/roles/es-azure-snapshot/tasks/main.yml

@@ -2,41 +2,42 @@
 
 - set_fact: base_path_date="{{ lookup('pipe','date +%Y-%m') }}"
 
-- name: Create azure snapshot
+- set_fact: snapshot_number="snapshot_{{ lookup('pipe','date +%s') }}"
+
+- name: Ensure backup folder exists in azure blob
+  shell: "az storage container create --name {{ es_azure_backup_folder_name }}"
+  environment:
+    AZURE_STORAGE_ACCOUNT: "{{ azure_management_storage_account_name }}"
+    AZURE_STORAGE_KEY: "{{ azure_management_storage_account_key }}"
+
+- name: Create Azure Repository
   uri:
-    url: "http://{{ es_snapshot_host }}:9200/_snapshot/azurebackup"
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}"
     method: PUT
     body: "{{ snapshot_create_request_body | to_json }}"
     headers:
       Content-Type: "application/json"
 
-- debug:
-    var: "{{ snapshot_base_path }}"
-
-- set_fact: snapshot_number="snapshot_{{ansible_date_time.epoch}}"
-
 - name: Take new snapshot
   uri:
-    url: "http://{{ es_snapshot_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}"
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
     method: PUT
-    body:  >
-      {"indices":"*","include_global_state":false}
     headers:
       Content-Type: "application/json"
 
 - name: Print all snapshots
   uri:
-    url: "http://{{ es_snapshot_host }}:9200/_snapshot/azurebackup/_all"
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/_all"
     method: GET
 
 - name: Print status of current snapshot
   uri:
-    url: "http://{{ es_snapshot_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}"
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
     method: GET
 
 - name: "Wait for backup to be completed"
   uri:
-    url: "http://{{ es_snapshot_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}"
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
     method: GET
     return_content: yes
     status_code: 200

ansible/roles/es-curator/defaults/main.yml

-# es_curator_major_version: 3
-# es_curator_version: 3.4.1-1
-
 es_curator_major_version: 5
-es_curator_version: 5.7.6
-python_es_curator_version: 3.4.1-1
+es_curator_version: 5.8.4
\ No newline at end of file

ansible/roles/es-curator/tasks/main.yml

@@ -7,11 +7,6 @@
 - debug:
     msg: "{{ es_curator_version }}"
 
-- name: Install python elasticsearch curator
-  apt:
-    name: python-elasticsearch-curator={{ python_es_curator_version }}
-    force: yes
-
 - name: Install elasticsearch curator
   apt:
     name: elasticsearch-curator={{ es_curator_version }}

ansible/roles/es-gcs-snapshot/defaults/main.yml

+snapshot_create_request_body: {
+    type: gcs,
+    settings: {
+      bucket: "{{ gcs_management_bucket_name }}",
+      base_path: "{{ es_gcs_backup_folder_name }}/{{ snapshot_base_path }}_{{ base_path_date }}"
+    }
+}
+
+# Override these values
+es_snapshot_host: "localhost"
+snapshot_base_path: "default"
+es_gcs_backup_folder_name: "elasticsearch-snapshots"
\ No newline at end of file

ansible/roles/es-gcs-snapshot/tasks/main.yml

+---
+
+- set_fact: base_path_date="{{ lookup('pipe','date +%Y-%m') }}"
+
+- set_fact: snapshot_number="snapshot_{{ lookup('pipe','date +%s') }}"
+
+- name: Create GCS Repository
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}"
+    method: PUT
+    body: "{{ snapshot_create_request_body | to_json }}"
+    headers:
+      Content-Type: "application/json"
+
+- name: Take new snapshot
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+    method: PUT
+    headers:
+      Content-Type: "application/json"
+
+- name: Print all snapshots
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/_all"
+    method: GET
+
+- name: Print status of current snapshot
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+    method: GET
+
+- name: "Wait for backup to be completed"
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+    method: GET
+    return_content: yes
+    status_code: 200
+    body_format: json
+  register: result
+  until: result.json.snapshots[0].state == 'SUCCESS'
+  retries: 120
+  delay: 10

ansible/roles/es-s3-snapshot/defaults/main.yml

+snapshot_create_request_body: {
+    type: s3,
+    settings: {
+      bucket: "{{ aws_management_bucket_name }}",
+      base_path: "{{ es_s3_backup_folder_name }}/{{ snapshot_base_path }}_{{ base_path_date }}"
+    }
+}
+
+# Override these values
+es_snapshot_host: "localhost"
+snapshot_base_path: "default"
+es_s3_backup_folder_name: "elasticsearch-snapshots"
\ No newline at end of file

ansible/roles/es-s3-snapshot/tasks/main.yml

+---
+
+- set_fact: base_path_date="{{ lookup('pipe','date +%Y-%m') }}"
+
+- set_fact: snapshot_number="snapshot_{{ lookup('pipe','date +%s') }}"
+
+- name: Create S3 Repository
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}"
+    method: PUT
+    body: "{{ snapshot_create_request_body | to_json }}"
+    headers:
+      Content-Type: "application/json"
+
+- name: Take new snapshot
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+    method: PUT
+    headers:
+      Content-Type: "application/json"
+
+- name: Print all snapshots
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/_all"
+    method: GET
+
+- name: Print status of current snapshot
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+    method: GET
+
+- name: "Wait for backup to be completed"
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+    method: GET
+    return_content: yes
+    status_code: 200
+    body_format: json
+  register: result
+  until: result.json.snapshots[0].state == 'SUCCESS'
+  retries: 120
+  delay: 10

ansible/roles/es2-snapshot-purge/defaults/main.yml

 es_snapshot_retention_days: 30
-es_snapshot_repository: azurebackup
+es_snapshot_repository: "{{ snapshot_base_path }}"

ansible/roles/es5-snapshot-purge/defaults/main.yml

 es_snapshot_host: localhost
-es_snapshot_repository: azurebackup
+es_snapshot_repository: "{{ snapshot_base_path }}"
 es_snapshot_retention_days: 30
 es_curator_config_dir: /etc/curator
 es_curator_config_file: "{{ es_curator_config_dir }}/curator.yml"

ansible/roles/es5-snapshot-purge/meta/main.yml

 ---
 dependencies:
-  - { role: es-curator, es_curator_major_version: 5, es_curator_version: 5.8.1, python_es_curator_version: 3.4.1-1 }
+  - { role: es-curator, es_curator_major_version: 5, es_curator_version: 5.8.4 }

ansible/roles/es6/defaults/main.yml

 ---
 es_major_version: "6.x"
-es_version: "6.2.3"
+es_version: "6.8.22"
 es_version_lock: false
 es_use_repository: true
 es_templates_fileglob: "files/templates/*.json"
@@ -11,7 +11,6 @@ es_start_service: true
 es_java_install: true
 update_java: false
 es_restart_on_change: true
-es_plugins_reinstall: false
 es_scripts: false
 es_templates: true
 es_user: elasticsearch
@@ -22,7 +21,7 @@ es_pid_dir: "/var/run/elasticsearch"
 es_data_dirs: "/var/lib/elasticsearch"
 es_log_dir: "/var/log/elasticsearch"
 es_max_open_files: 65536
-es_max_threads: "{{ 2048 if ( es_version | version_compare('6.0.0', '<')) else 8192 }}"
+es_max_threads: "{{ 2048 if ( es_version is version('6.0.0', '<')) else 8192 }}"
 es_max_map_count: 262144
 es_allow_downgrades: false
 es_enable_xpack: false
@@ -41,4 +40,9 @@ conf_dir: ''
 data_dirs: ''
 # JVM custom parameters
 es_jvm_custom_parameters: ''
-es_heap_size: "2g"
+es_heap_size: "1g"
+es_plugins_reinstall: true
+es_plugins:
+- plugin: "repository-azure"
+- plugin: "repository-s3"
+- plugin: "repository-gcs"
\ No newline at end of file

ansible/roles/es6/tasks/azure-plugin.yml

-- name: Check if plugin already installed or not
-  shell: "curl -s localhost:9200/_cat/plugins | awk '{print $2}' |head -1"
-  register: plugin_output
-
-- name: check the output
-  debug:
-    var: plugin_output
-
-- name: Install plugin
-  shell: ES_PATH_CONF=/etc/elasticsearch/"{{ es_instance_name }}" /usr/share/elasticsearch/bin/elasticsearch-plugin install repository-azure
-  when: plugin_output.stdout != "repository-azure"

ansible/roles/es6/tasks/elasticsearch-parameters.yml

@@ -58,7 +58,7 @@
 #Relies on elasticsearch distribution installing a serviced script to determine whether one should be copied.
 
 - name: set fact use_system_d
-  set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', '>=')) or (ansible_distribution in ['RedHat','CentOS'] and ansible_distribution_version | version_compare('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15', '>=')) }}
+  set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version is version('8', '>=')) or (ansible_distribution in ['RedHat','CentOS'] and ansible_distribution_version is version('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version is version('15', '>=')) }}
 
 - name: set fact instance_sysd_script
   set_fact: instance_sysd_script={{sysd_script | dirname }}/{{es_instance_name}}_{{sysd_script | basename}}

ansible/roles/es6/tasks/main.yml

@@ -31,16 +31,27 @@
   tags:
       - plugins
 
-# Install Elasticsearch plugin for backups
-- name: include elasticsearch plugin install
-  include: azure-plugin.yml
-
 #We always execute xpack as we may need to remove features
 - name: include xpack/elasticsearch-xpack.yml
   include: xpack/elasticsearch-xpack.yml
   tags:
       - xpack
 
+- name: include plugins/create-keystore.yml
+  include: plugins/create-keystore.yml
+
+- name: include plugins/repository-azure.yml
+  include: plugins/repository-azure.yml
+  when: cloud_service_provider == "azure"
+
+- name: include plugins/repository-s3.yml
+  include: plugins/repository-s3.yml
+  when: cloud_service_provider == "aws"
+
+- name: include plugins/repository-gcs.yml
+  include: plugins/repository-gcs.yml
+  when: cloud_service_provider == "gcloud"
+
 - name: flush handlers
   meta: flush_handlers
 
@@ -79,15 +90,3 @@
   when: es_templates
   tags:
       - templates
-
-# Add ES Plugin credentials for backups
-- name: Add ES Backup variables to elasticsearch to have backups enabled
-  lineinfile:
-    path: '{{conf_dir}}/elasticsearch.yml'
-    line: 'cloud.azure.storage.default.{{item.key}}: {{item.value}}'
-    regexp: "cloud.azure.storage.default.{{ item.key }}.*"
-  with_items:
-    - {key: 'account', value: '{{ sunbird_management_storage_account_name }}'}
-    - {key: 'key', value: '{{ sunbird_management_storage_account_key }}'}
-  when: sunbird_management_storage_account_name is defined and sunbird_management_storage_account_key is defined
-  notify: restart elasticsearch