Merge pull request #569 from project-sunbird/release-2.0.0

feat: Release 2.0.0 merge to master

ansible/artifacts/sunbird/login/login.ftl

@@ -47,6 +47,9 @@
                             <label id="passwordLabel" for="password" class="">
                                 ${msg("password")}
                             </label>
+                            <#if realm.resetPasswordAllowed>
+                                <a id="versionLink" class="ui right floated forgetPasswordLink" tabindex="1" onclick="javascript:storeLocation(); javascript:makeDivUnclickable()" href="${url.loginResetCredentialsUrl}">${msg("doForgotPassword")}</a>
+                            </#if>
                             <label id="passwordLabelPlaceholder" for="password" class="activeLabelColor hide">
                                 ${msg("placeholderForPassword")}
                             </label>
@@ -56,9 +59,7 @@
                     <div class="field">
                         <button id="login" class="mt-36 ui fluid button">${msg("doSignIn")}</button>
                     </div>
-                    <#if realm.resetPasswordAllowed>
-                        <a id="versionLink" class="ui right floated forgetPasswordLink" onclick="javascript:storeLocation(); javascript:makeDivUnclickable()" href="${url.loginResetCredentialsUrl}">${msg("doForgotPassword")}</a>
-                    </#if>
+                    
                     <div id="selfSingUp" class="hide">
                         <p class="or mb-30 mt-30 textCenter">OR</p>
                         <div class="field">

ansible/artifacts/sunbird/login/messages/messages_en.properties

@@ -134,7 +134,7 @@ client_broker=Broker
 invalidUserMessage=Invalid email ID/Mobile number or password. Please try again with valid credentials.
 invalidEmailMessage=Invalid email address.
 accountDisabledMessage=Account is disabled, contact admin.
-accountTemporarilyDisabledMessage=Invalid username or password. You can try again after 24 hours or ask your organisation administrator to reset your credentials.
+accountTemporarilyDisabledMessage=Your account has been locked due to too many incorrect login attempts. You can re-attempt to login after 24 hours. Please get in touch with the help desk team for support
 expiredCodeMessage=Sign in timeout. Please Sign In again.
 expiredActionMessage=Action expired. Please continue with Sign In now.
 expiredActionTokenNoSessionMessage=Action expired.

ansible/artifacts/sunbird/login/resources/css/login.css

@@ -106,9 +106,7 @@ input:focus{
     color: #9B9B9B !important;
     font-size: 12px;
     line-height: 17px;
-    position: absolute;
-    right: 0;
-    top: 85px;
+    float: right;
 }
 .forgetPasswordLink:hover,
 .forgetPasswordLink:focus{

ansible/cassandra-backup.yml

@@ -4,3 +4,4 @@
     - ['{{inventory_dir}}/secrets.yml']
   roles:
     - cassandra-backup
+  run_once: true

ansible/deploy-plugins.yml

@@ -7,6 +7,14 @@
         AZURE_STORAGE_ACCOUNT: "{{ azure_plugin_storage_account_name }}"
         AZURE_STORAGE_KEY: "{{ azure_plugin_storage_account_key }}"
   tasks:
+    - name: rename env_domain in preview_cdn.html for CDN
+      shell: |
+        echo "{{sunbird_portal_preview_cdn_url}}"
+        sed -i 's|cdn_url|{{sunbird_portal_preview_cdn_url}}|g' "{{currentws}}"/ansible/preview/preview_cdn.html
+      when: sunbird_portal_preview_cdn_url is defined
+      tags:
+        - preview
+
     - name: delete batch
       command: "az storage blob delete-batch -s {{ plugin_container_name }} --pattern {{ folder_name }}/*"
       async: 3600
@@ -28,6 +36,7 @@
         - preview
         - editor
         - core-plugins
+  
 
     - name: upload file
       command: "az storage blob upload --container-name {{ plugin_container_name }} --file {{ source_file_name }} --name artefacts/content-player/content-player-{{ player_version_number }}.zip"
@@ -41,4 +50,25 @@
       async: 3600
       poll: 10
       tags:
-        - plugins
\ No newline at end of file
+        - plugins
+
+- hosts: preview-vm
+  gather_facts: false
+  become: yes
+  tasks:
+    - name: removing dir
+      file:
+        path: /var/www/html/preview
+        state: absent
+    - name: creating dir
+      file:
+        path: /var/www/html/preview
+        state: directory
+
+    - name: copy zip to remote
+      copy:
+        src: preview/
+        dest: /var/www/html/preview/
+ # tags:
+  #  - preview
+   # - testupload

ansible/deploy.yml

@@ -23,7 +23,6 @@
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
   roles:
-#    - acr-docker
     - stack-sunbird
   tags:
     - stack-sunbird

ansible/docker-nodes-prune.yml

+- hosts: swarm-nodes
+  become: yes
+  gather_facts: no
+  tasks:
+  - command: docker system prune -af

ansible/inventory/env/group_vars/all.yml

@@ -45,14 +45,17 @@ sunbird_keycloak_required_action_link_expiration_seconds: 2592000
 sunbird_es_port: 9300
 mail_server_port: 587
 upstream_url: "{{azure_account_name}}.blob.core.windows.net/{{sunbird_content_azure_storage_container}}"
+
 ## Mailing list
 # !! Override for specific alert groups; Default will be alerts_mailing_list
-monitor_alerts_slack_channel:
-app_alerts_mailing_list: ""
-devops_alerts_mailing_list: ""
-site_alerts_mailing_list: ""
-keycloak_alerts_mailing_list: "{{devops_alerts_mailing_list}}"
-api_manager_alerts_mailing_list: "{{devops_alerts_mailing_list}}"
+# Enable below override below variables in common.yml for team specific alerts
+
+# monitor_alerts_slack_channel: ""
+# app_alerts_mailing_list: ""
+# devops_alerts_mailing_list: ""
+# site_alerts_mailing_list: ""
+# keycloak_alerts_mailing_list: ""
+# api_manager_alerts_mailing_list: ""
 
 # Learner
 sunbird_user_profile_field_default_visibility: private
@@ -262,7 +265,8 @@ proxy_prometheus: false
 enable_scraping_docker_metrics: false
 postgres_exporter_postgres_port: 5432
 postgres_exporter_user: postgres_exporter
-sunbird_cs_base_url: "{{proto}}://{{proxy_server_name}}/api"
+sunbird_cs_base_url: "http://{{groups['swarm-manager'][0]}}:5000"
+sunbird_user_service_base_url: "http://{{groups['swarm-manager'][0]}}:9000"
 kong_admin_api_url: http://localhost:8001
 
 postgresql_backup_azure_storage_account_name: "{{azure_account_name}}"
@@ -356,9 +360,9 @@ sunbird_device_register_api: "{{proto}}://{{api_proxy_name}}/v3/device/register/
 sunbird_course_batch_notification_signature: sunbird
 
 sunbird_authorization: "{{core_vault_sunbird_api_auth_token}}"
-sunbird_lms_base_url: "{{proto}}://{{proxy_server_name}}/api"
+sunbird_lms_base_url: "http://{{groups['swarm-manager'][0]}}:8000"
 sunbird_content_service_local_base_url: http://content-service:5000
-sunbird_kafka_host: "{{groups['kafka']|join(':9092,')}}:9092"
+sunbird_kafka_host: "{{sunbird_processing_kafka_host}}"
 sunbird_qrimage_topic: "{{env}}.qrimage.request"
 
 
@@ -435,7 +439,6 @@ content_service_blacklisted_resourcetype: ''
 content_service_whitelisted_resourcetype: ''
 content_service_whitelisted_mimetype: ''
 content_service_blacklisted_mimetype: ''
-prometheus_storage_retention_time: 72h0m0s
 sunbird_cloud_storage_urls: 'https://s3.ap-south-1.amazonaws.com/ekstep-public-{{ekstep_s3_env}}/,https://ekstep-public-{{ekstep_s3_env}}.s3-ap-south-1.amazonaws.com/,https://{{azure_account_name}}.blob.core.windows.net/{{sunbird_content_azure_storage_container}}/'
 sunbird_email_max_recipients_limit: 100
 sunbird_cassandra_consistency_level: one
@@ -565,3 +568,16 @@ sunbird_keycloak_user_federation_provider_id: "{{core_vault_sunbird_keycloak_use
 sunbird_course_metrics_base_url: https://{{dp_azure_account_name}}.blob.core.windows.net/
 sunbird_gzip_size_threshold: 262144
 prometheus_mount_point: "/root/dockerdata/prometheus/data/"
+
+## Release 2.0 ##
+swarm_dashboard: "{{ groups['swarm-dashboard-1'][0] }}"
+prometheus_stateful_mount_point: /root/dockerdata/prometheus_stateful/data/
+prometheus_fed_mount_point: /root/dockerdata/prometheus_fed/data/
+
+sunbird_processing_kafka_host:  "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092"
+sunbird_sso_kafka_topic: "{{env_name}}.lms.sso.events"
+__yarn_host__: "{{ groups['yarn-master'][0] }}"
+zookeepers: "{{groups['zookeeper']|join(':2181,')}}:2181"
+kafka_brokers: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092"
+__lms_host__: "http://{{ groups['swarm-manager'][0] }}:9000"
+sunbird_redis_host: "{{ groups['lp-redis'][0] }}"

ansible/jenkins-backup.yml

 ---
-- name: local
-  hosts: jenkins-backup
+- name: Jenkins backup
+  hosts: jenkins-master
   vars_files:
     - "{{inventory_dir}}/secrets.yml"
+  environment:
+    AZURE_STORAGE_ACCOUNT: "{{ backup_storage_name }}"
+    AZURE_STORAGE_KEY: "{{ backup_storage_key }}"
+
   roles:
     - jenkins-backup-upload
-  tags:
-    - jenkins-backup

ansible/jmeter-restart.yml

+---
+- hosts: "{{ hosts }}"
+  become: true
+  roles:
+    - stop-jmeter
+  tags:
+    - stop-cluster 
+    - restart-jmeter
+
+- hosts: "{{ hosts }}"
+  become: true
+  tasks:
+  - name: start jmeter
+    command: nohup ./bin/jmeter-server &
+    args:
+        chdir: /mnt/data/benchmark/apache-jmeter-4.0
+  tags:
+    - start-jmeter
+    - restart-jmeter
+
+

ansible/kong_migration.yml

+---
+
+# This script will migrate kong .9x to .10.3
+#
+# Prerequisites
+#
+# 1. remove api-manager_kong `docker service rm api-manager_kong`
+# 2. Create api_manager backup `pg_dump --host=<host> --port=5432 --username=<username> -F p <db_name> > api-manager.sql`
+# 3. Create new database out of the backup
+# 4. Add new database to kong_postgres_database variable in common.yml
+#
+# you can pass the inventory from where all docker related args will get scraped
+# Image name, you'll have to pass it as extra args
+# eg:
+# ansible-playbook -i /path/to/inventory --vault-password-file /path/to/vault/password -e kong_image=sunbird.azure.io/kong:2.0.0 kong_migration.yml
+
+- hosts: swarm-bootstrap-manager
+  gather_facts: false
+  become: yes
+  run_once: true
+  vars_files:
+    - ['{{ inventory_dir }}/secrets.yml']
+  tasks:
+  - name: Installing deps
+    pip:
+      name: docker-py
+      state: present
+  - name: login to registry
+    docker_login:
+      registry: "{{ vault_docker_registry_url }}"
+      username: "{{ vault_docker_registry_user }}"
+      password: "{{ vault_docker_registry_password }}"
+  - name: templating migration_api-manager env
+    template:
+      src: static-files/api-manager.env
+      dest: /home/deployer/env/migration_api-manager.env
+  - name: checking for api-manager_kong
+    shell: docker service ps api-manager_kong
+    register: service_status 
+    changed_when: false
+    failed_when: false
+  - name: failing if api-manager running
+    fail:
+      msg: api manager is still running. Please remove the service and run again !!!
+    when: service_status.rc == 0
+  - name: Creating  migration_api-manager with kong image version to .10.3
+    shell: "docker service create --env-file /home/deployer/env/migration_api-manager.env --with-registry-auth  --name migration_api-manager {{ kong_image }} kong migrations up"

ansible/logging.yml

+- name: Spray cluster name to nodes
+  hosts: swarm-nodes
+  become: yes
+  tasks:
+  - copy: dest=/home/deployer/cluster_name content="{{ cluster_name }}"
+    when: cluster_name is defined
+
 - hosts: swarm-bootstrap-manager
   become: yes
   vars_files:
@@ -6,23 +13,22 @@
     - stack-logger
   tags:
     - stack-logger
+  run_once: true
 
-- hosts: log-forwarder
+- hosts: swarm-agent-dashboard
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
   roles:
-    - vm-agents-filebeat
+    - stack-kibana
   tags:
-    - log-forwarder
+    - stack-kibana
 
-- hosts: log-es
+- hosts: log-forwarder
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
   roles:
-    - openjdk
-    - application_logger_elasticsearch
+    - vm-agents-filebeat
   tags:
-    - log-es
-
+    - log-forwarder

ansible/monitoring-visulaization.yml

+- hosts: monitoring-visualization
+  become: yes
+  vars_files:
+    - ['{{ inventory_dir }}/secrets.yml', 'secrets/{{ env }}.yml']
+  roles:
+    - prometheus-fed
+    - {role: 'monit', monit_checks: 'prometheus'}
+    - vm-agents-nodeexporter

ansible/monitoring.yml

-- hosts: swarm-agent-for-prometheus
+- name: Creating prometheus configuration folder
+  hosts: swarm-agent-for-prometheus
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
   tasks:
     - name: Create prometheus data dir
-      file: path="{{ prometheus_mount_point}}" state=directory mode=755
+      file:
+        path: "{{ prometheus_mount_point }}"
+        state: directory
+        mode: 0755
+        owner: 'nobody'
+        group: 'nogroup'
   tags:
     - prometheus
 
-- hosts: swarm-agent-for-grafana
+- name: Creating prometheus-stateful configuration folder
+  hosts: swarm-agent-for-prometheus-stateful
+  become: yes
+  vars_files:
+    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
+  tasks:
+    - name: Create prometheus data dir
+      file:
+        path: "{{ prometheus_stateful_mount_point }}"
+        state: directory
+        mode: 0755
+        owner: 'nobody'
+        group: 'nogroup'
+  tags:
+    - prometheus
+
+- hosts: swarm-agent-dashboard
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
   tasks:
     - name: Create grafana data dir
       file: path=/var/dockerdata/grafana state=directory mode=755
+    - name: Change ownership
+      command: chown -R 472 /var/dockerdata/grafana
   tags:
     - grafana
 
@@ -24,28 +48,73 @@
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
   tasks:
     - name: Create alertmanager data dir
-      file: path=/var/dockerdata/alertmanager/data state=directory mode=755
+      file:
+        path: /var/dockerdata/alertmanager/data
+        state: directory
+        mode: 0755
+  tags:
+    - alertmanager
+
+- hosts: alertmanager_stateful
+  become: yes
+  vars_files:
+    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
+  tasks:
+    - name: Create alertmanager data dir
+      file:
+        path: /var/dockerdata/stateful_alertmanager/data
+        state: directory
+        mode: 0755
   tags:
     - alertmanager
 
 - hosts: swarm-bootstrap-manager
+  gather_facts: false
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
   tasks:
     - name: Add label for prometheus node
-      command: "docker node update --label-add prometheus=1 {{ hostvars[groups['swarm-agent-for-prometheus'][0]]['ansible_hostname'] }}"
-    - name: Add label for grafana node
-      command: "docker node update --label-add grafana=1 {{ hostvars[groups['swarm-agent-for-grafana'][0]]['ansible_hostname'] }}"
-    - name: Remove old style label for alertmanager node
-      command: "docker node update --label-rm monitor {{ hostvars[groups['swarm-agent-for-alertmanager'][0]]['ansible_hostname'] }}"
-      ignore_errors: true
+      shell: "docker node update --label-add prometheus=1 {{ hostvars[groups['swarm-agent-for-prometheus'][0]]['ansible_hostname'] }}"
     - name: Add label for alertmanager node
-      command: "docker node update --label-add alertmanager=1 {{ hostvars[groups['swarm-agent-for-alertmanager'][0]]['ansible_hostname'] }}"
+      shell: "docker node update --label-add alertmanager=1 {{ hostvars[groups['swarm-agent-for-alertmanager'][0]]['ansible_hostname'] }}"
   tags:
     - monitor-labels
+  run_once: true
+
+- name: labelling grafana and stateful prom
+  hosts: swarm-dashboard
+  become: yes
+  vars_files:
+    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
+  tasks:
+    - name: Add label for grafana node
+      command: "docker node update --label-add grafana=1 {{ hostvars[groups['swarm-agent-dashboard'][0]]['ansible_hostname'] }}"
+    - name: Adding label for stateful prometheus
+      shell: "docker node update --label-add prometheus_stateful=1 {{ hostvars[groups['swarm-agent-for-prometheus-stateful'][0]]['ansible_hostname'] }}"
+    - name: Adding label for stateful alertmanager
+      shell: "docker node update --label-add alertmanager_stateful=1 {{ hostvars[groups['alertmanager_stateful'][0]]['ansible_hostname'] }}"
+  tags:
+    - grafana-label
+  run_once: true
+
+- name: labelling grafana and stateful prom
+  hosts: swarm-dashboard
+  become: yes
+  vars_files:
+    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
+  tasks:
+    - name: Add label for grafana node
+      command: "docker node update --label-add grafana=1 {{ hostvars[groups['swarm-agent-dashboard'][0]]['ansible_hostname'] }}"
+    - name: Adding label for stateful prometheus
+      shell: "docker node update --label-add prometheus_stateful=1 {{ hostvars[groups['swarm-agent-for-prometheus-stateful'][0]]['ansible_hostname'] }}"
+    - name: Adding label for stateful alertmanager
+      shell: "docker node update --label-add alertmanager_stateful=1 {{ hostvars[groups['alertmanager_stateful'][0]]['ansible_hostname'] }}"
+  tags:
+    - grafana-label
 
 - hosts: swarm-bootstrap-manager
+  gather_facts: false
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
@@ -54,9 +123,53 @@
     - {role: 'monit', monit_checks: ['alertmanager', 'prometheus']}
   tags:
     - stack-monitor
+  run_once: true
+
+- name: Deploying stateful monitoring
+  hosts: swarm-dashboard
+  gather_facts: false
+  become: yes
+  vars_files:
+    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
+  roles:
+    - stack-monitor-stateful
+  tags:
+    - stack-monitor-stateful
+  run_once: true
+
+- hosts: swarm-dashboard
+  become: yes
+  vars_files:
+    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
+  roles:
+    - stack-grafana
+  tags:
+    - stack-grafana
+  run_once: true
+
+- name: Deploying stateful monitoring
+  hosts: swarm-dashboard
+  gather_facts: false
+  become: yes
+  vars_files:
+    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
+  roles:
+    - stack-monitor-stateful
+  tags:
+    - stack-monitor-stateful
+
+- hosts: swarm-dashboard
+  become: yes
+  vars_files:
+    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
+  roles:
+    - stack-grafana
+  tags:
+    - stack-grafana
 
 - hosts: all
   become: yes
+  gather_facts: false
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
   roles:
@@ -73,7 +186,8 @@
   tags:
     - metrics-exporter
 
-- hosts: swarm-bootstrap-manager
+- hosts: swarm-dashboard
+  gather_facts: false
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml','secrets/{{env}}.yml']

ansible/postgres-managed-service-backup.yml

+- hosts: local
+  become: yes
+  vars_files:
+    - ['{{inventory_dir}}/secrets.yml']
+  roles:
+    - postgres-azure-managed-service
+  tags:
+    - postgres-azure-managed-service
\ No newline at end of file

ansible/prometheus-backup.yml

 - hosts: swarm-agent-for-prometheus
   become: yes
+  vars:
+    prometheus_data_dir: "{{prometheus_mount_point}}"
+    prometheus_backup_prefix: "prometheus_swarm_{{cluster_name|d('swarm_1')}}"
+    prometheus_url: "http://localhost:9090/prometheus"
   vars_files:
-    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
+    - ['{{ inventory_dir }}/secrets.yml']
+  environment:
+    AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}"
+    AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}"
   roles:
+    - prometheus-backup-v2
+  tags:
     - prometheus-backup
+
+- hosts: swarm-agent-for-prometheus-stateful
+  become: yes
+  vars:
+    prometheus_data_dir: "{{prometheus_fed_mount_point}}"
+    prometheus_backup_prefix: 'prometheus_fed'
+    prometheus_url: "http://localhost:29090/prometheus"
+  vars_files:
+    - ['{{inventory_dir}}/secrets.yml']
+  environment:
+    AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}"
+    AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}"
+  roles:
+    - prometheus-backup-v2
+  tags:
+    - federation-prometheus-backup
+
+
+- hosts: swarm-agent-for-prometheus-stateful
+  become: yes
+  vars:
+    prometheus_data_dir: "{{prometheus_stateful_mount_point}}"
+    prometheus_backup_prefix: 'prometheus_stateful'
+    prometheus_url: "http://localhost:19090/prometheus"
+  vars_files:
+    - ['{{inventory_dir}}/secrets.yml']
+  environment:
+    AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}"
+    AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}"
+  roles:
+    - prometheus-backup-v2
+  tags:
+    - stateful-prometheus-backup

ansible/prometheus-restore.yml

+- hosts: "{{prometheus_hosts}}"
+  become: yes
+  gather_facts: no
+  vars:
+    prometheus_data_dir: "{{prometheus_mount_point}}"
+    prometheus_service_name: "monitor_prometheus"
+  vars_files:
+    - ['{{ inventory_dir }}/secrets.yml']
+  environment:
+    AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}"
+    AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}"
+  roles:
+    - prometheus-restore
+  tags:
+    - swarm-prometheus-restore
+
+- hosts: "{{prometheus_hosts}}"
+  become: yes
+  gather_facts: no
+  vars:
+    prometheus_data_dir: "{{prometheus_fed_mount_point}}"
+    prometheus_service_name: "prometheus_fed_prometheus"
+  vars_files:
+    - ['{{ inventory_dir }}/secrets.yml']
+  environment:
+    AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}"
+    AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}"
+  roles:
+    - prometheus-restore
+  tags:
+    - federation-prometheus-restore
+
+- hosts: "{{prometheus_hosts}}"
+  become: yes
+  gather_facts: no
+  vars:
+    prometheus_data_dir: "{{prometheus_stateful_mount_point}}"
+    prometheus_service_name: "monitor_stateful_prometheus"
+  vars_files:
+    - ['{{ inventory_dir }}/secrets.yml']
+  environment:
+    AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}"
+    AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}"
+  roles:
+    - prometheus-restore
+  tags:
+    - statefull-prometheus-restore
\ No newline at end of file

ansible/provision.yml

@@ -29,7 +29,7 @@
           node.master: "{{ es_etc_node_master | default('true') }}",
           bootstrap.memory_lock: true,
         },
-        es_heap_size: "7g",
+        es_heap_size: "2g",
         es_etc_discovery_zen_ping_unicast_hosts: "{{ groups['es'] }}",
         es_etc_cluster_name: "{{ app_es_etc_cluster_name }}"
       }

ansible/redis_cluster_provision.yml

+---
+
+- hosts: redis-cluster
+  become: yes
+  roles:
+    - redis-cluster

ansible/reloadconfig.yml

+---
+- hosts: swarm-manager
+  gather_facts: no
+  become: yes
+  roles:
+    - reloadIngress