Issue #000 feat: updated prometheus backup role

ansible/prometheus-backup.yml

@@ -2,7 +2,7 @@
   become: yes
   vars:
     prometheus_data_dir: "{{prometheus_mount_point}}"
-    prometheus_backup_prefix: 'prometheus_swarm_{{cluster_name|d(swarm_1)}}'
+    prometheus_backup_prefix: 'prometheus_swarm_{{cluster_name|d('swarm_1')}}'
     prometheus_url: "http://localhost:9090/prometheus"
   vars_files:
     - ['{{ inventory_dir }}/secrets.yml']

ansible/prometheus-restore.yml

-- hosts: "{{prometheus_hosts}}"
+- hosts: localhost
   become: yes
   vars:
     prometheus_data_dir: "{{prometheus_mount_point}}"
     prometheus_service_name: "monitor_prometheus"
-  vars_files:
-    - ['{{ inventory_dir }}/secrets.yml']
+  #vars_files:
+  #  - ['{{ inventory_dir }}/secrets.yml']
   environment:
     AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}"
     AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}"
@@ -13,32 +13,32 @@
   tags:
     - swarm-prometheus-restore
 
-- hosts: "{{prometheus_hosts}}"
-  become: yes
-  vars:
-    prometheus_data_dir: "{{prometheus_fed_mount_point}}"
-    prometheus_service_name: "prometheus_fed_prometheus"
-  vars_files:
-    - ['{{ inventory_dir }}/secrets.yml']
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}"
-    AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}"
-  roles:
-    - prometheus-restore
-  tags:
-    - federation-prometheus-restore
-
-- hosts: "{{prometheus_hosts}}"
-  become: yes
-  vars:
-    prometheus_data_dir: "{{prometheus_stateful_mount_point}}"
-    prometheus_service_name: "monitor_stateful_prometheus"
-  vars_files:
-    - ['{{ inventory_dir }}/secrets.yml']
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}"
-    AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}"
-  roles:
-    - prometheus-restore
-  tags:
-    - statefull-prometheus-restore
\ No newline at end of file
+#- hosts: "{{prometheus_hosts}}"
+#  become: yes
+#  vars:
+#    prometheus_data_dir: "{{prometheus_fed_mount_point}}"
+#    prometheus_service_name: "prometheus_fed_prometheus"
+#  vars_files:
+#    - ['{{ inventory_dir }}/secrets.yml']
+#  environment:
+#    AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}"
+#    AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}"
+#  roles:
+#    - prometheus-restore
+#  tags:
+#    - federation-prometheus-restore
+#
+#- hosts: "{{prometheus_hosts}}"
+#  become: yes
+#  vars:
+#    prometheus_data_dir: "{{prometheus_stateful_mount_point}}"
+#    prometheus_service_name: "monitor_stateful_prometheus"
+#  vars_files:
+#    - ['{{ inventory_dir }}/secrets.yml']
+#  environment:
+#    AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}"
+#    AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}"
+#  roles:
+#    - prometheus-restore
+#  tags:
+#    - statefull-prometheus-restore
\ No newline at end of file

ansible/roles/prometheus-fed-stateful-backup/defaults/main.yml

-prometheus_backup_dir: /tmp/prometheus-backup
-#prometheus_backup_azure_container_name: federation-prometheus-backup
-
-# Set these vars per environment as show in example below
-# Override these values in group_vars
-backup_storage_name: backups
-backup_storage_key: '<backup_storage_key>'
\ No newline at end of file

ansible/roles/prometheus-fed-stateful-backup/meta/main.yml

-dependencies:
-  - azure-cli

ansible/roles/prometheus-fed-stateful-backup/tasks/main.yml

-- name: ensure prometheus is stopped before backup
-  shell: "docker service scale {{prometheus_service_name}}=0 && sleep 10"
-  delegate_to: "{{swarm_dashboard}}"
-
-- name: Starting Prometheus backup 
-  block:
-    - name: ensure backup dir exists
-      file: path="{{ prometheus_backup_dir }}" state=directory
-
-    - name: ensure backup dir is empty
-      shell: "rm -rf {{ prometheus_backup_dir }}/*"
-
-    - set_fact:
-        prometheus_backup_gzip_file_name: "prometheus_backup_{{ lookup('pipe', 'date +%Z-%Y-%m-%d-%H-%M-%S') }}.tar.gz"
-
-    - set_fact:
-        prometheus_backup_gzip_file_path: "{{ prometheus_backup_dir }}/{{ prometheus_backup_gzip_file_name }}"
-
-    - name: Save backup
-      archive:
-        path: "{{federation_prometheus_data_dir}}"
-        dest: "{{prometheus_backup_gzip_file_path}}"
-      async: 3600
-      poll: 10
-  always:
-    - name: ensure prometheus is started after backup
-      shell: "docker service scale {{prometheus_service_name}}=1"
-      delegate_to: "{{swarm_dashboard}}"
-
-- name: Ensure azure blob storage container exists
-  command: az storage container create  --name {{ prometheus_backup_azure_container_name }}
-  ignore_errors: true
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}"
-    AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}"
-
-- name: Upload to azure blob storage
-  command: az storage blob upload --name {{ prometheus_backup_gzip_file_name }} --file {{ prometheus_backup_gzip_file_path }} --container-name {{ prometheus_backup_azure_container_name }}
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}"
-    AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}"
-  async: 3600
-  poll: 10
-
-- name: clean up backup dir after upload
-  file: path="{{ prometheus_backup_dir }}" state=absent

ansible/roles/prometheus-restore/tasks/main.yml

@@ -10,7 +10,7 @@
 
 - name: ensure prometheus is stopped
   shell: "docker service scale {{prometheus_service_name}}=0 && sleep 10"
-  delegate_to: "{{manager_host}}"
+  delegate_to: "{{manager_host}}"  #variable is passed as extra vars from jenkins
 
 
 - name: Unarchive backup

pipelines/backup/prometheus-backup/Jenkinsfile

@@ -31,8 +31,20 @@ node() {
                 ansible_playbook_run(values)
             }
         }
+        currentBuild.result = 'SUCCESS'
     }
     catch (err) {
+        currentBuild.result = 'FAILURE'
+        slackSend (
+            channel: "${env.NOTIFY_SLACK_CHANNEL}",
+            color: 'danger',
+            message: "Build Failed - ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.BUILD_URL}|Open>)"
+        )
         throw err
+    } finally {
+        step([$class: 'Mailer',
+            notifyEveryUnstableBuild: true,
+            recipients: "${env.NOTIFY_EMAIL_ADDRESS}",
+            sendToIndividuals: true])
     }
 }

pipelines/backup/prometheus-fed-stateful-backup/Jenkinsfile

@@ -31,10 +31,22 @@ node() {
                 println values
                 ansible_playbook_run(values)
 
-                   }
+            }
         }
+        currentBuild.result = 'SUCCESS'
     }
     catch (err) {
+        currentBuild.result = 'FAILURE'
+        slackSend (
+            channel: "${env.NOTIFY_SLACK_CHANNEL}",
+            color: 'danger',
+            message: "Build Failed - ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.BUILD_URL}|Open>)"
+        )
         throw err
+    } finally {
+        step([$class: 'Mailer',
+            notifyEveryUnstableBuild: true,
+            recipients: "${env.NOTIFY_EMAIL_ADDRESS}",
+            sendToIndividuals: true])
     }
 }