Merge pull request #649 from project-sunbird/prom-fed

added ansible role and jenkins file for prometheus federation

ansible/monitoring-fed.yml

+- hosts: swarm-dashboard
+  become: yes
+  vars_files:
+    - ['{{ inventory_dir }}/secrets.yml']
+  roles:
+    - prometheus-fed

ansible/roles/prometheus-fed/defaults/main.yml

+prometheus_reservation_memory: 1G
+prometheus_limit_memory: 1G
+prometheus_storage_retention_time: 90d
+
+enable_scraping_docker_metrics: false
+docker_metrics_port: "2377"
+monitor_stack_files_dest_dir: /opt/docker/stacks/prom_fed/stacks
+monitor_config_files_dest_dir: /opt/docker/stacks/prom_fed/config
+
+monitor_config_templates:
+  - prometheus.yml
+
+prometheus_route_prefix: prometheus
+prometheus_web_external_url: "{{proto}}://{{api__host}}:29090/{{ prometheus_route_prefix }}"
+
+root_group: root
+root_owner: root
+backup_storage_name: prometheus_backup
+prometheus_storage_location: /root/dockerdata/prometheus_fed/data/

ansible/roles/prometheus-fed/tasks/main.yml

+---
+- name: Create prometheus data dir
+  file:
+    path: "{{ prometheus_storage_location }}"
+    state: directory
+    mode: 0755
+    owner: 'nobody'
+    group: 'nogroup'
+
+- name: Ensure stack directory exists
+  file:
+    path: "{{ monitor_stack_files_dest_dir }}"
+    state: directory
+    owner: '{{ root_owner }}'
+    group: '{{ root_group }}'
+
+- name: Ensure config directory exists
+  file:
+    path: "{{ monitor_config_files_dest_dir }}"
+    state: directory
+    owner: '{{ root_owner }}'
+    group: '{{ root_group }}'
+
+- name: Save stack file
+  template:
+    src: stack-monitor.yml
+    dest: "{{ monitor_stack_files_dest_dir }}/monitor.yml"
+    mode: 0644
+
+- name: Save prometheus config fed_{{ item }}
+  template:
+    src="{{ item }}"
+    dest="{{ monitor_config_files_dest_dir }}/{{ item }}" 
+    mode=0644
+  with_items: 
+    - "{{ monitor_config_templates }}"
+
+- name: Remove monitor stack
+  shell: "docker stack rm prometheus_fed"
+  ignore_errors: yes
+
+- name: Remove old docker config fed_{{ item }}
+  shell: "docker config rm fed_{{ item }}"
+  with_items: "{{ monitor_config_templates }}"
+  ignore_errors: yes
+
+- name: Save docker config fed_{{ item }}
+  shell: "docker config create fed_{{ item }} {{ monitor_config_files_dest_dir }}/{{ item }}"
+  with_items: "{{ monitor_config_templates }}"
+
+- name: Deploy stack
+  shell: "docker stack deploy -c monitor.yml prometheus_fed"
+  args:
+    chdir: "{{ monitor_stack_files_dest_dir }}"

ansible/roles/prometheus-fed/templates/prometheus.yml

+global:
+  scrape_interval: 1m
+  scrape_timeout: 10s
+  evaluation_interval: 1m
+  # Clustername is defined for multi swarm cluster
+  # can name first cluster as DS1 and 2nd as DS2 ... DSn
+  # all metrics will have label as cluster: < value >
+  # This value won't be accessible from GUI but for external tools like
+  # federation or alertmanager
+  external_labels:
+    federation: fed1
+
+
+scrape_configs:
+
+  {% if enable_scraping_docker_metrics %}
+  # This empty line ensures indentation is correct after ansible jinja2 template is materialized
+  - job_name: 'docker'
+    static_configs:
+      - targets: {{ groups['swarm-manager'] | map('regex_replace', '^(.*)$', '\\1:' + docker_metrics_port ) | list | to_yaml }}
+    metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: 'grpc_.*'
+        action: drop
+  {% endif %}
+
+
+  - job_name: 'federate'
+    metrics_path: /prometheus/federate
+    honor_labels: true
+    params:
+      match[]:
+        - '{__name__=~"nginx_.*"}'
+        - '{__name__=~"kong_.*"}'
+        - '{__name__=~"node_.*"}'
+    static_configs:
+      {# groups[alertmanager] and prometheus are same machines #}
+      {# and alertmanagers will be unique per swarm #}
+      # This empty line make sure the indentation is correct
+      - targets: [ '{{groups['alertmanager']|join(':9090\', \'')}}:9090', '{{groups['alertmanager_stateful']|join(':19090\', \'')}}:19090' ]
+
+  - job_name: 'vm-node-exporter'
+    static_configs:
+      - targets: ["{{ansible_host}}:9100"]
+

ansible/roles/prometheus-fed/templates/stack-monitor.yml

+version: "3.3"
+
+services:
+    prometheus:
+        image: quay.io/prometheus/prometheus:v2.8.1
+        ports:
+            - "29090:9090"
+        networks:
+            - application_default
+        volumes:
+            - {{prometheus_storage_location}}:/prometheus
+        command: "--config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --web.console.libraries=/etc/prometheus/console_libraries  --storage.tsdb.retention.time={{prometheus_storage_retention_time}} --web.console.templates=/etc/prometheus/consoles --web.route-prefix={{prometheus_route_prefix}} --web.external-url={{prometheus_web_external_url}} --web.enable-admin-api"
+        configs:
+            - source: fed_prometheus.yml
+              target: /etc/prometheus/prometheus.yml
+        deploy:
+            replicas: 1
+            resources:
+              reservations:
+                memory: "{{ prometheus_reservation_memory }}"
+              limits:
+                memory: "{{ prometheus_limit_memory }}"
+
+configs:
+  fed_prometheus.yml:
+    external: true
+
+networks:
+    application_default:
+        external: true
+

pipelines/deploy/monitor/prom-fed/Jenkinsfile

+@Library('deploy-conf') _
+node() {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        stage('checkout public repo') {
+            cleanWs()
+            checkout scm
+        }
+
+        ansiColor('xterm') {
+            stage('deploy'){
+                values = [:]
+                currentWs = sh(returnStdout: true, script: 'pwd').trim()
+                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+                ansiblePlaybook = "${currentWs}/ansible/monitoring-fed.yml"
+                ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass -v"
+		values.put('currentWs', currentWs)
+                values.put('env', envDir)
+                values.put('module', module)
+                values.put('jobName', jobName)
+                values.put('ansiblePlaybook', ansiblePlaybook)
+                values.put('ansibleExtraArgs', ansibleExtraArgs)
+                println values
+                ansible_playbook_run(values)
+            }
+        }
+    }
+    catch (err) {
+        throw err
+    }
+}
\ No newline at end of file