diff --git a/ansible/monitoring-fed.yml b/ansible/monitoring-fed.yml
new file mode 100644
index 0000000000000000000000000000000000000000..a247a8218769f9f3f5512c077879c34359b6736c
--- /dev/null
+++ b/ansible/monitoring-fed.yml
@@ -0,0 +1,6 @@
+- hosts: swarm-dashboard
+ become: yes
+ vars_files:
+ - ['{{ inventory_dir }}/secrets.yml']
+ roles:
+ - prometheus-fed
diff --git a/ansible/roles/prometheus-fed/defaults/main.yml b/ansible/roles/prometheus-fed/defaults/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..cccafed512e43086747471f5010338168e989f10
--- /dev/null
+++ b/ansible/roles/prometheus-fed/defaults/main.yml
@@ -0,0 +1,22 @@
+prometheus_reservation_memory: 1G
+prometheus_limit_memory: 1G
+prometheus_storage_retention_time: 90d
+
+# Defaults to 1KB. Good enough to catch errors mentioned in https://about.gitlab.com/2017/02/01/gitlab-dot-com-database-incident/
+expected_data_backup_size_in_bytes: 1024
+
+enable_scraping_docker_metrics: false
+docker_metrics_port: "2377"
+monitor_stack_files_dest_dir: /opt/docker/stacks/prom_fed/stacks
+monitor_config_files_dest_dir: /opt/docker/stacks/prom_fed/config
+
+monitor_config_templates:
+ - prometheus.yml
+
+prometheus_route_prefix: prometheus
+prometheus_web_external_url: "{{proto}}://{{api__host}}:9090/{{ prometheus_route_prefix }}"
+
+root_group: root
+root_owner: root
+backup_storage_name: prometheus_backup
+prometheus_storage_location: /root/dockerdata/prometheus_fed/data/
diff --git a/ansible/roles/prometheus-fed/tasks/main.yml b/ansible/roles/prometheus-fed/tasks/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..9962dafc0aa93681aeb5686bb64eb3c12fdf6a60
--- /dev/null
+++ b/ansible/roles/prometheus-fed/tasks/main.yml
@@ -0,0 +1,54 @@
+---
+- name: Create prometheus data dir
+ file:
+ path: "{{ prometheus_storage_location }}"
+ state: directory
+ mode: 0755
+ owner: 'nobody'
+ group: 'nogroup'
+
+- name: Ensure stack directory exists
+ file:
+ path: "{{ monitor_stack_files_dest_dir }}"
+ state: directory
+ owner: '{{ root_owner }}'
+ group: '{{ root_group }}'
+
+- name: Ensure config directory exists
+ file:
+ path: "{{ monitor_config_files_dest_dir }}"
+ state: directory
+ owner: '{{ root_owner }}'
+ group: '{{ root_group }}'
+
+- name: Save stack file
+ template:
+ src: stack-monitor.yml
+ dest: "{{ monitor_stack_files_dest_dir }}/monitor.yml"
+ mode: 0644
+
+- name: Save prometheus config fed_{{ item }}
+ template:
+ src="{{ item }}"
+ dest="{{ monitor_config_files_dest_dir }}/{{ item }}"
+ mode=0644
+ with_items:
+ - "{{ monitor_config_templates }}"
+
+- name: Remove monitor stack
+ shell: "docker stack rm prometheus_fed"
+ ignore_errors: yes
+
+- name: Remove old docker config fed_{{ item }}
+ shell: "docker config rm fed_{{ item }}"
+ with_items: "{{ monitor_config_templates }}"
+ ignore_errors: yes
+
+- name: Save docker config fed_{{ item }}
+ shell: "docker config create fed_{{ item }} {{ monitor_config_files_dest_dir }}/{{ item }}"
+ with_items: "{{ monitor_config_templates }}"
+
+- name: Deploy stack
+ shell: "docker stack deploy -c monitor.yml prometheus_fed"
+ args:
+ chdir: "{{ monitor_stack_files_dest_dir }}"
diff --git a/ansible/roles/prometheus-fed/templates/prometheus.yml b/ansible/roles/prometheus-fed/templates/prometheus.yml
new file mode 100644
index 0000000000000000000000000000000000000000..6857f3f4c1cb9877dbb7b2bf3ae0ba1fd790b3a1
--- /dev/null
+++ b/ansible/roles/prometheus-fed/templates/prometheus.yml
@@ -0,0 +1,45 @@
+global:
+ scrape_interval: 1m
+ scrape_timeout: 10s
+ evaluation_interval: 1m
+ # Clustername is defined for multi swarm cluster
+ # can name first cluster as DS1 and 2nd as DS2 ... DSn
+ # all metrics will have label as cluster: < value >
+ # This value won't be accessible from GUI but for external tools like
+ # federation or alertmanager
+ external_labels:
+ federation: fed1
+
+
+scrape_configs:
+
+ {% if enable_scraping_docker_metrics %}
+ # This empty line ensures indentation is correct after ansible jinja2 template is materialized
+ - job_name: 'docker'
+ static_configs:
+ - targets: {{ groups['swarm-manager'] | map('regex_replace', '^(.*)$', '\\1:' + docker_metrics_port ) | list | to_yaml }}
+ metric_relabel_configs:
+ - source_labels: [__name__]
+ regex: 'grpc_.*'
+ action: drop
+ {% endif %}
+
+
+ - job_name: 'federate'
+ metrics_path: /prometheus/federate
+ honor_labels: true
+ params:
+ match[]:
+ - '{__name__=~"nginx_.*"}'
+ - '{__name__=~"kong_.*"}'
+ - '{__name__=~"node_.*"}'
+ static_configs:
+ {# groups[alertmanager] and prometheus are same machines #}
+ {# and alertmanagers will be unique per swarm #}
+ # This empty line make sure the indentation is correct
+ - targets: [ '{{groups['alertmanager']|join(':9090\', \'')}}:9090', '{{groups['alertmanager_stateful']|join(':19090\', \'')}}:19090' ]
+
+ - job_name: 'vm-node-exporter'
+ static_configs:
+ - targets: ["{{ansible_host}}:9100"]
+
diff --git a/ansible/roles/prometheus-fed/templates/stack-monitor.yml b/ansible/roles/prometheus-fed/templates/stack-monitor.yml
new file mode 100644
index 0000000000000000000000000000000000000000..f9795b5ee2f925fe378441dcc94bfa1295ffcdbb
--- /dev/null
+++ b/ansible/roles/prometheus-fed/templates/stack-monitor.yml
@@ -0,0 +1,31 @@
+version: "3.3"
+
+services:
+ prometheus:
+ image: quay.io/prometheus/prometheus:v2.8.1
+ ports:
+ - "29090:9090"
+ networks:
+ - application_default
+ volumes:
+ - {{prometheus_storage_location}}:/prometheus
+ command: "--config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --web.console.libraries=/etc/prometheus/console_libraries --storage.tsdb.retention.time={{prometheus_storage_retention_time}} --web.console.templates=/etc/prometheus/consoles --web.route-prefix={{prometheus_route_prefix}} --web.external-url={{prometheus_web_external_url}} --web.enable-admin-api"
+ configs:
+ - source: fed_prometheus.yml
+ target: /etc/prometheus/prometheus.yml
+ deploy:
+ replicas: 1
+ resources:
+ reservations:
+ memory: "{{ prometheus_reservation_memory }}"
+ limits:
+ memory: "{{ prometheus_limit_memory }}"
+
+configs:
+ fed_prometheus.yml:
+ external: true
+
+networks:
+ application_default:
+ external: true
+
diff --git a/pipelines/deploy/monitor/prom-fed/Jenkinsfile b/pipelines/deploy/monitor/prom-fed/Jenkinsfile
new file mode 100644
index 0000000000000000000000000000000000000000..0294ea7c2127e02abc5ee26b1882e3d629ece38a
--- /dev/null
+++ b/pipelines/deploy/monitor/prom-fed/Jenkinsfile
@@ -0,0 +1,38 @@
+@Library('deploy-conf') _
+node() {
+ try {
+ String ANSI_GREEN = "\u001B[32m"
+ String ANSI_NORMAL = "\u001B[0m"
+ String ANSI_BOLD = "\u001B[1m"
+ String ANSI_RED = "\u001B[31m"
+ String ANSI_YELLOW = "\u001B[33m"
+
+ stage('checkout public repo') {
+ cleanWs()
+ checkout scm
+ }
+
+ ansiColor('xterm') {
+ stage('deploy'){
+ values = [:]
+ currentWs = sh(returnStdout: true, script: 'pwd').trim()
+ envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+ module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+ jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+ ansiblePlaybook = "${currentWs}/ansible/monitoring-fed.yml"
+ ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass -v"
+ values.put('currentWs', currentWs)
+ values.put('env', envDir)
+ values.put('module', module)
+ values.put('jobName', jobName)
+ values.put('ansiblePlaybook', ansiblePlaybook)
+ values.put('ansibleExtraArgs', ansibleExtraArgs)
+ println values
+ ansible_playbook_run(values)
+ }
+ }
+ }
+ catch (err) {
+ throw err
+ }
+}
\ No newline at end of file