Issue ekstep/sunbird-devops#332 feat: Use consistent service based alerting...

Issue ekstep/sunbird-devops#332 feat: Use consistent service based alerting with provision of dynamic team ownership

ansible/roles/stack-monitor/defaults/main.yml

@@ -83,13 +83,94 @@ monitor_config_templates:
   - monitor_logstash_grok_patterns
   - jmx_httpserver.yml
 
-teamwise_services:
+devops_alerts_mailing_list: ""
+
+service_teams:
+  - team: devops_team
+    alerts_mailing_list: "{{ devops_alerts_mailing_list }}"
+    services:
+      - monitor_.*
+      - monit
+      - logger_.*
+      - proxy_.*
   - team: app_team
+    alerts_mailing_list: "{{ app_alerts_mailing_list | default(devops_alerts_mailing_list) }}"
     services:
       - actor-service
       - learner-service
       - content_service_content_service
       - player_player
+      - cassandra
+      - composite_search
+      - page_assemble
+  - team: keycloak_team
+    alerts_mailing_list: "{{ keycloak_alerts_mailing_list | default(devops_alerts_mailing_list)}}"
+    services:
+      - keycloak
+  - team: api_manager_team
+    alerts_mailing_list: "{{ api_manager_alerts_mailing_list | default(devops_alerts_mailing_list)}}"
+    services:
+      - api-manager_.*
+      - adminutil_.*
+  - team: site_team
+    alerts_mailing_list: "{{ site_alerts_mailing_list | default(devops_alerts_mailing_list)}}"
+    services:
+      - sunbird_static_site
+
+service_blackbox_checks:
+  - service_name: 'adminutil_adminutil'
+    probe_module: [http_2xx]
+    targets:
+      - http://adminutil_adminutil:4000/health
+  - service_name: 'api-manager_echo'
+    probe_module: [http_2xx]
+    targets:
+      - http://api-manager_echo:9595/hello
+  - service_name: 'logger_kibana'
+    probe_module: [http_2xx]
+    targets:
+      - http://logger_kibana:5601/
+  - service_name: 'logger_grafana'
+    probe_module: [http_2xx]
+    targets:
+      - https://{{api__host}}/grafana
+  - service_name: 'player_player'
+    probe_module: [http_2xx]
+    targets:
+      - https://{{api__host}}
+      - http://player_player:3000
+  - service_name: 'content_service_content_service'
+    probe_module: [http_2xx]
+    targets:
+      - http://content_service_content_service:5000/health
+  - service_name: 'learner-service'
+    probe_module: [http_2xx]
+    targets:
+      - http://learner-service:9000/health
+  - service_name: 'monit'
+    probe_module: [http_2xx]
+    targets:
+      - http://{{ groups['swarm-bootstrap-manager'][0] }}:2812
+  - service_name: 'logger_oauth'
+    probe_module: [http_4xx_auth]
+    targets:
+      - https://{{api__host}}/dashboard
+  - service_name: 'cassandra'
+    probe_module: [tcp_connect]
+    targets:
+      - {{ sunbird_cassandra_host }}:{{ cassandra_port }}
+  - service_name: 'composite_search'
+    probe_module: [composite_search]
+    targets:
+      - https://{{api__host}}/api/composite/v1/search
+  - service_name: 'page_assemble'
+    probe_module: [page_assemble]
+    targets:
+      - https://{{api__host}}/api/data/v1/page/assemble
+  - service_name: 'keycloak'
+    probe_module: [http_2xx]
+    targets:
+      - https://{{api__host}}/auth/realms/sunbird/protocol/openid-connect/auth?client_id=portal&state=foo&redirect_uri=https%3A%2F%2F{{api__host}}%2Fprivate%2Findex%3Fauth_callback%3D1&scope=openid&response_type=code
 
 prometheus_route_prefix: prometheus
 prometheus_web_external_url: "https://{{api__host}}/{{ prometheus_route_prefix }}"

ansible/roles/stack-monitor/templates/alertmanagerconfig.yml

@@ -14,23 +14,14 @@ route:
   routes:
   - receiver: slack
     continue: true
-  - match:
-      owner: devops_team
-    receiver: devops_team
-  - match:
-      owner: app_team
-    receiver: app_team
-  - match:
-      owner: site_team
-    receiver: site_team
-  {% for item in teamwise_services %}
+  {% for item in service_teams %}
   # Dummy comment line to ensure yaml generated from ansible template has proper indentation
   - match_re:
       container_label_com_docker_swarm_service_name: "{{ item.services | join('|') }}"
-    receiver: {{ item.team }}
+    receiver: "{{ item.team }}"
   - match_re:
       service_name: "{{ item.services | join('|') }}"
-    receiver: {{ item.team }}
+    receiver: "{{ item.team }}"
   {% endfor %}
   # Dummy comment line to ensure yaml generated from ansible template has proper indentation
   - match:
@@ -41,33 +32,18 @@ receivers:
   - name: 'slack'
     slack_configs:
       - send_resolved: true
-        api_url: "{{monitor_alerts_slack_url}}"
+        api_url: "{{ monitor_alerts_slack_url }}"
         username: 'Monitor - Alerter'
-        channel: "{{monitor_alerts_slack_channel}}"
+        channel: "{{ monitor_alerts_slack_channel }}"
         text: "Instances: {% raw %}{{range .Alerts}}\n{{.Labels.instance}} : {{.Annotations.description}}{{end}}{% endraw %}"
         icon_emoji: ':dart:'
-
-  - name: 'devops_team'
-    email_configs:
-      - send_resolved: true
-        to: '{{ devops_alerts_mailing_list }}'
-        html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}'
-        headers:
-          subject: '[{{ env }}] {% raw %}{{ template "email.default.subject" . }}{% endraw %}'
-
-
-  - name: 'app_team'
-    email_configs:
-      - send_resolved: true
-        to: '{{ app_alerts_mailing_list }}'
-        html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}'
-        headers:
-          subject: '[{{ env }}] {% raw %}{{ template "email.default.subject" . }}{% endraw %}'
-
-  - name: 'site_team'
+  {% for item in service_teams %}
+  # Dummy comment line to ensure yaml generated from ansible template has proper indentation
+  - name: "{{ item.team }}"
     email_configs:
       - send_resolved: true
-        to: '{{ site_alerts_mailing_list }}'
+        to: '{{ item.alerts_mailing_list }}'
         html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}'
         headers:
           subject: '[{{ env }}] {% raw %}{{ template "email.default.subject" . }}{% endraw %}'
+  {% endfor %}

ansible/roles/stack-monitor/templates/prometheus.yml

@@ -91,109 +91,6 @@ scrape_configs:
     static_configs:
       - targets: ['monitor_data_backup_azure_blob_exporter:9358']
 
-  - job_name: 'availability_composite_search'
-    metrics_path: /probe
-    params:
-      module: [composite_search]  # Look for a HTTP 200 response.
-    static_configs:
-    - targets:
-      - https://{{api__host}}/api/composite/v1/search
-      labels:
-        "owner": "app_team"
-    relabel_configs:
-      - source_labels: [__address__]
-        target_label: __param_target
-      - source_labels: [__param_target]
-        target_label: instance
-      - target_label: __address__
-        replacement: monitor_blackbox-exporter:9115  # Blackbox exporter.
-
-  - job_name: 'availability_page_assemble'
-    metrics_path: /probe
-    params:
-      module: [page_assemble]  # Look for a HTTP 200 response.
-    static_configs:
-    - targets:
-      - https://{{api__host}}/api/data/v1/page/assemble
-      labels:
-        "owner": "app_team"
-    relabel_configs:
-      - source_labels: [__address__]
-        target_label: __param_target
-      - source_labels: [__param_target]
-        target_label: instance
-      - target_label: __address__
-        replacement: monitor_blackbox-exporter:9115  # Blackbox exporter.
-
-  - job_name: 'availability_general'
-    metrics_path: /probe
-    params:
-      module: [http_2xx]  # Look for a HTTP 200 response.
-    static_configs:
-    - targets:
-      - http://adminutil_adminutil:4000/health
-      - http://api-manager_echo:9595/hello
-      - http://kibana:5601/
-      - https://{{api__host}}/grafana
-      - https://{{api__host}}/auth/realms/sunbird/protocol/openid-connect/auth?client_id=portal&state=foo&redirect_uri=https%3A%2F%2F{{api__host}}%2Fprivate%2Findex%3Fauth_callback%3D1&scope=openid&response_type=code
-    relabel_configs:
-      - source_labels: [__address__]
-        target_label: __param_target
-      - source_labels: [__param_target]
-        target_label: instance
-      - target_label: __address__
-        replacement: monitor_blackbox-exporter:9115  # Blackbox exporter.
-
-  - job_name: 'availability_application'
-    metrics_path: /probe
-    params:
-      module: [http_2xx]  # Look for a HTTP 200 response.
-    static_configs:
-    - targets:
-      - https://{{api__host}}
-      - http://player_player:3000
-      - http://content_service_content_service:5000/health
-      - http://learner-service:9000/health
-      labels:
-        "owner": "app_team"
-    relabel_configs:
-      - source_labels: [__address__]
-        target_label: __param_target
-      - source_labels: [__param_target]
-        target_label: instance
-      - target_label: __address__
-        replacement: monitor_blackbox-exporter:9115  # Blackbox exporter.
-
-  - job_name: 'availability_monit'
-    metrics_path: /probe
-    params:
-      module: [http_2xx]  # Look for a HTTP 200 response.
-    static_configs:
-    - targets:
-      - http://{{ groups['swarm-bootstrap-manager'][0] }}:2812
-    relabel_configs:
-      - source_labels: [__address__]
-        target_label: __param_target
-      - source_labels: [__param_target]
-        target_label: instance
-      - target_label: __address__
-        replacement: monitor_blackbox-exporter:9115  # Blackbox exporter.
-
-  - job_name: 'availability_dashboard_oauth'
-    metrics_path: /probe
-    params:
-      module: [http_4xx_auth]
-    static_configs:
-    - targets:
-      - https://{{api__host}}/dashboard
-    relabel_configs:
-      - source_labels: [__address__]
-        target_label: __param_target
-      - source_labels: [__param_target]
-        target_label: instance
-      - target_label: __address__
-        replacement: monitor_blackbox-exporter:9115  # Blackbox exporter.
-
   - job_name: 'prometheus'
     metrics_path: /prometheus/metrics
     static_configs:
@@ -208,17 +105,16 @@ scrape_configs:
     static_configs:
       - targets: {{ groups['node-exporter'] | difference(["localhost"]) | map('regex_replace', '^(.*)$', '\\1:9101' ) | list | to_yaml }}
 
-  {% if static_site_url is defined %}
+  {% for item in service_blackbox_checks %}
   # This empty line ensures indentation is correct after ansible jinja2 template is materialized
-  - job_name: 'availability_static_site'
+  - job_name: 'availability_{{ item.service_name }}'
     metrics_path: /probe
     params:
-      module: [http_2xx]  # Look for a HTTP 200 response.
+      module: {{ item.probe_modules | to_json }}
     static_configs:
-    - targets:
-      - {{static_site_url}}
+    - targets: {{ item.targets | to_yaml }}
       labels:
-        "owner": "site_team"
+        "service_name": "{{ item.service_name }}"
     relabel_configs:
       - source_labels: [__address__]
         target_label: __param_target
@@ -226,17 +122,19 @@ scrape_configs:
         target_label: instance
       - target_label: __address__
         replacement: monitor_blackbox-exporter:9115  # Blackbox exporter.
-  {% endif %}
+  {% endfor %}
 
-  - job_name: 'availability_cassandra'
+  {% if static_site_url is defined %}
+  # This empty line ensures indentation is correct after ansible jinja2 template is materialized
+  - job_name: 'availability_sunbird_static_site'
     metrics_path: /probe
     params:
-      module: [tcp_connect]
+      module: [http_2xx]  # Look for a HTTP 200 response.
     static_configs:
     - targets:
-      - {{ sunbird_cassandra_host }}:{{ cassandra_port }}
+      - {{ static_site_url }}
       labels:
-        "owner": "app_team"
+        "service_name": "sunbird_static_site"
     relabel_configs:
       - source_labels: [__address__]
         target_label: __param_target
@@ -244,3 +142,4 @@ scrape_configs:
         target_label: instance
       - target_label: __address__
         replacement: monitor_blackbox-exporter:9115  # Blackbox exporter.
+  {% endif %}