Alert severity- filter (#606)

* Fixed identation error

* severity filters added

* severity filter added

ansible/roles/stack-monitor-stateful/defaults/main.yml

@@ -84,6 +84,7 @@ monitor_config_templates_postgres:
   - postgresslavequeries.yml
 
 devops_alerts_mailing_list: "{{ alerts_mailing_list }}"
+devops_alerts_mailing_severity_filter: "{{ alerts_mailing_severity_filter }}"
 
 service_teams:
   - team: devops_team
@@ -93,6 +94,7 @@ service_teams:
       - monit
       - logger_.*
       - proxy_.*
+    severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}"
   - team: app_team
     alerts_mailing_list: "{{ app_alerts_mailing_list | default(devops_alerts_mailing_list) }}"
     services:
@@ -108,19 +110,23 @@ service_teams:
       - logstash
       - search
       - neo4j
+    severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}"
   - team: keycloak_team
     alerts_mailing_list: "{{ keycloak_alerts_mailing_list | default(devops_alerts_mailing_list)}}"
     services:
       - keycloak
+    severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}"
   - team: api_manager_team
     alerts_mailing_list: "{{ api_manager_alerts_mailing_list | default(devops_alerts_mailing_list)}}"
     services:
       - api-manager_.*
       - adminutil_.*
+    severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}"
   - team: site_team
     alerts_mailing_list: "{{ site_alerts_mailing_list | default(devops_alerts_mailing_list)}}"
     services:
       - sunbird_static_site
+    severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}"
   - team: druid
     alerts_mailing_list: "{{ druid_alerts_mailing_list | default(devops_alerts_mailing_list)}}"
     services:
@@ -131,6 +137,7 @@ service_teams:
       - historical
       - broker
       - middlemanager
+    severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}"
 
 service_blackbox_checks:
   - service_name: 'analytics-api'

ansible/roles/stack-monitor-stateful/templates/alertmanagerconfig.yml

 global:
-  resolve_timeout: 1m
+  resolve_timeout: 5m
   smtp_from: "{{ monitor_alerts_mail_from_email }}"
   smtp_smarthost: "{{ monitor_alerts_mail_server_host }}:{{ monitor_alerts_mail_server_port}}"
   smtp_auth_username: "{{ monitor_alerts_mail_server_username }}"
@@ -8,6 +8,7 @@ global:
 
 route:
   receiver: 'devops_team'
+  group_by: ['alertname']
   group_wait: 30s
   group_interval: 5m
   repeat_interval: 4h
@@ -19,9 +20,31 @@ route:
   - match_re:
       container_label_com_docker_swarm_service_name: "{{ item.services | join('|') }}"
     receiver: "{{ item.team }}"
+    {% if item.severity_mailing_filter is defined and item.severity_mailing_filter|length %}
+    # Dummy line
+    routes:
+    {% for filter in item.severity_mailing_filter %}
+    # Dummy line
+    - match:
+        severity: "{{ filter.severity }}"
+      receiver: "{{ item.team }}_{{ filter.severity }}"
+    {% endfor %}
+    {% endif %}
+    # Dummy line
   - match_re:
       service_name: "{{ item.services | join('|') }}"
     receiver: "{{ item.team }}"
+    {% if item.severity_mailing_filter is defined and item.severity_mailing_filter|length %}
+    # Dummy line
+    routes:
+    {% for filter in item.severity_mailing_filter %}
+    # Dummy line
+    - match:
+        severity: "{{ filter.severity }}"
+      receiver: "{{ item.team }}_{{ filter.severity }}"
+    {% endfor %}
+    {% endif %}
+    # Dummy line
   {% endfor %}
   # Dummy comment line to ensure yaml generated from ansible template has proper indentation
   - match:
@@ -38,7 +61,7 @@ receivers:
         text: "[{{ env }}-STATEFUL]: {% raw %}{{range .Alerts}} {{.Annotations.description}}{{end}}{% endraw %}"
         icon_emoji: ':dart:'
   {% for item in service_teams %}
-  # comment line to ensure yaml generated from ansible template has proper indentation
+  # Dummy comment line to ensure yaml generated from ansible template has proper indentation
   - name: "{{ item.team }}"
     email_configs:
       - send_resolved: true
@@ -46,4 +69,16 @@ receivers:
         html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}'
         headers:
           subject: '[{{ env }}-STATEFUL] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}'
+  {% if item.severity_mailing_filter is defined and item.severity_mailing_filter|length %}
+  {% for filter in item.severity_mailing_filter %}
+  # Dummy line
+  - name: "{{ item.team }}_{{ filter.severity }}"
+    email_configs:
+      - send_resolved: true
+        to: '{{ filter.alerts_mailing_list }}'
+        html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}'
+        headers:
+          subject: '[{{ env }}-STATEFUL] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}'
+  {% endfor %}
+  {% endif %}
   {% endfor %}

ansible/roles/stack-monitor/defaults/main.yml

@@ -72,6 +72,7 @@ monitor_config_templates_postgres:
   - postgresslavequeries.yml
 
 devops_alerts_mailing_list: "{{ alerts_mailing_list }}"
+devops_alerts_mailing_severity_filter: "{{ alerts_mailing_severity_filter }}"
 
 service_teams:
   - team: devops_team
@@ -81,10 +82,10 @@ service_teams:
       - monit
       - logger_.*
       - proxy_.*
+    severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}"
   - team: app_team
     alerts_mailing_list: "{{ app_alerts_mailing_list | default(devops_alerts_mailing_list) }}"
     services:
-      - actor-service
       - learner-service
       - lms-service
       - content-service
@@ -92,19 +93,23 @@ service_teams:
       - cassandra
       - composite_search
       - analytics-api
+    severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}"
   - team: keycloak_team
     alerts_mailing_list: "{{ keycloak_alerts_mailing_list | default(devops_alerts_mailing_list)}}"
     services:
       - keycloak
+    severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}"
   - team: api_manager_team
     alerts_mailing_list: "{{ api_manager_alerts_mailing_list | default(devops_alerts_mailing_list)}}"
     services:
       - api-manager_.*
       - adminutil_.*
+    severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}"
   - team: site_team
     alerts_mailing_list: "{{ site_alerts_mailing_list | default(devops_alerts_mailing_list)}}"
     services:
       - sunbird_static_site
+    severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}"
   - team: druid
     alerts_mailing_list: "{{ druid_alerts_mailing_list | default(devops_alerts_mailing_list)}}"
     services:
@@ -115,6 +120,7 @@ service_teams:
       - historical
       - broker
       - middlemanager
+    severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}"
 
 service_blackbox_checks:
   - service_name: 'adminutil_adminutil'

ansible/roles/stack-monitor/templates/alertmanagerconfig.yml

 global:
-  resolve_timeout: 1m
+  resolve_timeout: 5m
   smtp_from: "{{ monitor_alerts_mail_from_email }}"
   smtp_smarthost: "{{ monitor_alerts_mail_server_host }}:{{ monitor_alerts_mail_server_port}}"
   smtp_auth_username: "{{ monitor_alerts_mail_server_username }}"
@@ -16,15 +16,37 @@ route:
   - receiver: slack
     continue: true
   {% for item in service_teams %}
-  # comment line to ensure yaml generated from ansible template has proper indentation
+  # Dummy comment line to ensure yaml generated from ansible template has proper indentation
   - match_re:
       container_label_com_docker_swarm_service_name: "{{ item.services | join('|') }}"
     receiver: "{{ item.team }}"
+    {% if item.severity_mailing_filter is defined and item.severity_mailing_filter|length %}
+    # Dummy line
+    routes:
+    {% for filter in item.severity_mailing_filter %}
+    # Dummy line
+    - match:
+        severity: "{{ filter.severity }}"
+      receiver: "{{ item.team }}_{{ filter.severity }}"
+    {% endfor %}
+    {% endif %}
+    # Dummy line
   - match_re:
       service_name: "{{ item.services | join('|') }}"
     receiver: "{{ item.team }}"
+    {% if item.severity_mailing_filter is defined and item.severity_mailing_filter|length %}
+    # Dummy line
+    routes:
+    {% for filter in item.severity_mailing_filter %}
+    # Dummy line
+    - match:
+        severity: "{{ filter.severity }}"
+      receiver: "{{ item.team }}_{{ filter.severity }}"
+    {% endfor %}
+    {% endif %}
+    # Dummy line
   {% endfor %}
-  # comment line to ensure yaml generated from ansible template has proper indentation
+  # Dummy comment line to ensure yaml generated from ansible template has proper indentation
   - match:
       owner:
     receiver: devops_team
@@ -39,7 +61,7 @@ receivers:
         text: "[{{ env }}-{{cluster_name}}]: {% raw %}{{range .Alerts}} {{.Annotations.description}}{{end}}{% endraw %}"
         icon_emoji: ':dart:'
   {% for item in service_teams %}
-  # comment line to ensure yaml generated from ansible template has proper indentation
+  # Dummy comment line to ensure yaml generated from ansible template has proper indentation
   - name: "{{ item.team }}"
     email_configs:
       - send_resolved: true
@@ -47,4 +69,16 @@ receivers:
         html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}'
         headers:
           subject: '[{{ env }}-{{cluster_name}}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}'
+  {% if item.severity_mailing_filter is defined and item.severity_mailing_filter|length %}
+  {% for filter in item.severity_mailing_filter %}
+  # Dummy line
+  - name: "{{ item.team }}_{{ filter.severity }}"
+    email_configs:
+      - send_resolved: true
+        to: '{{ filter.alerts_mailing_list }}'
+        html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}'
+        headers:
+          subject: '[{{ env }}-{{cluster_name}}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}'
+  {% endfor %}
+  {% endif %}
   {% endfor %}