diff --git a/ansible/roles/stack-monitor-stateful/defaults/main.yml b/ansible/roles/stack-monitor-stateful/defaults/main.yml index bdd773d33741b1d02ce571493f9131592899cea8..75d34a5c75b58159e2ee1edeac26ec752b08bbc6 100644 --- a/ansible/roles/stack-monitor-stateful/defaults/main.yml +++ b/ansible/roles/stack-monitor-stateful/defaults/main.yml @@ -84,6 +84,7 @@ monitor_config_templates_postgres: - postgresslavequeries.yml devops_alerts_mailing_list: "{{ alerts_mailing_list }}" +devops_alerts_mailing_severity_filter: "{{ alerts_mailing_severity_filter }}" service_teams: - team: devops_team @@ -93,6 +94,7 @@ service_teams: - monit - logger_.* - proxy_.* + severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}" - team: app_team alerts_mailing_list: "{{ app_alerts_mailing_list | default(devops_alerts_mailing_list) }}" services: @@ -108,19 +110,23 @@ service_teams: - logstash - search - neo4j + severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}" - team: keycloak_team alerts_mailing_list: "{{ keycloak_alerts_mailing_list | default(devops_alerts_mailing_list)}}" services: - keycloak + severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}" - team: api_manager_team alerts_mailing_list: "{{ api_manager_alerts_mailing_list | default(devops_alerts_mailing_list)}}" services: - api-manager_.* - adminutil_.* + severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}" - team: site_team alerts_mailing_list: "{{ site_alerts_mailing_list | default(devops_alerts_mailing_list)}}" services: - sunbird_static_site + severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}" - team: druid alerts_mailing_list: "{{ druid_alerts_mailing_list | default(devops_alerts_mailing_list)}}" services: @@ -131,6 +137,7 @@ service_teams: - historical - broker - middlemanager + severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}" service_blackbox_checks: - service_name: 'analytics-api' diff --git a/ansible/roles/stack-monitor-stateful/templates/alertmanagerconfig.yml b/ansible/roles/stack-monitor-stateful/templates/alertmanagerconfig.yml index 2fd41b0f8eaef38a91c772e6f2965e746d38963c..fe0ff15c2d3bf9844a1a28a73f907c34c65358aa 100644 --- a/ansible/roles/stack-monitor-stateful/templates/alertmanagerconfig.yml +++ b/ansible/roles/stack-monitor-stateful/templates/alertmanagerconfig.yml @@ -1,5 +1,5 @@ global: - resolve_timeout: 1m + resolve_timeout: 5m smtp_from: "{{ monitor_alerts_mail_from_email }}" smtp_smarthost: "{{ monitor_alerts_mail_server_host }}:{{ monitor_alerts_mail_server_port}}" smtp_auth_username: "{{ monitor_alerts_mail_server_username }}" @@ -8,6 +8,7 @@ global: route: receiver: 'devops_team' + group_by: ['alertname'] group_wait: 30s group_interval: 5m repeat_interval: 4h @@ -19,9 +20,31 @@ route: - match_re: container_label_com_docker_swarm_service_name: "{{ item.services | join('|') }}" receiver: "{{ item.team }}" + {% if item.severity_mailing_filter is defined and item.severity_mailing_filter|length %} + # Dummy line + routes: + {% for filter in item.severity_mailing_filter %} + # Dummy line + - match: + severity: "{{ filter.severity }}" + receiver: "{{ item.team }}_{{ filter.severity }}" + {% endfor %} + {% endif %} + # Dummy line - match_re: service_name: "{{ item.services | join('|') }}" receiver: "{{ item.team }}" + {% if item.severity_mailing_filter is defined and item.severity_mailing_filter|length %} + # Dummy line + routes: + {% for filter in item.severity_mailing_filter %} + # Dummy line + - match: + severity: "{{ filter.severity }}" + receiver: "{{ item.team }}_{{ filter.severity }}" + {% endfor %} + {% endif %} + # Dummy line {% endfor %} # Dummy comment line to ensure yaml generated from ansible template has proper indentation - match: @@ -38,7 +61,7 @@ receivers: text: "[{{ env }}-STATEFUL]: {% raw %}{{range .Alerts}} {{.Annotations.description}}{{end}}{% endraw %}" icon_emoji: ':dart:' {% for item in service_teams %} - # comment line to ensure yaml generated from ansible template has proper indentation + # Dummy comment line to ensure yaml generated from ansible template has proper indentation - name: "{{ item.team }}" email_configs: - send_resolved: true @@ -46,4 +69,16 @@ receivers: html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' headers: subject: '[{{ env }}-STATEFUL] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' + {% if item.severity_mailing_filter is defined and item.severity_mailing_filter|length %} + {% for filter in item.severity_mailing_filter %} + # Dummy line + - name: "{{ item.team }}_{{ filter.severity }}" + email_configs: + - send_resolved: true + to: '{{ filter.alerts_mailing_list }}' + html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' + headers: + subject: '[{{ env }}-STATEFUL] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' + {% endfor %} + {% endif %} {% endfor %} diff --git a/ansible/roles/stack-monitor/defaults/main.yml b/ansible/roles/stack-monitor/defaults/main.yml index cbd76f2da32beec644416cf0d8ca5b44b055c726..b53fc546d4b2a224ad062ccae4e5580426ac813f 100644 --- a/ansible/roles/stack-monitor/defaults/main.yml +++ b/ansible/roles/stack-monitor/defaults/main.yml @@ -72,6 +72,7 @@ monitor_config_templates_postgres: - postgresslavequeries.yml devops_alerts_mailing_list: "{{ alerts_mailing_list }}" +devops_alerts_mailing_severity_filter: "{{ alerts_mailing_severity_filter }}" service_teams: - team: devops_team @@ -81,10 +82,10 @@ service_teams: - monit - logger_.* - proxy_.* + severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}" - team: app_team alerts_mailing_list: "{{ app_alerts_mailing_list | default(devops_alerts_mailing_list) }}" services: - - actor-service - learner-service - lms-service - content-service @@ -92,19 +93,23 @@ service_teams: - cassandra - composite_search - analytics-api + severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}" - team: keycloak_team alerts_mailing_list: "{{ keycloak_alerts_mailing_list | default(devops_alerts_mailing_list)}}" services: - keycloak + severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}" - team: api_manager_team alerts_mailing_list: "{{ api_manager_alerts_mailing_list | default(devops_alerts_mailing_list)}}" services: - api-manager_.* - adminutil_.* + severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}" - team: site_team alerts_mailing_list: "{{ site_alerts_mailing_list | default(devops_alerts_mailing_list)}}" services: - sunbird_static_site + severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}" - team: druid alerts_mailing_list: "{{ druid_alerts_mailing_list | default(devops_alerts_mailing_list)}}" services: @@ -115,6 +120,7 @@ service_teams: - historical - broker - middlemanager + severity_mailing_filter: "{{ devops_alerts_mailing_severity_filter }}" service_blackbox_checks: - service_name: 'adminutil_adminutil' diff --git a/ansible/roles/stack-monitor/templates/alertmanagerconfig.yml b/ansible/roles/stack-monitor/templates/alertmanagerconfig.yml index 7be098b0c42ab61ae2f42a34c2f69211c6781c22..56b06ca0eaae86c12b9ebdedd33f5fd94747f835 100644 --- a/ansible/roles/stack-monitor/templates/alertmanagerconfig.yml +++ b/ansible/roles/stack-monitor/templates/alertmanagerconfig.yml @@ -1,5 +1,5 @@ global: - resolve_timeout: 1m + resolve_timeout: 5m smtp_from: "{{ monitor_alerts_mail_from_email }}" smtp_smarthost: "{{ monitor_alerts_mail_server_host }}:{{ monitor_alerts_mail_server_port}}" smtp_auth_username: "{{ monitor_alerts_mail_server_username }}" @@ -16,15 +16,37 @@ route: - receiver: slack continue: true {% for item in service_teams %} - # comment line to ensure yaml generated from ansible template has proper indentation + # Dummy comment line to ensure yaml generated from ansible template has proper indentation - match_re: container_label_com_docker_swarm_service_name: "{{ item.services | join('|') }}" receiver: "{{ item.team }}" + {% if item.severity_mailing_filter is defined and item.severity_mailing_filter|length %} + # Dummy line + routes: + {% for filter in item.severity_mailing_filter %} + # Dummy line + - match: + severity: "{{ filter.severity }}" + receiver: "{{ item.team }}_{{ filter.severity }}" + {% endfor %} + {% endif %} + # Dummy line - match_re: service_name: "{{ item.services | join('|') }}" receiver: "{{ item.team }}" + {% if item.severity_mailing_filter is defined and item.severity_mailing_filter|length %} + # Dummy line + routes: + {% for filter in item.severity_mailing_filter %} + # Dummy line + - match: + severity: "{{ filter.severity }}" + receiver: "{{ item.team }}_{{ filter.severity }}" + {% endfor %} + {% endif %} + # Dummy line {% endfor %} - # comment line to ensure yaml generated from ansible template has proper indentation + # Dummy comment line to ensure yaml generated from ansible template has proper indentation - match: owner: receiver: devops_team @@ -39,7 +61,7 @@ receivers: text: "[{{ env }}-{{cluster_name}}]: {% raw %}{{range .Alerts}} {{.Annotations.description}}{{end}}{% endraw %}" icon_emoji: ':dart:' {% for item in service_teams %} - # comment line to ensure yaml generated from ansible template has proper indentation + # Dummy comment line to ensure yaml generated from ansible template has proper indentation - name: "{{ item.team }}" email_configs: - send_resolved: true @@ -47,4 +69,16 @@ receivers: html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' headers: subject: '[{{ env }}-{{cluster_name}}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' + {% if item.severity_mailing_filter is defined and item.severity_mailing_filter|length %} + {% for filter in item.severity_mailing_filter %} + # Dummy line + - name: "{{ item.team }}_{{ filter.severity }}" + email_configs: + - send_resolved: true + to: '{{ filter.alerts_mailing_list }}' + html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' + headers: + subject: '[{{ env }}-{{cluster_name}}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' + {% endfor %} + {% endif %} {% endfor %}