feat: TG-1172 prometheus metrics for opa and envoy (#3155)

kubernetes/helm_charts/core/analytics/templates/deployment.yaml

@@ -159,5 +159,15 @@ spec:
   - name: http-{{ .Chart.Name }}
     protocol: TCP
     port: {{ .Values.network.targetport }}
+{{- if .Values.analytics_opa_enabled }}
+  - name: opa-metrics
+    port: 8181
+    protocol: TCP
+    targetPort: 8181
+  - name: envoy-metrics
+    port: 10000
+    protocol: TCP
+    targetPort: 10000
+{{- end }}
   selector:
-    app: {{ .Chart.Name }}
+    app: {{ .Chart.Name }}
\ No newline at end of file

kubernetes/helm_charts/core/analytics/templates/envoy-config.yaml

@@ -29,6 +29,15 @@ data:
                   domains:
                   - "*"
                   routes:
+                  - match:
+                      prefix: "/opa/metrics"
+                    route:
+                      prefix_rewrite: "/metrics"
+                      cluster: opa
+                    typed_per_filter_config:
+                      envoy.filters.http.ext_authz:
+                        "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute
+                        disabled: true
                   - match:
                       prefix: "{{ .Values.livenessProbe.httpGet.path }}"
                     route:
@@ -71,6 +80,18 @@ data:
                   socket_address:
                     address: 127.0.0.1
                     port_value: {{ .Values.network.targetport }}
+      - name: opa
+        connect_timeout: 5s
+        type: static
+        load_assignment:
+          cluster_name: opa
+          endpoints:
+          - lb_endpoints:
+            - endpoint:
+                address:
+                  socket_address:
+                    address: 127.0.0.1
+                    port_value: 8181
     admin:
       access_log_path: "/dev/stdout"
       address:

kubernetes/helm_charts/core/analytics/templates/serviceMonitor.yaml

@@ -15,6 +15,20 @@ spec:
     port: http-{{ .Chart.Name }}
     scheme: http
     scrapeTimeout: 10s
+{{- if .Values.analytics_opa_enabled }}
+  - honorLabels: true
+    interval: 30s
+    path: /opa/metrics
+    port: opa-metrics
+    scheme: http
+    scrapeTimeout: 10s
+  - honorLabels: true
+    interval: 30s
+    path: /stats/prometheus
+    port: envoy-metrics
+    scheme: http
+    scrapeTimeout: 10s
+{{- end }}
   jobLabel: {{ .Chart.Name }}
   namespaceSelector:
     matchNames:

kubernetes/helm_charts/core/certregistry/templates/deployment.yaml

@@ -147,5 +147,15 @@ spec:
   - name: http-{{ .Chart.Name }}
     protocol: TCP
     port: {{ .Values.network.targetport }}
+{{- if .Values.certregistry_opa_enabled }}
+  - name: opa-metrics
+    port: 8181
+    protocol: TCP
+    targetPort: 8181
+  - name: envoy-metrics
+    port: 10000
+    protocol: TCP
+    targetPort: 10000
+{{- end }}
   selector:
-    app: {{ .Chart.Name }}
+    app: {{ .Chart.Name }}
\ No newline at end of file

kubernetes/helm_charts/core/certregistry/templates/envoy-config.yaml

@@ -29,6 +29,15 @@ data:
                   domains:
                   - "*"
                   routes:
+                  - match:
+                      prefix: "/opa/metrics"
+                    route:
+                      prefix_rewrite: "/metrics"
+                      cluster: opa
+                    typed_per_filter_config:
+                      envoy.filters.http.ext_authz:
+                        "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute
+                        disabled: true
                   - match:
                       prefix: "{{ .Values.livenessProbe.httpGet.path }}"
                     route:
@@ -71,6 +80,18 @@ data:
                   socket_address:
                     address: 127.0.0.1
                     port_value: {{ .Values.network.targetport }}
+      - name: opa
+        connect_timeout: 5s
+        type: static
+        load_assignment:
+          cluster_name: opa
+          endpoints:
+          - lb_endpoints:
+            - endpoint:
+                address:
+                  socket_address:
+                    address: 127.0.0.1
+                    port_value: 8181
     admin:
       access_log_path: "/dev/stdout"
       address:

kubernetes/helm_charts/core/certregistry/templates/serviceMonitor.yaml

+{{- if .Values.certregistry_opa_enabled }}
+{{- if .Values.serviceMonitor.enabled }}
+{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) ( .Values.serviceMonitor.enabled ) }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ .Chart.Name }}-service
+  namespace: {{ .Values.namespace }}
+  labels:
+    {{- toYaml .Values.serviceMonitor.labels | nindent 4 }}
+spec:
+  endpoints:
+  - honorLabels: true
+    interval: 30s
+    path: /opa/metrics
+    port: opa-metrics
+    scheme: http
+    scrapeTimeout: 10s
+  - honorLabels: true
+    interval: 30s
+    path: /stats/prometheus
+    port: envoy-metrics
+    scheme: http
+    scrapeTimeout: 10s
+  jobLabel: {{ .Chart.Name }}
+  namespaceSelector:
+    matchNames:
+    - {{ default .Values.namespace .Release.Namespace  }}
+  selector:
+    matchLabels:
+      app: {{ .Chart.Name }}
+{{- end }}
+{{- end }}
+{{- end }}

kubernetes/helm_charts/core/certregistry/values.j2

@@ -56,6 +56,12 @@ initcontainer_resources:
     cpu: "{{ certregistry_initcontainer_cpu_limit | default('100m') }}"
     memory: "{{ certregistry_initcontainer_mem_limit | default('100Mi') }}"
 
+serviceMonitor:
+  enabled: true
+  labels: # labels with which the prometheus choose the serviceMonitor
+    app: prometheus-operator
+    release: prometheus-operator
+
 autoscaling:
   enabled: {{ certregistry_autoscaling_enabled | default('false') }}
   minReplicas: {{ certregistry_autoscaling_minReplicas|default(1) }}

kubernetes/helm_charts/core/content/templates/deployment.yaml

@@ -152,5 +152,15 @@ spec:
   - name: http-{{ .Chart.Name }}
     protocol: TCP
     port: {{ .Values.network.targetport }}
+{{- if .Values.content_opa_enabled }}
+  - name: opa-metrics
+    port: 8181
+    protocol: TCP
+    targetPort: 8181
+  - name: envoy-metrics
+    port: 10000
+    protocol: TCP
+    targetPort: 10000
+{{- end }}
   selector:
     app: {{ .Chart.Name }}

kubernetes/helm_charts/core/content/templates/envoy-config.yaml

@@ -29,6 +29,15 @@ data:
                   domains:
                   - "*"
                   routes:
+                  - match:
+                      prefix: "/opa/metrics"
+                    route:
+                      prefix_rewrite: "/metrics"
+                      cluster: opa
+                    typed_per_filter_config:
+                      envoy.filters.http.ext_authz:
+                        "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute
+                        disabled: true
                   - match:
                       prefix: "{{ .Values.livenessProbe.httpGet.path }}"
                     route:
@@ -71,6 +80,18 @@ data:
                   socket_address:
                     address: 127.0.0.1
                     port_value: {{ .Values.network.targetport }}
+      - name: opa
+        connect_timeout: 5s
+        type: static
+        load_assignment:
+          cluster_name: opa
+          endpoints:
+          - lb_endpoints:
+            - endpoint:
+                address:
+                  socket_address:
+                    address: 127.0.0.1
+                    port_value: 8181
     admin:
       access_log_path: "/dev/stdout"
       address:

kubernetes/helm_charts/core/content/templates/serviceMonitor.yaml

+{{- if .Values.content_opa_enabled }}
+{{- if .Values.serviceMonitor.enabled }}
+{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) ( .Values.serviceMonitor.enabled ) }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ .Chart.Name }}-service
+  namespace: {{ .Values.namespace }}
+  labels:
+    {{- toYaml .Values.serviceMonitor.labels | nindent 4 }}
+spec:
+  endpoints:
+  - honorLabels: true
+    interval: 30s
+    path: /opa/metrics
+    port: opa-metrics
+    scheme: http
+    scrapeTimeout: 10s
+  - honorLabels: true
+    interval: 30s
+    path: /stats/prometheus
+    port: envoy-metrics
+    scheme: http
+    scrapeTimeout: 10s
+  jobLabel: {{ .Chart.Name }}
+  namespaceSelector:
+    matchNames:
+    - {{ default .Values.namespace .Release.Namespace  }}
+  selector:
+    matchLabels:
+      app: {{ .Chart.Name }}
+{{- end }}
+{{- end }}
+{{- end }}

kubernetes/helm_charts/core/content/values.j2

@@ -56,6 +56,12 @@ initcontainer_resources:
     cpu: "{{ content_initcontainer_cpu_limit | default('100m') }}"
     memory: "{{ content_initcontainer_mem_limit | default('100Mi') }}"
 
+serviceMonitor:
+  enabled: true
+  labels: # labels with which the prometheus choose the serviceMonitor
+    app: prometheus-operator
+    release: prometheus-operator
+
 autoscaling:
   enabled: {{ content_autoscaling_enabled | default('false') }}
   minReplicas: {{ content_autoscaling_minReplicas|default(1) }}

kubernetes/helm_charts/core/knowledgemw/templates/deployment.yaml

@@ -143,5 +143,15 @@ spec:
   - name: http-{{ .Chart.Name }}
     protocol: TCP
     port: {{ .Values.network.targetport }}
+{{- if .Values.knowledgemw_opa_enabled }}
+  - name: opa-metrics
+    port: 8181
+    protocol: TCP
+    targetPort: 8181
+  - name: envoy-metrics
+    port: 10000
+    protocol: TCP
+    targetPort: 10000
+{{- end }}
   selector:
     app: {{ .Chart.Name }}

kubernetes/helm_charts/core/knowledgemw/templates/envoy-config.yaml

@@ -29,6 +29,15 @@ data:
                   domains:
                   - "*"
                   routes:
+                  - match:
+                      prefix: "/opa/metrics"
+                    route:
+                      prefix_rewrite: "/metrics"
+                      cluster: opa
+                    typed_per_filter_config:
+                      envoy.filters.http.ext_authz:
+                        "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute
+                        disabled: true
                   - match:
                       prefix: "{{ .Values.livenessProbe.httpGet.path }}"
                     route:
@@ -71,6 +80,18 @@ data:
                   socket_address:
                     address: 127.0.0.1
                     port_value: {{ .Values.network.targetport }}
+      - name: opa
+        connect_timeout: 5s
+        type: static
+        load_assignment:
+          cluster_name: opa
+          endpoints:
+          - lb_endpoints:
+            - endpoint:
+                address:
+                  socket_address:
+                    address: 127.0.0.1
+                    port_value: 8181
     admin:
       access_log_path: "/dev/stdout"
       address:

kubernetes/helm_charts/core/knowledgemw/templates/serviceMonitor.yaml

+{{- if .Values.knowledgemw_opa_enabled }}
+{{- if .Values.serviceMonitor.enabled }}
+{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) ( .Values.serviceMonitor.enabled ) }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ .Chart.Name }}-service
+  namespace: {{ .Values.namespace }}
+  labels:
+    {{- toYaml .Values.serviceMonitor.labels | nindent 4 }}
+spec:
+  endpoints:
+  - honorLabels: true
+    interval: 30s
+    path: /opa/metrics
+    port: opa-metrics
+    scheme: http
+    scrapeTimeout: 10s
+  - honorLabels: true
+    interval: 30s
+    path: /stats/prometheus
+    port: envoy-metrics
+    scheme: http
+    scrapeTimeout: 10s
+  jobLabel: {{ .Chart.Name }}
+  namespaceSelector:
+    matchNames:
+    - {{ default .Values.namespace .Release.Namespace  }}
+  selector:
+    matchLabels:
+      app: {{ .Chart.Name }}
+{{- end }}
+{{- end }}
+{{- end }}

kubernetes/helm_charts/core/knowledgemw/values.j2

@@ -55,6 +55,12 @@ initcontainer_resources:
 
 knowledgemw_access_basepath: {{ knowledgemw_access_basepath | default('/home/sunbird/mw/content/keys/') }}
 
+serviceMonitor:
+  enabled: true
+  labels: # labels with which the prometheus choose the serviceMonitor
+    app: prometheus-operator
+    release: prometheus-operator
+
 autoscaling:
   enabled: {{ knowledgemw_autoscaling_enabled | default('false') }}
   minReplicas: {{ knowledgemw_autoscaling_minReplicas|default(1) }}

kubernetes/helm_charts/core/learner/templates/deployment.yaml

@@ -157,5 +157,15 @@ spec:
   - name: http-{{ .Chart.Name }}
     protocol: TCP
     port: {{ .Values.network.targetport }}
+{{- if .Values.learner_opa_enabled }}
+  - name: opa-metrics
+    port: 8181
+    protocol: TCP
+    targetPort: 8181
+  - name: envoy-metrics
+    port: 10000
+    protocol: TCP
+    targetPort: 10000
+{{- end }}
   selector:
     app: {{ .Chart.Name }}

kubernetes/helm_charts/core/learner/templates/envoy-config.yaml

@@ -29,6 +29,15 @@ data:
                   domains:
                   - "*"
                   routes:
+                  - match:
+                      prefix: "/opa/metrics"
+                    route:
+                      prefix_rewrite: "/metrics"
+                      cluster: opa
+                    typed_per_filter_config:
+                      envoy.filters.http.ext_authz:
+                        "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute
+                        disabled: true
                   - match:
                       prefix: "{{ .Values.livenessProbe.httpGet.path }}"
                     route:
@@ -71,6 +80,18 @@ data:
                   socket_address:
                     address: 127.0.0.1
                     port_value: {{ .Values.network.targetport }}
+      - name: opa
+        connect_timeout: 5s
+        type: static
+        load_assignment:
+          cluster_name: opa
+          endpoints:
+          - lb_endpoints:
+            - endpoint:
+                address:
+                  socket_address:
+                    address: 127.0.0.1
+                    port_value: 8181
     admin:
       access_log_path: "/dev/stdout"
       address:

kubernetes/helm_charts/core/learner/templates/serviceMonitor.yaml

+{{- if .Values.learner_opa_enabled }}
+{{- if .Values.serviceMonitor.enabled }}
+{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) ( .Values.serviceMonitor.enabled ) }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ .Chart.Name }}-service
+  namespace: {{ .Values.namespace }}
+  labels:
+    {{- toYaml .Values.serviceMonitor.labels | nindent 4 }}
+spec:
+  endpoints:
+  - honorLabels: true
+    interval: 30s
+    path: /opa/metrics
+    port: opa-metrics
+    scheme: http
+    scrapeTimeout: 10s
+  - honorLabels: true
+    interval: 30s
+    path: /stats/prometheus
+    port: envoy-metrics
+    scheme: http
+    scrapeTimeout: 10s
+  jobLabel: {{ .Chart.Name }}
+  namespaceSelector:
+    matchNames:
+    - {{ default .Values.namespace .Release.Namespace  }}
+  selector:
+    matchLabels:
+      app: {{ .Chart.Name }}
+{{- end }}
+{{- end }}
+{{- end }}

kubernetes/helm_charts/core/learner/values.j2

@@ -59,6 +59,12 @@ initcontainer_resources:
 
 learner_access_basepath: {{ learner_access_basepath | default('/keys/') }}
 
+serviceMonitor:
+  enabled: true
+  labels: # labels with which the prometheus choose the serviceMonitor
+    app: prometheus-operator
+    release: prometheus-operator
+
 autoscaling:
   enabled: {{ learner_autoscaling_enabled | default('false') }}
   minReplicas: {{ learner_autoscaling_minReplicas|default(1) }}

kubernetes/helm_charts/core/lms/templates/deployment.yaml

@@ -162,5 +162,15 @@ spec:
   - name: http-{{ .Chart.Name }}
     protocol: TCP
     port: {{ .Values.network.targetport }}
+{{- if .Values.lms_opa_enabled }}
+  - name: opa-metrics
+    port: 8181
+    protocol: TCP
+    targetPort: 8181
+  - name: envoy-metrics
+    port: 10000
+    protocol: TCP
+    targetPort: 10000
+{{- end }}
   selector:
     app: {{ .Chart.Name }}