From fb7e07a2f55fbc4bf356dd737c3cb5cfd3b6c94f Mon Sep 17 00:00:00 2001
From: Rajesh Rajendran <rjshrjndrn@users.noreply.github.com>
Date: Fri, 12 Jun 2020 16:53:57 +0530
Subject: [PATCH] Nginx caching metrics implementation for prometheus (#1580)
* Adding default map for nginx http metrics cache_stats
Signed-off-by: Rajesh Rajendran <rjshrjndrn@gmail.com>
* Increasing prometheus shared memory space for metrics
Signed-off-by: Rajesh Rajendran <rjshrjndrn@gmail.com>
* prometheus lua code directly from submodule
Signed-off-by: Rajesh Rajendran <rjshrjndrn@gmail.com>
* Updating default nginx config
Signed-off-by: Rajesh Rajendran <rjshrjndrn@gmail.com>
* cloning git submodules for nginx build
Signed-off-by: Rajesh Rajendran <rjshrjndrn@gmail.com>
* fix: copy all lua files from prometheus lib and config update
Signed-off-by: Rajesh Rajendran <rjshrjndrn@gmail.com>
* Updating nginx config for new prometheus metrics
Signed-off-by: Rajesh Rajendran <rjshrjndrn@gmail.com>
* Adding nginx request method to prometheus metrics
Signed-off-by: Rajesh Rajendran <rjshrjndrn@gmail.com>
* Adding nginx request method to prometheus metrics
Signed-off-by: Rajesh Rajendran <rjshrjndrn@gmail.com>
---
.gitmodules | 3 ++
images/proxy/Dockerfile | 2 +-
images/proxy/Jenkinsfile | 2 +
images/proxy/nginx-lua-prometheus | 1 +
images/proxy/nginx.conf | 42 ++++++++++++-------
.../core/nginx-public-ingress/values.j2 | 35 +++++++++-------
6 files changed, 54 insertions(+), 31 deletions(-)
create mode 100644 .gitmodules
create mode 160000 images/proxy/nginx-lua-prometheus
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 000000000..e18365b30
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "images/proxy/nginx-lua-prometheus"]
+ path = images/proxy/nginx-lua-prometheus
+ url = https://github.com/knyar/nginx-lua-prometheus.git
diff --git a/images/proxy/Dockerfile b/images/proxy/Dockerfile
index e2c441b0e..da74f9731 100644
--- a/images/proxy/Dockerfile
+++ b/images/proxy/Dockerfile
@@ -157,7 +157,7 @@ RUN GPG_KEYS=B0F4253373F8F6F510D42178520A9993A1C052F8 \
&& ln -sf /dev/stderr /var/log/nginx/error.log
COPY nginx.conf /etc/nginx/nginx.conf
-copy prometheus.lua /etc/nginx/lua_modules/
+copy nginx-lua-prometheus/*.lua /etc/nginx/lua_modules/
EXPOSE 80
diff --git a/images/proxy/Jenkinsfile b/images/proxy/Jenkinsfile
index 6e42f8551..8b2839365 100644
--- a/images/proxy/Jenkinsfile
+++ b/images/proxy/Jenkinsfile
@@ -17,6 +17,7 @@ node('build-slave') {
cleanWs()
if (params.github_release_tag == "") {
checkout scm
+ sh("git submodule update --init --recursive")
commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim()
branch_name = sh(script: 'git name-rev --name-only HEAD | rev | cut -d "/" -f1| rev', returnStdout: true).trim()
build_tag = branch_name + "_" + commit_hash + "_" + env.BUILD_NUMBER
@@ -24,6 +25,7 @@ node('build-slave') {
} else {
def scmVars = checkout scm
checkout scm: [$class: 'GitSCM', branches: [[name: "refs/tags/$params.github_release_tag"]], userRemoteConfigs: [[url: scmVars.GIT_URL]]]
+ sh("git submodule update --init --recursive")
build_tag = params.github_release_tag + "_" + env.BUILD_NUMBER
println(ANSI_BOLD + ANSI_YELLOW + "Tag specified, building from tag: " + params.github_release_tag + ANSI_NORMAL)
}
diff --git a/images/proxy/nginx-lua-prometheus b/images/proxy/nginx-lua-prometheus
new file mode 160000
index 000000000..71ac90826
--- /dev/null
+++ b/images/proxy/nginx-lua-prometheus
@@ -0,0 +1 @@
+Subproject commit 71ac9082637ff1269f195e4f4f5e61836782b8d0
diff --git a/images/proxy/nginx.conf b/images/proxy/nginx.conf
index bd1f1af3d..fcf5621b7 100644
--- a/images/proxy/nginx.conf
+++ b/images/proxy/nginx.conf
@@ -23,24 +23,36 @@ http {
access_log /var/log/nginx/access.log main;
# Shared dictionary to store metrics
- lua_shared_dict prometheus_metrics 10M;
+ lua_shared_dict prometheus_metrics 100M;
lua_package_path "/etc/nginx/lua_modules/?.lua";
+
+ # Defining upstream cache status for nginx metrics
+ map $upstream_cache_status $cache_status {
+ default $upstream_cache_status;
+ '' "NONE";
+ }
+
# Defining metrics
- init_by_lua '
+ init_worker_by_lua_block {
prometheus = require("prometheus").init("prometheus_metrics")
metric_requests = prometheus:counter(
- "nginx_http_requests_total", "Number of HTTP requests", {"host", "status", "request_method"})
+ "nginx_http_requests_total", "Number of HTTP requests", {"host", "status", "request_method", "cache_status"})
metric_latency = prometheus:histogram(
"nginx_http_request_duration_seconds", "HTTP request latency", {"host"})
metric_connections = prometheus:gauge(
"nginx_http_connections", "Number of HTTP connections", {"state"})
- ';
-
- # Collecting metrics
- log_by_lua '
- metric_requests:inc(1, {ngx.var.server_name, ngx.var.status, ngx.var.request_method})
+ }
+ log_by_lua_block {
+ metric_requests:inc(1, {ngx.var.server_name, ngx.var.status, ngx.var.request_method, ngx.var.cache_status })
metric_latency:observe(tonumber(ngx.var.request_time), {ngx.var.server_name})
- ';
+ }
+
+ # local caching for images and files
+ proxy_cache_path /tmp/proxy_cache levels=1:2 keys_zone=proxy_cache:5m max_size=10m inactive=60m use_temp_path=off;
+
+ # cache framework
+ proxy_cache_path /tmp/framework_cache levels=1:2 keys_zone=framework_cache:5m max_size=700m inactive=60m use_temp_path=off;
+
header_filter_by_lua_block {
ngx.header["server"] = nil
@@ -60,12 +72,12 @@ http {
server {
listen 9145;
location /metrics {
- content_by_lua '
- metric_connections:set(ngx.var.connections_reading, {"reading"})
- metric_connections:set(ngx.var.connections_waiting, {"waiting"})
- metric_connections:set(ngx.var.connections_writing, {"writing"})
- prometheus:collect()
- ';
+ content_by_lua_block {
+ metric_connections:set(ngx.var.connections_reading, {"reading"})
+ metric_connections:set(ngx.var.connections_waiting, {"waiting"})
+ metric_connections:set(ngx.var.connections_writing, {"writing"})
+ prometheus:collect()
+ }
}
}
}
diff --git a/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 b/kubernetes/helm_charts/core/nginx-public-ingress/values.j2
index 6defc4f52..867ac7c2e 100644
--- a/kubernetes/helm_charts/core/nginx-public-ingress/values.j2
+++ b/kubernetes/helm_charts/core/nginx-public-ingress/values.j2
@@ -691,24 +691,29 @@ nginxconfig: |
access_log /var/log/nginx/access.log main;
# Shared dictionary to store metrics
- lua_shared_dict prometheus_metrics 10M;
+ lua_shared_dict prometheus_metrics 100M;
lua_package_path "/etc/nginx/lua_modules/?.lua";
+
+ # Defining upstream cache status for nginx metrics
+ map $upstream_cache_status $cache_status {
+ default $upstream_cache_status;
+ '' "NONE";
+ }
+
# Defining metrics
- init_by_lua '
+ init_worker_by_lua_block {
prometheus = require("prometheus").init("prometheus_metrics")
metric_requests = prometheus:counter(
- "nginx_http_requests_total", "Number of HTTP requests", {"host", "status", "request_method"})
+ "nginx_http_requests_total", "Number of HTTP requests", {"host", "status", "request_method", "cache_status"})
metric_latency = prometheus:histogram(
"nginx_http_request_duration_seconds", "HTTP request latency", {"host"})
metric_connections = prometheus:gauge(
"nginx_http_connections", "Number of HTTP connections", {"state"})
- ';
-
- # Collecting metrics
- log_by_lua '
- metric_requests:inc(1, {ngx.var.server_name, ngx.var.status, ngx.var.request_method})
+ }
+ log_by_lua_block {
+ metric_requests:inc(1, {ngx.var.server_name, ngx.var.status, ngx.var.request_method, ngx.var.cache_status })
metric_latency:observe(tonumber(ngx.var.request_time), {ngx.var.server_name})
- ';
+ }
header_filter_by_lua_block {
ngx.header["server"] = nil
@@ -752,12 +757,12 @@ nginxconfig: |
server {
listen 9145;
location /metrics {
- content_by_lua '
- metric_connections:set(ngx.var.connections_reading, {"reading"})
- metric_connections:set(ngx.var.connections_waiting, {"waiting"})
- metric_connections:set(ngx.var.connections_writing, {"writing"})
- prometheus:collect()
- ';
+ content_by_lua_block {
+ metric_connections:set(ngx.var.connections_reading, {"reading"})
+ metric_connections:set(ngx.var.connections_waiting, {"waiting"})
+ metric_connections:set(ngx.var.connections_writing, {"writing"})
+ prometheus:collect()
+ }
}
}
}
--
GitLab