From 88cb67e7b71a46240e13c6deb3659b2367bf95f9 Mon Sep 17 00:00:00 2001
From: Keshav Prasad <keshavprasadms@gmail.com>
Date: Mon, 15 Mar 2021 20:38:34 +0530
Subject: [PATCH] fix: ignore_older works only at type: log (#2321)

---
 .../templates/filebeat-inputs.yml.j2          | 49 +++++++++++++++++++
 .../templates/filebeat.yml.j2                 |  2 -
 2 files changed, 49 insertions(+), 2 deletions(-)

diff --git a/ansible/roles/vm-agents-filebeat/templates/filebeat-inputs.yml.j2 b/ansible/roles/vm-agents-filebeat/templates/filebeat-inputs.yml.j2
index 55edda034..639a99c16 100644
--- a/ansible/roles/vm-agents-filebeat/templates/filebeat-inputs.yml.j2
+++ b/ansible/roles/vm-agents-filebeat/templates/filebeat-inputs.yml.j2
@@ -7,6 +7,10 @@
   multiline.pattern: '^[INFO|ERROR|WARN|DEBUG]'
   multiline.negate: true
   multiline.match: after
+  ignore_older: 24h
+  close_inactive: 12h
+  clean_inactive: 25h
+
 
 - type: log
   enabled: true
@@ -15,6 +19,9 @@
   multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
   multiline.negate: true
   multiline.match: after
+  ignore_older: 24h
+  close_inactive: 12h
+  clean_inactive: 25h
 
 #=========================== Elasticsearch =============================
 - type: log
@@ -24,6 +31,9 @@
   multiline.pattern: '^\['
   multiline.negate: true
   multiline.match: after
+  ignore_older: 24h
+  close_inactive: 12h
+  clean_inactive: 25h
 
 #=========================== Kafka =============================
 - type: log
@@ -33,6 +43,9 @@
   multiline.pattern: '^\['
   multiline.negate: true
   multiline.match: after
+  ignore_older: 24h
+  close_inactive: 12h
+  clean_inactive: 25h
 
 #=========================== Keycloak =============================
 - type: log
@@ -42,6 +55,9 @@
   multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
   multiline.negate: true
   multiline.match: after
+  ignore_older: 24h
+  close_inactive: 12h
+  clean_inactive: 25h
 
 #=========================== Learning =============================
 - type: log
@@ -53,6 +69,9 @@
   multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
   multiline.negate: true
   multiline.match: after
+  ignore_older: 24h
+  close_inactive: 12h
+  clean_inactive: 25h
 
 - type: log
   enabled: true
@@ -61,6 +80,9 @@
   multiline.pattern: '^[0-9]{4}-[A-Za-z]{3}-[0-9]{2}'
   multiline.negate: true
   multiline.match: after
+  ignore_older: 24h
+  close_inactive: 12h
+  clean_inactive: 25h
 
 #=========================== Logstash =============================
 - type: log
@@ -73,6 +95,9 @@
   multiline.pattern: '^\['
   multiline.negate: true
   multiline.match: after
+  ignore_older: 24h
+  close_inactive: 12h
+  clean_inactive: 25h
 
 #=========================== Neo4j =============================
 - type: log
@@ -87,6 +112,9 @@
   multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
   multiline.negate: true
   multiline.match: after
+  ignore_older: 24h
+  close_inactive: 12h
+  clean_inactive: 25h
 
 #=========================== Redis =============================
 - type: log
@@ -97,6 +125,9 @@
   multiline.pattern: '^[0-9]*?:[M|S|C|X]'
   multiline.negate: true
   multiline.match: after
+  ignore_older: 24h
+  close_inactive: 12h
+  clean_inactive: 25h
 
 #=========================== Spark =============================
 - type: log
@@ -106,6 +137,9 @@
   multiline.pattern: '^[0-9]{2,4}[-|/][0-9]{2}[-|/][0-9]{2}'
   multiline.negate: true
   multiline.match: after
+  ignore_older: 24h
+  close_inactive: 12h
+  clean_inactive: 25h
 
 #=========================== Syslog =============================
 - type: log
@@ -113,6 +147,9 @@
   paths:
     - /var/log/*log
   exclude_files: ['/var/log/keycloak.err.log', '/var/log/keycloak.out.log']
+  ignore_older: 24h
+  close_inactive: 12h
+  clean_inactive: 25h
 
 #=========================== Zookeeper =============================
 - type: log
@@ -122,6 +159,9 @@
   multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
   multiline.negate: true
   multiline.match: after
+  ignore_older: 24h
+  close_inactive: 12h
+  clean_inactive: 25h
 
 #=========================== Dial =============================
 - type: log
@@ -133,6 +173,9 @@
   multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
   multiline.negate: true
   multiline.match: after
+  ignore_older: 24h
+  close_inactive: 12h
+  clean_inactive: 25h
 
 #=========================== Secor =============================
 - type: log
@@ -142,6 +185,9 @@
   multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
   multiline.negate: true
   multiline.match: after
+  ignore_older: 24h
+  close_inactive: 12h
+  clean_inactive: 25h
 
 #=========================== Postgres VM =============================
 - type: log    
@@ -151,3 +197,6 @@
   multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
   multiline.negate: true
   multiline.match: after
+  ignore_older: 24h
+  close_inactive: 12h
+  clean_inactive: 25h
diff --git a/ansible/roles/vm-agents-filebeat/templates/filebeat.yml.j2 b/ansible/roles/vm-agents-filebeat/templates/filebeat.yml.j2
index 87ac65a2b..5352309e5 100644
--- a/ansible/roles/vm-agents-filebeat/templates/filebeat.yml.j2
+++ b/ansible/roles/vm-agents-filebeat/templates/filebeat.yml.j2
@@ -74,5 +74,3 @@ output.elasticsearch:
 
 #----------------------------- General --------------------------------
 max_procs: 1
-ignore_older: 24h
-close_inactive: 12h
-- 
GitLab