diff --git a/ansible/logging.yml b/ansible/logging.yml
index fd3dc74c9538edf0d972fa05fef9b8fe59caed6d..426a6531d6c4193a034226e68e592c9844a925a5 100644
--- a/ansible/logging.yml
+++ b/ansible/logging.yml
@@ -1,126 +1,14 @@
 ---
-- hosts: swarm-nodes
-  become: yes
-  tasks:
-  - name: Spray cluster name to nodes
-    copy: dest=/home/deployer/cluster_name content="{{ cluster_name }}"
-    when: cluster_name is defined
-  tags:
-    - stack-logger
-
-- hosts: swarm-bootstrap-manager
-  become: yes
-  vars_files:
-    - ['{{inventory_dir}}/secrets.yml']
-  roles:
-    - stack-logger
-  tags:
-    - stack-logger
-  run_once: true
-
-- hosts: swarm-dashboard
+# You can choose host names on jenkins and run the job. If you have custom host groups, you can add them to jenkins job and run
+# To start or stop filebeat, choose the tag on jenkins job
+- hosts: "{{ hosts }}"
   become: yes
+  gather_facts: no
+  ignore_unreachable: yes
   vars_files:
-    - ['{{inventory_dir}}/secrets.yml']
-  roles:
-    - stack-kibana
-  tags:
-    - stack-kibana
-
-- hosts: swarm-dashboard
-  become: yes
-  vars_files:
-    - ['{{inventory_dir}}/secrets.yml']
-  roles:
-    - stack-oauth
-  tags:
-    - stack-oauth
-
-- hosts: log-forwarder
-  become: yes
-  vars_files:
-    - ['{{inventory_dir}}/secrets.yml']
-  roles:
-    - vm-agents-filebeat
-  tags:
-    - log-forwarder
-
-- hosts: lp-learning-ps
-  become: yes
-  vars_files:
-    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
-  roles:
-    - { role: vm-agents-filebeat, filebeat_log_path: '/data/logs/learning_service_mw.log' }
-  tags:
-    - learningall 
-    - filebeat
-                      
-- hosts: lp-search-ps
-  become: yes
-  vars_files:
-    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
-  roles:
-    - { role: vm-agents-filebeat, filebeat_log_path: '/data/logs/search_service_mw.log' }
-  tags:            
-    - searchall
-    - filebeat
-
-- hosts: dp-neo4j-ps
-  become: yes
-  vars_files:
-    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
-  roles:
-    - { role: vm-agents-filebeat, filebeat_log_path: '/home/learning/neo4j-learning/neo4j-enterprise-3.3.0/logs/neo4j.log' }
-  tags:            
-    - neo4j
-    - filebeat
-
-- hosts: cassandra-ps
-  become: yes
-  vars_files:
-    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
-  roles:
-    - { role: vm-agents-filebeat, filebeat_log_path: '/var/log/cassandra/debug.log' }
-  tags:            
-    - cassandra
-    - filebeat
-
-- hosts: dp-analytics-ps
-  become: yes
-  vars_files:
-    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
-  roles:
-    - { role: vm-agents-filebeat, filebeat_log_path: '/mount/data/analytics/logs/services/api-service.log, /var/log/logstash/logstash-plain.log' }
-  tags:            
-    - analytics
-    - filebeat
-
-- hosts: dp-kafkaindexer-ps
-  become: yes
-  vars_files:
-    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
-  roles:
-    - { role: vm-agents-filebeat, filebeat_log_path: '/var/log/logstash/logstash-plain.log' }
-  tags:            
-    - kafkaindexer
-    - filebeat
-      
-- hosts: zookeeper
-  become: yes
-  vars_files:
-    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
-  roles:
-    - { role: vm-agents-filebeat, filebeat_log_path: '/var/log/zookeeper/zookeeper.log' }
-  tags: 
-    - zookeeper
-    - filebeat
-
-- hosts: keycloak
-  become: yes
-  vars_files:
-    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
-  roles:
-    - { role: vm-agents-filebeat, filebeat_log_path: '/opt/keycloak/standalone/log/server.log' }                   
-  tags:     
-    - keycloak
-    - filebeat
+    - "{{inventory_dir}}/secrets.yml"
+  tasks:
+  - include_role:
+     name: vm-agents-filebeat
+    tags:
+    - "{{ tags }}"
diff --git a/ansible/roles/vm-agents-filebeat/defaults/main.yml b/ansible/roles/vm-agents-filebeat/defaults/main.yml
index 69eccee2faf389fa507a24e9153a8dbd442ebb29..28ea4936afde7fb4506c1105681b76333887697e 100644
--- a/ansible/roles/vm-agents-filebeat/defaults/main.yml
+++ b/ansible/roles/vm-agents-filebeat/defaults/main.yml
@@ -1,74 +1,2 @@
----
-# The version of filebeat to install
-filebeat_version: 1.3.1
-
-# `filebeat_config` is templated directly into filebeat.yml for the config.
-# You are expected to override this variable, as these configurations are
-# only suited for development purposes.
-# See https://github.com/elastic/beats/blob/master/filebeat/filebeat.yml for
-# an exhaustive list of configurations.
-filebeat_config:
-  filebeat:
-    prospectors:
-      - paths:
-          - /var/log/messages
-          - /var/log/*.log
-        input_type: log
-  output:
-    file:
-      path: /tmp/filebeat
-      filename: filebeat
-  logging:
-    to_syslog: true
-    level: error
-
-# The contents of this variable will be placed into the `filebeat_ca_path`
-# This should either be set to a string containing your CA certificate or
-# use a lookup plugin to retrieve it.
-#  ex:
-#   filebeat_ca_cert: "{{ lookup('file', '/path/to/ca.crt') }}"
-filebeat_ca_cert: null
-# Path to which the above certificate will be uploaded
-filebeat_ca_path: /etc/filebeat/ca.crt
-
-# Similar to the above but for ssl cert and ssl key
-filebeat_ssl_cert: null
-filebeat_ssl_cert_path: /etc/filebeat/ssl.crt
-filebeat_ssl_key: null
-filebeat_ssl_key_path: /etc/filebeat/ssl.key
-
-# Repository settings
-filebeat_gpg_url: https://packages.elastic.co/GPG-KEY-elasticsearch
-## Debian
-filebeat_apt_repo_v1: "deb https://packages.elastic.co/beats/apt stable main"
-filebeat_apt_repo_v5: "deb https://artifacts.elastic.co/packages/5.x/apt stable main"
-filebeat_apt_repo: "{{ filebeat_version|version_compare('5', '<')|ternary(filebeat_apt_repo_v1, filebeat_apt_repo_v5) }}"
-## Redhat
-filebeat_repo_url_v1: https://packages.elastic.co/beats/yum/el/$basearch
-filebeat_repo_url_v5: https://artifacts.elastic.co/packages/5.x/yum
-filebeat_repo_url: "{{ filebeat_version|version_compare('5', '<')|ternary(filebeat_repo_url_v1, filebeat_repo_url_v5) }}"
-
-filebeath_log_path: "{{ filebeath_log_path }}"
-filebeat_logstash_port: 5044
-filebeat_logstash_host: "{{groups['swarm-manager'][0]}}"
-filebeat:
-  base_path: /etc/filebeat
-  config:
-    filebeat:
-      prospectors:
-        - paths:
-            - "{{ filebeat_log_path }}" 
-          input_type: log
-    output:
-      logstash:
-        hosts:
-          - "{{ filebeat_logstash_host }}:{{ filebeat_logstash_port }}"
-        timeout: 15
-    logging:
-        level: debug
-        to_files: true
-        to_syslog: false
-        files:
-            path: /tmp/filebeat
-            name: filebeat.log
-            keepfiles: 7
+filebeat_version: "6.8.13"
+filebeat_config_directory: "/etc/filebeat/configs"
diff --git a/ansible/roles/vm-agents-filebeat/tasks/main.yml b/ansible/roles/vm-agents-filebeat/tasks/main.yml
index 5c00c0a308c2d0f9989eb64ffa7fdd513702b8b1..8dab7da33babba38abb5f680529d9630ee519266 100644
--- a/ansible/roles/vm-agents-filebeat/tasks/main.yml
+++ b/ansible/roles/vm-agents-filebeat/tasks/main.yml
@@ -1,66 +1,97 @@
 ---
-- debug:
-    msg: "Installing on {{ansible_os_family}}"
+- name: Gather package facts
+  package_facts:
+    manager: auto
+  tags:
+   - default
 
-- include: redhat.yml
-  when: ansible_os_family == 'RedHat'
+- name: Check the filebeat version if its already installed
+  set_fact:
+    filebeat_installed_version: "{{ansible_facts.packages['filebeat'][0].version}}"
+  when: "ansible_facts.packages is defined and 'filebeat' in ansible_facts.packages and (ansible_facts.packages['filebeat'] | length) == 1"
+  tags:
+   - default
 
-- include: debian.yml
-  when: ansible_os_family == 'Debian'
+- name: Remove filebeat package if it does not match the version which is being installed
+  apt:
+    name: filebeat
+    state: absent
+  when: filebeat_installed_version is defined and filebeat_installed_version != filebeat_version
+  tags:
+   - default
 
-- include: darwin.yml
-  when: ansible_os_family == 'Darwin'
+- name: Remove filebeat registry directory if it does not match the version which is being installed
+  file: 
+    path: /var/lib/filebeat/registry
+    state: absent
+  when: filebeat_installed_version is defined and filebeat_installed_version != filebeat_version
+  tags:
+   - default
 
-- name: list
-  debug:
-    msg: "{{ filebeat_log_path }}"
+- name: Add elastic gpg key
+  apt_key:
+    url: https://artifacts.elastic.co/GPG-KEY-elasticsearch
+    state: present
+  tags:
+   - default
 
-- name: create filebeat.yml
-  template:
-    src: filebeat.yml.j2
-    dest: "{{filebeat.base_path}}/filebeat.yml"
-  notify:
-    - restart filebeat
+- name: Add elastic repository
+  apt_repository:
+    repo: deb https://artifacts.elastic.co/packages/6.x/apt stable main
+    state: present
+    filename: /etc/apt/sources.list.d/elastic-6.x.list
+  tags:
+   - default
 
-- name: copy ca certificate if required
-  copy:
-    content: "{{ filebeat_ca_cert }}"
-    dest: "{{ filebeat_ca_path }}"
-    owner: root
-    group: root
-    mode: 0400
-  when: filebeat_ca_cert != None
+- name: Install the filebeat and required packages
+  apt:
+    name: ['apt-transport-https', 'filebeat={{filebeat_version}}']
+    state: present
+    update_cache: yes
+    cache_valid_time: 3600
+  tags:
+   - default
 
-- name: copy ssl certificate if required
-  copy:
-    content: "{{ filebeat_ssl_cert }}"
-    dest: "{{ filebeat_ssl_cert_path }}"
-    owner: root
-    group: root
-    mode: 0400
-  when: filebeat_ssl_cert != None
+- name: Create the filebeat configs directory
+  file:
+    path: "{{filebeat_config_directory}}"
+    state: directory
+  tags:
+   - default
 
-- name: copy ssl key if required
-  copy:
-    content: "{{ filebeat_ssl_key }}"
-    dest: "{{ filebeat_ssl_key_path }}"
-    owner: root
-    group: root
-    mode: 0400
-  when: filebeat_ssl_key != None
+- name: Copy the filebeat input config template(s)
+  template:
+    src: "filebeat-inputs.yml.j2"
+    dest: "{{filebeat_config_directory}}/filebeat-inputs.yml"
+  tags:
+   - default
 
-- name: flush handlers to prevent start then restart
-  meta: flush_handlers
+- name: Copy the filebeat outout config template
+  template:
+    src: "filebeat.yml.j2"
+    dest: "/etc/filebeat/filebeat.yml"
+  tags:
+   - default
 
-- name: start and enable filebeat
-  service:
+- name: Enable and start filebeat as a daemon process
+  systemd:
     name: filebeat
-    state: started
-#    enabled: true
-  when: ansible_os_family == 'RedHat' or ansible_os_family == 'Debian'
+    enabled: yes
+    state: restarted
+    daemon_reload: yes
+  tags:
+   - default
 
-- name: start and enable filebeat
-  command: ./filebeat -c filebeat.yml -d "publish"
-  args:
-    chdir: "{{filebeat.base_path}}"
-  when: ansible_os_family == 'Darwin'
+- name: Stop filebeat
+  systemd:
+    name: filebeat
+    state: stopped
+  tags:
+   - stop-filebeat
+
+- name: Start filebeat
+  systemd:
+    name: filebeat
+    state: started
+  tags:
+   - start-filebeat
diff --git a/ansible/roles/vm-agents-filebeat/templates/filebeat-inputs.yml.j2 b/ansible/roles/vm-agents-filebeat/templates/filebeat-inputs.yml.j2
new file mode 100644
index 0000000000000000000000000000000000000000..55edda0342a9de7007c8f921696fef6657b735d7
--- /dev/null
+++ b/ansible/roles/vm-agents-filebeat/templates/filebeat-inputs.yml.j2
@@ -0,0 +1,153 @@
+#=========================== Cassandra =============================
+- type: log
+  enabled: true
+  paths:
+    - /var/log/cassandra/system.log
+    - /var/log/cassandra/debug.log
+  multiline.pattern: '^[INFO|ERROR|WARN|DEBUG]'
+  multiline.negate: true
+  multiline.match: after
+
+- type: log
+  enabled: true
+  paths:
+  - /var/log/cassandra/gc.log.0
+  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
+  multiline.negate: true
+  multiline.match: after
+
+#=========================== Elasticsearch =============================
+- type: log
+  enabled: true
+  paths:
+    - /var/log/elasticsearch/*/*.log
+  multiline.pattern: '^\['
+  multiline.negate: true
+  multiline.match: after
+
+#=========================== Kafka =============================
+- type: log
+  enabled: true
+  paths:
+    - /var/log/kafka/*.log
+  multiline.pattern: '^\['
+  multiline.negate: true
+  multiline.match: after
+
+#=========================== Keycloak =============================
+- type: log
+  enabled: true
+  paths:
+    - /opt/keycloak/standalone/log/server.log
+  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
+  multiline.negate: true
+  multiline.match: after
+
+#=========================== Learning =============================
+- type: log
+  enabled: true
+  paths:
+    - /data/logs/learning_service_mw.log
+    - /data/logs/learning_telemetry_event_mw.log
+    - /data/logs/learning_perf_mw.log
+  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
+  multiline.negate: true
+  multiline.match: after
+
+- type: log
+  enabled: true
+  paths:
+  - /home/learning/apache-tomcat-8.0.36/logs/catalina.out
+  multiline.pattern: '^[0-9]{4}-[A-Za-z]{3}-[0-9]{2}'
+  multiline.negate: true
+  multiline.match: after
+
+#=========================== Logstash =============================
+- type: log
+  enabled: true
+  paths:
+  - /home/learning/logstash-6.3.1/logs/logstash-plain.log
+  - /home/learning/logstash-6.3.1/logs/logstash-slowlog-plain.log
+  - /var/log/logstash/logstash-plain.log
+  - /var/log/logstash/logstash-slowlog-plain.log
+  multiline.pattern: '^\['
+  multiline.negate: true
+  multiline.match: after
+
+#=========================== Neo4j =============================
+- type: log
+  enabled: true
+  paths:
+    - /home/learning/neo4j-learning/*/logs/debug.log
+    - /home/learning/neo4j-learning/*/logs/gc.log.0.current
+    - /home/learning/neo4j-learning/*/logs/neo4j.log
+    - /home/learning/neo4j-learning/*/logs/query.log
+    - /data/logs/learning_graph_event_neo4j.log
+    - /data/logs/learning_service_neo4j.log
+  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
+  multiline.negate: true
+  multiline.match: after
+
+#=========================== Redis =============================
+- type: log
+  enabled: true
+  paths:
+    - /home/analytics/redis-stable/redis.log
+    - /home/learning/redis-stable/redis.log
+  multiline.pattern: '^[0-9]*?:[M|S|C|X]'
+  multiline.negate: true
+  multiline.match: after
+
+#=========================== Spark =============================
+- type: log
+  enabled: true
+  paths:
+    - /mount/data/analytics/logs/*/*.log
+  multiline.pattern: '^[0-9]{2,4}[-|/][0-9]{2}[-|/][0-9]{2}'
+  multiline.negate: true
+  multiline.match: after
+
+#=========================== Syslog =============================
+- type: log
+  enabled: true
+  paths:
+    - /var/log/*log
+  exclude_files: ['/var/log/keycloak.err.log', '/var/log/keycloak.out.log']
+
+#=========================== Zookeeper =============================
+- type: log
+  enabled: true
+  paths:
+    - /var/log/zookeeper/*.log
+  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
+  multiline.negate: true
+  multiline.match: after
+
+#=========================== Dial =============================
+- type: log
+  enabled: true
+  paths:
+    - /data/logs/dial_graph_event_mw.log
+    - /data/logs/dial_service_mw.log
+    - /data/logs/dial_telemetry_event_mw.log
+  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
+  multiline.negate: true
+  multiline.match: after
+
+#=========================== Secor =============================
+- type: log
+  enabled: true
+  paths:
+    - /mount/secor/logs/*.log
+  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
+  multiline.negate: true
+  multiline.match: after
+
+#=========================== Postgres VM =============================
+- type: log    
+  enabled: true
+  paths:
+    - /var/log/postgresql/*.log
+  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
+  multiline.negate: true
+  multiline.match: after
diff --git a/ansible/roles/vm-agents-filebeat/templates/filebeat.yml.j2 b/ansible/roles/vm-agents-filebeat/templates/filebeat.yml.j2
index e31090f7b6bbde8b8ffccadb3861a9df80b6a0eb..87ac65a2bd6923ea4554b637af07f74d906e2591 100644
--- a/ansible/roles/vm-agents-filebeat/templates/filebeat.yml.j2
+++ b/ansible/roles/vm-agents-filebeat/templates/filebeat.yml.j2
@@ -1,13 +1,78 @@
-filebeat:
-  prospectors:
-  - input_type: log
-    paths: [{{ filebeat_log_path }}]
-logging:
-  files: {keepfiles: 7, name: filebeat.log, path: /tmp/filebeat}
-  level: debug
-  to_files: true
-  to_syslog: false
-output:
-  logstash:
-    hosts: ['{{ filebeat_logstash_host }}:{{ filebeat_logstash_port }}']
-    timeout: 15
+#=========================== Filebeat inputs =============================
+filebeat.config.inputs:
+  enabled: true
+  path: "{{filebeat_config_directory}}/*.yml"
+
+#==================== Elasticsearch template setting ==========================
+setup.template.name: "filebeat"
+setup.template.pattern: "filebeat-*"
+setup.template.overwrite: false
+setup.template.settings:
+  index.number_of_shards: 3
+
+#================================ Outputs =====================================
+
+#-------------------------- Elasticsearch output ------------------------------
+output.elasticsearch:
+  # Array of hosts to connect to.
+  hosts: ["{{ groups['log-es']|join(':9200\", \"')}}:9200"]
+  indices:
+  - index: "redis-dp-%{+yyyy.MM.dd}"
+    when.contains:
+      source: analytics/redis-stable/
+  - index: "redis-kp-%{+yyyy.MM.dd}"
+    when.contains:
+      source: learning/redis-stable/
+  - index: "spark-%{+yyyy.MM.dd}"
+    when.contains:
+      source: /mount/data/analytics/logs/
+  - index: "cassandra-%{+yyyy.MM.dd}"
+    when.contains:
+      source: cassandra
+  - index: "composite-es-%{+yyyy.MM.dd}"
+    when.regexp:
+      source: elasticsearch/.*cs-node.*
+  - index: "log-es-%{+yyyy.MM.dd}"
+    when.regexp:
+      source: elasticsearch/.*log-es.*/
+  - index: "lms-es-%{+yyyy.MM.dd}"
+    when.regexp:
+      source: elasticsearch/.*es.*/
+  - index: "keycloak-%{+yyyy.MM.dd}"
+    when.contains:
+      source: /opt/keycloak/standalone/log/
+  - index: "neo4j-%{+yyyy.MM.dd}"
+    when.contains:
+      source: neo4j
+  - index: "dial-%{+yyyy.MM.dd}"
+    when.contains:
+      source: dial
+  - index: "logstash-vm-%{+yyyy.MM.dd}"
+    when.contains:
+      source: logstash
+  - index: "learning-%{+yyyy.MM.dd}"
+    when.contains:
+      source: /data/logs/learning
+  - index: "kafka-%{+yyyy.MM.dd}"
+    when.contains:
+      source: kafka
+  - index: "zookeeper-%{+yyyy.MM.dd}"
+    when.contains:
+      source: zookeeper
+  - index: "secor-%{+yyyy.MM.dd}"
+    when.contains:
+      source: secor
+  - index: "postgres-%{+yyyy.MM.dd}"
+    when.contains:
+      source: postgres
+  - index: "syslogs-%{+yyyy.MM.dd}"
+
+#----------------------------- Logstash output --------------------------------
+#output.logstash:
+  # The Logstash hosts
+  #hosts: ["localhost:5044"]
+
+#----------------------------- General --------------------------------
+max_procs: 1
+ignore_older: 24h
+close_inactive: 12h