diff --git a/ansible/logging.yml b/ansible/logging.yml index fd3dc74c9538edf0d972fa05fef9b8fe59caed6d..426a6531d6c4193a034226e68e592c9844a925a5 100644 --- a/ansible/logging.yml +++ b/ansible/logging.yml @@ -1,126 +1,14 @@ --- -- hosts: swarm-nodes - become: yes - tasks: - - name: Spray cluster name to nodes - copy: dest=/home/deployer/cluster_name content="{{ cluster_name }}" - when: cluster_name is defined - tags: - - stack-logger - -- hosts: swarm-bootstrap-manager - become: yes - vars_files: - - ['{{inventory_dir}}/secrets.yml'] - roles: - - stack-logger - tags: - - stack-logger - run_once: true - -- hosts: swarm-dashboard +# You can choose host names on jenkins and run the job. If you have custom host groups, you can add them to jenkins job and run +# To start or stop filebeat, choose the tag on jenkins job +- hosts: "{{ hosts }}" become: yes + gather_facts: no + ignore_unreachable: yes vars_files: - - ['{{inventory_dir}}/secrets.yml'] - roles: - - stack-kibana - tags: - - stack-kibana - -- hosts: swarm-dashboard - become: yes - vars_files: - - ['{{inventory_dir}}/secrets.yml'] - roles: - - stack-oauth - tags: - - stack-oauth - -- hosts: log-forwarder - become: yes - vars_files: - - ['{{inventory_dir}}/secrets.yml'] - roles: - - vm-agents-filebeat - tags: - - log-forwarder - -- hosts: lp-learning-ps - become: yes - vars_files: - - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] - roles: - - { role: vm-agents-filebeat, filebeat_log_path: '/data/logs/learning_service_mw.log' } - tags: - - learningall - - filebeat - -- hosts: lp-search-ps - become: yes - vars_files: - - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] - roles: - - { role: vm-agents-filebeat, filebeat_log_path: '/data/logs/search_service_mw.log' } - tags: - - searchall - - filebeat - -- hosts: dp-neo4j-ps - become: yes - vars_files: - - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] - roles: - - { role: vm-agents-filebeat, filebeat_log_path: '/home/learning/neo4j-learning/neo4j-enterprise-3.3.0/logs/neo4j.log' } - tags: - - neo4j - - filebeat - -- hosts: cassandra-ps - become: yes - vars_files: - - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] - roles: - - { role: vm-agents-filebeat, filebeat_log_path: '/var/log/cassandra/debug.log' } - tags: - - cassandra - - filebeat - -- hosts: dp-analytics-ps - become: yes - vars_files: - - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] - roles: - - { role: vm-agents-filebeat, filebeat_log_path: '/mount/data/analytics/logs/services/api-service.log, /var/log/logstash/logstash-plain.log' } - tags: - - analytics - - filebeat - -- hosts: dp-kafkaindexer-ps - become: yes - vars_files: - - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] - roles: - - { role: vm-agents-filebeat, filebeat_log_path: '/var/log/logstash/logstash-plain.log' } - tags: - - kafkaindexer - - filebeat - -- hosts: zookeeper - become: yes - vars_files: - - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] - roles: - - { role: vm-agents-filebeat, filebeat_log_path: '/var/log/zookeeper/zookeeper.log' } - tags: - - zookeeper - - filebeat - -- hosts: keycloak - become: yes - vars_files: - - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] - roles: - - { role: vm-agents-filebeat, filebeat_log_path: '/opt/keycloak/standalone/log/server.log' } - tags: - - keycloak - - filebeat + - "{{inventory_dir}}/secrets.yml" + tasks: + - include_role: + name: vm-agents-filebeat + tags: + - "{{ tags }}" diff --git a/ansible/roles/vm-agents-filebeat/defaults/main.yml b/ansible/roles/vm-agents-filebeat/defaults/main.yml index 69eccee2faf389fa507a24e9153a8dbd442ebb29..28ea4936afde7fb4506c1105681b76333887697e 100644 --- a/ansible/roles/vm-agents-filebeat/defaults/main.yml +++ b/ansible/roles/vm-agents-filebeat/defaults/main.yml @@ -1,74 +1,2 @@ ---- -# The version of filebeat to install -filebeat_version: 1.3.1 - -# `filebeat_config` is templated directly into filebeat.yml for the config. -# You are expected to override this variable, as these configurations are -# only suited for development purposes. -# See https://github.com/elastic/beats/blob/master/filebeat/filebeat.yml for -# an exhaustive list of configurations. -filebeat_config: - filebeat: - prospectors: - - paths: - - /var/log/messages - - /var/log/*.log - input_type: log - output: - file: - path: /tmp/filebeat - filename: filebeat - logging: - to_syslog: true - level: error - -# The contents of this variable will be placed into the `filebeat_ca_path` -# This should either be set to a string containing your CA certificate or -# use a lookup plugin to retrieve it. -# ex: -# filebeat_ca_cert: "{{ lookup('file', '/path/to/ca.crt') }}" -filebeat_ca_cert: null -# Path to which the above certificate will be uploaded -filebeat_ca_path: /etc/filebeat/ca.crt - -# Similar to the above but for ssl cert and ssl key -filebeat_ssl_cert: null -filebeat_ssl_cert_path: /etc/filebeat/ssl.crt -filebeat_ssl_key: null -filebeat_ssl_key_path: /etc/filebeat/ssl.key - -# Repository settings -filebeat_gpg_url: https://packages.elastic.co/GPG-KEY-elasticsearch -## Debian -filebeat_apt_repo_v1: "deb https://packages.elastic.co/beats/apt stable main" -filebeat_apt_repo_v5: "deb https://artifacts.elastic.co/packages/5.x/apt stable main" -filebeat_apt_repo: "{{ filebeat_version|version_compare('5', '<')|ternary(filebeat_apt_repo_v1, filebeat_apt_repo_v5) }}" -## Redhat -filebeat_repo_url_v1: https://packages.elastic.co/beats/yum/el/$basearch -filebeat_repo_url_v5: https://artifacts.elastic.co/packages/5.x/yum -filebeat_repo_url: "{{ filebeat_version|version_compare('5', '<')|ternary(filebeat_repo_url_v1, filebeat_repo_url_v5) }}" - -filebeath_log_path: "{{ filebeath_log_path }}" -filebeat_logstash_port: 5044 -filebeat_logstash_host: "{{groups['swarm-manager'][0]}}" -filebeat: - base_path: /etc/filebeat - config: - filebeat: - prospectors: - - paths: - - "{{ filebeat_log_path }}" - input_type: log - output: - logstash: - hosts: - - "{{ filebeat_logstash_host }}:{{ filebeat_logstash_port }}" - timeout: 15 - logging: - level: debug - to_files: true - to_syslog: false - files: - path: /tmp/filebeat - name: filebeat.log - keepfiles: 7 +filebeat_version: "6.8.13" +filebeat_config_directory: "/etc/filebeat/configs" diff --git a/ansible/roles/vm-agents-filebeat/tasks/main.yml b/ansible/roles/vm-agents-filebeat/tasks/main.yml index 5c00c0a308c2d0f9989eb64ffa7fdd513702b8b1..8dab7da33babba38abb5f680529d9630ee519266 100644 --- a/ansible/roles/vm-agents-filebeat/tasks/main.yml +++ b/ansible/roles/vm-agents-filebeat/tasks/main.yml @@ -1,66 +1,97 @@ --- -- debug: - msg: "Installing on {{ansible_os_family}}" +- name: Gather package facts + package_facts: + manager: auto + tags: + - default -- include: redhat.yml - when: ansible_os_family == 'RedHat' +- name: Check the filebeat version if its already installed + set_fact: + filebeat_installed_version: "{{ansible_facts.packages['filebeat'][0].version}}" + when: "ansible_facts.packages is defined and 'filebeat' in ansible_facts.packages and (ansible_facts.packages['filebeat'] | length) == 1" + tags: + - default -- include: debian.yml - when: ansible_os_family == 'Debian' +- name: Remove filebeat package if it does not match the version which is being installed + apt: + name: filebeat + state: absent + when: filebeat_installed_version is defined and filebeat_installed_version != filebeat_version + tags: + - default -- include: darwin.yml - when: ansible_os_family == 'Darwin' +- name: Remove filebeat registry directory if it does not match the version which is being installed + file: + path: /var/lib/filebeat/registry + state: absent + when: filebeat_installed_version is defined and filebeat_installed_version != filebeat_version + tags: + - default -- name: list - debug: - msg: "{{ filebeat_log_path }}" +- name: Add elastic gpg key + apt_key: + url: https://artifacts.elastic.co/GPG-KEY-elasticsearch + state: present + tags: + - default -- name: create filebeat.yml - template: - src: filebeat.yml.j2 - dest: "{{filebeat.base_path}}/filebeat.yml" - notify: - - restart filebeat +- name: Add elastic repository + apt_repository: + repo: deb https://artifacts.elastic.co/packages/6.x/apt stable main + state: present + filename: /etc/apt/sources.list.d/elastic-6.x.list + tags: + - default -- name: copy ca certificate if required - copy: - content: "{{ filebeat_ca_cert }}" - dest: "{{ filebeat_ca_path }}" - owner: root - group: root - mode: 0400 - when: filebeat_ca_cert != None +- name: Install the filebeat and required packages + apt: + name: ['apt-transport-https', 'filebeat={{filebeat_version}}'] + state: present + update_cache: yes + cache_valid_time: 3600 + tags: + - default -- name: copy ssl certificate if required - copy: - content: "{{ filebeat_ssl_cert }}" - dest: "{{ filebeat_ssl_cert_path }}" - owner: root - group: root - mode: 0400 - when: filebeat_ssl_cert != None +- name: Create the filebeat configs directory + file: + path: "{{filebeat_config_directory}}" + state: directory + tags: + - default -- name: copy ssl key if required - copy: - content: "{{ filebeat_ssl_key }}" - dest: "{{ filebeat_ssl_key_path }}" - owner: root - group: root - mode: 0400 - when: filebeat_ssl_key != None +- name: Copy the filebeat input config template(s) + template: + src: "filebeat-inputs.yml.j2" + dest: "{{filebeat_config_directory}}/filebeat-inputs.yml" + tags: + - default -- name: flush handlers to prevent start then restart - meta: flush_handlers +- name: Copy the filebeat outout config template + template: + src: "filebeat.yml.j2" + dest: "/etc/filebeat/filebeat.yml" + tags: + - default -- name: start and enable filebeat - service: +- name: Enable and start filebeat as a daemon process + systemd: name: filebeat - state: started -# enabled: true - when: ansible_os_family == 'RedHat' or ansible_os_family == 'Debian' + enabled: yes + state: restarted + daemon_reload: yes + tags: + - default -- name: start and enable filebeat - command: ./filebeat -c filebeat.yml -d "publish" - args: - chdir: "{{filebeat.base_path}}" - when: ansible_os_family == 'Darwin' +- name: Stop filebeat + systemd: + name: filebeat + state: stopped + tags: + - stop-filebeat + +- name: Start filebeat + systemd: + name: filebeat + state: started + tags: + - start-filebeat diff --git a/ansible/roles/vm-agents-filebeat/templates/filebeat-inputs.yml.j2 b/ansible/roles/vm-agents-filebeat/templates/filebeat-inputs.yml.j2 new file mode 100644 index 0000000000000000000000000000000000000000..55edda0342a9de7007c8f921696fef6657b735d7 --- /dev/null +++ b/ansible/roles/vm-agents-filebeat/templates/filebeat-inputs.yml.j2 @@ -0,0 +1,153 @@ +#=========================== Cassandra ============================= +- type: log + enabled: true + paths: + - /var/log/cassandra/system.log + - /var/log/cassandra/debug.log + multiline.pattern: '^[INFO|ERROR|WARN|DEBUG]' + multiline.negate: true + multiline.match: after + +- type: log + enabled: true + paths: + - /var/log/cassandra/gc.log.0 + multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}' + multiline.negate: true + multiline.match: after + +#=========================== Elasticsearch ============================= +- type: log + enabled: true + paths: + - /var/log/elasticsearch/*/*.log + multiline.pattern: '^\[' + multiline.negate: true + multiline.match: after + +#=========================== Kafka ============================= +- type: log + enabled: true + paths: + - /var/log/kafka/*.log + multiline.pattern: '^\[' + multiline.negate: true + multiline.match: after + +#=========================== Keycloak ============================= +- type: log + enabled: true + paths: + - /opt/keycloak/standalone/log/server.log + multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}' + multiline.negate: true + multiline.match: after + +#=========================== Learning ============================= +- type: log + enabled: true + paths: + - /data/logs/learning_service_mw.log + - /data/logs/learning_telemetry_event_mw.log + - /data/logs/learning_perf_mw.log + multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}' + multiline.negate: true + multiline.match: after + +- type: log + enabled: true + paths: + - /home/learning/apache-tomcat-8.0.36/logs/catalina.out + multiline.pattern: '^[0-9]{4}-[A-Za-z]{3}-[0-9]{2}' + multiline.negate: true + multiline.match: after + +#=========================== Logstash ============================= +- type: log + enabled: true + paths: + - /home/learning/logstash-6.3.1/logs/logstash-plain.log + - /home/learning/logstash-6.3.1/logs/logstash-slowlog-plain.log + - /var/log/logstash/logstash-plain.log + - /var/log/logstash/logstash-slowlog-plain.log + multiline.pattern: '^\[' + multiline.negate: true + multiline.match: after + +#=========================== Neo4j ============================= +- type: log + enabled: true + paths: + - /home/learning/neo4j-learning/*/logs/debug.log + - /home/learning/neo4j-learning/*/logs/gc.log.0.current + - /home/learning/neo4j-learning/*/logs/neo4j.log + - /home/learning/neo4j-learning/*/logs/query.log + - /data/logs/learning_graph_event_neo4j.log + - /data/logs/learning_service_neo4j.log + multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}' + multiline.negate: true + multiline.match: after + +#=========================== Redis ============================= +- type: log + enabled: true + paths: + - /home/analytics/redis-stable/redis.log + - /home/learning/redis-stable/redis.log + multiline.pattern: '^[0-9]*?:[M|S|C|X]' + multiline.negate: true + multiline.match: after + +#=========================== Spark ============================= +- type: log + enabled: true + paths: + - /mount/data/analytics/logs/*/*.log + multiline.pattern: '^[0-9]{2,4}[-|/][0-9]{2}[-|/][0-9]{2}' + multiline.negate: true + multiline.match: after + +#=========================== Syslog ============================= +- type: log + enabled: true + paths: + - /var/log/*log + exclude_files: ['/var/log/keycloak.err.log', '/var/log/keycloak.out.log'] + +#=========================== Zookeeper ============================= +- type: log + enabled: true + paths: + - /var/log/zookeeper/*.log + multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}' + multiline.negate: true + multiline.match: after + +#=========================== Dial ============================= +- type: log + enabled: true + paths: + - /data/logs/dial_graph_event_mw.log + - /data/logs/dial_service_mw.log + - /data/logs/dial_telemetry_event_mw.log + multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}' + multiline.negate: true + multiline.match: after + +#=========================== Secor ============================= +- type: log + enabled: true + paths: + - /mount/secor/logs/*.log + multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}' + multiline.negate: true + multiline.match: after + +#=========================== Postgres VM ============================= +- type: log + enabled: true + paths: + - /var/log/postgresql/*.log + multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}' + multiline.negate: true + multiline.match: after diff --git a/ansible/roles/vm-agents-filebeat/templates/filebeat.yml.j2 b/ansible/roles/vm-agents-filebeat/templates/filebeat.yml.j2 index e31090f7b6bbde8b8ffccadb3861a9df80b6a0eb..87ac65a2bd6923ea4554b637af07f74d906e2591 100644 --- a/ansible/roles/vm-agents-filebeat/templates/filebeat.yml.j2 +++ b/ansible/roles/vm-agents-filebeat/templates/filebeat.yml.j2 @@ -1,13 +1,78 @@ -filebeat: - prospectors: - - input_type: log - paths: [{{ filebeat_log_path }}] -logging: - files: {keepfiles: 7, name: filebeat.log, path: /tmp/filebeat} - level: debug - to_files: true - to_syslog: false -output: - logstash: - hosts: ['{{ filebeat_logstash_host }}:{{ filebeat_logstash_port }}'] - timeout: 15 +#=========================== Filebeat inputs ============================= +filebeat.config.inputs: + enabled: true + path: "{{filebeat_config_directory}}/*.yml" + +#==================== Elasticsearch template setting ========================== +setup.template.name: "filebeat" +setup.template.pattern: "filebeat-*" +setup.template.overwrite: false +setup.template.settings: + index.number_of_shards: 3 + +#================================ Outputs ===================================== + +#-------------------------- Elasticsearch output ------------------------------ +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["{{ groups['log-es']|join(':9200\", \"')}}:9200"] + indices: + - index: "redis-dp-%{+yyyy.MM.dd}" + when.contains: + source: analytics/redis-stable/ + - index: "redis-kp-%{+yyyy.MM.dd}" + when.contains: + source: learning/redis-stable/ + - index: "spark-%{+yyyy.MM.dd}" + when.contains: + source: /mount/data/analytics/logs/ + - index: "cassandra-%{+yyyy.MM.dd}" + when.contains: + source: cassandra + - index: "composite-es-%{+yyyy.MM.dd}" + when.regexp: + source: elasticsearch/.*cs-node.* + - index: "log-es-%{+yyyy.MM.dd}" + when.regexp: + source: elasticsearch/.*log-es.*/ + - index: "lms-es-%{+yyyy.MM.dd}" + when.regexp: + source: elasticsearch/.*es.*/ + - index: "keycloak-%{+yyyy.MM.dd}" + when.contains: + source: /opt/keycloak/standalone/log/ + - index: "neo4j-%{+yyyy.MM.dd}" + when.contains: + source: neo4j + - index: "dial-%{+yyyy.MM.dd}" + when.contains: + source: dial + - index: "logstash-vm-%{+yyyy.MM.dd}" + when.contains: + source: logstash + - index: "learning-%{+yyyy.MM.dd}" + when.contains: + source: /data/logs/learning + - index: "kafka-%{+yyyy.MM.dd}" + when.contains: + source: kafka + - index: "zookeeper-%{+yyyy.MM.dd}" + when.contains: + source: zookeeper + - index: "secor-%{+yyyy.MM.dd}" + when.contains: + source: secor + - index: "postgres-%{+yyyy.MM.dd}" + when.contains: + source: postgres + - index: "syslogs-%{+yyyy.MM.dd}" + +#----------------------------- Logstash output -------------------------------- +#output.logstash: + # The Logstash hosts + #hosts: ["localhost:5044"] + +#----------------------------- General -------------------------------- +max_procs: 1 +ignore_older: 24h +close_inactive: 12h