From 5e3fe3cf80b410a5d2147d3b914d759c9251b795 Mon Sep 17 00:00:00 2001
From: SMY ALTAMASH <30286162+SMYALTAMASH@users.noreply.github.com>
Date: Thu, 23 May 2019 15:17:10 +0530
Subject: [PATCH] Issue #000 heap: getting the heap size of server dynamicly
and having a provision to override (#490)
* Issue #000 vars: grafana url changes
* Issue #000 heap: getting the heap size of server dynamicly and having a provision to override (#489)
* Issue #000 host: default variable for content preview
* Issue #000 heap: taking the heap size of elasticsearch as dynamic and having a provision to override
* Issue #000 host: default variable for content preview
* Issue #000 host: default variable for content preview
* Issue #000 host: change trampoline secret
* Issue #000 host: change trampoline secret
* Issue #000 host: change trampoline secret
* Issue #000 host: change trampoline secret
* Issue #000 host: change trampoline secret
* Issue #000 host: change trampoline secret
* Issue #000 host: change trampoline secret
* Issue #000 host: change trampoline secret
* Issue #000 host: change trampoline secret
* Issue #000 host: change trampoline secret
* Issue #000 host: cdn url changes
* Issue #000 host: default variable for heap size
* Issue #000 host: default variable for heap size
* Issue #000 host: default variable for heap size
* Issue #000 host: default variable for heap size
* Issue #000 host: es changes
* Issue #000 host: vars updation
* Issue #000 host: vars updation
* Issue #000 host: vars updation
* Issue #000 host: vars updation
* Issue #000 host: taking ram dynamicly
* es fix
* es fix
* es fix
* Issue #000 es: make the es run the mappings in the initial time
* Issue #000 es: Test es command
* Issue #000 es: run curl command
* Issue #000 es: run curl command
* Issue #000 es: run curl command with changes
---
ansible/inventory/env/group_vars/all.yml | 2 +-
ansible/provision.yml | 3 ++-
.../roles/curl_es_command/templates/curl_commands.j2 | 12 ++++++------
ansible/roles/elasticsearch_old/defaults/main.yml | 1 -
ansible/roles/elasticsearch_old/tasks/debian.yml | 6 ++++++
ansible/roles/es6/tasks/elasticsearch-config.yml | 6 +++++-
6 files changed, 20 insertions(+), 10 deletions(-)
diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml
index 375d89c2b..cf464e69b 100644
--- a/ansible/inventory/env/group_vars/all.yml
+++ b/ansible/inventory/env/group_vars/all.yml
@@ -553,7 +553,7 @@ backup_azure_storage_access_key: "{{core_vault_azure_storage_secret}}"
app_es_snapshot_host: "{{ groups['es'][0] }}"
app_snapshot_base_path: applicationelasticsearch
-sunbird-es-host: "{{ groups['es'][0] }}"
+sunbird_es_host: "{{ groups['es'][0] }}"
#log es backup
log_es_snapshot_host: "{{ groups['log-es'][0] }}"
diff --git a/ansible/provision.yml b/ansible/provision.yml
index 6367cc671..0546293a3 100644
--- a/ansible/provision.yml
+++ b/ansible/provision.yml
@@ -19,7 +19,7 @@
es_instance_name: "{% for servername in play_hosts %}{% if inventory_hostname==servername %}es-{{ loop.index }}{% endif %}{% endfor %}"
roles:
- openjdk
- - { role: es6,
+ - { role: es6,
es_config: {
cluster.name: "{{ app_es_etc_cluster_name }}",
discovery.zen.ping.unicast.hosts: "{{ groups['es'] }}",
@@ -38,6 +38,7 @@
- es
- hosts: log-es
+ gather_facts: true
become: yes
vars_files:
- ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
diff --git a/ansible/roles/curl_es_command/templates/curl_commands.j2 b/ansible/roles/curl_es_command/templates/curl_commands.j2
index 04a11dd7d..d0de39f1f 100755
--- a/ansible/roles/curl_es_command/templates/curl_commands.j2
+++ b/ansible/roles/curl_es_command/templates/curl_commands.j2
@@ -2,32 +2,32 @@
-curl -XPUT '{{sunbird-es-host}}:9200/searchindex?pretty' -H 'Content-Type: application/json' -d'
+curl -XPUT '{{sunbird_es_host}}:9200/searchindex?pretty' -H 'Content-Type: application/json' -d'
{
"settings" : {"analysis": {"analyzer": {"cs_index_analyzer": {"type": "custom","tokenizer": "standard","filter": ["lowercase","mynGram"]},"cs_search_analyzer": {"type": "custom","tokenizer": "standard","filter": ["lowercase","standard"]},"keylower": {"type": "custom","tokenizer": "keyword","filter": "lowercase"}},"filter": {"mynGram": {"type": "ngram","min_gram": 1,"max_gram": 20,"token_chars": ["letter", "digit","whitespace","punctuation","symbol"]} }}}
}'
-curl -XPUT '{{sunbird-es-host}}:9200/searchindex/user/_mapping?pretty' -H 'Content-Type: application/json' -d'
+curl -XPUT '{{sunbird_es_host}}:9200/searchindex/user/_mapping?pretty' -H 'Content-Type: application/json' -d'
{
"user" : { "dynamic_templates": [ {"longs": {"match_mapping_type": "long", "mapping": {"type": "long", "fields": { "raw": {"type": "long" } }}}},{"booleans": {"match_mapping_type": "boolean", "mapping": {"type": "boolean", "fields": { "raw": { "type": "boolean" }} }}},{"doubles": {"match_mapping_type": "double","mapping": {"type": "double","fields":{"raw": { "type": "double" } }}}},{ "dates": {"match_mapping_type": "date", "mapping": { "type": "date","fields": {"raw": { "type": "date" } } }}},{"strings": {"match_mapping_type": "string","mapping": {"type": "text","fielddata": true,"copy_to": "all_fields","analyzer": "cs_index_analyzer","search_analyzer": "cs_search_analyzer","fields": {"raw": {"type": "text","fielddata": true,"analyzer": "keylower"}}}}}],"properties": {"all_fields": {"type": "text","analyzer": "cs_index_analyzer","search_analyzer": "cs_search_analyzer","fields": { "raw": { "type": "text","analyzer": "keylower" } }} }}
}'
-curl -XPUT '{{sunbird-es-host}}:9200/searchindex/org/_mapping?pretty' -H 'Content-Type: application/json' -d'
+curl -XPUT '{{sunbird_es_host}}:9200/searchindex/org/_mapping?pretty' -H 'Content-Type: application/json' -d'
{
"org" : { "dynamic_templates": [ {"longs": {"match_mapping_type": "long", "mapping": {"type": "long", "fields": { "raw": {"type": "long" } }}}},{"booleans": {"match_mapping_type": "boolean", "mapping": {"type": "boolean", "fields": { "raw": { "type": "boolean" }} }}},{"doubles": {"match_mapping_type": "double","mapping": {"type": "double","fields":{"raw": { "type": "double" } }}}},{ "dates": {"match_mapping_type": "date", "mapping": { "type": "date","fields": {"raw": { "type": "date" } } }}},{"strings": {"match_mapping_type": "string","mapping": {"type": "text","fielddata": true,"copy_to": "all_fields","analyzer": "cs_index_analyzer","search_analyzer": "cs_search_analyzer","fields": {"raw": {"type": "text","fielddata": true,"analyzer": "keylower"}}}}}],"properties": {"all_fields": {"type": "text","analyzer": "cs_index_analyzer","search_analyzer": "cs_search_analyzer","fields": { "raw": { "type": "text","analyzer": "keylower" } }} }}
}'
-curl -XPUT '{{sunbird-es-host}}:9200/searchindex/course/_mapping?pretty' -H 'Content-Type: application/json' -d'
+curl -XPUT '{{sunbird_es_host}}:9200/searchindex/course/_mapping?pretty' -H 'Content-Type: application/json' -d'
{
"course" : { "dynamic_templates": [ {"longs": {"match_mapping_type": "long", "mapping": {"type": "long", "fields": { "raw": {"type": "long" } }}}},{"booleans": {"match_mapping_type": "boolean", "mapping": {"type": "boolean", "fields": { "raw": { "type": "boolean" }} }}},{"doubles": {"match_mapping_type": "double","mapping": {"type": "double","fields":{"raw": { "type": "double" } }}}},{ "dates": {"match_mapping_type": "date", "mapping": { "type": "date","fields": {"raw": { "type": "date" } } }}},{"strings": {"match_mapping_type": "string","mapping": {"type": "text","fielddata": true,"copy_to": "all_fields","analyzer": "cs_index_analyzer","search_analyzer": "cs_search_analyzer","fields": {"raw": {"type": "text","fielddata": true,"analyzer": "keylower"}}}}}],"properties": {"all_fields": {"type": "text","analyzer": "cs_index_analyzer","search_analyzer": "cs_search_analyzer","fields": { "raw": { "type": "text","analyzer": "keylower" } }} }}
}'
-curl -XPUT '{{sunbird-es-host}}:9200/searchindex/usercourses/_mapping?pretty' -H 'Content-Type: application/json' -d'
+curl -XPUT '{{sunbird_es_host}}:9200/searchindex/usercourses/_mapping?pretty' -H 'Content-Type: application/json' -d'
{
"usercourses" : { "dynamic_templates": [ {"longs": {"match_mapping_type": "long", "mapping": {"type": "long", "fields": { "raw": {"type": "long" } }}}},{"booleans": {"match_mapping_type": "boolean", "mapping": {"type": "boolean", "fields": { "raw": { "type": "boolean" }} }}},{"doubles": {"match_mapping_type": "double","mapping": {"type": "double","fields":{"raw": { "type": "double" } }}}},{ "dates": {"match_mapping_type": "date", "mapping": { "type": "date","fields": {"raw": { "type": "date" } } }}},{"strings": {"match_mapping_type": "string","mapping": {"type": "text","fielddata": true,"copy_to": "all_fields","analyzer": "cs_index_analyzer","search_analyzer": "cs_search_analyzer","fields": {"raw": {"type": "text","fielddata": true,"analyzer": "keylower"}}}}}],"properties": {"all_fields": {"type": "text","analyzer": "cs_index_analyzer","search_analyzer": "cs_search_analyzer","fields": { "raw": { "type": "text","analyzer": "keylower" } }} }}
}'
-curl -XPUT '{{sunbird-es-host}}:9200/searchindex/cbatch/_mapping?pretty' \
+curl -XPUT '{{sunbird_es_host}}:9200/searchindex/cbatch/_mapping?pretty' \
-H 'content-type: application/json' \
-d '{"dynamic":false,"properties":{"all_fields":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower"}},"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"countDecrementDate":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fielddata":true},"countDecrementStatus":{"type":"boolean","fields":{"raw":{"type":"boolean"}}},"countIncrementDate":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fielddata":true},"countIncrementStatus":{"type":"boolean","fields":{"raw":{"type":"boolean"}}},"courseCreator":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fielddata":true},"courseId":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fielddata":true},"batchId":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fielddata":true},"createdBy":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fielddata":true},"createdDate":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fielddata":true},"createdFor":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fielddata":true},"description":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fielddata":true},"endDate":{"type":"date","fields":{"raw":{"type":"date"}}},"enrollmentType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fielddata":true},"hashTagId":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fielddata":true},"id":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fielddata":true},"identifier":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fielddata":true},"mentors":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fielddata":true},"name":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fielddata":true},"startDate":{"type":"date","fields":{"raw":{"type":"date"}}},"status":{"type":"long","fields":{"raw":{"type":"long"}}},"updatedDate":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fielddata":true}}}'
diff --git a/ansible/roles/elasticsearch_old/defaults/main.yml b/ansible/roles/elasticsearch_old/defaults/main.yml
index d33c97d2b..32f111f44 100644
--- a/ansible/roles/elasticsearch_old/defaults/main.yml
+++ b/ansible/roles/elasticsearch_old/defaults/main.yml
@@ -7,7 +7,6 @@ es_logging_level: INFO
es_etc_path_data: /var/lib/elasticsearch
es_default_max_open_files: 65535
es_etc_index_number_of_shards: 1
-es_default_es_heap_size: 512m
es_version: "{{ es_release }}.{{es_minor_release}}"
backup_storage_name:
backup_storage_key:
diff --git a/ansible/roles/elasticsearch_old/tasks/debian.yml b/ansible/roles/elasticsearch_old/tasks/debian.yml
index 4e237570f..8a375e384 100644
--- a/ansible/roles/elasticsearch_old/tasks/debian.yml
+++ b/ansible/roles/elasticsearch_old/tasks/debian.yml
@@ -7,6 +7,12 @@
repo: 'deb http://packages.elasticsearch.org/elasticsearch/2.x/debian stable main'
state: present
+- name: Get information on the RAM
+ set_fact:
+ es_default_es_heap_size: "{% if ((ansible_memory_mb.real.total/(2*1024))|int) > 30000 %}26g{% else %}{{(((ansible_memory_mb.real.total/(2*1024))+1)|int)}}g{% endif %}"
+ when: es_default_es_heap_size is not defined
+
+
- name: Copy /etc/default/elasticsearch
template: src=elasticsearch dest=/etc/default/elasticsearch
tags: update_es_config
diff --git a/ansible/roles/es6/tasks/elasticsearch-config.yml b/ansible/roles/es6/tasks/elasticsearch-config.yml
index 987de718d..4f8ecc672 100644
--- a/ansible/roles/es6/tasks/elasticsearch-config.yml
+++ b/ansible/roles/es6/tasks/elasticsearch-config.yml
@@ -16,7 +16,6 @@
with_items:
- "{{data_dirs}}"
-
#Copy the config template
- name: Copy Configuration File
become: yes
@@ -59,6 +58,11 @@
template: src={{es_config_log4j2}} dest={{conf_dir}}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
notify: restart elasticsearch
+- name: Get information on the RAM
+ set_fact:
+ es_heap_size: "{% if ((ansible_memory_mb.real.total/(2*1024))|int) > 26 %}26g{% else %}{{(((ansible_memory_mb.real.total/(2*1024))+1)|int)}}g{% endif %}"
+ when: es_heap_size is not defined
+
- name: Copy jvm.options File for Instance
become: yes
template: src=jvm.options.j2 dest={{conf_dir}}/jvm.options owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
--
GitLab