diff --git a/ansible/elasticsearch-api-query.yml b/ansible/elasticsearch-api-query.yml index ae4e7648365395312801d309edf438c3551b086c..884c7fe6f92df4647562f0c76d86090d01d36fe5 100644 --- a/ansible/elasticsearch-api-query.yml +++ b/ansible/elasticsearch-api-query.yml @@ -1,9 +1,20 @@ -- hosts: log-es-1 +- hosts: log-es-2 vars_files: - ['{{inventory_dir}}/secrets.yml'] - tasks: + become: yes + roles: + - azure-cli + tasks: + - name: install jq + apt: + name: apt + state: present + - name: Copy the elasticsearch api query Shell script + template: + src: static-files/api_count_query.sh + dest: /tmp/ + mode: 0775 - name: Running the shell script to get the data from elasticsearch - command: ./api_count.sh "{{ mail_server_username }}" "{{ core_vault_mail_server_password }}" "{{ analytics_report_container_name }}" "{{ analytics_report_azure_account_name }}" "{{ core_vault_analytics_report_azure_account_key }}" + command: ./api_count_query.sh "{{ mail_server_username }}" "{{ core_vault_mail_server_password }}" "{{ analytics_report_container_name }}" "{{ analytics_report_azure_account_name }}" "{{ core_vault_analytics_report_azure_account_key }}" args: - chdir: /home/ops - become: yes + chdir: /tmp diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml index 839e4e702b9eb7519d0efc7979e82d0b12a546e5..a43301dfb04f2a06e324ee0dde47a7edc7969de5 100644 --- a/ansible/inventory/env/group_vars/all.yml +++ b/ansible/inventory/env/group_vars/all.yml @@ -667,4 +667,7 @@ job_names: lms.sso-account-updater_1: job_file_name: 'sso-account-updater' lms.indexer_1: - job_file_name: 'indexer' + job_file_name: 'indexer' + +### api call report mailing list #### +api_report_mailing_list: "" ## This mailing list to send the daily api count report. if adaptor want to use they have to override in common.yml diff --git a/ansible/roles/stack-proxy/templates/proxy-default.conf b/ansible/roles/stack-proxy/templates/proxy-default.conf index 4374009c55aae0b325ef7b35f5d3992afcccac42..8152bb168649166a170f44313653404a08d20502 100644 --- a/ansible/roles/stack-proxy/templates/proxy-default.conf +++ b/ansible/roles/stack-proxy/templates/proxy-default.conf @@ -111,6 +111,11 @@ server { proxy_send_timeout 60; proxy_read_timeout 70; proxy_http_version 1.1; + header_filter_by_lua_block { + local h = ngx.req.get_headers() + + ngx.log(ngx.WARN, "Deviceid: ", h["x-device-id"], " Channelid: ", h["x-channel-id"], " Appid: ", h["x-app-id"]) + } proxy_pass http://kong; } diff --git a/ansible/static-files/api_count_query.sh b/ansible/static-files/api_count_query.sh new file mode 100755 index 0000000000000000000000000000000000000000..ce22a25494cfa6a07d5ce524bfd1d4073b5c176e --- /dev/null +++ b/ansible/static-files/api_count_query.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +###-----------------------------------------------------### +# Author:: Kaliraja +# Description:: This script is to query the api calls data +# from log-es and send as a report to email and uploading +# the same to azure storage. +###-----------------------------------------------------### + + +#date variables +prev_day=`date "+%s" -d "yesterday 03:30:00"` +today=`date "+%s" -d "today 03:30:00"` +date=`date +"%m-%d-%Y"` + +#prev_day=`date "+%s" -d "yesterday -7 day 03:30:00"` +#today=`date "+%s" -d "yesterday -6 day 03:30:00"` +#date=`date +%m-%d-%y --date="yesterday -6 day" | sed 's/19/2019/'` + +#api variable +contentsearch="/api/composite/v1/search" +contentread="/api/content/v1/read/" +telemetry="/api/data/v1/telemetry" +registermobile="/api/api-manager/v1/consumer/mobile_device/credential/register" + +#filename variable +contentsearch_filename=contentsearch-$date.txt +contentread_filename=contentread-$date.txt +telemetry_filename=telemetry-$date.txt +mobiledevice_registerfilename=registermobile-$date.txt + +#sedngrid variable +sguser="$1" +sgpass="$2" +container_name="$3" +account_name="$4" +storage_key="$5" + + +query(){ + curl -H 'Content-Type:application/json' -s -XPOST 'localhost:9200/logstash-*/_search?pretty' -d '{"query":{"bool":{"must":{"query_string":{"analyze_wildcard":true,"query":"\"'$1'\""}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"'"$prev_day"'","lte":"'"$today"'","format":"epoch_second"}}}],"must_not":[]}}}},"size":0,"aggs":{"2":{"date_histogram":{"field":"@timestamp","interval":"15m","time_zone":"Asia/Kolkata","min_doc_count":1,"extended_bounds":{"min": 0,"max": 500}}}}}' | jq -r '.aggregations."2".buckets[]|.key_as_string+" "+ (.doc_count|tostring)' | column -t > $2 +} + +#Executing content search query + +query $contentsearch $contentsearch_filename + +#Execurting the contentread query + +query $contentread $contentread_filename + +#Executing the telemetry query + +query $telemetry $telemetry_filename + +#Executing the registermobiledevice query + +query $registermobile $mobiledevice_registerfilename + +#sending an email with an attachment + +curl https://api.sendgrid.com/api/mail.send.json \ + {{ api_report_mailing_list }} -F subject="Data for Diksha api calls" \ + -F text="Data" --form-string html="<strong>Hi Team, PFA.</strong>" \ + -F from=reports@diksha.in -F api_user="$sguser" -F api_key="$sgpass" \ + -F files\[contentsearch.txt\]=@contentsearch-$date.txt -F files\[contentread.txt\]=@contentread-$date.txt -F files\[telemetry.txt]=@telemetry-$date.txt -F files\[registermobile.txt]=@registermobile-$date.txt + + +# uploading the reports to storage + +az storage blob upload \ +--container-name $container_name \ +--file contentsearch-$date.txt \ +--name contentsearch-$date.txt \ +--account-name $account_name \ +--account-key $storage_key + +az storage blob upload \ +--container-name $container_name \ +--file contentread-$date.txt \ +--name contentread-$date.txt \ +--account-name $account_name \ +--account-key $storage_key + + +az storage blob upload \ +--container-name $container_name \ +--file telemetry-$date.txt \ +--name telemetry-$date.txt \ +--account-name $account_name \ +--account-key $storage_key + +az storage blob upload \ +--container-name $container_name \ +--file registermobile-$date.txt \ +--name registermobile-$date.txt \ +--account-name $account_name \ +--account-key $storage_key + + +# deleting files + +rm *-$date.txt