Merge branch 'master' into release-2.0.0

ansible/artifacts/sunbird/login/info.ftl

+<#import "template.ftl" as layout>
+<@layout.registrationLayout displayMessage=false; section>
+    <#if section = "title">
+    ${message.summary}
+    <#elseif section = "form">
+    <div class="ui raised shadow container segment fullpage-background-image">
+        <div class="ui three column grid stackable">
+            <div class="ui column tablet only computer only"></div>
+            <div class="ui column height-fix">
+                <div id="kc-info-message">
+                    <p class="instruction">${message.summary}</p>
+                    <#if skipLink??>
+                    <#else>
+                        <#if pageRedirectUri??>
+                            <p><a href="${pageRedirectUri}">${msg("backToApplication")}</a></p>
+                        <#elseif client.baseUrl??>
+                            <p><a href="${client.baseUrl}">${msg("backToApplication")}</a></p>
+                        </#if>
+                    </#if>
+                </div>
+            </div>
+            <div class="ui column tablet only computer only"></div>
+        </div>
+    </div>
+    </#if>
+</@layout.registrationLayout>
\ No newline at end of file

ansible/inventory/env/group_vars/all.yml

@@ -84,8 +84,6 @@ kong_postgres_user: "{{core_vault_postgres_username}}"
 kong_postgres_password: "{{core_vault_postgres_password}}"
 enc_postgres_user: "{{core_vault_postgres_username}}"
 badger_postgres_user: "{{core_vault_postgres_username}}"
-backup_storage_account: "{{azure_account_name}}"
-backup_storage_key: "{{core_vault_sunbird_azure_storage_key}}"
 sunbird_account_name: "{{azure_account_name}}"
 sunbird_account_key: "{{core_vault_sunbird_azure_storage_key}}"
 ansible_vault_password: "{{ core_vault_ansible_vault_password }}"
@@ -550,7 +548,8 @@ config_service_enabled: false
 config_cache_ttl: 600
 
 #### backup storage secret
-backup_azure_storage_access_key: "{{core_vault_azure_storage_secret}}"
+backup_azure_storage_access_key: "{{core_vault_sunbird_azure_storage_key}}"
+backup_azure_storage_account_name: "{{azure_account_name}}"
 
 #es-backup
 app_es_snapshot_host: "{{ groups['es'][0] }}"

ansible/roles/docker-ce/defaults/main.yml

 ---
 # defaults file for docker-ce
 docker_version: "18.03.1*"
+

ansible/roles/keycloak/tasks/provision.yml

@@ -10,6 +10,12 @@
 - name: Copy service file to init.d
   template: src=keycloak-service.j2 dest=/etc/init.d/keycloak mode=0755
 
+- name: Installing unzip
+  become: yes
+  apt:
+    name: unzip
+    state: present
+
 - name: Adding ENV Vars to bashrc
   lineinfile: 
     path: '/etc/environment'

ansible/roles/sunbird-auth-provision/tasks/install-keycloak.yml

@@ -29,7 +29,7 @@
   file: path=/opt owner={{wildfly_user}} group={{wildfly_group}} mode=0755 recurse=yes
 
 - name: Paste sunbird auth package on server
-  copy: src="sunbird_auth_{{sunbird_auth_version}}.zip" dest="/opt/" force=no owner={{wildfly_user}} group={{wildfly_group}}
+  copy: src="../../../sunbird_auth_{{sunbird_auth_version}}.zip" dest="/opt/" force=no owner={{wildfly_user}} group={{wildfly_group}}
 
 - name: extarct the folder
   unarchive: src="/opt/sunbird_auth_{{sunbird_auth_version}}.zip" dest=/opt/ copy=no owner={{wildfly_user}} group={{wildfly_group}}

ansible/roles/sunbird-auth-provision/tasks/prerequisite.yml

@@ -9,3 +9,4 @@
 
 - name: Copy service file to init.d
   template: src=roles/sunbird-auth-provision/templates/keycloak-service.j2 dest=/etc/init.d/keycloak mode=0755
+

ansible/sunbird-auth-deploy.yml

 ---
-- hosts: all
+- hosts: keycloak
   become: true
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']

ansible/sunbird-auth-provision.yml

 ---
-- hosts: all
+- hosts: keycloak
   become: true
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']

pipelines/backup/jenkins-backup-upload/Jenkinsfile

+@Library('deploy-conf') _
+node() {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        stage('checkout public repo') {
+            cleanWs()
+            checkout scm
+        }
+
+        ansiColor('xterm') {
+            stage('deploy'){
+                values = [:]
+                currentWs = sh(returnStdout: true, script: 'pwd').trim()
+                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+                ansiblePlaybook = "${currentWs}/ansible/jenkins-backup.yml"
+                ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass"
+                values.put('currentWs', currentWs)
+                values.put('env', envDir)
+                values.put('module', module)
+                values.put('jobName', jobName)
+                values.put('ansiblePlaybook', ansiblePlaybook)
+                values.put('ansibleExtraArgs', ansibleExtraArgs)
+                println values
+                ansible_playbook_run(values)
+            }
+        }
+        currentBuild.result = 'SUCCESS'
+    }
+    catch (err) {
+        currentBuild.result = 'FAILURE'
+        slackSend (
+            channel: "${env.NOTIFY_SLACK_CHANNEL}",
+            color: 'danger',
+            message: "Build Failed - ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.BUILD_URL}|Open>)"
+        )
+        throw err
+    } finally {
+        step([$class: 'Mailer',
+            notifyEveryUnstableBuild: true,
+            recipients: "${env.NOTIFY_EMAIL_ADDRESS}",
+            sendToIndividuals: true])
+    }
+}

private_repo/ansible/inventory/dev/Core/common.yml

@@ -66,3 +66,5 @@ keycloak_ekstep_sunbird_login_theme_file: "login.tar.gz"
 keycloak_ekstep_sunbird_login_theme_dest: "/opt/keycloak/themes/sunbird/login"
 keycloak_postgresql: "postgresql-9.4.1212.jar"
 sunbird_auth_version: 1.0v
+
+

private_repo/ansible/inventory/dev/Core/hosts

@@ -18,35 +18,35 @@ swarm-agent-for-prometheus
 [swarm-manager:children]
 swarm-manager-1
 
-[keycloak-1]  
+[keycloak-1]
 18.0.0.9
 
 [keycloak:children]
 keycloak-1
 
-[log-es-1] 
-18.0.0.15 node_name=refactor-log-es-1 es_etc_node_master=true es_etc_node_data=true 
+[log-es-1]
+18.0.0.15 node_name=refactor-log-es-1 es_etc_node_master=true es_etc_node_data=true
 
 [log-es:children]
 log-es-1
 
-[swarm-agent-for-prometheus-1] 
-18.0.0.15 
+[swarm-agent-for-prometheus-1]
+18.0.0.15
 
 [swarm-agent-for-prometheus:children]
 swarm-agent-for-prometheus-1
 
-[swarm-agent-for-grafana-1] 
+[swarm-agent-for-grafana-1]
 18.0.0.15
 
 [swarm-agent-for-grafana:children]
 swarm-agent-for-grafana-1
 
-[swarm-agent-for-alertmanager-1] 
+[swarm-agent-for-alertmanager-1]
 18.0.0.15
 
 
-[es-1] 
+[es-1]
 18.0.0.9 es_instance_name=es-1 es_etc_node_master=true es_etc_node_data=true
 
 [es-backup:children]
@@ -58,20 +58,20 @@ log-es-1
 [es:children]
 es-1
 
-[cassandra-1] 
+[cassandra-1]
 18.0.0.9
 
 [cassandra:children]
 cassandra-1
 
-[postgresql-master-1] 
-18.0.0.9 
+[postgresql-master-1]
+18.0.0.9
 
 [postgresql-master:children]
 postgresql-master-1
 
-[postgresql-slave-1] 
-18.0.0.9 
+[postgresql-slave-1]
+18.0.0.9
 
 [postgresql-slave:children]
 postgresql-slave-1
@@ -81,7 +81,7 @@ postgresql-slave
 postgresql-master
 
 [kafka-1]
-18.0.0.13 kafka_id=1 
+18.0.0.13 kafka_id=1
 
 [processing-cluster-kafka]
 18.0.0.13

private_repo/ansible/inventory/dev/DataPipeline/common.yml

@@ -7,27 +7,21 @@ domain_name:
 ssh_public_key_deployer:  
 bootstrap_user: 
 bootstrap_key_path: 
-ansible_ssh_private_key_file: 
 
 #################### DP ##########################
 
-
-
-# Every where the value is telemetry-data-store
-# Shall we change the value to telemetry-data-store in dev also?
-# What's the implication
-# Is it azure blob or s3 bucket??
 # Is it same as `channel_data_exhaust_bucket`
-secor_azure_container_name: 
+secor_azure_container_name: ""
+secor_alerts_slack_channel: ""
 
 dp_azure_account_name:  # azure storage account name
-default_org_hash_id: # channel id of the organization
+default_org_hash_id: "" # channel id of the organization
 
 # Data products monitoring web hooks
-data_exhaust_webhook_url: ""
-data_exhaust_Channel: ""
-data_exhaust_name: 
-bucket: 
+data_exhaust_webhook_url: ""   ####Slack webhook
+data_exhaust_Channel: ""      ### salck channel for data products alerts
+data_exhaust_name:   
+bucket:   ####  same as secor_azure_container_name
 stream_base_url: ""
 
 # Media content
@@ -38,3 +32,11 @@ media_service_azure_account_name: ""
 media_service_azure_resource_group_name: ""
 media_service_azure_token_client_key: ""
 media_service_azure_token_client_secret: ""
+
+# kafka topic required by core #
+# Please do not change this #
+ingestion_kafka_topics:
+  - name: telemetry.ingestion
+    num_of_partitions: 4
+    replication_factor: 1
+    retention_time: 604800000

private_repo/ansible/inventory/dev/DataPipeline/hosts

@@ -4,6 +4,10 @@
 
 # Sunbird zookeeper
 [ingestion-cluster-zookeepers]
+15.0.0.8
+
+[core-cassandra]
+15.0.0.7
 
 
 ################# LP ##########################
@@ -108,7 +112,7 @@ dp
 
 [env:vars]
 ansible_ssh_user=deployer
-ansible_ssh_private_key_file=/run/secrets/deployer_ssh_key
+ansible_ssh_private_key_file=/var/lib/jenkins/secrets/deployer_ssh_key
 
 
 ########################### Monitoring ##########################
@@ -142,4 +146,4 @@ kafka-ps
 
 [all:vars]
 ansible_ssh_user=deployer
-ansible_ssh_private_key_file=/run/secrets/deployer_ssh_key
+ansible_ssh_private_key_file=/var/lib/jenkins/secrets/deployer_ssh_key

private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml

@@ -32,9 +32,11 @@ artifacts_container:
 
 
 neo4j_zip: neo4j-enterprise-3.3.9-unix.tar.gz       # neo4j enterprise version.zip present in the artifacts folder
+neo4j_home: "{{learner_user_home}}/{{neo4j_dir}}/neo4j-enterprise-3.3.9"   # Change this path to the version of neo4j you will be using
 neo4j_enterprise: "true"
 
-## Cassandra-backup
-cassandra_backup_azure_container_name: 
-default_org_hash_id: # channel id of the organization
+## backup
+backup_azure_storage_account_name: sunbirdbackupsdev
+### overriding backup secret
+backup_azure_storage_access_key: "{{lp_vault_backup_azure_storage_secret}}"
 

private_repo/ansible/inventory/dev/KnowledgePlatform/hosts

@@ -83,8 +83,8 @@ lp
 swarm-bootstrap-manager
 
 [env:vars]
-ansible_ssh_user="deployer"
-ansible_ssh_private_key_file=/run/secrets/deployer_ssh_key
+ansible_ssh_user=deployer
+ansible_ssh_private_key_file=/var/lib/jenkins/secrets/deployer_ssh_key
 
 ####################### Monitoring ########################
 [logstash-ps]
@@ -98,9 +98,9 @@ cassandra-ps
 15.0.2.7
 
 [process-exporter:vars]
-ansible_ssh_user="deployer"
-ansible_ssh_private_key_file=/run/secrets/deployer_ssh_key
+ansible_ssh_user=deployer
+ansible_ssh_private_key_file=/var/lib/jenkins/secrets/deployer_ssh_key
 
 [all:vars]
 ansible_ssh_user=deployer
-ansible_ssh_private_key_file=/run/secrets/deployer_ssh_key
+ansible_ssh_private_key_file=/var/lib/jenkins/secrets/deployer_ssh_key