Sprint3 to master branch merge (#372)

* Update artifacts-download.yml

* Update artifacts-upload.yml

* Update all.yml

* Update common.yml

* Update common.yml

* Update common.yml

* Update all.yml

* Issue #000: change keycloak provision playbook

* Issue #000: change keycloak role to import

* Issue #000: change hosts to keycloak

* Issue #000: typo in items

* Issue #000: change from import to include

* Issue #000: change back to include

* Issue #000: Removed tags

* Update keycloak provision

* Update keycloak provision playbook

* Update keycloak provision playbook

* Update keycloak provision

* Update keycloak provision

* Update keycloak provision

* Update keycloak provision

* Update keycloak deploy

* Revert changes to Jenkinsfile

* Revert changes to provision keycloak

* Issue #000 keycloak: deploy build package to keyclok server with custom changes

* Issue #000 keycloak: installing unzip to extract keycloak

* Issue #000: updated jobs setup for env order

* Issue #000 feat: Moving cassandra repo url and key (#361)

* Issue #000 feat: Moving cassandra repo url and key

* Updated jenkins config, variables, templates, setup scripts (#367)

* feat #000: Updated jenkins config files

* feat #000: Removed test file

* Revert adminutils config

* Revert "Revert adminutils config"

This reverts commit b055da7383d14cbc6fc0ebb98a180eabde07d410.

* Revert "Revert "Revert adminutils config""

This reverts commit 46c72250d556bd51087dff8fa63ca33f74377fb9.

* Revert "Revert adminutils config"

This reverts commit b055da7383d14cbc6fc0ebb98a180eabde07d410.

* Revert "feat #000: Removed test file"

This reverts commit 49936eb13c39fef129ef9552e9f6a4313e3c9838.

* Revert "feat #000: Updated jenkins config files"

This reverts commit 69af673dfd771ef14dbb37fef81403664492bd21.

* Revert "Issue #000: updated jobs setup for env order"

This reverts commit 9573bdd9.

* Revert "Revert "Issue #000: updated jobs setup for env order""

This reverts commit 2965328f592513517aa8c1d9ede741c2e63fdba8.

* feat: Added updated jenkins config files

* Issue #00: Changed incorrect update for neo4j branch

* Removed auto trigger for deploy jobs

* feat: Remove all downstream trigger for deploy

* fix: Typo names in variable reference

* fix: Updated jenkins secrets path

* fix: Adding kafka topic, empty quotes for not mandatory vars

* fix: Adding neo4j home, this value will be removed from all.yml

* fix: Adding core cassandra group and dummy IP's

* chore: uniformity - without quotes for deployer

* feat: Removed meta, adding azcli to jenkins setup

* fix: remove unused variable (#380)

ansible/artifacts-download.yml

@@ -4,7 +4,7 @@
   vars_files:
     - "{{inventory_dir}}/secrets.yml"
   environment:
-    AZURE_STORAGE_ACCOUNT: "{{ azure_account_name }}"
-    AZURE_STORAGE_KEY: "{{ azure_account_key }}"
+    AZURE_STORAGE_ACCOUNT: "{{ artifact_azure_account_name }}"
+    AZURE_STORAGE_KEY: "{{ artifact_azure_account_key }}"
   roles:
     - artifacts-download-azure

ansible/artifacts-upload.yml

@@ -4,7 +4,7 @@
   vars_files:
     - "{{inventory_dir}}/secrets.yml"
   environment:
-    AZURE_STORAGE_ACCOUNT: "{{ azure_account_name }}"
-    AZURE_STORAGE_KEY: "{{ azure_account_key }}"
+    AZURE_STORAGE_ACCOUNT: "{{ artifact_azure_account_name }}"
+    AZURE_STORAGE_KEY: "{{ artifact_azure_account_key }}"
   roles:
     - artifacts-upload-azure

ansible/inventory/env/group_vars/all.yml

@@ -18,6 +18,10 @@ sunbird_env: "{{env}}"                   #Ekstep environment to connect to. Use
 sunbird_app_name: "{{env}}"
 azure_account_key: "{{core_vault_sunbird_azure_storage_key}}"
 
+#artifact upload
+artifact_azure_account_name: "{{core_vault_sunbird_azure_storage_account}}"
+artifact_azure_account_key: "{{core_vault_sunbird_azure_storage_key}}"
+
 # Keycloak
 keycloak_api_management_user_first_name: "admin"
 keycloak_api_management_user_last_name: "admin"
@@ -72,8 +76,6 @@ kong_postgres_user: "{{core_vault_postgres_username}}"
 kong_postgres_password: "{{core_vault_postgres_password}}"
 enc_postgres_user: "{{core_vault_postgres_username}}"
 badger_postgres_user: "{{core_vault_postgres_username}}"
-backup_storage_account: "{{azure_account_name}}"
-backup_storage_key: "{{core_vault_sunbird_azure_storage_key}}"
 sunbird_account_name: "{{azure_account_name}}"
 sunbird_account_key: "{{core_vault_sunbird_azure_storage_key}}"
 ansible_vault_password: "{{ core_vault_ansible_vault_password }}"
@@ -538,7 +540,8 @@ config_service_enabled: false
 config_cache_ttl: 600
 
 #### backup storage secret
-backup_azure_storage_access_key: "{{core_vault_azure_storage_secret}}"
+backup_azure_storage_access_key: "{{core_vault_sunbird_azure_storage_key}}"
+backup_azure_storage_account_name: "{{azure_account_name}}"
 
 #es-backup
 app_es_snapshot_host: "{{ groups['es'][0] }}"

ansible/roles/artifacts-download-azure/meta/main.yml

-dependencies:
-  - azure-cli

ansible/roles/artifacts-upload-azure/meta/main.yml

-dependencies:
-  - azure-cli

ansible/roles/cassandra/defaults/main.yml

-user_home: /home/deployer
\ No newline at end of file
+user_home: /home/deployer
+cassandra_repo_key: 'https://www.apache.org/dist/cassandra/KEYS'
+cassandra_repo: 'deb http://www.apache.org/dist/cassandra/debian 39x main'

ansible/roles/keycloak/tasks/provision.yml

@@ -10,6 +10,12 @@
 - name: Copy service file to init.d
   template: src=keycloak-service.j2 dest=/etc/init.d/keycloak mode=0755
 
+- name: Installing unzip
+  become: yes
+  apt:
+    name: unzip
+    state: present
+
 - name: Adding ENV Vars to bashrc
   lineinfile: 
     path: '/etc/environment'

ansible/roles/sunbird-auth-provision/tasks/install-keycloak.yml

@@ -29,7 +29,7 @@
   file: path=/opt owner={{wildfly_user}} group={{wildfly_group}} mode=0755 recurse=yes
 
 - name: Paste sunbird auth package on server
-  copy: src="sunbird_auth_{{sunbird_auth_version}}.zip" dest="/opt/" force=no owner={{wildfly_user}} group={{wildfly_group}}
+  copy: src="../../../sunbird_auth_{{sunbird_auth_version}}.zip" dest="/opt/" force=no owner={{wildfly_user}} group={{wildfly_group}}
 
 - name: extarct the folder
   unarchive: src="/opt/sunbird_auth_{{sunbird_auth_version}}.zip" dest=/opt/ copy=no owner={{wildfly_user}} group={{wildfly_group}}

ansible/roles/sunbird-auth-provision/tasks/prerequisite.yml

@@ -9,3 +9,4 @@
 
 - name: Copy service file to init.d
   template: src=roles/sunbird-auth-provision/templates/keycloak-service.j2 dest=/etc/init.d/keycloak mode=0755
+

ansible/sunbird-auth-deploy.yml

 ---
-- hosts: all
+- hosts: keycloak
   become: true
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']

ansible/sunbird-auth-provision.yml

 ---
-- hosts: all
+- hosts: keycloak
   become: true
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']

deploy/jenkins/jenkins-jobs-setup.sh

@@ -22,6 +22,7 @@ setupJobs(){
       find $JENKINS_TMP/Deploy/jobs/${arr[0]} -type f -name config.xml -exec sed -i "s#ArtifactUpload/dev/#ArtifactUpload/${arr[0]}/#g" {} \;
       find $JENKINS_TMP/Deploy/jobs/${arr[0]} -type f -name config.xml -exec sed -i "s#Deploy/dev/#Deploy/${arr[0]}/#g" {} \;
    fi
+   find $JENKINS_TMP/Deploy/jobs/${arr[0]} -type d -path "*Summary*" -prune -o -name config.xml -exec sed -i 's/<upstreamProjects>.*//g' {} \;
    echo -e "\e[0;33m${bold}Jobs created for ${arr[0]}${normal}"
 
    for key in "${!arr[@]}"; do
@@ -31,8 +32,8 @@ setupJobs(){
       cp -r $JENKINS_TMP/Provision/jobs/${arr[0]} $JENKINS_TMP/Provision/jobs/${arr[$key]}
       cp -r $JENKINS_TMP/OpsAdministration/jobs/${arr[0]} $JENKINS_TMP/OpsAdministration/jobs/${arr[$key]}
       cp -r $JENKINS_TMP/Deploy/jobs/${arr[0]} $JENKINS_TMP/Deploy/jobs/${arr[$key]}
-      find $JENKINS_TMP/Deploy/jobs/${arr[$key]} -type f -name config.xml -exec sed -i "s#ArtifactUpload/${arr[0]}/#Deploy/${arr[$(($key - 1))]}/#g" {} \;
-      find $JENKINS_TMP/Deploy/jobs/${arr[$key]} -type d -path "*Summary*" -prune -o -type f -name config.xml -exec sed -i "/<triggers>/,/<\/triggers>/d" {} \;
+      find $JENKINS_TMP/Deploy/jobs/${arr[$key]} -type f -name config.xml -exec bash -c 'configPath=$0; jobPath=$(dirname $configPath); jobName=$(basename $jobPath); modulePath=${jobPath%/*/*}; moduleName=$(basename $modulePath); sed -i "s#ArtifactUpload/$1/$moduleName/.*<#Deploy/$2/$moduleName/$jobName<#g" $0' {} ${arr[0]} ${arr[$(($key - 1))]} \;
+      find $JENKINS_TMP/Deploy/jobs/${arr[$key]} -type d -path "*Summary*" -prune -o -name config.xml -exec sed -i 's/<upstreamProjects>.*//g' {} \;
       find $JENKINS_TMP/Deploy/jobs/${arr[$key]}/jobs/Summary/jobs/DeployedVersions -type f -name config.xml -exec sed -i "s#Deploy/${arr[0]}/#Deploy/${arr[$key]}/#g" {} \;
       echo -e "\e[0;33m${bold}Jobs created for ${arr[$key]}${normal}"
    done

deploy/jenkins/jenkins-server-setup.sh

@@ -55,6 +55,17 @@ ln -s /usr/local/lib/node-v6.1.0-linux-x64/bin/gulp /usr/bin/gulp
 echo -e "\n\e[0;32m${bold}Installating Ansible${normal}"
 pip install ansible==2.5.0
 
+echo -e "\n\e[0;32m${bold}Installating azure cli${normal}"
+apt-get install curl apt-transport-https lsb-release gpg
+curl -sL https://packages.microsoft.com/keys/microsoft.asc | \
+    gpg --dearmor | \
+    sudo tee /etc/apt/trusted.gpg.d/microsoft.asc.gpg > /dev/null
+AZ_REPO=$(lsb_release -cs)
+echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $AZ_REPO main" | \
+    sudo tee /etc/apt/sources.list.d/azure-cli.list
+sudo apt-get update
+sudo apt-get install azure-cli
+
 echo -e "\n\e[0;32m${bold}Installating Docker-py${normal}"
 pip install docker-py
 

deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/AdminUtils/config.xml

@@ -7,23 +7,31 @@
         <string>com.sonyericsson.rebuild.RebuildSettings</string>
       </jobPropertyDescriptors>
     </org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction>
+    <org.jenkinsci.plugins.pipeline.modeldefinition.actions.DeclarativeJobAction plugin="pipeline-model-definition@1.3.4.1"/>
+    <org.jenkinsci.plugins.pipeline.modeldefinition.actions.DeclarativeJobPropertyTrackerAction plugin="pipeline-model-definition@1.3.4.1">
+      <jobProperties/>
+      <triggers/>
+      <parameters/>
+      <options/>
+    </org.jenkinsci.plugins.pipeline.modeldefinition.actions.DeclarativeJobPropertyTrackerAction>
   </actions>
   <description></description>
   <keepDependencies>false</keepDependencies>
   <properties>
-    <jenkins.model.BuildDiscarderProperty>
-      <strategy class="hudson.tasks.LogRotator">
-        <daysToKeep>-1</daysToKeep>
-        <numToKeep>-1</numToKeep>
-        <artifactDaysToKeep>-1</artifactDaysToKeep>
-        <artifactNumToKeep>5</artifactNumToKeep>
-      </strategy>
-    </jenkins.model.BuildDiscarderProperty>
-    <org.jenkinsci.plugins.workflow.job.properties.DisableConcurrentBuildsJobProperty/>
-    <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.29">
-      <autoRebuild>false</autoRebuild>
-      <rebuildDisabled>false</rebuildDisabled>
-    </com.sonyericsson.rebuild.RebuildSettings>
+    <org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+      <triggers>
+        <jenkins.triggers.ReverseBuildTrigger>
+          <spec></spec>
+          <upstreamProjects>/Build/Core/AdminUtils</upstreamProjects>
+          <threshold>
+            <name>SUCCESS</name>
+            <ordinal>0</ordinal>
+            <color>BLUE</color>
+            <completeBuild>true</completeBuild>
+          </threshold>
+        </jenkins.triggers.ReverseBuildTrigger>
+      </triggers>
+    </org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
     <hudson.model.ParametersDefinitionProperty>
       <parameterDefinitions>
         <hudson.model.StringParameterDefinition>
@@ -51,6 +59,19 @@ ArtifactRepo - Push the docker image to container registry.
         </hudson.model.ChoiceParameterDefinition>
       </parameterDefinitions>
     </hudson.model.ParametersDefinitionProperty>
+    <jenkins.model.BuildDiscarderProperty>
+      <strategy class="hudson.tasks.LogRotator">
+        <daysToKeep>-1</daysToKeep>
+        <numToKeep>-1</numToKeep>
+        <artifactDaysToKeep>-1</artifactDaysToKeep>
+        <artifactNumToKeep>5</artifactNumToKeep>
+      </strategy>
+    </jenkins.model.BuildDiscarderProperty>
+    <org.jenkinsci.plugins.workflow.job.properties.DisableConcurrentBuildsJobProperty/>
+    <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.29">
+      <autoRebuild>false</autoRebuild>
+      <rebuildDisabled>false</rebuildDisabled>
+    </com.sonyericsson.rebuild.RebuildSettings>
     <hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.0.1">
       <maxConcurrentPerNode>0</maxConcurrentPerNode>
       <maxConcurrentTotal>0</maxConcurrentTotal>
@@ -59,21 +80,8 @@ ArtifactRepo - Push the docker image to container registry.
       <throttleOption>project</throttleOption>
       <limitOneJobWithMatchingParams>false</limitOneJobWithMatchingParams>
       <paramsToUseForLimit></paramsToUseForLimit>
+      <configVersion>1</configVersion>
     </hudson.plugins.throttleconcurrents.ThrottleJobProperty>
-    <org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
-      <triggers>
-        <jenkins.triggers.ReverseBuildTrigger>
-          <spec></spec>
-          <upstreamProjects>/Build/Core/AdminUtils</upstreamProjects>
-          <threshold>
-            <name>SUCCESS</name>
-            <ordinal>0</ordinal>
-            <color>BLUE</color>
-            <completeBuild>true</completeBuild>
-          </threshold>
-        </jenkins.triggers.ReverseBuildTrigger>
-      </triggers>
-    </org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
   </properties>
   <definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.63">
     <scm class="hudson.plugins.git.GitSCM" plugin="git@3.9.3">

deploy/jenkins/jobs/Build/jobs/Core/jobs/Config/config.xml

@@ -42,6 +42,7 @@
       <throttleOption>project</throttleOption>
       <limitOneJobWithMatchingParams>false</limitOneJobWithMatchingParams>
       <paramsToUseForLimit></paramsToUseForLimit>
+      <configVersion>1</configVersion>
     </hudson.plugins.throttleconcurrents.ThrottleJobProperty>
     <org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
       <triggers>
@@ -80,5 +81,5 @@
     <lightweight>false</lightweight>
   </definition>
   <triggers/>
-  <disabled>false</disabled>
+  <disabled>true</disabled>
 </flow-definition>
\ No newline at end of file

deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/FuntionalTestCases/config.xml

@@ -32,6 +32,8 @@
       <rebuildDisabled>false</rebuildDisabled>
     </com.sonyericsson.rebuild.RebuildSettings>
     <hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.0.1">
+      <maxConcurrentPerNode>0</maxConcurrentPerNode>
+      <maxConcurrentTotal>0</maxConcurrentTotal>
       <categories class="java.util.concurrent.CopyOnWriteArrayList"/>
       <throttleEnabled>false</throttleEnabled>
       <throttleOption>project</throttleOption>

deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/KeycloakRealm/config.xml

@@ -25,6 +25,8 @@
       <rebuildDisabled>false</rebuildDisabled>
     </com.sonyericsson.rebuild.RebuildSettings>
     <hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.0.1">
+      <maxConcurrentPerNode>0</maxConcurrentPerNode>
+      <maxConcurrentTotal>0</maxConcurrentTotal>
       <categories class="java.util.concurrent.CopyOnWriteArrayList"/>
       <throttleEnabled>false</throttleEnabled>
       <throttleOption>project</throttleOption>
@@ -45,7 +47,7 @@
       </userRemoteConfigs>
       <branches>
         <hudson.plugins.git.BranchSpec>
-          <name>keycloak-deployment</name>
+          <name>${public_repo_branch}</name>
         </hudson.plugins.git.BranchSpec>
       </branches>
       <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>

deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/OnboardConsumers/config.xml

@@ -18,6 +18,8 @@
       <rebuildDisabled>false</rebuildDisabled>
     </com.sonyericsson.rebuild.RebuildSettings>
     <hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.0.1">
+      <maxConcurrentPerNode>0</maxConcurrentPerNode>
+      <maxConcurrentTotal>0</maxConcurrentTotal>
       <categories class="java.util.concurrent.CopyOnWriteArrayList"/>
       <throttleEnabled>false</throttleEnabled>
       <throttleOption>project</throttleOption>

deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsFetchLogs/config.xml

@@ -89,5 +89,5 @@
     <lightweight>false</lightweight>
   </definition>
   <triggers/>
-  <disabled>true</disabled>
+  <disabled>false</disabled>
 </flow-definition>
\ No newline at end of file

deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsReplayJobs/config.xml

@@ -109,5 +109,5 @@
     <lightweight>false</lightweight>
   </definition>
   <triggers/>
-  <disabled>true</disabled>
+  <disabled>false</disabled>
 </flow-definition>
\ No newline at end of file