From 3d52b12b04ecf83a9777ffee0d3d46cc37958347 Mon Sep 17 00:00:00 2001
From: Keshav Prasad <msknext@gmail.com>
Date: Tue, 9 Jul 2019 17:41:35 +0530
Subject: [PATCH] fix: codebase from 2.1.0 (#604)
* Issue #000 feat: postgres restore role
* feat: Adding vars of relase-2.0.0 for adopters
* feat: vars of relase-2.0.0 and 1.15 for adopters
* Release 2.0 and 1.15 vars for adopters (#580)
* feat: Adding vars of relase-2.0.0 for adopters
* fix: Fixing jenkins version and Yarn config file
* fix: Fixing private_Repo vars
* fix: Adding new host group for monitoring and logging
* Create Jenkinsfile
* Create plugin.yml
* Update plugin.yml (#591)
* fix: updating badger and apimanager configs (#593)
* fix: Adding redis group
* Issue #000 feat: updated postgres restore role
* Issue #000 feat: added postgres restore jenkins file
* fix: added missing vars
* fix: updating base url value
---
ansible/inventory/env/group_vars/all.yml | 3 +-
ansible/plugin.yml | 22 ++++++
ansible/postgresql-restore.yml | 6 +-
.../defaults/main.yml | 14 ++++
.../tasks/main.yml | 70 +++++++++++++++++++
ansible/roles/stack-sunbird/defaults/main.yml | 1 +
deploy/jenkins/jenkins-server-setup.sh | 2 +-
.../jobs/Core/jobs/APIManager/config.xml | 38 +++++-----
.../Build/jobs/Core/jobs/Badger/config.xml | 65 ++++++-----------
.../jobs/DataPipeline/jobs/Yarn/config.xml | 24 ++++---
.../backup/postgresql-restore/Jenkinsfile | 2 +-
.../org_sunbird_questionunit_quml/Jenkinsfile | 66 +++++++++++++++++
.../ansible/inventory/dev/Core/common.yml | 5 +-
private_repo/ansible/inventory/dev/Core/hosts | 32 +++++++--
.../ansible/inventory/dev/Core/secrets.yml | 4 +-
.../ansible/inventory/dev/DataPipeline/hosts | 13 +++-
.../dev/KnowledgePlatform/common.yml | 2 +-
.../inventory/dev/KnowledgePlatform/hosts | 5 +-
18 files changed, 282 insertions(+), 92 deletions(-)
create mode 100644 ansible/plugin.yml
create mode 100644 ansible/roles/postgres-azure-managed-service-restore/defaults/main.yml
create mode 100644 ansible/roles/postgres-azure-managed-service-restore/tasks/main.yml
create mode 100644 pipelines/deploy/org_sunbird_questionunit_quml/Jenkinsfile
diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml
index 941d617ad..17b712b29 100644
--- a/ansible/inventory/env/group_vars/all.yml
+++ b/ansible/inventory/env/group_vars/all.yml
@@ -335,7 +335,8 @@ sunbird_open_saber_bridge_enable: 'false'
sunbird_content_service_enable_logging: 'true'
sunbird_language_service_api_key: "{{core_vault_sunbird_ekstep_api_key}}"
sunbird_installation_display_name: "{{sunbird_app_name}} {{env}}"
-sunbird_ekstep_api_base_url: "https://{{ekstep_s3_env}}.ekstep.in/api" #API base URL of the Ekstep environment. Use `https://qa.ekstep.in/api` for non-prod deployments, and use `https://api.ekstep.in/` for prod deployment.
+sunbird_ekstep_proxy_base_url: "https://{{ekstep_s3_env}}.ekstep.in" #Base URL of the Ekstep environment. Use `https://qa.ekstep.in/` for non-prod deployments, and `https://community.ekstep.in/` for prod deployment.
+sunbird_ekstep_api_base_url: "{{sunbird_ekstep_proxy_base_url}}/api" #API base URL of the Ekstep environment. Use `https://qa.ekstep.in/api` for non-prod deployments, and use `https://api.ekstep.in/` for prod deployment.
sunbird_language_service_api_base_url: '{{sunbird_ekstep_api_base_url}}/language'
kong_version: 1.5.0-gold
diff --git a/ansible/plugin.yml b/ansible/plugin.yml
new file mode 100644
index 000000000..2276d95b9
--- /dev/null
+++ b/ansible/plugin.yml
@@ -0,0 +1,22 @@
+- hosts: local
+ become: yes
+ gather_facts: no
+ vars_files:
+ - "{{inventory_dir}}/secrets.yml"
+ environment:
+ AZURE_STORAGE_ACCOUNT: "{{ azure_plugin_storage_account_name }}"
+ AZURE_STORAGE_KEY: "{{ azure_plugin_storage_account_key }}"
+ tasks:
+ - name: delte plugin org_sunbird_questionunit_quml
+ command: "az storage blob delete-batch --source {{ plugin_container_name }} --pattern content-plugins/{{ plugins_name }}"
+ async: 3600
+ poll: 10
+ tags:
+ - org_sunbird_questionunit_quml
+
+ - name: upload plugin org_sunbird_questionunit_quml
+ command: "az storage blob upload-batch --destination {{ plugin_container_name }}/content-plugins/{{ plugins_name }} --source {{ source_file }}"
+ async: 3600
+ poll: 10
+ tags:
+ - org_sunbird_questionunit_quml
diff --git a/ansible/postgresql-restore.yml b/ansible/postgresql-restore.yml
index 7faf9af55..e2d80770d 100644
--- a/ansible/postgresql-restore.yml
+++ b/ansible/postgresql-restore.yml
@@ -1,8 +1,8 @@
-- hosts: postgresql-restore
+- hosts: local
become: yes
vars_files:
- - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
+ - ['{{inventory_dir}}/secrets.yml']
roles:
- - postgresql-restore
+ - postgres-azure-managed-service-restore
tags:
- postgresql-restore
diff --git a/ansible/roles/postgres-azure-managed-service-restore/defaults/main.yml b/ansible/roles/postgres-azure-managed-service-restore/defaults/main.yml
new file mode 100644
index 000000000..f90d20c9f
--- /dev/null
+++ b/ansible/roles/postgres-azure-managed-service-restore/defaults/main.yml
@@ -0,0 +1,14 @@
+postgresql_restore_dir: /tmp/postgres-restore
+postgres_backup_azure_container_name: postgresql-backup
+
+db:
+ name: ['keycloak', 'api_manager_{{ postgres_env }}', 'badger', 'quartz']
+ role: ['keycloak', 'api_manager_{{ postgres_env }}', 'badger', 'quartz']
+ user: ['azure_superuser, {{ env_user_name }}']
+
+#these variables are passed as extra vars
+postgres_backup_filename:
+postgres_user:
+postgres_password:
+postgres_hostname:
+postgres_env:
diff --git a/ansible/roles/postgres-azure-managed-service-restore/tasks/main.yml b/ansible/roles/postgres-azure-managed-service-restore/tasks/main.yml
new file mode 100644
index 000000000..ba413e794
--- /dev/null
+++ b/ansible/roles/postgres-azure-managed-service-restore/tasks/main.yml
@@ -0,0 +1,70 @@
+- name: install psycopg2
+ package:
+ name: python-psycopg2
+ state: present
+
+- name: ensure restore dir exists
+ file: path="{{ postgresql_restore_dir }}" state=directory mode=0777
+
+- set_fact:
+ postgres_backup_filepath: "{{ postgresql_restore_dir }}/{{ postgres_backup_filename }}"
+
+- name: Download backup from azure
+ command: az storage blob download -c {{ postgres_backup_azure_container_name }} --name {{ postgres_backup_filename }} -f {{ postgres_backup_filepath }}
+ args:
+ chdir: "{{ postgres_restore_dir }}"
+ async: 100
+ poll: 10
+
+- name: unarchive artifact
+ unarchive: src={{ postgresql_restore_dir }}/{{ postgres_backup_filename }} dest={{ postgresql_restore_dir }}/ copy=no
+
+- name: Create db's
+ postgresql_db:
+ login_user: "{{ postgres_user }}"
+ login_password: "{{ postgres_password }}"
+ login_host: "{{ postgres_hostname }}"
+ name: "{{ item }}"
+ state: present
+ with_items: "{{ db.name }}"
+ async: 1000
+ poll: 10
+
+- name: Create role and grant access to db's
+ postgresql_user:
+ login_user: "{{ postgres_user }}"
+ login_password: "{{ postgres_password }}"
+ login_host: "{{ postgres_hostname }}"
+ db: "{{ item[0] }}"
+ name: "{{ item[1] }}"
+ priv: ALL
+ state: present
+ role_attr_flags: CREATEROLE
+ with_nested:
+ - "{{ db.name }}"
+ - "{{ db.role }}"
+ async: 1000
+ poll: 10
+
+- name: create user
+ postgresql_user:
+ login_user: "{{ postgres_user }}"
+ login_password: "{{ postgres_password }}"
+ login_host: "{{ postgres_hostname }}"
+ name: "{{ item }}"
+ with_items: "{{ db.user }}"
+ async: 1000
+ poll: 10
+
+- name: Restore db's
+ postgresql_db:
+ login_user: "{{ postgres_user }}"
+ login_password: "{{ postgres_password }}"
+ login_host: "{{ postgres_hostname }}"
+ name: "{{ item }}"
+ state: restore
+ target: "{{ item }}.sql"
+ args:
+ chdir: "{{ postgres_restore_dir }}"
+ with_items: "{{ db.name }}"
+
diff --git a/ansible/roles/stack-sunbird/defaults/main.yml b/ansible/roles/stack-sunbird/defaults/main.yml
index 86b134617..43cb794b5 100644
--- a/ansible/roles/stack-sunbird/defaults/main.yml
+++ b/ansible/roles/stack-sunbird/defaults/main.yml
@@ -110,3 +110,4 @@ sunbird_analytics_blob_account_key:
sunbird_portal_player_cdn_enabled:
sunbird_portal_preview_cdn_url:
cdn_file_path:
+sunbird_portal_cdn_blob_url:
diff --git a/deploy/jenkins/jenkins-server-setup.sh b/deploy/jenkins/jenkins-server-setup.sh
index 6ccb350fd..c7dd7d61a 100755
--- a/deploy/jenkins/jenkins-server-setup.sh
+++ b/deploy/jenkins/jenkins-server-setup.sh
@@ -12,7 +12,7 @@ echo -e "\n\e[0;32m${bold}Installating Jenkins${normal}"
wget -q -O - https://pkg.jenkins.io/debian-stable/jenkins.io.key | apt-key add -
sudo apt-add-repository "deb https://pkg.jenkins.io/debian-stable binary/"
sudo apt-get update
-sudo apt-get install -y jenkins=2.164.2
+sudo apt-get install -y jenkins=2.164.3
echo -e "\n\e[0;32m${bold}Installating PIP${normal}"
apt-get install -y python-pip
diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/APIManager/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/APIManager/config.xml
index 4a039c448..da41923c8 100644
--- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/APIManager/config.xml
+++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/APIManager/config.xml
@@ -1,5 +1,5 @@
<?xml version='1.1' encoding='UTF-8'?>
-<flow-definition plugin="workflow-job@2.32">
+<flow-definition plugin="workflow-job@2.33">
<actions/>
<description></description>
<keepDependencies>false</keepDependencies>
@@ -18,26 +18,22 @@
<paramsToUseForLimit></paramsToUseForLimit>
</hudson.plugins.throttleconcurrents.ThrottleJobProperty>
</properties>
- <definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.67">
- <scm class="hudson.plugins.git.GitSCM" plugin="git@3.9.3">
- <configVersion>2</configVersion>
- <userRemoteConfigs>
- <hudson.plugins.git.UserRemoteConfig>
- <url>https://github.com/project-sunbird/sunbird-devops</url>
- </hudson.plugins.git.UserRemoteConfig>
- </userRemoteConfigs>
- <branches>
- <hudson.plugins.git.BranchSpec>
- <name>${github_release_tag}</name>
- </hudson.plugins.git.BranchSpec>
- </branches>
- <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
- <submoduleCfg class="list"/>
- <extensions/>
- </scm>
- <scriptPath>images/kong/Jenkinsfile.build</scriptPath>
- <lightweight>false</lightweight>
+ <definition class="org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition" plugin="workflow-cps@2.70">
+ <script>node(){
+ build_tag="1.8.0-gold"
+ name="kong"
+ node=env.NODE_NAME
+ org=env.hub_org
+
+ sh """
+ docker pull sunbird/kong:1.8.0-gold
+ docker tag sunbird/kong:1.8.0-gold ${org}/${name}:${build_tag}
+ """
+writeFile file: 'metadata.json', text: '{ "image_name" : "' + name + '", "image_tag" : "' + build_tag + '", "node_name" : "' + node + '"}'
+
+ archiveArtifacts artifacts: 'metadata.json', onlyIfSuccessful: true
+}</script>
+ <sandbox>true</sandbox>
</definition>
<triggers/>
<disabled>false</disabled>
-</flow-definition>
diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Badger/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Badger/config.xml
index a19a4fe04..6595d9f53 100644
--- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Badger/config.xml
+++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Badger/config.xml
@@ -1,7 +1,7 @@
<?xml version='1.1' encoding='UTF-8'?>
-<flow-definition plugin="workflow-job@2.31">
+<flow-definition plugin="workflow-job@2.33">
<actions>
- <org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction plugin="workflow-multibranch@2.20">
+ <org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction plugin="workflow-multibranch@2.21">
<jobPropertyDescriptors>
<string>hudson.model.ParametersDefinitionProperty</string>
<string>com.sonyericsson.rebuild.RebuildSettings</string>
@@ -20,20 +20,10 @@
</strategy>
</jenkins.model.BuildDiscarderProperty>
<org.jenkinsci.plugins.workflow.job.properties.DisableConcurrentBuildsJobProperty/>
- <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.29">
+ <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.31">
<autoRebuild>false</autoRebuild>
<rebuildDisabled>false</rebuildDisabled>
</com.sonyericsson.rebuild.RebuildSettings>
- <hudson.model.ParametersDefinitionProperty>
- <parameterDefinitions>
- <hudson.model.StringParameterDefinition>
- <name>github_release_tag</name>
- <description><font color=red size=2><b>CAUTION: If the value is blank, latest code will be built. Specify github tag name to build from a tag.</b></font></description>
- <defaultValue></defaultValue>
- <trim>false</trim>
- </hudson.model.StringParameterDefinition>
- </parameterDefinitions>
- </hudson.model.ParametersDefinitionProperty>
<hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.0.1">
<maxConcurrentPerNode>0</maxConcurrentPerNode>
<maxConcurrentTotal>0</maxConcurrentTotal>
@@ -44,40 +34,25 @@
<paramsToUseForLimit></paramsToUseForLimit>
</hudson.plugins.throttleconcurrents.ThrottleJobProperty>
<org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
- <triggers>
- <com.cloudbees.jenkins.GitHubPushTrigger plugin="github@1.29.4">
- <spec></spec>
- </com.cloudbees.jenkins.GitHubPushTrigger>
- </triggers>
+ <triggers/>
</org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
</properties>
- <definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.63">
- <scm class="hudson.plugins.git.GitSCM" plugin="git@3.9.3">
- <configVersion>2</configVersion>
- <userRemoteConfigs>
- <hudson.plugins.git.UserRemoteConfig>
- <url>https://github.com/project-sunbird/sunbird-devops.git</url>
- </hudson.plugins.git.UserRemoteConfig>
- </userRemoteConfigs>
- <branches>
- <hudson.plugins.git.BranchSpec>
- <name>${github_release_tag}</name>
- </hudson.plugins.git.BranchSpec>
- </branches>
- <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
- <submoduleCfg class="list"/>
- <extensions>
- <hudson.plugins.git.extensions.impl.CloneOption>
- <shallow>true</shallow>
- <noTags>false</noTags>
- <reference></reference>
- <depth>0</depth>
- <honorRefspec>false</honorRefspec>
- </hudson.plugins.git.extensions.impl.CloneOption>
- </extensions>
- </scm>
- <scriptPath>images/openbadger/Jenkinsfile</scriptPath>
- <lightweight>false</lightweight>
+ <definition class="org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition" plugin="workflow-cps@2.70">
+ <script>node(){
+ build_tag="1.8.0-gold"
+ name="badger"
+ node=env.NODE_NAME
+ org=env.hub_org
+
+ sh """
+ docker pull sunbird/${name}:${build_tag}
+ docker tag sunbird/${name}:${build_tag} ${org}/${name}:${build_tag}
+ """
+writeFile file: 'metadata.json', text: '{ "image_name" : "' + name + '", "image_tag" : "' + build_tag + '", "node_name" : "' + node + '"}'
+
+ archiveArtifacts artifacts: 'metadata.json', onlyIfSuccessful: true
+}</script>
+ <sandbox>true</sandbox>
</definition>
<triggers/>
<disabled>false</disabled>
diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/Yarn/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/Yarn/config.xml
index a36f5c0d7..bb720de72 100644
--- a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/Yarn/config.xml
+++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/Yarn/config.xml
@@ -1,5 +1,5 @@
<?xml version='1.1' encoding='UTF-8'?>
-<flow-definition plugin="workflow-job@2.32">
+<flow-definition plugin="workflow-job@2.33">
<actions>
<org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction plugin="workflow-multibranch@2.21">
<jobPropertyDescriptors>
@@ -20,7 +20,7 @@
</strategy>
</jenkins.model.BuildDiscarderProperty>
<org.jenkinsci.plugins.workflow.job.properties.DisableConcurrentBuildsJobProperty/>
- <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.30">
+ <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.31">
<autoRebuild>false</autoRebuild>
<rebuildDisabled>false</rebuildDisabled>
</com.sonyericsson.rebuild.RebuildSettings>
@@ -32,7 +32,7 @@
<randomName>choice-parameter-196500122471866</randomName>
<visibleItemCount>1</visibleItemCount>
<script class="org.biouno.unochoice.model.GroovyScript">
- <secureScript plugin="script-security@1.57">
+ <secureScript plugin="script-security@1.60">
<script>if (override_private_branch.equals("true")) {
return """<input name="value" value="${private_repo_branch}" class="setting-input" type="text"><br> <font color=dimgray size=2><b>Change this value to checkout a different branch from private repository.</b></font>"""
}
@@ -40,7 +40,7 @@ else
return """<b>This parameter is not used</b>"""</script>
<sandbox>false</sandbox>
</secureScript>
- <secureFallbackScript plugin="script-security@1.57">
+ <secureFallbackScript plugin="script-security@1.60">
<script>return """<b>This parameter is not used</b>"""</script>
<sandbox>false</sandbox>
</secureFallbackScript>
@@ -57,7 +57,7 @@ return """<b>This parameter is not used</b>"&quo
<randomName>choice-parameter-196500128714203</randomName>
<visibleItemCount>1</visibleItemCount>
<script class="org.biouno.unochoice.model.GroovyScript">
- <secureScript plugin="script-security@1.57">
+ <secureScript plugin="script-security@1.60">
<script>if (override_public_branch.equals("true")) {
return """<input name="value" value="" class="setting-input" type="text"><br> <font color=dimgray size=2><b>Provide the tag or branch name to checkout the Jenkinsfile and codebase.<br>Note: The tag or branch name for this job should be taken from <a href="https://github.com/project-sunbird/sunbird-data-pipeline">project-sunbird/sunbird-data-pipeline</a></b></font></b></font>""";
@@ -66,7 +66,7 @@ else
return """<b>This parameter is not used</b>"""</script>
<sandbox>false</sandbox>
</secureScript>
- <secureFallbackScript plugin="script-security@1.57">
+ <secureFallbackScript plugin="script-security@1.60">
<script>return """<b>This parameter is not used</b>"""</script>
<sandbox>false</sandbox>
</secureFallbackScript>
@@ -77,9 +77,17 @@ return """<b>This parameter is not used</b>"&quo
<choiceType>ET_FORMATTED_HTML</choiceType>
<omitValueField>true</omitValueField>
</org.biouno.unochoice.DynamicReferenceParameter>
+ <hudson.model.StringParameterDefinition>
+ <name>hosts</name>
+ <description><font color=dimgray size=2><b>Specify IP or group to limit the run on specific IP or host</b></font></description>
+ <defaultValue>all</defaultValue>
+ <trim>false</trim>
+ </hudson.model.StringParameterDefinition>
</parameterDefinitions>
</hudson.model.ParametersDefinitionProperty>
<hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.0.1">
+ <maxConcurrentPerNode>0</maxConcurrentPerNode>
+ <maxConcurrentTotal>0</maxConcurrentTotal>
<categories class="java.util.concurrent.CopyOnWriteArrayList"/>
<throttleEnabled>false</throttleEnabled>
<throttleOption>project</throttleOption>
@@ -87,8 +95,8 @@ return """<b>This parameter is not used</b>"&quo
<paramsToUseForLimit></paramsToUseForLimit>
</hudson.plugins.throttleconcurrents.ThrottleJobProperty>
</properties>
- <definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.66">
- <scm class="hudson.plugins.git.GitSCM" plugin="git@3.9.3">
+ <definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.70">
+ <scm class="hudson.plugins.git.GitSCM" plugin="git@3.10.0">
<configVersion>2</configVersion>
<userRemoteConfigs>
<hudson.plugins.git.UserRemoteConfig>
diff --git a/pipelines/backup/postgresql-restore/Jenkinsfile b/pipelines/backup/postgresql-restore/Jenkinsfile
index 53effdd0c..ad24fdf9b 100644
--- a/pipelines/backup/postgresql-restore/Jenkinsfile
+++ b/pipelines/backup/postgresql-restore/Jenkinsfile
@@ -22,7 +22,7 @@ node() {
jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
currentWs = sh(returnStdout: true, script: 'pwd').trim()
ansiblePlaybook = "${currentWs}/ansible/postgresql-restore.yml"
- ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass"
+ ansibleExtraArgs = "--extra-vars \"postgres_backup_filename=${params.postgres_backup_filename} postgres_user=${params.postgres_user} postgres_password=${params.postgres_password} postgres_hostname=${params.postgres_hostname} postgres_env=${params.postgres_env}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass"
values.put('currentWs', currentWs)
values.put('env', envDir)
values.put('module', module)
diff --git a/pipelines/deploy/org_sunbird_questionunit_quml/Jenkinsfile b/pipelines/deploy/org_sunbird_questionunit_quml/Jenkinsfile
new file mode 100644
index 000000000..bad14b81a
--- /dev/null
+++ b/pipelines/deploy/org_sunbird_questionunit_quml/Jenkinsfile
@@ -0,0 +1,66 @@
+@Library('deploy-conf') _
+node() {
+ try {
+ String ANSI_GREEN = "\u001B[32m"
+ String ANSI_NORMAL = "\u001B[0m"
+ String ANSI_BOLD = "\u001B[1m"
+ String ANSI_RED = "\u001B[31m"
+ String ANSI_YELLOW = "\u001B[33m"
+
+ stage('checkout public repo') {
+ cleanWs()
+ checkout scm
+ }
+
+ ansiColor('xterm') {
+ values = lp_dp_params()
+ values.put('module', 'Core')
+ stage('get artifact') {
+ currentWs = sh(returnStdout: true, script: 'pwd').trim()
+ artifact = values.artifact_name + ":" + values.artifact_version
+ values.put('currentWs', currentWs)
+ if (params.artifact_source == "ArtifactRepo") {
+ println(ANSI_BOLD + ANSI_YELLOW + '''\
+ Option chosen is ArtifactRepo, ignoring any previously copied artifacts and new artifacts will be downloaded from remote source
+ '''.stripIndent().replace("\n", " ") + ANSI_NORMAL)
+ ansiblePlaybook = "${currentWs}/ansible/artifacts-download.yml"
+ ansibleExtraArgs = """\
+ --extra-vars "artifact=${artifact}
+ artifact_path=${currentWs}/${artifact}"
+ --vault-password-file /var/lib/jenkins/secrets/vault-pass
+ """.stripIndent().replace("\n", " ")
+ values.put('ansiblePlaybook', ansiblePlaybook)
+ values.put('ansibleExtraArgs', ansibleExtraArgs)
+ ansible_playbook_run(values)
+ }
+ else{
+ println(ANSI_BOLD + ANSI_YELLOW + '''\
+ Option chosen is JenkinsJob, using the artifacts copied
+ '''.stripIndent().replace("\n", " ") + ANSI_NORMAL)
+ }
+ }
+ stage('deploy artifact'){
+ sh """
+ unzip ${artifact}
+ unzip content-plugins.zip
+ chmod a+x content-plugins/az_copy.sh
+ mv content-plugins ansible
+ """
+
+ ansiblePlaybook = "${currentWs}/ansible/plugin.yml"
+ ansibleExtraArgs = "--tags org_sunbird_questionunit_quml --extra-vars \" plugins_name=${params.plugin_name} source_file=${currentWs}/ansible/content-plugins/${params.plugin_name}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass"
+ values.put('ansiblePlaybook', ansiblePlaybook)
+ values.put('ansibleExtraArgs', ansibleExtraArgs)
+ println values
+ ansible_playbook_run(values)
+
+ archiveArtifacts artifacts: "${artifact}", fingerprint: true, onlyIfSuccessful: true
+ archiveArtifacts artifacts: 'metadata.json', onlyIfSuccessful: true
+ currentBuild.description = "${values.artifact_version}"
+ }
+ }
+ }
+ catch (err) {
+ throw err
+ }
+}
diff --git a/private_repo/ansible/inventory/dev/Core/common.yml b/private_repo/ansible/inventory/dev/Core/common.yml
index df24b8ef8..c30e17dbd 100644
--- a/private_repo/ansible/inventory/dev/Core/common.yml
+++ b/private_repo/ansible/inventory/dev/Core/common.yml
@@ -1,13 +1,14 @@
##########################################
# Starting Of Core VARS
env: #Name of the environment, e.g. dev, staging or production.
+env_name: "{{ env }}"
domain_name: #Domain on which the portal will be accessed. e.g. staging.{implementation-name}.org
proto: # Protocol
bootstrap_user:
bootstrap_key_path:
ssh_public_key_deployer: "" # SSH Public key for deployer user, with which ansible will execute tasks
-ekstep_s3_env: qa # dev or qa of ekstep for plugins
+ekstep_s3_env: qa # dev or qa or prod of ekstep for plugins
learningservice_ip: # Load balancer IP for learning server
searchservice_ip: # search service Load balancer IP
analyticsapi_ip: # Analytics Service Load Balancer IP
@@ -67,4 +68,6 @@ keycloak_ekstep_sunbird_login_theme_dest: "/opt/keycloak/themes/sunbird/login"
keycloak_postgresql: "postgresql-9.4.1212.jar"
sunbird_auth_version: 1.0v
+############## Release-1.15.0 & 2.0.0 ##################
+#sunbird_cassandra_replication_strategy: '{"class":"NetworkTopologyStrategy","datacenter1":2}' # If using cluster give this value and choose datacenter and replication factor as required '{"class":"NetworkTopologyStrategy","datacenter1":2}' if not using cluster, leave this variable commented
diff --git a/private_repo/ansible/inventory/dev/Core/hosts b/private_repo/ansible/inventory/dev/Core/hosts
index 6a7d19569..dbf41b480 100644
--- a/private_repo/ansible/inventory/dev/Core/hosts
+++ b/private_repo/ansible/inventory/dev/Core/hosts
@@ -36,15 +36,26 @@ log-es-1
[swarm-agent-for-prometheus:children]
swarm-agent-for-prometheus-1
-[swarm-agent-for-grafana-1]
+[swarm-agent-for-prometheus-stateful:children]
+swarm-agent-for-prometheus
+
+[swarm-dashboard-1]
18.0.0.15
-[swarm-agent-for-grafana:children]
-swarm-agent-for-grafana-1
+[swarm-dashboard:children]
+swarm-dashboard-1
-[swarm-agent-for-alertmanager-1]
+[swarm-agent-dashboard-1]
18.0.0.15
+[swarm-agent-dashboard:children]
+swarm-agent-dashboard-1
+
+[alertmanager_stateful:children]
+swarm-agent-dashboard
+
+[swarm-agent-for-alertmanager-1]
+18.0.0.15
[es-1]
18.0.0.9 es_instance_name=es-1 es_etc_node_master=true es_etc_node_data=true
@@ -92,6 +103,15 @@ postgresql-master
[kafka:children]
kafka-1
+[processing-cluster-zookeepers]
+18.0.0.50 # Zookeeper IP of processing cluster in Data pipeline
+
+[zookeeper:children]
+processing-cluster-zookeepers
+
+[lp-redis]
+18.0.0.51 # Redis master IP of Knowledge platform
+
[local]
localhost
@@ -127,6 +147,8 @@ swarm-nodes
swarm-manager
kafka
keycloak
+swarm-dashboard
+swarm-agent-dashboard
[env:children]
core
@@ -142,4 +164,4 @@ swarm-manager
# cluster_name=DC1
#
ansible_ssh_user=deployer
-ansible_ssh_private_key_file=/run/secrets/deployer_ssh_key
+ansible_ssh_private_key_file=/var/lib/jenkins/secrets/deployer_ssh_key
diff --git a/private_repo/ansible/inventory/dev/Core/secrets.yml b/private_repo/ansible/inventory/dev/Core/secrets.yml
index f1a8f2cd4..fdb8ad4af 100644
--- a/private_repo/ansible/inventory/dev/Core/secrets.yml
+++ b/private_repo/ansible/inventory/dev/Core/secrets.yml
@@ -1,6 +1,6 @@
# !! Might deprecate
core_vault_sunbird_ekstep_api_key:
-core_vault_sunbird_encryption_key:
+core_vault_sunbird_encryption_key: #Random hash to encrypt data
core_vault_kong__test_jwt:
############# Core #######################
@@ -21,7 +21,7 @@ core_vault_ansible_vault_password:
################### DONT FILL THESE VARIABLES INITIALLY ###################
core_vault_sunbird_sso_publickey: # Get After Keycloak Deployment
-core_vault_sunbird_api_auth_token: #Authorization key (JWT) to access Sunbird APIs. This will be in the output of deploy-apis.sh script, extracting it out is documented in the deployment wiki.
+core_vault_sunbird_api_auth_token: #Take the jwt token api-management-test-user from Jenkins job OnboardConsumers and update core_vault_sunbird_api_auth_token if using KP and DP along with core
core_vault_sunbird_keycloak_user_federation_provider_id: # Get after keycloak deployment
######################## Not mandatory ###################################
diff --git a/private_repo/ansible/inventory/dev/DataPipeline/hosts b/private_repo/ansible/inventory/dev/DataPipeline/hosts
index 919631f7f..5243a3e7a 100644
--- a/private_repo/ansible/inventory/dev/DataPipeline/hosts
+++ b/private_repo/ansible/inventory/dev/DataPipeline/hosts
@@ -9,6 +9,11 @@
[core-cassandra]
15.0.0.7
+[core-es-1]
+15.0.0.50 # This should be elasticsearch master IP of Core
+
+[core-es:children]
+core-es-1
################# LP ##########################
[search]
@@ -56,9 +61,14 @@ cassandra
15.0.3.8
[influxdb]
-#15.0.3.8
15.0.3.6
+[redis]
+15.0.3.5
+
+[redisall:children]
+redis
+
[processing-cluster-kafka]
15.0.3.8
15.0.3.7
@@ -70,7 +80,6 @@ cassandra
[telemetry-search-cluster1]
15.0.2.6
-
[telemetry-search-cluster:children]
telemetry-search-cluster1
diff --git a/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml b/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml
index 05ff5d786..a4f306758 100644
--- a/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml
+++ b/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml
@@ -38,5 +38,5 @@ neo4j_enterprise: "true"
## backup
backup_azure_storage_account_name: sunbirdbackupsdev
### overriding backup secret
-backup_azure_storage_access_key: "{{lp_vault_backup_azure_storage_secret}}"
+backup_azure_storage_access_key: "{{lp_vault_azure_storage_secret}}"
diff --git a/private_repo/ansible/inventory/dev/KnowledgePlatform/hosts b/private_repo/ansible/inventory/dev/KnowledgePlatform/hosts
index d67f201cf..a0aad45ea 100644
--- a/private_repo/ansible/inventory/dev/KnowledgePlatform/hosts
+++ b/private_repo/ansible/inventory/dev/KnowledgePlatform/hosts
@@ -31,9 +31,12 @@ redis1
[learning-neo4j-cluster:children]
learning-neo4j-node1
-[cassandra]
+[cassandra-node-1] # The old cassandra group has been renamed to cassandra-node-* to accomadate for clusters
15.0.2.7
+[cassandra:children]
+cassandra-node-1
+
[dp-cassandra]
15.0.2.7
--
GitLab