Merge pull request #602 from project-sunbird/release-2.1.0

fix: code base from 2.1.0

ansible/inventory/env/group_vars/all.yml

@@ -340,7 +340,8 @@ sunbird_open_saber_bridge_enable: 'false'
 sunbird_content_service_enable_logging: 'true'
 sunbird_language_service_api_key: "{{core_vault_sunbird_ekstep_api_key}}"
 sunbird_installation_display_name: "{{sunbird_app_name}} {{env}}"
-sunbird_ekstep_api_base_url: "https://{{ekstep_s3_env}}.ekstep.in/api"   #API base URL of the Ekstep environment. Use `https://qa.ekstep.in/api` for non-prod deployments, and use `https://api.ekstep.in/` for prod deployment.
+sunbird_ekstep_proxy_base_url: "https://{{ekstep_s3_env}}.ekstep.in"  #Base URL of the Ekstep environment. Use `https://qa.ekstep.in/` for non-prod deployments, and `https://community.ekstep.in/` for prod deployment.
+sunbird_ekstep_api_base_url: "{{sunbird_ekstep_proxy_base_url}}/api"   #API base URL of the Ekstep environment. Use `https://qa.ekstep.in/api` for non-prod deployments, and use `https://api.ekstep.in/` for prod deployment.
 sunbird_language_service_api_base_url: '{{sunbird_ekstep_api_base_url}}/language'
 
 kong_version: 1.5.0-gold

ansible/postgresql-restore.yml

-- hosts: postgresql-restore
+- hosts: local
   become: yes
   vars_files:
-    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
+    - ['{{inventory_dir}}/secrets.yml']
   roles:
-    - postgresql-restore
+    - postgres-azure-managed-service-restore
   tags:
     - postgresql-restore

ansible/roles/postgres-azure-managed-service-restore/defaults/main.yml

+postgresql_restore_dir: /tmp/postgres-restore
+postgres_backup_azure_container_name: postgresql-backup
+
+db:
+  name: ['keycloak', 'api_manager_{{ postgres_env }}', 'badger', 'quartz']
+  role: ['keycloak', 'api_manager_{{ postgres_env }}', 'badger', 'quartz']
+  user: ['azure_superuser, {{ env_user_name }}']
+
+#these variables are passed as extra vars
+postgres_backup_filename:
+postgres_user: 
+postgres_password: 
+postgres_hostname:
+postgres_env:

ansible/roles/postgres-azure-managed-service-restore/tasks/main.yml

+- name: install psycopg2
+  package:
+    name: python-psycopg2
+    state: present
+   
+- name: ensure restore dir exists
+  file: path="{{ postgresql_restore_dir }}" state=directory mode=0777
+
+- set_fact:
+    postgres_backup_filepath: "{{ postgresql_restore_dir }}/{{ postgres_backup_filename }}"
+
+- name: Download backup from azure
+  command: az storage blob download -c {{ postgres_backup_azure_container_name }} --name {{ postgres_backup_filename }} -f {{ postgres_backup_filepath }}
+  args:
+    chdir: "{{ postgres_restore_dir }}"
+  async: 100
+  poll: 10
+
+- name: unarchive artifact
+  unarchive: src={{ postgresql_restore_dir }}/{{ postgres_backup_filename }} dest={{ postgresql_restore_dir }}/ copy=no
+
+- name: Create db's
+  postgresql_db:
+    login_user: "{{ postgres_user }}"
+    login_password: "{{ postgres_password }}"
+    login_host: "{{ postgres_hostname }}"
+    name: "{{ item }}"
+    state: present
+  with_items: "{{ db.name }}"
+  async: 1000
+  poll: 10
+
+- name: Create role and grant access to db's
+  postgresql_user:
+    login_user: "{{ postgres_user }}"
+    login_password: "{{ postgres_password }}"
+    login_host: "{{ postgres_hostname }}"
+    db: "{{ item[0] }}"
+    name: "{{ item[1] }}"
+    priv: ALL
+    state: present
+    role_attr_flags: CREATEROLE
+  with_nested:
+    - "{{ db.name }}"
+    - "{{ db.role }}"
+  async: 1000
+  poll: 10
+
+- name: create user
+  postgresql_user:
+    login_user: "{{ postgres_user }}"
+    login_password: "{{ postgres_password }}"
+    login_host: "{{ postgres_hostname }}"
+    name: "{{ item }}"
+  with_items: "{{ db.user }}"
+  async: 1000
+  poll: 10
+
+- name: Restore db's
+  postgresql_db:
+    login_user: "{{ postgres_user }}"
+    login_password: "{{ postgres_password }}"
+    login_host: "{{ postgres_hostname }}"
+    name: "{{ item }}"
+    state: restore
+    target: "{{ item }}.sql"
+  args:
+    chdir: "{{ postgres_restore_dir }}"
+  with_items: "{{ db.name }}"
+

pipelines/backup/postgresql-restore/Jenkinsfile

@@ -22,7 +22,7 @@ node() {
                 jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
                 currentWs = sh(returnStdout: true, script: 'pwd').trim()
                 ansiblePlaybook = "${currentWs}/ansible/postgresql-restore.yml"
-                ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass"
+                ansibleExtraArgs = "--extra-vars \"postgres_backup_filename=${params.postgres_backup_filename} postgres_user=${params.postgres_user} postgres_password=${params.postgres_password} postgres_hostname=${params.postgres_hostname} postgres_env=${params.postgres_env}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass"
                 values.put('currentWs', currentWs)
                 values.put('env', envDir)
                 values.put('module', module)

private_repo/ansible/inventory/dev/Core/common.yml

@@ -8,7 +8,7 @@ bootstrap_user:
 bootstrap_key_path: 
 ssh_public_key_deployer: ""             # SSH Public key for deployer user, with which ansible will execute tasks
 
-ekstep_s3_env: qa                       # dev or qa of ekstep for plugins
+ekstep_s3_env: qa                       # dev or qa or prod of ekstep for plugins
 learningservice_ip:                     # Load balancer IP for learning server
 searchservice_ip:                       # search service Load balancer IP
 analyticsapi_ip:                        # Analytics Service Load Balancer IP

private_repo/ansible/inventory/dev/Core/hosts

@@ -164,4 +164,4 @@ swarm-manager
 # cluster_name=DC1
 #
 ansible_ssh_user=deployer
-ansible_ssh_private_key_file=/run/secrets/deployer_ssh_key
+ansible_ssh_private_key_file=/var/lib/jenkins/secrets/deployer_ssh_key

private_repo/ansible/inventory/dev/DataPipeline/hosts

@@ -61,9 +61,14 @@ cassandra
 15.0.3.8
 
 [influxdb]
-#15.0.3.8
 15.0.3.6
 
+[redis]
+15.0.3.5
+
+[redisall:children]
+redis
+
 [processing-cluster-kafka]
 15.0.3.8
 15.0.3.7
@@ -75,7 +80,6 @@ cassandra
 [telemetry-search-cluster1]
 15.0.2.6
 
-
 [telemetry-search-cluster:children]
 telemetry-search-cluster1