`cassandra state validated` script changes for cassandra migration and es sync (#839)

* cassandra state validated script

* ES query sunbird_sync_block_size change

* Pipeline changes to create folder to get build artifacts

ansible/roles/cassandra-state-validated/defaults/main.yml

@@ -4,7 +4,7 @@ sunbird_cassandra_server: "{{groups['cassandra'][0]}}"
 sunbird_cassandra_port: 9042
 sunbird_sync_api_endpoint: "{{proto}}://{{domain_name}}/api/data/v1/index/sync"
 sunbird_sync_api_key: "{{sunbird_api_auth_token}}"
-sunbird_sync_block_size: 150
+sunbird_sync_block_size: 50
 sunbird_sync_sleep_time: 10000
 sunbird_cassandra_username: cassandra
 sunbird_cassandra_password: password

ansible/roles/cassandra-state-validated/tasks/main.yml

@@ -9,6 +9,11 @@
   shell: java -jar statevalidator-util-1.0-SNAPSHOT-jar-with-dependencies.jar
   run_once: true
 
+- name: cloning the sunbird-utils repo
+  file:
+    state: absent
+    path: /tmp/sunbird-utils/
+
 - name: cloning the sunbird-utils repo
   git:
     repo: 'https://github.com/project-sunbird/sunbird-utils.git'
@@ -18,19 +23,41 @@
 
 - name: Unarchive a file that is already on the remote machine
   unarchive:
-    src: /tmp/sunbird-utils/cassandra-migration-etl/common/SyncBin.zip
-    dest: /tmp/sunbird-utils/cassandra-migration-etl/common/
+    src: /tmp/sunbird-utils/sunbird-cassandra-migration/cassandra-migration-etl/common/UserSyncBin.zip
+    dest: /tmp/sunbird-utils/sunbird-cassandra-migration/cassandra-migration-etl/common/
     remote_src: yes
 
-- name: change ownership
-  file:
-    path: /tmp/sunbird-utils
-    owner: deployer
-    group: deployer
-    mode: '0644'
-    recurse: yes
-
 - name: run the script to synch es data
-  shell: bash Sync_run.sh --context_param sunbird_cassandra_server="{{sunbird_cassandra_server}}" --context_param sunbird_cassandra_port="{{sunbird_cassandra_port}}" --context_param sunbird_cassandra_username="{{sunbird_cassandra_username}}" --context_param sunbird_cassandra_password="{{sunbird_cassandra_password}}" --context_param sunbird_sync_api_endpoint="{{sunbird_sync_api_endpoint}}" --context_param sunbird_sync_api_key="{{sunbird_sync_api_key}}" --context_param sunbird_sync_block_size="{{sunbird_sync_block_size}}" --context_param sunbird_sync_sleep_time="{{sunbird_sync_sleep_time}}" --context_param sunbird_cassandra_table="user" --context_param sunbird_sync_object_type="user"
+  shell: bash UserSync_run.sh --context_param sunbird_cassandra_server="{{sunbird_cassandra_server}}" --context_param sunbird_cassandra_port="{{sunbird_cassandra_port}}" --context_param sunbird_cassandra_username="{{sunbird_cassandra_username}}" --context_param sunbird_cassandra_password="{{sunbird_cassandra_password}}" --context_param sunbird_user_sync_api_endpoint="{{sunbird_sync_api_endpoint}}" --context_param sunbird_user_sync_api_key="{{sunbird_sync_api_key}}" --context_param sunbird_user_sync_block_size="{{sunbird_sync_block_size}}" --context_param sunbird_user_sync_sleep_time="{{sunbird_sync_sleep_time}}" --context_param sunbird_cassandra_table="user" --context_param sunbird_sync_object_type="user"
   args:
-    chdir: /tmp/sunbird-utils/cassandra-migration-etl/common/Sync/
+    chdir: /tmp/sunbird-utils/sunbird-cassandra-migration/cassandra-migration-etl/common/UserSync/
+
+- name: Fetching the cassandra log files from server to jenkins
+  fetch:
+    src: "{{ item }}"
+    dest: "{{artifactFolder}}/"
+    flat: yes
+  with_items:
+    - /home/deployer/flagsdatavalue.log
+    - /home/deployer/preProcessedRecords.txt
+    - /tmp/sunbird-utils/sunbird-cassandra-migration/cassandra-migration-etl/common/UserSync/userIds.csv
+
+- name: Find the log files for the es sync job
+  find:
+    paths: "/tmp/sunbird-utils/sunbird-cassandra-migration/cassandra-migration-etl/common/UserSync/"
+    patterns: "{{item}}*.log"
+    recurse: yes
+  with_items:
+    - userSyncFailure
+    - userSyncSuccess
+  register: es_sync_log_files
+
+- debug:
+    var: es_sync_log_files
+
+- name: Fetching the es sync log files from server to jenkins
+  fetch:
+    src: "{{ item  }}"
+    dest: "{{artifactFolder}}/"
+    flat: yes
+  with_items: "{{ es_sync_log_files | json_query('results[].files[].path')}}"

pipelines/deploy/cassandra-deploy/Jenkinsfile.stateValidated

@@ -23,15 +23,17 @@ node() {
                 stage('deploy artifact') {
                     sh """
                        unzip ${artifact}
+                       mkdir -p ${currentWs}/artifactsForSync
                     """
                     ansiblePlaybook = "${currentWs}/ansible/cassandra-state-validated-util.yml"
-                    ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass -v"
+                    ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass -e artifactFolder=\"${currentWs}/artifactsForSync\" -v"
                     values.put('ansiblePlaybook', ansiblePlaybook)
                     values.put('ansibleExtraArgs', ansibleExtraArgs)
                     println values
                     ansible_playbook_run(values)
                     currentBuild.result = 'SUCCESS'
                     archiveArtifacts artifacts: "${artifact}", fingerprint: true, onlyIfSuccessful: true
+                    archiveArtifacts artifacts: "artifactsForSync/*", fingerprint: true, onlyIfSuccessful: true
                     archiveArtifacts artifacts: 'metadata.json', onlyIfSuccessful: true
                     currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
                 }