Merge pull request #1 from project-sunbird/release-2.2.0

Release 2.2.0

ansible/roles/postgresql-data-update/tasks/databases.yml

@@ -2,20 +2,13 @@
 - name: Ensure PostgreSQL databases are present.
   postgresql_db:
     name: "{{ item.name }}"
-    lc_collate: "{{ item.lc_collate | default('en_US.UTF-8') }}"
-    lc_ctype: "{{ item.lc_ctype | default('en_US.UTF-8') }}"
-    encoding: "{{ item.encoding | default('UTF-8') }}"
-    template: "{{ item.template | default('template0') }}"
     login_host: "{{ item.login_host | default('localhost') }}"
     login_password: "{{ item.login_password | default(omit) }}"
     login_user: "{{ item.login_user | default(postgresql_user) }}"
     login_unix_socket: "{{ item.login_unix_socket | default(postgresql_unix_socket_directories[0]) }}"
     port: "{{ item.port | default(omit) }}"
-    owner: "{{ item.owner | default(postgresql_user) }}"
+    owner: '{{ item.owner.split("@")[0] | default(postgresql_user) }}'
     state: "{{ item.state | default('present') }}"
   with_items: "{{ postgresql_databases }}"
-  become: yes
-  become_user: "{{ postgresql_user }}"
-  # See: https://github.com/ansible/ansible/issues/16048#issuecomment-229012509
   vars:
     ansible_ssh_pipelining: true

ansible/roles/postgresql-data-update/tasks/main.yml

+- name: install psycopg2
+  package:
+    name: python-psycopg2
+    state: present
+
 - name: Verifying db users are present else creating them
   include: databases.yml
 
@@ -9,9 +14,28 @@
   template: src={{item}} dest=/tmp/{{item}}
   with_items:
     - tables_postgres.sql
+    - enc_postgres.sql
 
-- name: Run the postgresql command
-  become: yes
-  environment:
-    PGPASSWORD: "{{ application_postgres_password }}"
-  command: "psql -h 127.0.0.1 -U {{application_postgres_user}} -d {{application_postgres_database}} -a -f /tmp/tables_postgres.sql"
+- name: Install postgres client to create schema from file
+  apt:
+    name: postgresql-client
+    state: present
+    update_cache: yes
+
+- name: Dump api manager database to a file
+  postgresql_db:
+    login_user: "{{ application_postgres_user }}"
+    login_password: "{{ application_postgres_password }}"
+    login_host: "{{ application_postgres_host }}"
+    name: "{{ application_postgres_database }}"
+    state: restore
+    target: "/tmp/tables_postgres.sql"
+
+- name: Create the schema for encryption service
+  postgresql_db:
+    login_user: "{{ enc_postgres_user }}"
+    login_password: "{{ enc_postgres_password }}"
+    login_host: "{{ enc_postgres_host }}"
+    name: "{{ enc_postgres_database }}"
+    state: restore
+    target: "/tmp/enc_postgres.sql"

ansible/roles/postgresql-data-update/tasks/users.yml

@@ -9,13 +9,9 @@
     login_host: "{{ item.login_host | default('localhost') }}"
     login_password: "{{ item.login_password | default(omit) }}"
     login_user: "{{ item.login_user | default(postgresql_user) }}"
-    login_unix_socket: "{{ item.login_unix_socket | default(postgresql_unix_socket_directories[0]) }}"
-    port: "{{ item.port | default(omit) }}"
+    encrypted: "{{ item.encrypted | default('yes')}}"
     state: "{{ item.state | default('present') }}"
   with_items: "{{ postgresql_users }}"
-  no_log: true
-  become: yes
-  become_user: "{{ postgresql_user }}"
-  # See: https://github.com/ansible/ansible/issues/16048#issuecomment-229012509
   vars:
     ansible_ssh_pipelining: true
+

ansible/roles/postgresql-data-update/templates/enc_postgres.sql

+
+CREATE TYPE "enum_Keys_type" AS ENUM ('MASTER','OTHER');
+
+CREATE TABLE "Keys" (
+  id SERIAL PRIMARY KEY,
+  public text NOT NULL,
+  private text NOT NULL,
+  type "enum_Keys_type" NOT NULL,
+  active boolean DEFAULT true NOT NULL,
+  "createdAt" timestamp with time zone NOT NULL,
+  "updatedAt" timestamp with time zone NOT NULL
+);
+commit;

ansible/roles/postgresql-data-update/templates/tables_postgres.sql

@@ -178,5 +178,4 @@ create index idx_qrtz_ft_jg on qrtz_fired_triggers(SCHED_NAME,JOB_GROUP);
 create index idx_qrtz_ft_t_g on qrtz_fired_triggers(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP);
 create index idx_qrtz_ft_tg on qrtz_fired_triggers(SCHED_NAME,TRIGGER_GROUP);
 
-
 commit;

ansible/roles/prometheus-backup-v2/tasks/main.yml

 ---
 # tasks file for ansible/roles/prometheus-backup-v2
 # Issue: https://github.com/prometheus/prometheus/issues/5686
-- name: preparing snapshot
-  uri:
-    url: "{{ prometheus_url }}/api/v1/admin/tsdb/snapshot"
-    method: POST
-  register: temp_snap
-
 - name: taking snapshot
   uri:
     url: "{{ prometheus_url }}/api/v1/admin/tsdb/snapshot?skip_head=True"
@@ -32,11 +26,8 @@
 
 - name: Deleting snapshot
   file:
-    path: "{{ prometheus_data_dir }}/snapshots/{{ item }}"
+    path: "{{ prometheus_data_dir }}/snapshots/{{ snapshot_name }}"
     state: absent
-  with_items:
-    - "{{ snapshot_name }}"
-    - "{{ temp_snap.json.data.name }}"
 
 - name: Deleting archive snapshot
   file:

ansible/roles/reset-docker/tasks/main.yml

@@ -10,8 +10,8 @@
   shell: "docker stack rm player"
   ignore_errors: yes
 
-- name: Remove learner service
-  shell: "docker stack rm player"
+- name: Remove lms service
+  shell: "docker stack rm lms-service"
   ignore_errors: yes
  
 - name: Copy the restart command script

ansible/roles/stack-monitor-stateful/defaults/main.yml

@@ -107,6 +107,7 @@ service_teams:
     services:
       - actor-service
       - learner-service
+      - lms-service
       - content-service
       - player_player
       - cassandra

ansible/roles/stack-monitor/defaults/main.yml

@@ -96,6 +96,7 @@ service_teams:
     services:
       - actor-service
       - learner-service
+      - lms-service
       - content-service
       - player_player
       - cassandra

ansible/roles/stack-monitor/templates/prometheus.yml

@@ -62,6 +62,16 @@ scrape_configs:
         action: drop
   {% endif %}
 
+  - job_name: 'nginx'
+    metrics_path: /metrics
+    scrape_interval: 10s
+    scrape_timeout: 5s
+    dns_sd_configs:
+    - names:
+      - 'tasks.proxy_proxy'
+      type: 'A'
+      port: 9145
+
   - job_name: 'statsd-exporter'
     static_configs:
       - targets: ['monitor_statsd_exporter:9102']

ansible/roles/stack-proxy/defaults/main.yml

@@ -3,7 +3,8 @@ hub_org: sunbird
 
 proxy_replicas: 1
 proxy_reservation_memory: 32M
-proxy_limit_memory: 128M
+proxy_limit_memory: 64M
+nginx_per_ip_connection_limit: 400
 
 proxy_prometheus: false
 

ansible/roles/stack-proxy/templates/nginx.conf

@@ -5,7 +5,7 @@ error_log  /var/log/nginx/error.log warn;
 pid        /var/run/nginx.pid;
 
 events {
-    worker_connections  2048;
+    worker_connections  10000;
 }
 
 
@@ -13,29 +13,66 @@ http {
     include       /etc/nginx/mime.types;
     default_type  application/octet-stream;
 
-    log_format  main  '$http_x_forwarded_for - $remote_addr - $remote_user [$time_local] '
+    lua_load_resty_core off;
+    log_format  main  '$remote_addr - $remote_user [$time_local] '
                       '"$request" $status $body_bytes_sent '
                       '$request_time $upstream_response_time $pipe'
                       '"$http_referer" "$http_user_agent"';
 
     access_log  /var/log/nginx/access.log  main;
 
+    # Shared dictionary to store metrics
+    lua_shared_dict prometheus_metrics 10M;
+    lua_package_path "/etc/nginx/lua_modules/?.lua";
+    # Defining metrics
+    init_by_lua '
+      prometheus = require("prometheus").init("prometheus_metrics")
+      metric_requests = prometheus:counter(
+        "nginx_http_requests_total", "Number of HTTP requests", {"host", "status", "request_method"})
+      metric_latency = prometheus:histogram(
+        "nginx_http_request_duration_seconds", "HTTP request latency", {"host"})
+      metric_connections = prometheus:gauge(
+        "nginx_http_connections", "Number of HTTP connections", {"state"})
+    ';
+
+    # Collecting metrics
+    log_by_lua '
+      metric_requests:inc(1, {ngx.var.server_name, ngx.var.status, ngx.var.request_method})
+      metric_latency:observe(tonumber(ngx.var.request_time), {ngx.var.server_name})
+    ';
+
     sendfile        on;
     #tcp_nopush     on;
     client_max_body_size 60M;
 
-    keepalive_timeout  65;
+    keepalive_timeout  500s;
+    keepalive_requests 200;
+
+    # Nginx connection limit per ip
+    limit_conn_zone $binary_remote_addr zone=limitbyaddr:10m;
+    limit_conn_status 429;
 
     upstream kong {
         server api-manager_kong:8000;
         keepalive 1000;
     }
 
-    #gzip  on;
     upstream player {
         server player_player:3000;
         keepalive 1000;
     }
 
     include /etc/nginx/conf.d/*.conf;
+
+   server {
+     listen 9145;
+     location /metrics {
+       content_by_lua '
+         metric_connections:set(ngx.var.connections_reading, {"reading"})
+         metric_connections:set(ngx.var.connections_waiting, {"waiting"})
+         metric_connections:set(ngx.var.connections_writing, {"writing"})
+         prometheus:collect()
+       ';
+     }
+   }
 }

ansible/roles/stack-proxy/templates/proxy-default.conf

@@ -3,6 +3,8 @@ server {
   listen 80;
   listen [::]:80;
   server_name {{ proxy_server_name }};
+  # Limitting open connection per ip
+  limit_conn limitbyaddr {{ nginx_per_ip_connection_limit }};
 
   return 301 https://{{ proxy_server_name }}$request_uri;
 }
@@ -18,6 +20,8 @@ server {
 {% endif  %}
   server_name           {{ proxy_server_name }};
 
+  # Limitting open connection per ip
+  limit_conn limitbyaddr {{ nginx_per_ip_connection_limit }};
   proxy_set_header    Host              $host;
   proxy_set_header    X-Real-IP         $remote_addr;
   proxy_set_header    X-Forwarded-For   $proxy_add_x_forwarded_for;
@@ -45,7 +49,10 @@ server {
   location ~* ^/auth/realms/(.+)/clients-registrations/ {
     return 301 {{proto}}://$host/api/auth/v1/realms/$1/clients-registrations/;
   }
-  
+  location ~* ^/auth/v1/refresh/token  {
+    return 301 {{proto}}://{{ proxy_server_name }};
+  }
+ 
   location ~* ^/auth/admin/master/console/ {
     return 301 {{proto}}://{{ proxy_server_name }};
   }

ansible/roles/stack-proxy/templates/stack-proxy.yml

@@ -3,11 +3,18 @@ version: '3.3'
 services:
   proxy:
     image: "{{hub_org}}/{{image_name}}:{{image_tag}}"
+    # This is a workaround for remote ip address is not visible nginx
     ports:
-      - "443:443"
-      - "80:80"
+      - mode: host
+        target: 80
+        published: 80
+        protocol: TCP
+      - mode: host
+        target: 443
+        published: 443
+        protocol: TCP
     deploy:
-      replicas: {{ proxy_replicas }}
+      mode: global
       resources:
         reservations:
           memory: "{{ proxy_reservation_memory }}"

ansible/roles/stack-sunbird/defaults/main.yml

@@ -75,11 +75,21 @@ content_limit_cpu: 1
 content_reservation_cpu: 1
 
 user_org_replicas: 1
-user_org_reservation_memory: 750m
-user_org_limit_memory: 1g
+user_org_reservation_memory: 750MB
+user_org_limit_memory: 800MB
 user_org_limit_cpu: 1
 user_org_reservation_cpu: 1
 
+# Encryption service vars
+enc_replicas: 1
+enc_reservation_memory: 750MB
+enc_limit_memory: 800MB
+enc_limit_cpu: 1
+enc_reservation_cpu: 1
+postgres_port: 5432
+enc_dialect: postgres
+enc_entry_passwod: password
+
 
 telemetry_service_threads:
 telemetry_local_storage_enabled:
@@ -88,10 +98,6 @@ telemetry_kafka_broker_list:
 telemetry_kafka_topic:
 
 # Encryption service
-encryption_replicas: 1
-encryption_reservation_memory: "256M"
-encryption_limit_memory: "512M"
-
 # Learner
 sunbird_keycloak_required_action_link_expiration_seconds: 2592000
 sunbird_time_zone: "Asia/Kolkata"
@@ -110,3 +116,4 @@ sunbird_analytics_blob_account_key:
 sunbird_portal_player_cdn_enabled: 
 sunbird_portal_preview_cdn_url:
 cdn_file_path:
+sunbird_portal_cdn_blob_url:

ansible/roles/stack-sunbird/tasks/enc_service.yml

+---
+- name: Remove enc service
+  shell: "docker service rm enc-service"
+  ignore_errors: yes
+
+- name: Deploy enc service
+  shell: "docker service create --with-registry-auth --replicas {{ enc_replicas }} -p 9010:8013  --name enc-service --hostname enc-service --reserve-memory {{ enc_reservation_memory }} --limit-memory {{ enc_limit_memory }} --limit-cpu {{ enc_limit_cpu }} --reserve-cpu {{ enc_reservation_cpu }} --network application_default --env-file /home/deployer/env/sunbird_enc-service.env  {{hub_org}}/{{image_name}}:{{image_tag}}"

ansible/roles/stack-sunbird/tasks/encryption_service.yml

----
-- name: Remove encryption service
-  shell: "docker service rm encryption-service"
-  ignore_errors: yes
-
-- name: Deploy encryption service
-  shell: "docker service create --replicas {{ encryption_replicas }} -p 8013:8013  --name encryption-service --hostname encryption-service --reserve-memory {{ encryption_reservation_memory }} --limit-memory {{ encryption_limit_memory }} --network application_default --env-file /home/deployer/env/sunbird_encryption-service.env  {{hub_org}}/{{image_name}}:{{image_tag}} {{application_postgres_password}}"
-  args:
-    chdir: /home/deployer/stack

ansible/roles/stack-sunbird/tasks/lms_service.yml

+---
+- name: Remove lms service
+  shell: "docker service rm lms-service"
+  ignore_errors: yes
+
+- name: Deploy lms service
+  shell: "docker service create --with-registry-auth --replicas {{ learner_replicas }} -p 9005:9000  --name lms-service --hostname lms-service --reserve-memory {{ learner_reservation_memory }} --limit-memory {{ learner_limit_memory }} --limit-cpu {{ learner_limit_cpu }} --reserve-cpu {{ learner_reservation_cpu }} --health-cmd 'wget -qO- lms-service:9000/service/health || exit 1' --health-timeout 3s --health-retries 3  --network application_default --env-file /home/deployer/env/sunbird_lms-service.env  {{hub_org}}/{{image_name}}:{{image_tag}}"
+  args:
+    chdir: /home/deployer/stack
\ No newline at end of file

ansible/roles/stack-sunbird/tasks/main.yml

@@ -6,6 +6,12 @@
 
   - include: learner_service.yml
     when: deploy_learner is defined
+  
+  - include: lms_service.yml
+    when: deploy_lms is defined
+
+  - include: enc_service.yml
+    when: deploy_enc is defined
 
   - include: user_org_service.yml
     when: deploy_user_org is defined
@@ -24,6 +30,3 @@
 
   - include: telemetry_logstash_datapipeline.yml
     when: deploy_telemetry_logstash_datapipeline is defined
-
-  - include: encryption_service.yml
-    when: deploy_encryption is defined

ansible/roles/stack-sunbird/tasks/user_org_service.yml

@@ -5,5 +5,3 @@
 
 - name: Deploy user org service
   shell: "docker service create --with-registry-auth --replicas {{ user_org_replicas }} -p 9009:9000  --name user-org-service --hostname user-org-service --reserve-memory {{ user_org_reservation_memory }} --limit-memory {{ user_org_limit_memory }} --limit-cpu {{ user_org_limit_cpu }} --reserve-cpu {{ user_org_reservation_cpu }} --network application_default --env-file /home/deployer/env/sunbird_user-org-service.env  {{hub_org}}/{{image_name}}:{{image_tag}}"
-  args:
-    chdir: /home/deployer/stack