Merge pull request #45 from Samagra-Development/dev

Dev

.env

@@ -5,10 +5,42 @@ DATABASE_PASSWORD=hkaLBM3RVnyYeYeqE3WI1w2e4Avpy0Wd5O3s3
 ES_JAVA_OPTS="-Xms512m -Xmx512m"
 FUSIONAUTH_APP_MEMORY=512M
 
+#minio
 MINIO_USERNAME=workflow-dev
 MINIO_PASSWORD=workflow-dev
 MINIO_BUCKETNAME=affiliationbucket
 
 # central envs
 CENTRAL_USER_EMAIL=chakshu@samagragovernance.in
-CENTRAL_USER_PASS=StrongPassword@1234
\ No newline at end of file
+CENTRAL_USER_PASS=StrongPassword@1234
+
+#enketo-express
+ENCRYPTION_KEY=somestrongencrytionkey
+LESS_SECURE_ENCRYPTION_KEY=ssssomestrongencrytionkey
+FORM_MANAGER_BASE_URI=IP:PORT_OF_DEPLOYED_INSTANCE
+
+# form manager
+FORM_MANAGER_URL=http://form-manager:3006
+MINIO_URL=IP:PORT_OF_DEPLOYED_MINIO
+MINIO_ENDPOINT=minio
+MINIO_PORT=9000
+REDIS_HOST=fm-cache
+MINIO_USE_SSL=false
+REDIS_PORT=6379
+
+# TSDB 
+TSDB_POSTGRES_USER=timescaledb
+TSDB_POSTGRES_PASSWORD=postgrespassword
+
+# GQL
+HASURA_GRAPHQL_ADMIN_SECRET=myadminsecretkey
+HASURA_GRAPHQL_JWT_SECRET={"type":"RS256","jwk_url":"http://fusionauth:9011/.well-known/jwks.json"}
+HASURA_GRAPHQL_DATABASE_URL=postgres://$TSDB_POSTGRES_USER:$TSDB_POSTGRES_PASSWORD@tsdb:5432/postgres?sslmode=disable
+
+#react wrapper 
+# http://service-name is not going to work as we are building a static build
+# and serving it through nginx
+# use actual IP address instead
+REACT_APP_ENKETO_URL=https://8065-tushar5526-workflow-0qffaj3egws.ws-us93.gitpod.io
+REACT_APP_FORM_MANAGER_URL=https://3006-tushar5526-workflow-0qffaj3egws.ws-us93.gitpod.io
+REACT_APP_HASURA_URL=https://8080-tushar5526-workflow-0qffaj3egws.ws-us93.gitpod.io
\ No newline at end of file

.gitpod.yml

@@ -14,41 +14,25 @@ tasks:
     init: |
       gp sync-done dev-setup
       bash install-central.sh
-  - name: Setup Enketo Express
+  - name: Start Services
     openMode: tab-after
     init: |
       gp sync-done dev-setup
-      bash install-node.sh
-      nvm use 14
-      cd packages/enketo-express && pnpm i && pnpm i -g grunt
-      bash update-config.sh
+      # Not needed now, we can use docker-compose networking.. but one can use this in a script to repalce env vars at runtime in .env
+      # sed -i~ "/^FORM_MANAGER_URL=/s/=.*/=https\:\/\/3006-${GITPOD_WORKSPACE_URL:8}/" .env
+      sed -i~ "/^MINIO_URL=/s/=.*/=https\:\/\/9000-${GITPOD_WORKSPACE_URL:8}/" .env
+      # sed -i~ "/^MINIO_ENDPOINT=/s/=.*/=9000-${GITPOD_WORKSPACE_URL:8}/" .env
+      # sed -i~ "/^MINIO_PORT=/s/=.*/=80/" .env
+      # sed -i~ "/^MINIO_USE_SSL=/s/=.*/=true/" .env
+      # Replace env vars for react wrapper
+      sed -i~ "/^REACT_APP_ENKETO_URL=/s/=.*/=https\:\/\/8065-${GITPOD_WORKSPACE_URL:8}/" .env
+      sed -i~ "/^REACT_APP_FORM_MANAGER_URL=/s/=.*/=https\:\/\/3006-${GITPOD_WORKSPACE_URL:8}/" .env
+      sed -i~ "/^REACT_APP_HASURA_URL=/s/=.*/=https\:\/\/8080-${GITPOD_WORKSPACE_URL:8}/" .env
+      # for enketo-express
+      sed -i~ "/^FORM_MANAGER_BASE_URI=/s/=.*/=https\:\/\/3006-${GITPOD_WORKSPACE_URL:8}/" .env
+      
+      docker-compose up -d
       gp sync-done setup
-    command: |
-      grunt develop
-  - name: Setup Form Manager
-    openMode: tab-after
-    init: |
-      gp sync-await setup
-      source .env
-      docker run --name enketo-redis-main -p 6369:6379 -d redis 
-      cd packages/form-manager
-      echo "MINIO_USERNAME=$MINIO_USERNAME"  >> .env
-      echo "MINIO_PASSWORD=$MINIO_PASSWORD"  >> .env
-      echo "MINIO_BUCKETNAME=$MINIO_BUCKETNAME"  >> .env
-      nvm use 16
-      pnpm install
-    command: |
-      npm run start:dev
-  - name: Setup React Wrapper
-    openMode: tab-after
-    init: |
-      gp sync-await setup
-      cd apps/wrapper
-      nvm use 16
-      pnpm install
-      bash make-envs-for-react.sh
-    command: |
-      npm run start
       
 ports:
   - port: 8065

README.md

@@ -221,3 +221,10 @@ TODO: Add details on the specifications
 ## Possible Attack Vectors
 1. XSS (High Priority) - Simple form
 2. SQL Injection (High Priority) - needs to be fixed.
+
+NOTE:
+-----
+```
+For local development enketo-express needs node 14 and pnpm@7
+Run nvm use 14 && npm i -g pnpm@7 if developing in enketo-express
+```

apps/wrapper/.dockerignore

+node_modules

apps/wrapper/Dockerfile

+#get the latest alpine image from node registry
+FROM node:16-alpine AS build-stage
+RUN npm i -g pnpm
+#set the working directory
+WORKDIR /app
+
+#copy the package and package lock files
+#from local to container work directory /app
+COPY package.json /app/
+COPY pnpm-lock.yaml /app/
+
+#Run command npm install to install packages
+RUN pnpm install
+
+#copy all the folder contents from local to container
+COPY . .
+
+#specify env variables at runtime
+ARG REACT_APP_ENKETO_URL
+ARG REACT_APP_FORM_MANAGER_URL
+ARG REACT_APP_HASURA_URL
+
+ENV REACT_APP_ENKETO_URL $REACT_APP_ENKETO_URL
+ENV REACT_APP_FORM_MANAGER_URL $REACT_APP_FORM_MANAGER_URL
+ENV REACT_APP_HASURA_URL $REACT_APP_HASURA_URL
+
+
+#create a react production build
+RUN npm run build
+
+#get the latest alpine image from nginx registry
+FROM nginx:alpine
+
+#we copy the output from first stage that is our react build
+#into nginx html directory where it will serve our index file
+COPY --from=build-stage /app/build/ /usr/share/nginx/html
\ No newline at end of file

apps/wrapper/make-envs-for-react.sh

-#!/bin/bash
-
-echo "REACT_APP_GITPOD_WORKSPACE_URL=${GITPOD_WORKSPACE_URL}" > .env
\ No newline at end of file

apps/wrapper/src/api/index.js

-const GITPOD_URL = process.env.REACT_APP_GITPOD_WORKSPACE_URL
+const HASURA_URL = process.env.REACT_APP_HASURA_URL
+const FORM_MANAGER_URL = process.env.REACT_APP_FORM_MANAGER_URL
 
 const makeHasuraCalls = async (query) => {
     // const userData = getCookie("userData");
-    return fetch(`${GITPOD_URL.slice(0, GITPOD_URL.indexOf('/') + 2) + "8080-" + GITPOD_URL.slice(GITPOD_URL.indexOf('/') + 2)}/v1/graphql`, {
+    return fetch(`${HASURA_URL}/v1/graphql`, {
         method: "POST",
         headers: {
             Accept: "application/json",
@@ -43,7 +44,7 @@ export const saveFormSubmission = (data) => {
 
 export const getPrefillXML = async (form, onFormSuccessData, prefillXML, imageUrls) => {
     try {
-        let res = await fetch(`${GITPOD_URL.slice(0, GITPOD_URL.indexOf('/') + 2) + "3006-" + GITPOD_URL.slice(GITPOD_URL.indexOf('/') + 2)}/prefillXML?form=${form}&onFormSuccessData=${encodeURI(
+        let res = await fetch(`${FORM_MANAGER_URL}/prefillXML?form=${form}&onFormSuccessData=${encodeURI(
             JSON.stringify(onFormSuccessData)
         )}`, {
             method: 'POST',

apps/wrapper/src/components/GenericForm/index.js

@@ -3,7 +3,9 @@ import styles from './index.module.css';
 import beautify from "xml-beautifier";
 import { getPrefillXML, saveFormSubmission } from '../../api';
 
-const GITPOD_URL = process.env.REACT_APP_GITPOD_WORKSPACE_URL
+const ENKETO_URL = process.env.REACT_APP_ENKETO_URL
+const FORM_MANAGER_URL = process.env.REACT_APP_FORM_MANAGER_URL
+const HASURA_URL = process.env.REACT_APP_HASURA_URL
 
 const GenericForm = (props) => {
   const { selectedFlow, setSelectedFlow } = props;
@@ -77,7 +79,7 @@ const GenericForm = (props) => {
   }
 
   const parseFormData = async (formData) => {
-    let jsonRes = await fetch(`${GITPOD_URL.slice(0, GITPOD_URL.indexOf('/') + 2) + "3006-" + GITPOD_URL.slice(GITPOD_URL.indexOf('/') + 2)}/parse`, {
+    let jsonRes = await fetch(`${FORM_MANAGER_URL}/parse`, {
       method: 'POST',
       body: JSON.stringify({
         xml: formData,
@@ -114,13 +116,13 @@ const GenericForm = (props) => {
         selectedFlow.offline && <p className='animate__animated animate__fadeIn' style={{ color: '#fff', fontSize: '1.5rem' }}>Disable internet and try submitting the form</p>
       }
       {
-        selectedFlow.submitToHasura && <p className='animate__animated animate__fadeIn' style={{ color: '#fff', fontSize: '1.5rem' }}>Submit the form and check <a style={{color: '#ffc119'}} target="_blank" href={`${GITPOD_URL.slice(0, GITPOD_URL.indexOf('/') + 2) + "8080-" + GITPOD_URL.slice(GITPOD_URL.indexOf('/') + 2)}`}>Hasura</a></p>
+        selectedFlow.submitToHasura && <p className='animate__animated animate__fadeIn' style={{ color: '#fff', fontSize: '1.5rem' }}>Submit the form and check <a style={{ color: '#ffc119' }} target="_blank" href={`${HASURA_URL}`}>Hasura</a></p>
       }
       <div className={styles.formContainer}>
         <iframe title='current-form'
           className={styles.odkForm}
           src={
-            `${GITPOD_URL.slice(0, GITPOD_URL.indexOf('/') + 2) + "8065-" + GITPOD_URL.slice(GITPOD_URL.indexOf('/') + 2)}/preview?formSpec=${encodedFormSpec}&xform=${encodedFormURI}`
+            `${ENKETO_URL}/preview?formSpec=${encodedFormSpec}&xform=${encodedFormURI}`
           }
         />
         <div className={styles.jsonResponse}>

apps/wrapper/src/workflow_first.json

 {
   "forms": {
-    "jumping_form_1": {
+    "test_form": {
       "skipOnSuccessMessage": true,
       "prefill": {},
       "submissionURL": "",
@@ -49,6 +49,6 @@
       }
     }
   },
-  "startingForm": "jumping_form_1",
+  "startingForm": "test_form",
   "metaData": {}
 }
\ No newline at end of file

docker-compose.gitpod.yaml → docker-compose.yml

@@ -20,7 +20,6 @@ services:
 
   search:
     image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0
-    container_name: search
     environment:
       cluster.name: fusionauth
       bootstrap.memory_lock: "true"
@@ -82,19 +81,13 @@ services:
 
   redis_main:
     image: redis:5
-    ports:
-      - "6381:6379"
     volumes:
-      - ../redis/conf/redis-enketo-main.conf:/etc/redis/redis.conf:ro
       - ./redis_main_data/:/data/
     restart: unless-stopped
 
   redis_cache:
     image: redis:5
-    ports:
-      - "6382:6379"
     volumes:
-      - ../redis/conf/redis-enketo-cache.conf:/etc/redis/redis.conf:ro
       - ./redis_cache_data/:/data/
     restart: unless-stopped
 
@@ -107,8 +100,8 @@ services:
     volumes:
       - ./pgdata:/var/lib/postgresql/data
     environment:
-      POSTGRES_USER: timescaledb
-      POSTGRES_PASSWORD: postgrespassword
+      POSTGRES_USER: ${TSDB_POSTGRES_USER}
+      POSTGRES_PASSWORD: ${TSDB_POSTGRES_PASSWORD}
 
   gql:
     image: hasura/graphql-engine:latest
@@ -118,17 +111,18 @@ services:
       - ./data/migrations:/hasura-migrations
       - ./data/metadata:/hasura-metadata
     depends_on:
-      - "tsdb"
+      - tsdb
+      - fusionauth
     restart: always
     environment:
-      - HASURA_GRAPHQL_DATABASE_URL=postgres://timescaledb:postgrespassword@tsdb:5432/postgres?sslmode=disable
+      - HASURA_GRAPHQL_DATABASE_URL=${HASURA_GRAPHQL_DATABASE_URL}
       - HASURA_GRAPHQL_ENABLE_CONSOLE=true # set to "false" to disable console
       - HASURA_GRAPHQL_DEV_MODE=true
       - HASURA_GRAPHQL_ENABLED_LOG_TYPES=startup,http-log,webhook-log,websocket-log,query-log
       - HASURA_GRAPHQL_MIGRATIONS_DISABLE_TRANSACTION=true
       - HASURA_GRAPHQL_CONSOLE_ASSETS_DIR=/srv/console-assets
-      - HASURA_GRAPHQL_ADMIN_SECRET=myadminsecretkey
-      - HASURA_GRAPHQL_JWT_SECRET={"type":"RS256","jwk_url":"http://fusionauth:9011/.well-known/jwks.json"}
+      - HASURA_GRAPHQL_ADMIN_SECRET=${HASURA_GRAPHQL_ADMIN_SECRET}
+      - HASURA_GRAPHQL_JWT_SECRET=${HASURA_GRAPHQL_JWT_SECRET}
 
   minio:
     image: minio/minio
@@ -155,6 +149,65 @@ services:
       exit 0;
       "
 
+  enketo-express:
+    image: enketo-express
+    build:
+      dockerfile: enketo-express/Dockerfile
+      context: packages
+    depends_on:
+      - redis_main
+      - redis_cache
+    ports:
+      - 8065:8065
+    environment:
+      ENCRYPTION_KEY: ${ENCRYPTION_KEY}
+      LESS_SECURE_ENCRYPTION_KEY: ${LESS_SECURE_ENCRYPTION_KEY}
+      REDIS_MAIN_HOST: redis_main
+      REDIS_MAIN_PORT: 6379
+      REDIS_CACHE_HOST: redis_cache
+      REDIS_CACHE_PORT: 6379
+      FORM_MANAGER_BASE_URI: ${FORM_MANAGER_BASE_URI}
+  
+  fm-cache:
+    image: redis:7
+    volumes:
+      - ./fm_redis_data/:/data/
+    restart: unless-stopped
+
+  form-manager:
+    image: form-manager
+    build: packages/form-manager
+    depends_on:
+      - fm-cache
+      - minio
+      - createbuckets
+    ports: 
+      - 3006:3006
+    environment:
+      MINIO_USERNAME: ${MINIO_USERNAME}
+      MINIO_ENDPOINT: ${MINIO_ENDPOINT}
+      MINIO_PORT: ${MINIO_PORT}
+      MINIO_URL: ${MINIO_URL}
+      MINIO_PASSWORD: ${MINIO_PASSWORD}
+      MINIO_BUCKETNAME: ${MINIO_BUCKETNAME} 
+      REDIS_HOST: ${REDIS_HOST}
+      REDIS_PORT: ${REDIS_PORT}
+      FORM_MANAGER_URL: ${http://form-manager:3006}
+      MINIO_USE_SSL: ${MINIO_USE_SSL}
+
+
+  wrapper:
+    image: wrapper
+    build:
+      context: apps/wrapper
+      # env vars to be passed at build time as it will build a static file
+      args:
+        REACT_APP_ENKETO_URL: ${REACT_APP_ENKETO_URL}
+        REACT_APP_FORM_MANAGER_URL: ${REACT_APP_FORM_MANAGER_URL}
+        REACT_APP_HASURA_URL: ${REACT_APP_HASURA_URL}
+    ports:
+      - 3000:80
+      
 networks:
   db_net:
     driver: bridge

install-node.sh

-#!/bin/bash
-
-docker-compose -f docker-compose.gitpod.yaml up -d
-
-curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.3/install.sh | bash
-export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"
-[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm
-
-nvm install v14.19.1
-nvm install 16
-nvm use 14
-
-wget -qO- https://get.pnpm.io/install.sh | ENV="$HOME/.bashrc" SHELL="$(which bash)" bash -
\ No newline at end of file

packages/.dockerignore

+# Ignore node modules and form manager
+
+enketo-express/node_modules
+enketo-core/node_modules
+form-manager
\ No newline at end of file

packages/enketo-express/Dockerfile

-FROM node:12
-
-ENV ENKETO_SRC_DIR=/srv/src/enketo_express
-
-WORKDIR ${ENKETO_SRC_DIR}
-
-RUN npm install -g grunt-cli pm2
-
-COPY . ${ENKETO_SRC_DIR}
-
-RUN npm install --production
-
-# Persist the `secrets` directory so the encryption key remains consistent.
-RUN mkdir -p ${ENKETO_SRC_DIR}/setup/docker/secrets
-VOLUME ${ENKETO_SRC_DIR}/setup/docker/secrets
-
-EXPOSE 8005
-
-CMD ["/bin/bash", "-c", "${ENKETO_SRC_DIR}/setup/docker/start.sh"]
+FROM ubuntu:20.04
+
+RUN apt update && apt install -y build-essential curl wget jq
+RUN curl -fsSL https://deb.nodesource.com/setup_14.x | bash -
+RUN apt-get install -y nodejs
+RUN npm i -g pnpm@7 
+RUN npm i -g pm2
+WORKDIR /app
+COPY enketo-core enketo-core
+COPY enketo-express enketo-express
+WORKDIR enketo-express
+# TODO: builds can be made faster if we only copy files needed to do pnpm i. pnpm i will be cached or explore multistage builds
+RUN pnpm i 
+EXPOSE 8065
+RUN chmod +x start.sh
+ENTRYPOINT ["./start.sh"]
\ No newline at end of file

packages/enketo-express/config/default-config.json

@@ -111,4 +111,4 @@
     "payload limit": "100kb",
     "text field character limit": 2000,
     "formManagerBaseURI": "https://enketo-manager-ratings-tech.samagra.io/"
-}
+}
\ No newline at end of file

packages/enketo-express/start.sh

+#!/bin/bash
+
+echo "$(jq -r --arg ENCRYPTION_KEY $ENCRYPTION_KEY '."encryption key" |= $ENCRYPTION_KEY' config/default-config.json)" > config/default-config.json
+echo "$(jq -r --arg LESS_SECURE_ENCRYPTION_KEY $LESS_SECURE_ENCRYPTION_KEY '."less secure encryption key" |= $LESS_SECURE_ENCRYPTION_KEY' config/default-config.json)" > config/default-config.json
+echo "$(jq -r --arg REDIS_MAIN_HOST $REDIS_MAIN_HOST '.redis.main.host |= $REDIS_MAIN_HOST' config/default-config.json)" > config/default-config.json
+echo "$(jq -r --arg REDIS_MAIN_PORT $REDIS_MAIN_PORT '.redis.main.port |= $REDIS_MAIN_PORT' config/default-config.json)" > config/default-config.json
+echo "$(jq -r --arg REDIS_CACHE_HOST $REDIS_CACHE_HOST '.redis.cache.host |= $REDIS_CACHE_HOST' config/default-config.json)" > config/default-config.json
+echo "$(jq -r --arg REDIS_CACHE_PORT $REDIS_CACHE_PORT '.redis.cache.port |= $REDIS_CACHE_PORT' config/default-config.json)" > config/default-config.json
+echo "$(jq -r --arg FORM_MANAGER_BASE_URI $FORM_MANAGER_BASE_URI '.formManagerBaseURI |= $FORM_MANAGER_BASE_URI' config/default-config.json)" > config/default-config.json
+
+node app.js
\ No newline at end of file

packages/enketo-express/update-config.sh

-#!/bin/bash
-
-cd ./config
-formManagerBaseURI=${GITPOD_WORKSPACE_URL:8}
-echo "$( jq '."formManagerBaseURI" = "'"https://3006-$formManagerBaseURI"'"' default-config.json )" > default-config.json   
-cd ..
\ No newline at end of file

packages/form-manager/.dockerignore

+.env
+node_modules
+dist
\ No newline at end of file