Merge pull request #1066 from UniversityOfHelsinkiCS/trunk

DB split, Fix rounding

.gitignore

@@ -1,5 +1,6 @@
 data
 pgdata
+pgdata_kone
 user_pgdata
 analytics_pgdata
 usage_pgdata
@@ -26,4 +27,4 @@ services/updater_scheduler/active_student_numbers.txt
 services/oodikone2-backend/debug.log
 datastore
 services/updater_api/debug.log
-services/backend/updater_writer/dumps
+services/backend/updater_writer/dumps

.travis.yml

@@ -56,12 +56,12 @@ jobs:
         - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record
         - docker ps -a
         - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi
-      deploy: # deploy is always skipped for PR
-        provider: script
-        skip_cleanup: true
-        script: "docker-compose -f docker-compose.lateste2e.production.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}"
-        on:
-          tags: true
+      # deploy: # deploy is always skipped for PR
+      #   provider: script
+      #   skip_cleanup: true
+      #   script: "docker-compose -f docker-compose.lateste2e.production.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}"
+      #   on:
+      #     tags: true
 
 after_failure:
   - "curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${IMAGE} buildi paskana. STOP AND FIX tai tulee kenkää! *${AUTHOR_NAME}* ${TRAVIS_BUILD_WEB_URL}\\\"}\" ${SLACKBOT_URL}"

adminer-plugins/login-servers.php

@@ -6,6 +6,6 @@
     * @param string
     */
 return new AdminerLoginServers(
-    $servers = array("db" => "db", "user_db" => "user_db", "analytics_db" => "analytics_db"),
+    $servers = array("db" => "db", "db_kone" => "db_kone", "user_db" => "user_db", "analytics_db" => "analytics_db"),
     $driver = "pgsql"
 );

docker-compose.lateste2e.production.yml

@@ -42,6 +42,18 @@ services:
       - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test
       - POSTGRES_USER=postgres
 
+  db_kone:
+    image: postgres:9.6.3
+    ports:
+      - "5425:5432"
+    volumes:
+      - ./pgdata_kone:/var/lib/postgresql/data
+      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
+    container_name: db_kone
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=db_kone
+      - POSTGRES_USER=postgres
+
   redis:
     image: redis
     command: ["redis-server", "--appendonly", "yes"]
@@ -69,10 +81,12 @@ services:
       - "8080:8080"
       - "9229:9229"
     environment:
-      WAIT_HOSTS: redis:6379,db:5432,nats:4222
+      WAIT_HOSTS: redis:6379,db:5432,db_kone:5432,nats:4222
       REDIS: redis
       DB_URL: postgres://postgres@db:5432/tkt_oodi
+      DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone
       TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
+      TEST_DB_KONE: postgres://postgres@db_kone:5432/db_kone_test
       FRONT_URL: http://localhost:1337
       TOKEN_SECRET: IWannaBeTheVeryBest
       OODILEARN_URL: http://oodilearn:5000
@@ -83,7 +97,6 @@ services:
       ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
       ANALYTICS_URL: http://analytics:4568
       NATS_URI: nats://nats:4222
-      DB_SCHEMA_KONE: kone_data
     container_name: backend
     depends_on:
       - db

docker-compose.lateste2e.yml

@@ -44,6 +44,18 @@ services:
       - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test
       - POSTGRES_USER=postgres
 
+  db_kone:
+    image: postgres:9.6.3
+    ports:
+      - "5425:5432"
+    volumes:
+      - ./pgdata_kone:/var/lib/postgresql/data
+      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
+    container_name: db_kone
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=db_kone
+      - POSTGRES_USER=postgres
+
   redis:
     image: redis
     command: ["redis-server", "--appendonly", "yes"]
@@ -71,10 +83,12 @@ services:
       - "8080:8080"
       - "9229:9229"
     environment:
-      WAIT_HOSTS: redis:6379,db:5432,nats:4222
+      WAIT_HOSTS: redis:6379,db:5432,db_kone:5432,nats:4222
       REDIS: redis
       DB_URL: postgres://postgres@db:5432/tkt_oodi
+      DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone
       TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
+      TEST_DB_KONE: postgres://postgres@db_kone:5432/db_kone_test
       FRONT_URL: http://localhost:1337
       TOKEN_SECRET: IWannaBeTheVeryBest
       OODILEARN_URL: http://oodilearn:5000
@@ -85,7 +99,6 @@ services:
       ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
       ANALYTICS_URL: http://analytics:4568
       NATS_URI: nats://nats:4222
-      DB_SCHEMA_KONE: kone_data
     container_name: backend
     depends_on:
       - db
@@ -105,7 +118,7 @@ services:
       BACKEND_ADDR: backend
       ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres
     container_name: frontend
-    
+
   userservice:
     image: toska/oodikone2-userservice:staging
     build:

docker-compose.yml

@@ -84,6 +84,18 @@ services:
       - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test,tkt_oodi_real
       - POSTGRES_USER=postgres
 
+  db_kone:
+    image: postgres:9.6.3
+    ports:
+      - "5425:5432"
+    volumes:
+      - ./pgdata_kone:/var/lib/postgresql/data
+      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
+    container_name: db_kone
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=db_kone
+      - POSTGRES_USER=postgres
+
   redis:
     image: redis
     command: ["redis-server", "--appendonly", "yes"]
@@ -114,10 +126,12 @@ services:
     ports:
       - "8080:8080"
     environment:
-      WAIT_HOSTS: redis:6379,db:5432,nats:4222
+      WAIT_HOSTS: redis:6379,db:5432,db_kone:5432,nats:4222
       REDIS: redis
       DB_URL: postgres://postgres@db:5432/tkt_oodi
+      DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone
       TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
+      TEST_DB_KONE: postgres://postgres@db_kone:5432/db_kone_test
       FRONT_URL: http://localhost:8000
       TOKEN_SECRET: IWannaBeTheVeryBest
       OODILEARN_URL: http://oodilearn:5000
@@ -127,7 +141,6 @@ services:
       USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       ANALYTICS_URL: http://analytics:4568
       NATS_URI: nats://nats:4222
-      DB_SCHEMA_KONE: kone_data
     volumes:
       - ./services/backend/oodikone2-backend/:/usr/src/app
       - ./services/backend/shared/models:/usr/src/app/src/models
@@ -156,6 +169,7 @@ services:
       USER_ADMINER_URL: http://localhost:5050/?pgsql=user_db&username=postgres
       ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
       USAGE_ADMINER_URL: http://localhost:5050/?pgsql=usage_db&username=postgres
+      KONE_ADMINER_URL: http://localhost:5050/?pgsql=db_kone&username=postgres
     volumes:
       - ./services/oodikone2-frontend/:/usr/src/app
     container_name: frontend

scripts.sh

@@ -5,6 +5,7 @@ ANONDB_DIR=anonyymioodi
 BACKUP_DIR=backups
 PSQL_DB_BACKUP="$ANONDB_DIR/anon.sqz"
 USER_DB_BACKUP="$ANONDB_DIR/user-dump.sqz"
+KONE_DB_BACKUP="$ANONDB_DIR/anon_kone.sqz"
 PSQL_REAL_DB_BACKUP="$BACKUP_DIR/latest-pg.sqz"
 USER_REAL_DB_BACKUP="$BACKUP_DIR/latest-user-pg.sqz"
 
@@ -70,6 +71,10 @@ restore_real_userdb_from_backup () {
     time pg_restore -U postgres -h localhost -p 5422 --no-owner -F c --dbname=user_db_real -j4 $USER_REAL_DB_BACKUP
 }
 
+restore_dbkone_from_backup () {
+    time pg_restore -U postgres -h localhost -p 5425 --no-owner -F c --dbname=db_kone -j4 $KONE_DB_BACKUP
+}
+
 # oodilearn
 # restore_mongodb_from_backup () {
 #     docker exec -t mongo_db mongorestore -d oodilearn "/dump"
@@ -97,6 +102,9 @@ db_anon_setup_full () {
     ping_psql "oodi_db" "tkt_oodi"
     ping_psql "oodi_db" "tkt_oodi_test"
     retry restore_psql_from_backup
+    ping_psql "db_kone" "db_kone"
+    ping_psql "db_kone" "db_kone_test"
+    retry restore_dbkone_from_backup
     # echo "Restoring MongoDB from backup"
     # retry restore_mongodb_from_backup
     echo "Restore user db from backup"
@@ -120,11 +128,15 @@ reset_real_db () {
 
 reset_db () {
     docker-compose down
-    docker-compose up -d db user_db
+    docker-compose up -d db user_db db_kone
     ping_psql "oodi_db" "tkt_oodi"
     docker exec -u postgres oodi_db dropdb "tkt_oodi"
     ping_psql "oodi_db" "tkt_oodi_test"
     docker exec -u postgres oodi_db dropdb "tkt_oodi_test"
+    ping_psql "db_kone" "db_kone"
+    docker exec -u postgres db_kone dropdb "db_kone"
+    ping_psql "db_kone" "db_kone_test"
+    docker exec -u postgres db_kone dropdb "db_kone_test"
     ping_psql "oodi_user_db" "user_db"
     docker exec -u postgres oodi_user_db dropdb "user_db"
     db_anon_setup_full
@@ -155,7 +167,7 @@ run_full_setup () {
     echo "Building images"
     docker-compose build
     echo "Setup oodikone db from dump."
-    docker-compose up -d db user_db
+    docker-compose up -d db user_db db_kone
     db_setup_full
     db_anon_setup_full
     docker-compose down
@@ -172,7 +184,7 @@ run_anon_full_setup () {
     echo "Building images"
     docker-compose build
     echo "Setup oodikone db from dump."
-    docker-compose up -d db user_db
+    docker-compose up -d db user_db db_kone
     db_anon_setup_full
     docker-compose down
     show_instructions
@@ -188,7 +200,7 @@ run_e2e_setup () {
     echo "Building images"
     docker-compose -f $1 build
     echo "Setup oodikone db from dump."
-    docker-compose -f $1 up -d db user_db
+    docker-compose -f $1 up -d db user_db db_kone
     db_anon_setup_full
     echo "Starting services."
     docker-compose -f $1 up -d

services/backend/oodikone2-backend/src/conf-backend.js

@@ -1,19 +1,22 @@
 require('dotenv').config()
 
 const { NODE_ENV } = process.env
-let DB_URL = process.env.DB_URL
 const isTest = process.env.NODE_ENV === 'test'
+let DB_URL = process.env.DB_URL
+let DB_URL_KONE = process.env.DB_URL_KONE
 if (isTest) {
   DB_URL = process.env.TEST_DB
+  DB_URL_KONE = process.env.TEST_DB_KONE
 }
 else if (NODE_ENV === 'anon') {
   DB_URL = process.env.ANON_DB
+  DB_URL_KONE = process.env.ANON_DB_KONE
 }
 const frontend_addr = process.env.FRONT_URL
 const redis = process.env.REDIS
 const TOKEN_SECRET = process.env.TOKEN_SECRET
 const DB_SCHEMA = process.env.DB_SCHEMA || 'public'
-const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'kone_data'
+const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'public'
 const CERT_PATH = process.env.CERT_PATH // production/staging only
 const KEY_PATH = process.env.KEY_PATH // production/staging only
 const OODILEARN_URL = process.env.OODILEARN_URL
@@ -58,10 +61,25 @@ if (process.env.NODE_ENV === 'dev' || isTest) {
 }
 
 module.exports = {
-  frontend_addr, DB_URL, redis, TOKEN_SECRET, DB_SCHEMA, DB_SCHEMA_KONE, OODI_ADDR, CERT_PATH, KEY_PATH, FEATURES, OODILEARN_URL,
-  USERSERVICE_URL: formatURL(USERSERVICE_URL), ACCESS_TOKEN_HEADER_KEY, PORT,
+  frontend_addr,
+  DB_URL,
+  DB_URL_KONE,
+  redis,
+  TOKEN_SECRET,
+  DB_SCHEMA,
+  DB_SCHEMA_KONE,
+  OODI_ADDR,
+  CERT_PATH,
+  KEY_PATH,
+  FEATURES,
+  OODILEARN_URL,
+  USERSERVICE_URL: formatURL(USERSERVICE_URL),
+  ACCESS_TOKEN_HEADER_KEY,
+  PORT,
   ANALYTICS_URL: formatURL(ANALYTICS_URL),
   USAGESERVICE_URL,
-  requiredGroup, OODI_SECRET, OODI_SECRET_HEADER_KEY,
+  requiredGroup,
+  OODI_SECRET,
+  OODI_SECRET_HEADER_KEY,
   isTest
 }

services/backend/oodikone2-backend/src/database/connection.js

@@ -10,14 +10,13 @@ const sequelize = new Sequelize(conf.DB_URL, {
 })
 sequelize.query(`SET SESSION search_path to ${conf.DB_SCHEMA}`)
 
-const sequelizeKone = new Sequelize(conf.DB_URL, {
+const sequelizeKone = new Sequelize(conf.DB_URL_KONE, {
   schema: conf.DB_SCHEMA_KONE,
   searchPath: conf.DB_SCHEMA_KONE,
   logging: false,
   operatorsAliases: false
 })
 sequelizeKone.query(`SET SESSION search_path to ${conf.DB_SCHEMA_KONE}`)
-// See https://github.com/sequelize/sequelize/issues/10875
 
 const runMigrations = async () => {
   try {
@@ -74,7 +73,8 @@ const runMigrationsKone = async () => {
   }
 }
 
-const migrationPromise = !conf.isTest ? runMigrations().then(() => runMigrationsKone()) : Promise.resolve()
+const migrationPromise = !conf.isTest ? runMigrations() : Promise.resolve()
+const migrationPromiseKone = !conf.isTest ? runMigrationsKone() : Promise.resolve()
 
 const forceSyncDatabase = async () => {
   try {
@@ -95,5 +95,6 @@ module.exports = {
   sequelize,
   sequelizeKone,
   migrationPromise,
+  migrationPromiseKone,
   forceSyncDatabase
 }

services/backend/oodikone2-backend/src/database/seed_migrations.js

@@ -1,5 +1,6 @@
 const fs = require('fs')
-const { Migration, MigrationKone } = require('../models/index')
+const { Migration } = require('../models/index')
+const { MigrationKone } = require('../models/models_kone')
 
 const DEFAULT_PATH = `${process.cwd()}/src/database/migrations`
 const DEFAULT_PATH_KONE = `${process.cwd()}/src/database/migrations_kone`

services/backend/oodikone2-backend/src/events.js

@@ -73,6 +73,7 @@ const StartNats = () => {
                 e
               )
             }
+            console.error(e)
             console.log(
               `Failed to update throughput stats for code: ${code}, reason: ${
                 e.message
@@ -93,6 +94,7 @@ const StartNats = () => {
             } catch (e) {
               console.error(e)
             }
+            console.error(e)
             console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
           }
           ready += 1

services/backend/oodikone2-backend/src/routes/elementdetails.js

@@ -89,6 +89,7 @@ router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
         console.error(e)
         return
       }
+      console.error(e)
       console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
     }
     ready += 1
@@ -149,6 +150,7 @@ router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
         console.error(e)
         return
       }
+      console.error(e)
       console.log(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
     }
     ready += 1