Merge pull request #1070 from UniversityOfHelsinkiCS/trunk

Changed pg_restore commands to use docker instead and edited ping_psq…
UniversityOfHelsinkiCS · Jul 3, 2019 · 05de857 · 05de857
2 parents 3755149 + c91d633
commit 05de857
Show file tree

Hide file tree

Showing 4 changed files with 48 additions and 64 deletions.
diff --git a/docker-compose.real.yml b/docker-compose.real.yml
@@ -4,6 +4,7 @@ services:
   backend:
     environment:
       DB_URL: postgres://postgres@db:5432/tkt_oodi_real
+      DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone_real
   userservice:
     environment:
       DB_URL: postgres://postgres@user_db:5432/user_db_real
diff --git a/run.sh b/run.sh
@@ -26,7 +26,6 @@ options=(
     "Reset real db"
     "Download latest anon db"
     "Download latest real db"
-    "Full reset"
     "Quit"
 )
 select opt in "${options[@]}"
@@ -56,9 +55,6 @@ do
         "Download latest real db")
             get_oodikone_server_backup
             ;;
-        "Full reset")
-            purge
-            ;;
         "Quit")
             break
             ;;

diff --git a/scripts.sh b/scripts.sh
@@ -7,6 +7,7 @@ PSQL_DB_BACKUP="$ANONDB_DIR/anon.sqz"
 USER_DB_BACKUP="$ANONDB_DIR/user-dump.sqz"
 KONE_DB_BACKUP="$ANONDB_DIR/anon_kone.sqz"
 PSQL_REAL_DB_BACKUP="$BACKUP_DIR/latest-pg.sqz"
+KONE_REAL_DB_BACKUP="$BACKUP_DIR/latest-kone-pg.sqz"
 USER_REAL_DB_BACKUP="$BACKUP_DIR/latest-user-pg.sqz"
 
 retry () {
@@ -26,18 +27,6 @@ echo_path () {
     echo $(pwd)
 }
 
-purge () {
-    docker-compose down || echo "docker-compose down failed"
-    git clean -f -fdX
-}
-
-megapurge () {
-    git clean -f -fdX
-    docker stop $(docker ps -q)
-    docker container prune
-    docker rmi $(docker images -q)
-}
-
 get_oodikone_server_backup() {
     scp -r -o ProxyCommand="ssh -W %h:%p melkki.cs.helsinki.fi" oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/* "$BACKUP_DIR/"
 }
@@ -56,23 +45,8 @@ get_anon_oodikone() {
 }
 
 restore_psql_from_backup () {
-    time pg_restore -U postgres -h localhost -p 5421 --no-owner -F c --dbname=tkt_oodi -j4 $PSQL_DB_BACKUP
-}
-
-restore_real_psql_from_backup () {
-    time pg_restore -U postgres -h localhost -p 5421 --no-owner -F c --dbname=tkt_oodi_real -j4 $PSQL_REAL_DB_BACKUP
-}
-
-restore_userdb_from_backup () {
-    time pg_restore -U postgres -h localhost -p 5422 --no-owner -F c --dbname=user_db -j4 $USER_DB_BACKUP
-}
-
-restore_real_userdb_from_backup () {
-    time pg_restore -U postgres -h localhost -p 5422 --no-owner -F c --dbname=user_db_real -j4 $USER_REAL_DB_BACKUP
-}
-
-restore_dbkone_from_backup () {
-    time pg_restore -U postgres -h localhost -p 5425 --no-owner -F c --dbname=db_kone -j4 $KONE_DB_BACKUP
+    docker cp $1 $2:/asd.sqz
+    docker exec -it $2 pg_restore -U postgres --no-owner -F c --dbname=$3 -j4 /asd.sqz
 }
 
 # oodilearn
@@ -81,64 +55,56 @@ restore_dbkone_from_backup () {
 # }
 
 ping_psql () {
-    echo "Pinging psql in container $1 with db name $2"
+    drop_psql $1 $2
+    echo "Creating psql in container $1 with db name $2"
     retry docker exec -u postgres $1 pg_isready --dbname=$2
     docker exec -u postgres $1 psql -c "CREATE DATABASE $2" || echo "container $1 DB $2 already exists"
 }
 
+drop_psql () {
+    echo "Dropping psql in container $1 with db name $2"
+    retry docker exec -u postgres $1 pg_isready --dbname=$2
+    docker exec -u postgres $1 psql -c "DROP DATABASE $2" || echo "container $1 DB $2 doesn't exists"
+}
+
 db_setup_full () {
     echo "Restoring PostgreSQL from backup"
-    retry restore_real_psql_from_backup
+    ping_psql "oodi_db" "tkt_oodi_real"
+    restore_psql_from_backup $PSQL_REAL_DB_BACKUP oodi_db tkt_oodi_real
+    ping_psql "db_kone" "db_kone_real"
+    restore_psql_from_backup $KONE_REAL_DB_BACKUP db_kone db_kone_real
+    ping_psql "oodi_user_db" "user_db_real"
+    restore_psql_from_backup $USER_REAL_DB_BACKUP oodi_user_db user_db_real
     # echo "Restoring MongoDB from backup"
     # retry restore_mongodb_from_backup
-    echo "Restore user db from backup"
-    ping_psql "oodi_user_db" "user_db_real"
-    retry restore_real_userdb_from_backup
     echo "Database setup finished"
 }
 
 db_anon_setup_full () {
     echo "Restoring PostgreSQL from backup"
     ping_psql "oodi_db" "tkt_oodi"
     ping_psql "oodi_db" "tkt_oodi_test"
-    retry restore_psql_from_backup
+    restore_psql_from_backup $PSQL_DB_BACKUP oodi_db tkt_oodi
     ping_psql "db_kone" "db_kone"
     ping_psql "db_kone" "db_kone_test"
-    retry restore_dbkone_from_backup
+    restore_psql_from_backup $KONE_DB_BACKUP db_kone db_kone
+    ping_psql "oodi_user_db" "user_db"
+    restore_psql_from_backup $USER_DB_BACKUP oodi_user_db user_db
     # echo "Restoring MongoDB from backup"
     # retry restore_mongodb_from_backup
-    echo "Restore user db from backup"
-    ping_psql "oodi_user_db" "user_db"
-    retry restore_userdb_from_backup
     echo "Database setup finished"
 }
 
 reset_real_db () {
     docker-compose down
-    docker-compose up -d db user_db
-    ping_psql "oodi_db" "tkt_oodi_real"
-    docker exec -u postgres oodi_db dropdb "tkt_oodi_real"
-    docker exec -u postgres oodi_db psql -c "CREATE DATABASE tkt_oodi_real"
-    ping_psql "oodi_user_db" "user_db_real"
-    docker exec -u postgres oodi_user_db dropdb "user_db_real"
-    docker exec -u postgres oodi_user_db psql -c "CREATE DATABASE user_db_real"
+    docker-compose up -d db user_db db_kone
     db_setup_full
     docker-compose down
 }
 
 reset_db () {
     docker-compose down
     docker-compose up -d db user_db db_kone
-    ping_psql "oodi_db" "tkt_oodi"
-    docker exec -u postgres oodi_db dropdb "tkt_oodi"
-    ping_psql "oodi_db" "tkt_oodi_test"
-    docker exec -u postgres oodi_db dropdb "tkt_oodi_test"
-    ping_psql "db_kone" "db_kone"
-    docker exec -u postgres db_kone dropdb "db_kone"
-    ping_psql "db_kone" "db_kone_test"
-    docker exec -u postgres db_kone dropdb "db_kone_test"
-    ping_psql "oodi_user_db" "user_db"
-    docker exec -u postgres oodi_user_db dropdb "user_db"
     db_anon_setup_full
     docker-compose down
 }
@@ -147,10 +113,6 @@ install_cli_npm_packages () {
     npm ci
 }
 
-docker_build () {
-    docker-compose up -d --build
-}
-
 show_instructions () {
     cat ./assets/instructions.txt
 }

diff --git a/services/backend/shared/migrations_kone/20190703_00_add_usage_table.js b/services/backend/shared/migrations_kone/20190703_00_add_usage_table.js
@@ -0,0 +1,25 @@
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.sequelize.query(`
+-- Adminer 4.6.2 PostgreSQL dump
+
+CREATE TABLE IF NOT EXISTS "public"."usage_statistics" (
+    "id" character varying(255) NOT NULL,
+    "username" character varying(255),
+    "name" character varying(255),
+    "time" integer,
+    "admin" boolean,
+    "method" character varying(255),
+    "URL" character varying(255),
+    "status" integer,
+    "data" jsonb,
+    CONSTRAINT "usage_statistics_pkey" PRIMARY KEY ("id")
+) WITH (oids = false);
+
+
+-- 2019-07-03 15:39:09.675736+00
+`)
+  },
+  down: async () => {
+  }
+}