From f1df5630d4e663d9fa4c96ef2041be13d646e384 Mon Sep 17 00:00:00 2001 From: esakemp Date: Mon, 8 Jul 2019 10:28:06 +0300 Subject: [PATCH 1/3] lol sasu --- .../oodikone2-frontend/src/components/StudentDetails/index.jsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/oodikone2-frontend/src/components/StudentDetails/index.jsx b/services/oodikone2-frontend/src/components/StudentDetails/index.jsx index 4c33258c93..8dfeb2487a 100644 --- a/services/oodikone2-frontend/src/components/StudentDetails/index.jsx +++ b/services/oodikone2-frontend/src/components/StudentDetails/index.jsx @@ -74,7 +74,8 @@ class StudentDetails extends Component { } else { icon = } - const year = -(moment(new Date('1.1.1950')).diff(new Date(date), 'years')) // :D + const year = (moment(new Date(date)).diff(new Date('1950-1-1'), 'years')) + return [ reformatDate(date, 'DD.MM.YYYY'), `${isStudyModuleCredit ? `${getTextIn(course.name, language)} [Study Module]` : getTextIn(course.name, language)} (${course.code})`, From 477e39d38d26195489e6789f2096bc00a8c506c8 Mon Sep 17 00:00:00 2001 From: Warro Date: Mon, 8 Jul 2019 13:22:21 +0300 Subject: [PATCH 2/3] Use extend in travis docker-compose files --- .travis.yml | 12 +- README.md | 14 +- docker-compose.lateste2e.production.yml | 249 ----------------- docker-compose.lateste2e.yml | 251 ------------------ docker-compose.yml | 153 +---------- .../docker-compose.dev.real.yml | 0 docker/docker-compose.dev.yml | 170 ++++++++++++ .../docker-compose.lateste2e.production.yml | 62 +++++ docker/docker-compose.lateste2e.yml | 62 +++++ package.json | 12 +- run.sh | 2 +- scripts.sh | 34 +-- 12 files changed, 347 insertions(+), 674 deletions(-) delete mode 100644 docker-compose.lateste2e.production.yml delete mode 100644 docker-compose.lateste2e.yml rename docker-compose.real.yml => docker/docker-compose.dev.real.yml (100%) create mode 100644 docker/docker-compose.dev.yml create mode 100644 docker/docker-compose.lateste2e.production.yml create mode 100644 docker/docker-compose.lateste2e.yml diff --git a/.travis.yml b/.travis.yml index 4cb784ee4a..31cd499f7f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,10 +26,10 @@ jobs: - IMAGE=STAGING script: - docker login -u $DOCKER_USER -p $DOCKER_PASS - - source scripts.sh && time run_e2e_setup docker-compose.lateste2e.yml + - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml - docker ps - cat nginx/log - - docker-compose logs -f & + - docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f & - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done - time npm run test_services - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record @@ -38,7 +38,7 @@ jobs: deploy: # deploy is always skipped for PR provider: script skip_cleanup: true - script: "docker-compose -f docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}" + script: "docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}" on: branch: master - # in parallel @@ -47,10 +47,10 @@ jobs: - IMAGE=PRODUCTION script: - docker login -u $DOCKER_USER -p $DOCKER_PASS - - source scripts.sh && time run_e2e_setup docker-compose.lateste2e.production.yml + - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.production.yml - docker ps - cat nginx/log - - docker-compose logs -f & + - docker-compose -f docker-compose.yml -f ./docker/ocker-compose.lateste2e.production.yml logs -f & - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done - time npm run test_services - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record @@ -59,7 +59,7 @@ jobs: deploy: # deploy is always skipped for PR provider: script skip_cleanup: true - script: "docker-compose -f docker-compose.lateste2e.production.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}" + script: "docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.production.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}" on: tags: true diff --git a/README.md b/README.md index e3c3a2bbd7..5f4080ddf3 100644 --- a/README.md +++ b/README.md @@ -58,27 +58,27 @@ The mapping of ports and environment variables are also defined in the docker-co #### With anonymized data: ``` -docker-compose up -d +npm run docker:up ``` #### With real data: ``` -docker-compose -f docker-compose.yml -f docker-compose.real.yml up -d +docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml -f ./docker/docker-compose.dev.real.yml up -d ``` ### Stop the development environment ``` -docker-compose down +npm run docker:down ``` ### Restart container(s) ``` -docker-compose restart # all +npm run docker:restart # all -docker-compose restart backend # just backend +npm run docker:restart:backend # just backend ``` ### View the containers in the running environment @@ -90,9 +90,9 @@ docker-compose ps ``` -docker-compose logs -f # all +npm run docker:logs # all -docker-compose logs -f backend # just backend +npm run docker:logs:backend # just backend ``` diff --git a/docker-compose.lateste2e.production.yml b/docker-compose.lateste2e.production.yml deleted file mode 100644 index fd1e60b903..0000000000 --- a/docker-compose.lateste2e.production.yml +++ /dev/null @@ -1,249 +0,0 @@ -version: '3' - -services: - nats: - image: nats-streaming - command: -cid updaterNATS - expose: - - "4222" - container_name: nats - - analytics_db: - image: postgres:9.6.3 - ports: - - "5423:5432" - volumes: - - ./analytics_pgdata:/var/lib/postgresql/data - container_name: oodi_analytics_db - environment: - - POSTGRES_USER=postgres - - POSTGRES_DB=analytics_db - - user_db: - image: postgres:9.6.3 - ports: - - "5422:5432" - volumes: - - ./user_pgdata:/var/lib/postgresql/data - container_name: oodi_user_db - environment: - - POSTGRES_USER=postgres - - POSTGRES_DB=user_db - - db: - image: postgres:9.6.3 - ports: - - "5421:5432" - volumes: - - ./pgdata:/var/lib/postgresql/data - - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro - container_name: oodi_db - environment: - - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test - - POSTGRES_USER=postgres - - db_kone: - image: postgres:9.6.3 - ports: - - "5425:5432" - volumes: - - ./pgdata_kone:/var/lib/postgresql/data - - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro - container_name: db_kone - environment: - - POSTGRES_MULTIPLE_DATABASES=db_kone - - POSTGRES_USER=postgres - - redis: - image: redis - command: ["redis-server", "--appendonly", "yes"] - ports: - - "6379:6379" - volumes: - - ./redis-data:/data - container_name: redis - - analytics_redis: - image: redis - command: ["redis-server", "--appendonly", "yes"] - ports: - - "6380:6379" - volumes: - - ./analytics-redis-data:/data - container_name: analytics_redis - - backend: - image: toska/oodikone2-backend:latest - build: - context: services/backend - dockerfile: oodikone2-backend/Dockerfile - ports: - - "8080:8080" - - "9229:9229" - environment: - WAIT_HOSTS: redis:6379,db:5432,db_kone:5432,nats:4222 - REDIS: redis - DB_URL: postgres://postgres@db:5432/tkt_oodi - DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone - TEST_DB: postgres://postgres@db:5432/tkt_oodi_test - TEST_DB_KONE: postgres://postgres@db_kone:5432/db_kone_test - FRONT_URL: http://localhost:1337 - TOKEN_SECRET: IWannaBeTheVeryBest - OODILEARN_URL: http://oodilearn:5000 - USERSERVICE_URL: http://userservice:4567 - USAGESERVICE_URL: http://usageservice:545 - USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 - USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 - ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres - ANALYTICS_URL: http://analytics:4568 - NATS_URI: nats://nats:4222 - container_name: backend - depends_on: - - db - - redis - - nats - - frontend: - image: toska/oodikone2-frontend:latest - build: - context: services/oodikone2-frontend - args: - BASE_PATH: / - ports: - - "5000:5000" - environment: - WAIT_HOSTS: user_db:5432 - BACKEND_ADDR: backend - ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres - container_name: frontend - - userservice: - image: toska/oodikone2-userservice:latest - build: - context: services/oodikone2-userservice - ports: - - 4567:4567 - environment: - - WAIT_HOSTS=user_db:5432 - - DB_URL=postgres://postgres@user_db:5432/user_db - - TEST_DB=postgres://postgres@user_db:5432/user_db_test - - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 - - TOKEN_SECRET=IWannaBeTheVeryBest - depends_on: - - user_db - container_name: userservice - - usageservice: - image: toska/oodikone2-usageservice:latest - build: - context: services/oodikone2-usageservice - ports: - - 545:545 - environment: - - WAIT_HOSTS=usage_db:5432 - - PORT=545 - - DB_URL=postgres://postgres@usage_db:5432/usage_db - - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 - depends_on: - - usage_db - container_name: usageservice - - usage_db: - image: postgres:9.6.3 - ports: - - "5424:5432" - volumes: - - ./usage_pgdata:/var/lib/postgresql/data - container_name: oodi_usage_db - environment: - - POSTGRES_USER=postgres - - POSTGRES_DB=usage_db - - analytics: - image: toska/oodikone2-analytics:latest - build: - context: services/oodikone2-analytics - ports: - - 4568:4568 - environment: - - WAIT_HOSTS=analytics_db:5432,analytics_redis:6379 - - REDIS=analytics_redis - - DB_URL=postgres://postgres@analytics_db:5432/analytics_db - - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test - depends_on: - - analytics_db - - analytics_redis - container_name: analytics - - updater_api: - image: toska/updater_api:latest - build: - context: services/updater_api - depends_on: - - nats - environment: - - WAIT_HOSTS=nats:4222 - - NATS_URI=nats://nats:4222 - - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi - - TOKEN=toskab1gs8cret5 - container_name: updater_api - - updater_writer: - image: toska/updater_writer:latest - build: - context: services/backend - dockerfile: updater_writer/Dockerfile - depends_on: - - nats - - db - environment: - WAIT_HOSTS: nats:4222,db:5432 - NATS_URI: nats://nats:4222 - DB_URL: postgres://postgres@db:5432/tkt_oodi - TEST_DB: postgres://postgres@db:5432/tkt_oodi_test - TEST_DB_SCHEMA: updater_test - container_name: updater_writer - - scheduler_mongo: - image: mongo - volumes: - - ./mongodata:/data/db - expose: - - "27017" - container_name: scheduler_db - - updater_scheduler: - image: toska/updater_scheduler:latest - build: - context: services/updater_scheduler - command: ["echo", "mluukkai on orjuuttaja"] - depends_on: - - nats - - scheduler_mongo - environment: - - WAIT_HOSTS=nats:4222,scheduler_mongo:27017 - - NATS_URI=nats://nats:4222 - - MONGO_URI=mongodb://scheduler_mongo:27017 - - STUDENT_NUMBERS=./all_student_numbers.txt - - ACTIVE_STUDENTS=./active_student_numbers.txt - - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi - - TOKEN=toskab1gs8cret5 - container_name: updater_scheduler - - nginx: - image: nginx:latest - volumes: - - ./nginx.production.conf:/etc/nginx/nginx.conf - - ./nginx/error.log:/etc/nginx/error_log.log - - ./nginx/log:/var/log/nginx/localhost.error_log - - ./nginx/cache/:/etc/nginx/cache - - /etc/letsencrypt/:/etc/letsencrypt/ - ports: - - 1337:80 - depends_on: - - frontend - - backend - - userservice - - analytics - - container_name: nginx diff --git a/docker-compose.lateste2e.yml b/docker-compose.lateste2e.yml deleted file mode 100644 index 3dbb38f905..0000000000 --- a/docker-compose.lateste2e.yml +++ /dev/null @@ -1,251 +0,0 @@ -version: '3' - -services: - nats: - image: nats-streaming - command: -cid updaterNATS --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore - expose: - - "4222" - volumes: - - ./datastore:/datastore - container_name: nats - - analytics_db: - image: postgres:9.6.3 - ports: - - "5423:5432" - volumes: - - ./analytics_pgdata:/var/lib/postgresql/data - container_name: oodi_analytics_db - environment: - - POSTGRES_USER=postgres - - POSTGRES_DB=analytics_db - - user_db: - image: postgres:9.6.3 - ports: - - "5422:5432" - volumes: - - ./user_pgdata:/var/lib/postgresql/data - container_name: oodi_user_db - environment: - - POSTGRES_USER=postgres - - POSTGRES_DB=user_db - - db: - image: postgres:9.6.3 - ports: - - "5421:5432" - volumes: - - ./pgdata:/var/lib/postgresql/data - - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro - container_name: oodi_db - environment: - - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test - - POSTGRES_USER=postgres - - db_kone: - image: postgres:9.6.3 - ports: - - "5425:5432" - volumes: - - ./pgdata_kone:/var/lib/postgresql/data - - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro - container_name: db_kone - environment: - - POSTGRES_MULTIPLE_DATABASES=db_kone - - POSTGRES_USER=postgres - - redis: - image: redis - command: ["redis-server", "--appendonly", "yes"] - ports: - - "6379:6379" - volumes: - - ./redis-data:/data - container_name: redis - - analytics_redis: - image: redis - command: ["redis-server", "--appendonly", "yes"] - ports: - - "6380:6379" - volumes: - - ./analytics-redis-data:/data - container_name: analytics_redis - - backend: - image: toska/oodikone2-backend:staging - build: - context: services/backend - dockerfile: oodikone2-backend/Dockerfile - ports: - - "8080:8080" - - "9229:9229" - environment: - WAIT_HOSTS: redis:6379,db:5432,db_kone:5432,nats:4222 - REDIS: redis - DB_URL: postgres://postgres@db:5432/tkt_oodi - DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone - TEST_DB: postgres://postgres@db:5432/tkt_oodi_test - TEST_DB_KONE: postgres://postgres@db_kone:5432/db_kone_test - FRONT_URL: http://localhost:1337 - TOKEN_SECRET: IWannaBeTheVeryBest - OODILEARN_URL: http://oodilearn:5000 - USERSERVICE_URL: http://userservice:4567 - USAGESERVICE_URL: http://usageservice:545 - USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 - USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 - ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres - ANALYTICS_URL: http://analytics:4568 - NATS_URI: nats://nats:4222 - container_name: backend - depends_on: - - db - - redis - - nats - - frontend: - image: toska/oodikone2-frontend:staging - build: - context: services/oodikone2-frontend - args: - BASE_PATH: / - ports: - - "5000:5000" - environment: - WAIT_HOSTS: user_db:5432 - BACKEND_ADDR: backend - ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres - container_name: frontend - - userservice: - image: toska/oodikone2-userservice:staging - build: - context: services/oodikone2-userservice - ports: - - 4567:4567 - environment: - - WAIT_HOSTS=user_db:5432 - - DB_URL=postgres://postgres@user_db:5432/user_db - - TEST_DB=postgres://postgres@user_db:5432/user_db_test - - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 - - TOKEN_SECRET=IWannaBeTheVeryBest - depends_on: - - user_db - container_name: userservice - - usageservice: - image: toska/oodikone2-usageservice:staging - build: - context: services/oodikone2-usageservice - ports: - - 545:545 - environment: - - WAIT_HOSTS=usage_db:5432 - - PORT=545 - - DB_URL=postgres://postgres@usage_db:5432/usage_db - - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 - depends_on: - - usage_db - container_name: usageservice - - usage_db: - image: postgres:9.6.3 - ports: - - "5424:5432" - volumes: - - ./usage_pgdata:/var/lib/postgresql/data - container_name: oodi_usage_db - environment: - - POSTGRES_USER=postgres - - POSTGRES_DB=usage_db - - analytics: - image: toska/oodikone2-analytics:staging - build: - context: services/oodikone2-analytics - ports: - - 4568:4568 - environment: - - WAIT_HOSTS=analytics_db:5432,analytics_redis:6379 - - REDIS=analytics_redis - - DB_URL=postgres://postgres@analytics_db:5432/analytics_db - - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test - depends_on: - - analytics_db - - analytics_redis - container_name: analytics - - updater_api: - image: toska/updater_api:staging - build: - context: services/updater_api - depends_on: - - nats - environment: - - WAIT_HOSTS=nats:4222 - - NATS_URI=nats://nats:4222 - - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi - - TOKEN=toskab1gs8cret5 - container_name: updater_api - - updater_writer: - image: toska/updater_writer:staging - build: - context: services/backend - dockerfile: updater_writer/Dockerfile - depends_on: - - nats - - db - environment: - WAIT_HOSTS: nats:4222,db:5432 - NATS_URI: nats://nats:4222 - DB_URL: postgres://postgres@db:5432/tkt_oodi - TEST_DB: postgres://postgres@db:5432/tkt_oodi_test - TEST_DB_SCHEMA: updater_test - container_name: updater_writer - - scheduler_mongo: - image: mongo - volumes: - - ./mongodata:/data/db - expose: - - "27017" - container_name: scheduler_db - - updater_scheduler: - image: toska/updater_scheduler:staging - build: - context: services/updater_scheduler - command: ["echo", "mluukkai on orjuuttaja"] - depends_on: - - nats - - scheduler_mongo - environment: - - WAIT_HOSTS=nats:4222,scheduler_mongo:27017 - - NATS_URI=nats://nats:4222 - - MONGO_URI=mongodb://scheduler_mongo:27017 - - STUDENT_NUMBERS=./all_student_numbers.txt - - ACTIVE_STUDENTS=./active_student_numbers.txt - - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi - - TOKEN=toskab1gs8cret5 - container_name: updater_scheduler - - nginx: - image: nginx:latest - volumes: - - ./nginx.staging.conf:/etc/nginx/nginx.conf - - ./nginx/error.log:/etc/nginx/error_log.log - - ./nginx/log:/var/log/nginx/localhost.error_log - - ./nginx/cache/:/etc/nginx/cache - - /etc/letsencrypt/:/etc/letsencrypt/ - ports: - - 1337:80 - depends_on: - - frontend - - backend - - userservice - - analytics - - container_name: nginx diff --git a/docker-compose.yml b/docker-compose.yml index d2a82df6e1..5b66eb5217 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,43 +1,17 @@ version: '3' services: - # zookeeper: - # image: wurstmeister/zookeeper - # ports: - # - "2181:2181" - # environment: - # - KAFKA_BROKER_ID=1 - # container_name: zookeeper - - # kafka: - # image: wurstmeister/kafka - # container_name: kafka - # depends_on: - # - zookeeper - # ports: - # - "9092:9092" - # environment: - # - HOSTNAME_COMMAND=route -n | awk '/UG[ \t]/{print $$2}' - # - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - # - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true - # - KAFKA_DELETE_TOPIC_ENABLE=true - # - KAFKA_CREATE_TOPICS=studentnumbers:1:1,priority_studentnumbers:1:1 - # - KAFKA_LOG_DIRS=/kafka/kafka-logs-1 - # volumes: - # - ./kafka/docker.sock:/var/run/docker.sock - # - ./kafka/logs:/kafka/kafka-logs-1 - nats: image: nats-streaming - command: -cid updaterNATS --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore + command: -cid updaterNATS --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore expose: - "4222" - volumes: - - ./datastore:/datastore ports: - "8222:8222" - "4222:4222" container_name: nats + volumes: + - ./datastore:/datastore analytics_db: image: postgres:9.6.3 @@ -59,7 +33,6 @@ services: container_name: oodi_user_db environment: - POSTGRES_USER=postgres - - POSTGRES_MULTIPLE_DATABASES=user_db,user_db_real usage_db: image: postgres:9.6.3 @@ -81,7 +54,6 @@ services: - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro container_name: oodi_db environment: - - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test,tkt_oodi_real - POSTGRES_USER=postgres db_kone: @@ -105,24 +77,10 @@ services: - ./redis-data:/data container_name: redis - # mongo_db: - # image: mongo - # restart: unless-stopped - # volumes: - # - ./backups/mongo/oodilearn:/dump - # - ./mongodata:/data/db - # ports: - # - "27017:27017" - # container_name: mongo_db - backend: build: context: services/backend dockerfile: oodikone2-backend/Dockerfile - args: - NODE_ENV: 'development' - command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] - restart: unless-stopped ports: - "8080:8080" environment: @@ -132,7 +90,6 @@ services: DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone TEST_DB: postgres://postgres@db:5432/tkt_oodi_test TEST_DB_KONE: postgres://postgres@db_kone:5432/db_kone_test - FRONT_URL: http://localhost:8000 TOKEN_SECRET: IWannaBeTheVeryBest OODILEARN_URL: http://oodilearn:5000 USERSERVICE_URL: http://userservice:4567 @@ -141,12 +98,6 @@ services: USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 ANALYTICS_URL: http://analytics:4568 NATS_URI: nats://nats:4222 - volumes: - - ./services/backend/oodikone2-backend/:/usr/src/app - - ./services/backend/shared/models:/usr/src/app/src/models - - ./services/backend/shared/migrations:/usr/src/app/src/database/migrations - - ./services/backend/shared/migrations_kone:/usr/src/app/src/database/migrations_kone - container_name: backend depends_on: - db @@ -156,49 +107,14 @@ services: frontend: build: context: services/oodikone2-frontend - args: - NODE_ENV: 'development' - command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run docker'] - restart: unless-stopped - ports: - - "8081:8081" - - "7777:7777" environment: BACKEND_ADDR: backend ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres - USER_ADMINER_URL: http://localhost:5050/?pgsql=user_db&username=postgres - ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres - USAGE_ADMINER_URL: http://localhost:5050/?pgsql=usage_db&username=postgres - KONE_ADMINER_URL: http://localhost:5050/?pgsql=db_kone&username=postgres - volumes: - - ./services/oodikone2-frontend/:/usr/src/app container_name: frontend - # oodilearn: - # build: - # context: services/oodilearn/server - # dockerfile: Dockerfile - # restart: unless-stopped - # volumes: - # - ./services/oodilearn/data:/data - # - ./services/oodilearn/server:/src - # - ./services/oodilearn/models:/models - # ports: - # - "5000:5000" - # environment: - # - ENV=development - # - MONGO_URI=mongodb://mongo_db:27017/oodilearn - # container_name: oodilearn - userservice: build: context: services/oodikone2-userservice - args: - NODE_ENV: 'development' - command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] - restart: unless-stopped - volumes: - - ./services/oodikone2-userservice/:/usr/src/app ports: - 4567:4567 environment: @@ -214,10 +130,6 @@ services: usageservice: build: context: services/oodikone2-usageservice - args: - NODE_ENV: 'development' - command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] - restart: unless-stopped ports: - 545:545 environment: @@ -234,12 +146,6 @@ services: updater_api: build: context: services/updater_api - args: - NODE_ENV: 'development' - command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] - restart: unless-stopped - volumes: - - ./services/updater_api/:/usr/src/app depends_on: - nats environment: @@ -253,16 +159,6 @@ services: build: context: services/backend dockerfile: updater_writer/Dockerfile - args: - NODE_ENV: 'development' - command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] - restart: unless-stopped - volumes: - - ./services/backend/updater_writer/:/usr/src/app - - ./services/backend/shared/models:/usr/src/app/models - - ./services/backend/shared/migrations:/usr/src/app/database/migrations - - ./services/backend/shared/migrations_kone:/usr/src/app/database/migrations_kone - depends_on: - nats - db @@ -272,28 +168,19 @@ services: DB_URL: postgres://postgres@db:5432/tkt_oodi TEST_DB: postgres://postgres@db:5432/tkt_oodi_test TEST_DB_SCHEMA: updater_test - DB_SCHEMA_KONE: kone_data - - + container_name: updater_writer scheduler_mongo: image: mongo - restart: unless-stopped volumes: - ./mongodata:/data/db expose: - "27017" - container_name: scheduler_db + container_name: scheduler_mongo updater_scheduler: build: context: services/updater_scheduler - args: - NODE_ENV: 'development' - command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] - restart: unless-stopped - volumes: - - ./services/updater_scheduler/:/usr/src/app depends_on: - nats - scheduler_mongo @@ -305,29 +192,18 @@ services: - ACTIVE_STUDENTS=./active_student_numbers.txt - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi - TOKEN=toskab1gs8cret5 - - LOG_HOST=toska.cs.helsinki.fi - - LOG_PORT=9501 - - LOG_HOSTNAME=updater_scheduler - container_name: updater_scheduler - analytics: build: context: services/oodikone2-analytics - args: - NODE_ENV: 'development' - command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] - restart: unless-stopped - volumes: - - ./services/oodikone2-analytics/:/usr/src/app ports: - 4568:4568 environment: - WAIT_HOSTS: analytics_db:5432,analytics_redis:6379 - REDIS: analytics_redis - DB_URL: postgres://postgres@analytics_db:5432/analytics_db - TEST_DB: postgres://postgres@analytics_db:5432/analytics_db_test + - WAIT_HOSTS=analytics_db:5432,analytics_redis:6379 + - REDIS=analytics_redis + - DB_URL=postgres://postgres@analytics_db:5432/analytics_db + - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test depends_on: - analytics_db - analytics_redis @@ -341,14 +217,3 @@ services: volumes: - ./analytics-redis-data:/data container_name: analytics_redis - - adminer: - image: adminer:4.6.2 - environment: - - ADMINER_DESIGN=pepa-linha - volumes: - - ./adminer-plugins:/var/www/html/plugins-enabled - ports: - - 5050:8080 - restart: always - container_name: adminer diff --git a/docker-compose.real.yml b/docker/docker-compose.dev.real.yml similarity index 100% rename from docker-compose.real.yml rename to docker/docker-compose.dev.real.yml diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml new file mode 100644 index 0000000000..e13691d297 --- /dev/null +++ b/docker/docker-compose.dev.yml @@ -0,0 +1,170 @@ +version: '3' + +services: + user_db: + environment: + - POSTGRES_MULTIPLE_DATABASES=user_db,user_db_real + + db: + environment: + - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test,tkt_oodi_real + + backend: + build: + args: + NODE_ENV: 'development' + command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] + restart: unless-stopped + environment: + FRONT_URL: http://localhost:8000 + volumes: + - ./services/backend/oodikone2-backend/:/usr/src/app + - ./services/backend/shared/models:/usr/src/app/src/models + - ./services/backend/shared/migrations:/usr/src/app/src/database/migrations + - ./services/backend/shared/migrations_kone:/usr/src/app/src/database/migrations_kone + + frontend: + build: + args: + NODE_ENV: 'development' + command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run docker'] + restart: unless-stopped + ports: + - "8081:8081" + - "7777:7777" + environment: + USER_ADMINER_URL: http://localhost:5050/?pgsql=user_db&username=postgres + ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres + USAGE_ADMINER_URL: http://localhost:5050/?pgsql=usage_db&username=postgres + KONE_ADMINER_URL: http://localhost:5050/?pgsql=db_kone&username=postgres + volumes: + - ./services/oodikone2-frontend/:/usr/src/app + + userservice: + build: + args: + NODE_ENV: 'development' + command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] + restart: unless-stopped + volumes: + - ./services/oodikone2-userservice/:/usr/src/app + + usageservice: + build: + args: + NODE_ENV: 'development' + command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] + restart: unless-stopped + volumes: + - ./services/oodikone2-usageservice/:/usr/src/app + + updater_api: + build: + args: + NODE_ENV: 'development' + command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] + restart: unless-stopped + volumes: + - ./services/updater_api/:/usr/src/app + + updater_writer: + build: + args: + NODE_ENV: 'development' + command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] + restart: unless-stopped + volumes: + - ./services/backend/updater_writer/:/usr/src/app + - ./services/backend/shared/models:/usr/src/app/models + - ./services/backend/shared/migrations:/usr/src/app/database/migrations + - ./services/backend/shared/migrations_kone:/usr/src/app/database/migrations_kone + environment: + DB_SCHEMA_KONE: kone_data + + scheduler_mongo: + restart: unless-stopped + + updater_scheduler: + build: + args: + NODE_ENV: 'development' + command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] + restart: unless-stopped + volumes: + - ./services/updater_scheduler/:/usr/src/app + environment: + - LOG_HOST=toska.cs.helsinki.fi + - LOG_PORT=9501 + - LOG_HOSTNAME=updater_scheduler + + analytics: + build: + args: + NODE_ENV: 'development' + command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] + restart: unless-stopped + volumes: + - ./services/oodikone2-analytics/:/usr/src/app + + adminer: + image: adminer:4.6.2 + environment: + - ADMINER_DESIGN=pepa-linha + volumes: + - ./adminer-plugins:/var/www/html/plugins-enabled + ports: + - 5050:8080 + restart: always + container_name: adminer + + # zookeeper: + # image: wurstmeister/zookeeper + # ports: + # - "2181:2181" + # environment: + # - KAFKA_BROKER_ID=1 + # container_name: zookeeper + + # kafka: + # image: wurstmeister/kafka + # container_name: kafka + # depends_on: + # - zookeeper + # ports: + # - "9092:9092" + # environment: + # - HOSTNAME_COMMAND=route -n | awk '/UG[ \t]/{print $$2}' + # - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + # - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true + # - KAFKA_DELETE_TOPIC_ENABLE=true + # - KAFKA_CREATE_TOPICS=studentnumbers:1:1,priority_studentnumbers:1:1 + # - KAFKA_LOG_DIRS=/kafka/kafka-logs-1 + # volumes: + # - ./kafka/docker.sock:/var/run/docker.sock + # - ./kafka/logs:/kafka/kafka-logs-1 + + # mongo_db: + # image: mongo + # restart: unless-stopped + # volumes: + # - ./backups/mongo/oodilearn:/dump + # - ./mongodata:/data/db + # ports: + # - "27017:27017" + # container_name: mongo_db + + # oodilearn: + # build: + # context: services/oodilearn/server + # dockerfile: Dockerfile + # restart: unless-stopped + # volumes: + # - ./services/oodilearn/data:/data + # - ./services/oodilearn/server:/src + # - ./services/oodilearn/models:/models + # ports: + # - "5000:5000" + # environment: + # - ENV=development + # - MONGO_URI=mongodb://mongo_db:27017/oodilearn + # container_name: oodilearn diff --git a/docker/docker-compose.lateste2e.production.yml b/docker/docker-compose.lateste2e.production.yml new file mode 100644 index 0000000000..fc5eea51ec --- /dev/null +++ b/docker/docker-compose.lateste2e.production.yml @@ -0,0 +1,62 @@ +version: '3' + +services: + user_db: + environment: + - POSTGRES_DB=user_db + + db: + environment: + - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test + + backend: + image: toska/oodikone2-backend:latest + ports: + - "9229:9229" + environment: + FRONT_URL: http://localhost:1337 + ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres + + frontend: + image: toska/oodikone2-frontend:latest + build: + args: + BASE_PATH: / + ports: + - "5000:5000" + + userservice: + image: toska/oodikone2-userservice:latest + + usageservice: + image: toska/oodikone2-usageservice:latest + + updater_api: + image: toska/updater_api:latest + + updater_writer: + image: toska/updater_writer:latest + + updater_scheduler: + image: toska/updater_scheduler:latest + command: ["echo", "mluukkai on orjuuttaja"] + + analytics: + image: toska/oodikone2-analytics:latest + + nginx: + image: nginx:latest + volumes: + - ./nginx.production.conf:/etc/nginx/nginx.conf + - ./nginx/error.log:/etc/nginx/error_log.log + - ./nginx/log:/var/log/nginx/localhost.error_log + - ./nginx/cache/:/etc/nginx/cache + - /etc/letsencrypt/:/etc/letsencrypt/ + ports: + - 1337:80 + depends_on: + - frontend + - backend + - userservice + - analytics + container_name: nginx diff --git a/docker/docker-compose.lateste2e.yml b/docker/docker-compose.lateste2e.yml new file mode 100644 index 0000000000..e8553d34bb --- /dev/null +++ b/docker/docker-compose.lateste2e.yml @@ -0,0 +1,62 @@ +version: '3' + +services: + user_db: + environment: + - POSTGRES_DB=user_db + + db: + environment: + - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test + + backend: + image: toska/oodikone2-backend:staging + ports: + - "9229:9229" + environment: + FRONT_URL: http://localhost:1337 + ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres + + frontend: + image: toska/oodikone2-frontend:staging + build: + args: + BASE_PATH: / + ports: + - "5000:5000" + + userservice: + image: toska/oodikone2-userservice:staging + + usageservice: + image: toska/oodikone2-usageservice:staging + + updater_api: + image: toska/updater_api:staging + + updater_writer: + image: toska/updater_writer:staging + + updater_scheduler: + image: toska/updater_scheduler:staging + command: ["echo", "mluukkai on orjuuttaja"] + + analytics: + image: toska/oodikone2-analytics:staging + + nginx: + image: nginx:latest + volumes: + - ./nginx.staging.conf:/etc/nginx/nginx.conf + - ./nginx/error.log:/etc/nginx/error_log.log + - ./nginx/log:/var/log/nginx/localhost.error_log + - ./nginx/cache/:/etc/nginx/cache + - /etc/letsencrypt/:/etc/letsencrypt/ + ports: + - 1337:80 + depends_on: + - frontend + - backend + - userservice + - analytics + container_name: nginx diff --git a/package.json b/package.json index 0ac6ceecb9..4ef320e3bd 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,17 @@ "cypress:record": "cypress run --config videoUploadOnPasses=false -P ./", "concurrently": "concurrently", "test": "./run_all_tests.sh", - "test_services": "./run_service_tests.sh" + "test_services": "./run_service_tests.sh", + "docker": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml", + "docker:build": "npm run docker -- build", + "docker:up": "npm run docker -- up -d", + "docker:down": "npm run docker -- down", + "docker:logs": "npm run docker -- logs -f", + "docker:logs:backend": "npm run docker -- logs -f backend", + "docker:restart": "npm run docker -- restart", + "docker:restart:backend": "npm run docker -- restart backend", + "docker:obliterate": "docker stop $(docker ps -aq) && docker system prune", + "start": "npm run docker:up && npm run docker:logs" }, "husky": { "hooks": { diff --git a/run.sh b/run.sh index 6a20f8addd..47d42cac2f 100755 --- a/run.sh +++ b/run.sh @@ -33,7 +33,7 @@ do case $opt in "e2e setup") mopo - run_e2e_setup docker-compose.lateste2e.yml + run_e2e_setup ./docker/docker-compose.lateste2e.yml ;; "Anon setup") mopo diff --git a/scripts.sh b/scripts.sh index 8bca1a0a42..fccdd5c8b3 100644 --- a/scripts.sh +++ b/scripts.sh @@ -10,6 +10,10 @@ PSQL_REAL_DB_BACKUP="$BACKUP_DIR/latest-pg.sqz" KONE_REAL_DB_BACKUP="$BACKUP_DIR/latest-kone-pg.sqz" USER_REAL_DB_BACKUP="$BACKUP_DIR/latest-user-pg.sqz" +docker-compose-dev () { + docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml "$@" +} + retry () { for i in {1..60} do @@ -96,17 +100,17 @@ db_anon_setup_full () { } reset_real_db () { - docker-compose down - docker-compose up -d db user_db db_kone + docker-compose-dev down + docker-compose-dev up -d db user_db db_kone db_setup_full - docker-compose down + docker-compose-dev down } reset_db () { - docker-compose down - docker-compose up -d db user_db db_kone + docker-compose-dev down + docker-compose-dev up -d db user_db db_kone db_anon_setup_full - docker-compose down + docker-compose-dev down } install_cli_npm_packages () { @@ -127,12 +131,12 @@ run_full_setup () { echo "Getting anon backups from the private repository. " get_anon_oodikone echo "Building images" - docker-compose build + docker-compose-dev build echo "Setup oodikone db from dump." - docker-compose up -d db user_db db_kone + docker-compose-dev up -d db user_db db_kone db_setup_full db_anon_setup_full - docker-compose down + docker-compose-dev down show_instructions } @@ -144,11 +148,11 @@ run_anon_full_setup () { echo "Getting anon backups from the private repository. " get_anon_oodikone echo "Building images" - docker-compose build + docker-compose-dev build echo "Setup oodikone db from dump." - docker-compose up -d db user_db db_kone + docker-compose-dev up -d db user_db db_kone db_anon_setup_full - docker-compose down + docker-compose-dev down show_instructions } @@ -160,10 +164,10 @@ run_e2e_setup () { echo "Getting anon backups from the private repository. " get_anon_oodikone echo "Building images" - docker-compose -f $1 build + docker-compose -f docker-compose.yml -f $1 build echo "Setup oodikone db from dump." - docker-compose -f $1 up -d db user_db db_kone + docker-compose -f docker-compose.yml -f $1 up -d db user_db db_kone db_anon_setup_full echo "Starting services." - docker-compose -f $1 up -d + docker-compose -f docker-compose.yml -f $1 up -d } From 68df47a04d241338acb9579abe11bfd0e49ea240 Mon Sep 17 00:00:00 2001 From: Warro Date: Mon, 8 Jul 2019 14:00:12 +0300 Subject: [PATCH 3/3] Removed volume usageservice in docker-compose root file --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5b66eb5217..67f4a3bdf5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -137,8 +137,6 @@ services: PORT: '545' DB_URL: postgres://postgres@usage_db:5432/usage_db SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 - volumes: - - ./services/oodikone2-usageservice/:/usr/src/app depends_on: - usage_db container_name: usageservice