From b1245da77c2740f2f25f6c85eff91ec16dbeda27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A4kinen=20Sasu=20S?= Date: Mon, 8 Jul 2019 14:35:47 +0300 Subject: [PATCH 01/15] systems --- docker-compose.yml | 3 + .../oodikone2-backend/src/conf-backend.js | 4 +- .../src/services/updaterService.js | 21 + services/updater_scheduler/index.js | 2 + services/updater_scheduler/package-lock.json | 388 ++++++++++++++++++ services/updater_scheduler/package.json | 1 + services/updater_scheduler/src/api.js | 12 + 7 files changed, 430 insertions(+), 1 deletion(-) create mode 100644 services/backend/oodikone2-backend/src/services/updaterService.js create mode 100644 services/updater_scheduler/src/api.js diff --git a/docker-compose.yml b/docker-compose.yml index d2a82df6e1..d40b756613 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -141,6 +141,8 @@ services: USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 ANALYTICS_URL: http://analytics:4568 NATS_URI: nats://nats:4222 + UPDATER_URL: http://updater_scheduler:3678 + volumes: - ./services/backend/oodikone2-backend/:/usr/src/app - ./services/backend/shared/models:/usr/src/app/src/models @@ -308,6 +310,7 @@ services: - LOG_HOST=toska.cs.helsinki.fi - LOG_PORT=9501 - LOG_HOSTNAME=updater_scheduler + - PORT=3678 container_name: updater_scheduler diff --git a/services/backend/oodikone2-backend/src/conf-backend.js b/services/backend/oodikone2-backend/src/conf-backend.js index d18da0a3ab..c6bb7dd8ed 100644 --- a/services/backend/oodikone2-backend/src/conf-backend.js +++ b/services/backend/oodikone2-backend/src/conf-backend.js @@ -21,6 +21,7 @@ const CERT_PATH = process.env.CERT_PATH // production/staging only const KEY_PATH = process.env.KEY_PATH // production/staging only const OODILEARN_URL = process.env.OODILEARN_URL const USERSERVICE_URL = process.env.USERSERVICE_URL +const UPDATER_URL = process.env.UPDATER_URL const USAGESERVICE_URL = process.env.USAGESERVICE_URL const ANALYTICS_URL = process.env.ANALYTICS_URL const PORT = isTest ? 8079 : 8080 @@ -81,5 +82,6 @@ module.exports = { requiredGroup, OODI_SECRET, OODI_SECRET_HEADER_KEY, - isTest + isTest, + UPDATER_URL } \ No newline at end of file diff --git a/services/backend/oodikone2-backend/src/services/updaterService.js b/services/backend/oodikone2-backend/src/services/updaterService.js new file mode 100644 index 0000000000..1fdeb76f28 --- /dev/null +++ b/services/backend/oodikone2-backend/src/services/updaterService.js @@ -0,0 +1,21 @@ +const axios = require('axios') +const { UPDATER_URL } = require('../conf-backend') + +const client = axios.create({ baseURL: UPDATER_URL }) + +const ping = async () => { + const url = '/ping' + const response = await axios.get(url) + return response.data +} + +const updateStudents = async (studentNumbers) => { + const response = await client.post('/update', studentNumbers) + return response.data +} + + +module.exports = { + ping, + updateStudents +} \ No newline at end of file diff --git a/services/updater_scheduler/index.js b/services/updater_scheduler/index.js index 7cce2984f5..e25499637b 100644 --- a/services/updater_scheduler/index.js +++ b/services/updater_scheduler/index.js @@ -6,6 +6,8 @@ const logger = require('./logger') const { updateStudentNumberList } = require('./src/student_list_updater') const { scheduleActiveStudents, scheduleAllStudentsAndMeta } = require('./src/schedule_students') const { getOldestTasks, getCurrentStatus } = require('./src/SchedulingStatistics') +const { app } = require('./src/api') +console.log(app) let updatedCount = 0 let scheduledCount = 0 diff --git a/services/updater_scheduler/package-lock.json b/services/updater_scheduler/package-lock.json index 6dacfba8aa..f367346e49 100644 --- a/services/updater_scheduler/package-lock.json +++ b/services/updater_scheduler/package-lock.json @@ -9,6 +9,15 @@ "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, + "accepts": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", + "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", + "requires": { + "mime-types": "~2.1.24", + "negotiator": "0.6.2" + } + }, "ansi-align": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-2.0.0.tgz", @@ -64,6 +73,11 @@ "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", "integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=" }, + "array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + }, "array-unique": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz", @@ -173,6 +187,33 @@ "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz", "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA==" }, + "body-parser": { + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", + "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", + "requires": { + "bytes": "3.1.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", + "on-finished": "~2.3.0", + "qs": "6.7.0", + "raw-body": "2.4.0", + "type-is": "~1.6.17" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + } + } + }, "boxen": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/boxen/-/boxen-1.3.0.tgz", @@ -228,6 +269,11 @@ "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.1.tgz", "integrity": "sha512-jCGVYLoYMHDkOsbwJZBCqwMHyH4c+wzgI9hG7Z6SZJRXWr+x58pdIbm2i9a/jFGCkRJqRUr8eoI7lDWa0hTkxg==" }, + "bytes": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", + "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" + }, "cache-base": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", @@ -396,6 +442,29 @@ "xdg-basedir": "^3.0.0" } }, + "content-disposition": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", + "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", + "requires": { + "safe-buffer": "5.1.2" + } + }, + "content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + }, + "cookie": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", + "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==" + }, + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, "copy-descriptor": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz", @@ -491,6 +560,16 @@ } } }, + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + }, + "destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + }, "diagnostics": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/diagnostics/-/diagnostics-1.1.1.tgz", @@ -514,6 +593,11 @@ "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=" }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, "enabled": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/enabled/-/enabled-1.0.2.tgz", @@ -522,16 +606,31 @@ "env-variable": "0.0.x" } }, + "encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + }, "env-variable": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.5.tgz", "integrity": "sha512-zoB603vQReOFvTg5xMl9I1P2PnHsHQQKTEowsKKD7nseUfJq6UWzK+4YtlWUO1nhiQUxe6XMkk+JleSZD1NZFA==" }, + "escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, + "etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + }, "execa": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", @@ -586,6 +685,53 @@ } } }, + "express": { + "version": "4.17.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", + "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", + "requires": { + "accepts": "~1.3.7", + "array-flatten": "1.1.1", + "body-parser": "1.19.0", + "content-disposition": "0.5.3", + "content-type": "~1.0.4", + "cookie": "0.4.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.1.2", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.5", + "qs": "6.7.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.1.2", + "send": "0.17.1", + "serve-static": "1.14.1", + "setprototypeof": "1.1.1", + "statuses": "~1.5.0", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + } + } + }, "extend-shallow": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz", @@ -695,6 +841,30 @@ } } }, + "finalhandler": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", + "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "requires": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", + "unpipe": "~1.0.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + } + } + }, "follow-redirects": { "version": "1.5.10", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", @@ -718,6 +888,11 @@ "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", "integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=" }, + "forwarded": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", + "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" + }, "fragment-cache": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", @@ -726,6 +901,11 @@ "map-cache": "^0.2.2" } }, + "fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + }, "fsevents": { "version": "1.2.9", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.9.tgz", @@ -1287,6 +1467,26 @@ } } }, + "http-errors": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", + "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.1", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.0" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, "ignore-by-default": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", @@ -1312,6 +1512,11 @@ "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" }, + "ipaddr.js": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.0.tgz", + "integrity": "sha512-M4Sjn6N/+O6/IXSJseKqHoFc+5FdGJ22sXqnjTpdZweHK64MzEPAyQZyEU3R/KRv2GLoa7nNtg/C2Ev6m7z+eA==" + }, "is-accessor-descriptor": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", @@ -1587,12 +1792,27 @@ "object-visit": "^1.0.0" } }, + "media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + }, "memory-pager": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", "optional": true }, + "merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + }, + "methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + }, "micromatch": { "version": "3.1.10", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz", @@ -1613,6 +1833,24 @@ "to-regex": "^3.0.2" } }, + "mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" + }, + "mime-db": { + "version": "1.40.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz", + "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==" + }, + "mime-types": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz", + "integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==", + "requires": { + "mime-db": "1.40.0" + } + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -1767,6 +2005,11 @@ "ts-nkeys": "^1.0.8" } }, + "negotiator": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", + "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" + }, "node-cron": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-2.0.3.tgz", @@ -1873,6 +2116,14 @@ "isobject": "^3.0.1" } }, + "on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "requires": { + "ee-first": "1.1.1" + } + }, "one-time": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/one-time/-/one-time-0.0.4.tgz", @@ -1899,6 +2150,11 @@ "semver": "^5.1.0" } }, + "parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" + }, "pascalcase": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz", @@ -1924,6 +2180,11 @@ "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=" }, + "path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, "pify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", @@ -1944,6 +2205,15 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" }, + "proxy-addr": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz", + "integrity": "sha512-t/7RxHXPH6cJtP0pRG6smSr9QJidhB+3kXu0KgXnbGYMgzEnUxRQ4/LDdfOwZEMyIh3/xHb8PX3t+lfL9z+YVQ==", + "requires": { + "forwarded": "~0.1.2", + "ipaddr.js": "1.9.0" + } + }, "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -1954,6 +2224,27 @@ "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.6.tgz", "integrity": "sha512-NdF35+QsqD7EgNEI5mkI/X+UwaxVEbQaz9f4IooEmMUv6ZPmlTQYGjBPJGgrlzNdjSvIy4MWMg6Q6vCgBO2K+w==" }, + "qs": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", + "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" + }, + "range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" + }, + "raw-body": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", + "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", + "requires": { + "bytes": "3.1.0", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, "rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", @@ -2072,6 +2363,11 @@ "ret": "~0.1.10" } }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, "saslprep": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", @@ -2094,6 +2390,59 @@ "semver": "^5.0.3" } }, + "send": { + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", + "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", + "requires": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "~1.7.2", + "mime": "1.6.0", + "ms": "2.1.1", + "on-finished": "~2.3.0", + "range-parser": "~1.2.1", + "statuses": "~1.5.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } + } + }, + "ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + } + } + }, + "serve-static": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", + "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", + "requires": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.17.1" + } + }, "set-value": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz", @@ -2115,6 +2464,11 @@ } } }, + "setprototypeof": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", + "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" + }, "shebang-command": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", @@ -2319,6 +2673,11 @@ } } }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" + }, "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", @@ -2418,6 +2777,11 @@ "repeat-string": "^1.6.1" } }, + "toidentifier": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", + "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" + }, "touch": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz", @@ -2444,6 +2808,15 @@ "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-1.0.1.tgz", "integrity": "sha512-kcoMoKTPYnoeS50tzoqjPY3Uv9axeuuFAZY9M/9zFnhoVvRfxz9K29IMPD7jGmt2c8SW7i3gT9WqDl2+nV7p4A==" }, + "type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + } + }, "tz-offset": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/tz-offset/-/tz-offset-0.0.1.tgz", @@ -2507,6 +2880,11 @@ "crypto-random-string": "^1.0.0" } }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + }, "unset-value": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz", @@ -2593,6 +2971,16 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, + "utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + }, + "vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", diff --git a/services/updater_scheduler/package.json b/services/updater_scheduler/package.json index 66560b9594..9246a33587 100644 --- a/services/updater_scheduler/package.json +++ b/services/updater_scheduler/package.json @@ -16,6 +16,7 @@ "license": "ISC", "dependencies": { "axios": "^0.19.0", + "express": "^4.17.1", "moment": "^2.24.0", "mongoose": "^5.5.8", "node-cron": "^2.0.3", diff --git a/services/updater_scheduler/src/api.js b/services/updater_scheduler/src/api.js new file mode 100644 index 0000000000..f2a154cae9 --- /dev/null +++ b/services/updater_scheduler/src/api.js @@ -0,0 +1,12 @@ +const express = require('express') +const app = express() +const port = process.env.PORT +const bodyParser = require('body-parser') + +app.use(bodyParser.json()) + + +app.get('/ping', (req, res) => res.json({ message: 'pong '})) + +app.listen(port, () => console.log(`listening on port ${port}!`)) +module.exports = { app } \ No newline at end of file From 95d5bb5a93c5d078d5c09f5b93c395d861ac6abf Mon Sep 17 00:00:00 2001 From: Tuomo Torppa Date: Mon, 8 Jul 2019 15:38:43 +0300 Subject: [PATCH 02/15] Sad push of non-implement course page - population --- .../src/routes/population.js | 72 +++++++++++++++++++ .../services/course_yearly_stats_counter.js | 4 +- .../oodikone2-backend/src/services/courses.js | 4 +- .../src/services/populations.js | 65 +++++++++-------- .../src/services/students.js | 47 +++++++++++- .../src/redux/populationCourses.js | 14 ++++ 6 files changed, 175 insertions(+), 31 deletions(-) diff --git a/services/backend/oodikone2-backend/src/routes/population.js b/services/backend/oodikone2-backend/src/routes/population.js index 307adb2f13..28c29e580d 100644 --- a/services/backend/oodikone2-backend/src/routes/population.js +++ b/services/backend/oodikone2-backend/src/routes/population.js @@ -1,8 +1,10 @@ const router = require('express').Router() const Population = require('../services/populations') const Filters = require('../services/filters') +const Student = require('../services/students') const { updateStudents } = require('../services/doo_api_database_updater/database_updater') const StudyrightService = require('../services/studyrights') +const UserService = require('../services/userService') // POST instead of GET because of too long params and "sensitive" data router.post('/v2/populationstatistics/courses', async (req, res) => { @@ -31,6 +33,37 @@ router.post('/v2/populationstatistics/courses', async (req, res) => { } }) +router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) => { + try { + if (!req.body.yearcode || !req.body.coursecode) { + res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' }) + return + } + const { coursecode, yearcode } = req.body + let studentnumberlist + const studentnumbers = await Student.findByCourseAndSemesters(coursecode, yearcode) + + const { roles, userId } = req.decodedToken + if (roles && roles.map(r => r.group_code).includes('admin')) { + studentnumberlist = studentnumbers + } else { + const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId) + const codes = unitsUserCanAccess.map(unit => unit.id) + studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes) + } + const result = await Population.bottlenecksOf(req.body, studentnumberlist) + + if (result.error) { + res.status(400).json(result) + return + } + + res.json(result) + } catch (e) { + res.status(400).json({ error: e }) + } +}) + router.get('/v3/populationstatistics', async (req, res) => { const { startYear, semesters, studyRights: studyRightsJSON } = req.query try { @@ -73,6 +106,45 @@ router.get('/v3/populationstatistics', async (req, res) => { } }) +router.get('/v3/populationstatisticsbycourse', async (req, res) => { + const { coursecode, yearcode } = req.query + const semesters = ['FALL', 'SPRING'] + let studentnumberlist + console.log(coursecode, yearcode) + const studentnumbers = await Student.findByCourseAndSemesters(coursecode, yearcode) + console.log(studentnumbers) + const { roles, userId } = req.decodedToken + if (roles && roles.map(r => r.group_code).includes('admin')) { + studentnumberlist = studentnumbers + } else { + const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId) + const codes = unitsUserCanAccess.map(unit => unit.id) + studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes) + } + try { + const result = await Population.optimizedStatisticsOf({ + startYear: 1900, + endYear: 2200, + studyRights: [], + semesters, + months: 1000 + }, studentnumberlist) + + if (result.error) { + console.log(result.error) + res.status(400).end() + return + } + + console.log(`request completed ${new Date()}`) + res.json(result) + } catch (e) { + console.log(e) + res.status(400).json({ error: e }) + } +}) + + router.get('/v2/populationstatistics/filters', async (req, res) => { let results = [] diff --git a/services/backend/oodikone2-backend/src/services/course_yearly_stats_counter.js b/services/backend/oodikone2-backend/src/services/course_yearly_stats_counter.js index cec78b98c9..09407b99c7 100644 --- a/services/backend/oodikone2-backend/src/services/course_yearly_stats_counter.js +++ b/services/backend/oodikone2-backend/src/services/course_yearly_stats_counter.js @@ -127,12 +127,14 @@ class CourseYearlyStatsCounter { formatStudentStatistics(students) { const grades = {} const classes = {} + const studentnumbers = [] Object.entries(students).forEach(([studentnumber, stat]) => { const { grade, category } = stat grades[grade] = grades[grade] ? grades[grade].concat(studentnumber) : [studentnumber] classes[category] = classes[category] ? classes[category].concat(studentnumber) : [studentnumber] + studentnumbers.includes(studentnumber) ? null : studentnumbers.push(studentnumber) }) - return { grades, classes } + return { grades, classes, studentnumbers } } formatGroupStatistics() { diff --git a/services/backend/oodikone2-backend/src/services/courses.js b/services/backend/oodikone2-backend/src/services/courses.js index 5488b736dd..d6ed736640 100644 --- a/services/backend/oodikone2-backend/src/services/courses.js +++ b/services/backend/oodikone2-backend/src/services/courses.js @@ -196,6 +196,7 @@ const creditsOf = async (codes) => { fail: credits.filter(Credit.failed).length, pass: credits.filter(Credit.passed).length, students: credits.length, + studentnumbers: credits.map(cr => cr.student_studentnumber) } } @@ -238,7 +239,7 @@ const oneYearStats = (instances, year, separate, allInstancesUntilYear) => { return { studentsThatPassedThisYear, studentsThatFailedThisYear, allStudentsThatFailedEver, passedStudentsThatFailedBefore, passedStudentsOnFirstTry, failedStudentsThatFailedBefore, - failedStudentsOnFirstTry, gradeDistribution + failedStudentsOnFirstTry, gradeDistribution, studentnumbers: thisSemester.studentnumbers } } const stats = [] @@ -345,6 +346,7 @@ const yearlyStatsOf = async (code, year, separate, language) => { if (yearInst) { for (let year = start; year < end; year++) { stats = oneYearStats(yearInst, year, separate, allInstancesUntilYear) + console.logs(stats) if (stats.length > 0) { resultStats.push(...stats) resultProgrammes = getProgrammesFromStats(stats) diff --git a/services/backend/oodikone2-backend/src/services/populations.js b/services/backend/oodikone2-backend/src/services/populations.js index 5ff7a0515f..52676e6b2a 100644 --- a/services/backend/oodikone2-backend/src/services/populations.js +++ b/services/backend/oodikone2-backend/src/services/populations.js @@ -143,7 +143,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa creditsOfStudentLaakis : creditsOfStudentOther const students = await Student.findAll({ - attributes: ['firstnames', 'lastname', 'studentnumber', + attributes: ['firstnames', 'lastname', 'studentnumber', 'home_country_en', 'dateofuniversityenrollment', 'creditcount', 'matriculationexamination', 'abbreviatedname', 'email', 'updatedAt', 'gender_code', 'gender_fi', 'gender_sv', 'gender_en'], include: [ @@ -423,7 +423,7 @@ const formatStudentsForApi = async (students, startDate, endDate, { studyRights } } -const optimizedStatisticsOf = async (query) => { +const optimizedStatisticsOf = async (query, studentnumberlist) => { if (!query.semesters.map(semester => semester === 'FALL' || semester === 'SPRING').every(e => e === true)) { return { error: 'Semester should be either SPRING OR FALL' } } @@ -438,7 +438,8 @@ const optimizedStatisticsOf = async (query) => { studyRights, startDate, months, endDate, exchangeStudents, cancelledStudents, nondegreeStudents } = parseQueryParams(query) - const studentnumbers = + const studentnumbers = studentnumberlist ? + studentnumberlist : await studentnumbersWithAllStudyrightElements( studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents ) @@ -509,26 +510,39 @@ const parseCreditInfo = credit => ({ date: credit.attainment_date }) -const bottlenecksOf = async (query) => { - if (!query.semesters.map(semester => semester === 'FALL' || semester === 'SPRING').every(e => e === true)) { - return { error: 'Semester should be either SPRING OR FALL' } - } - if (query.studentStatuses && - !query.studentStatuses.map( - status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE' - ).every(e => e === true)) { - return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE' } - } - const { studyRights, startDate, endDate, months, exchangeStudents, cancelledStudents } = parseQueryParams(query) - - if (query.selectedStudents) { - const allStudents = - await studentnumbersWithAllStudyrightElements( - studyRights, startDate, endDate, exchangeStudents, cancelledStudents - ) - const disallowedRequest = - checkThatSelectedStudentsAreUnderRequestedStudyright(query.selectedStudents, allStudents) - if (disallowedRequest) return { error: 'Trying to request unauthorized students data' } +const bottlenecksOf = async (query, studentnumberlist) => { + let allstudents + let courses + if (!studentnumberlist) { + if (!query.semesters.map(semester => semester === 'FALL' || semester === 'SPRING').every(e => e === true)) { + return { error: 'Semester should be either SPRING OR FALL' } + } + if (query.studentStatuses && + !query.studentStatuses.map( + status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE' + ).every(e => e === true)) { + return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE' } + } + const { studyRights, startDate, endDate, months, exchangeStudents, cancelledStudents } = parseQueryParams(query) + + if (query.selectedStudents) { + const allStudents = + await studentnumbersWithAllStudyrightElements( + studyRights, startDate, endDate, exchangeStudents, cancelledStudents + ) + const disallowedRequest = + checkThatSelectedStudentsAreUnderRequestedStudyright(query.selectedStudents, allStudents) + if (disallowedRequest) return { error: 'Trying to request unauthorized students data' } + } + + const studentnumbers = query.selectedStudents || + await studentnumbersWithAllStudyrightElements(studyRights, startDate, endDate, exchangeStudents, cancelledStudents) + allstudents = studentnumbers.reduce((numbers, num) => ({ ...numbers, [num]: true }), {}) + courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months)) + + } else { + allstudents = studentnumberlist + courses = await findCourses(allstudents, new Date()) } const bottlenecks = { @@ -538,11 +552,6 @@ const bottlenecksOf = async (query) => { const codeduplicates = await getMainCodesMap() - const studentnumbers = query.selectedStudents || - await studentnumbersWithAllStudyrightElements(studyRights, startDate, endDate, exchangeStudents, cancelledStudents) - const allstudents = studentnumbers.reduce((numbers, num) => ({ ...numbers, [num]: true }), {}) - const courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months)) - const allcoursestatistics = await courses.reduce(async (coursestatistics, course) => { const stats = await coursestatistics let { code, name, disciplines, course_type } = course diff --git a/services/backend/oodikone2-backend/src/services/students.js b/services/backend/oodikone2-backend/src/services/students.js index f4b1a923e2..49f304504d 100644 --- a/services/backend/oodikone2-backend/src/services/students.js +++ b/services/backend/oodikone2-backend/src/services/students.js @@ -1,4 +1,5 @@ const Sequelize = require('sequelize') +const { sequelize } = require('../database/connection') const moment = require('moment') const { Student, Credit, Course, Studyright, StudyrightElement, ElementDetails } = require('../models') const Op = Sequelize.Op @@ -41,6 +42,24 @@ const byId = async (id) => Student.findByPk(id, { ] }) + +const findByCourseAndSemesters = async (coursecode, yearcode) => + sequelize.query(` + SELECT + studentnumber, credit.course_code, attainment_date + FROM student + INNER JOIN credit ON + student.studentnumber=credit.student_studentnumber + WHERE + course_code=:coursecode AND + attainment_date + BETWEEN + (select startdate FROM semesters where yearcode=:yearcode ORDER BY semestercode LIMIT 1) AND + (select enddate FROM semesters where yearcode=:yearcode ORDER BY semestercode DESC LIMIT 1); + `, + { replacements: { coursecode, yearcode }, type: sequelize.QueryTypes.SELECT }) + .map(st => st.studentnumber) + const byAbreviatedNameOrStudentNumber = (searchTerm) => { return Student.findAll({ where: { @@ -280,10 +299,36 @@ const bySearchTermAndElementsNew = async (searchterm, codes) => { return matches.map(formatStudent) } +const filterStudentnumbersByAccessrights = async (studentnumbers, codes) => { + const students = await Student.findAll({ + include: { + model: StudyrightElement, + required: true, + where: { + code: { + [Op.in]: codes + } + } + }, + where: { + studentnumber: { + [Op.in]: studentnumbers + } + } + }) + return students.map(student => student.studentnumber) +} + const NEW_SEARCH = true const bySearchTerm = NEW_SEARCH ? bySearchTermNew : bySearchTermOld const bySearchTermAndElements = NEW_SEARCH ? bySearchTermAndElementsNew : bySearchTermAndElementsOld module.exports = { - withId, bySearchTerm, createStudent, updateStudent, bySearchTermAndElements + withId, + bySearchTerm, + createStudent, + updateStudent, + bySearchTermAndElements, + filterStudentnumbersByAccessrights, + findByCourseAndSemesters } \ No newline at end of file diff --git a/services/oodikone2-frontend/src/redux/populationCourses.js b/services/oodikone2-frontend/src/redux/populationCourses.js index 719c28b8a1..002cd844ea 100644 --- a/services/oodikone2-frontend/src/redux/populationCourses.js +++ b/services/oodikone2-frontend/src/redux/populationCourses.js @@ -20,6 +20,20 @@ export const getPopulationCourses = ({ return callController(route, prefix, body, 'post', query) } +export const getPopulationCoursesByCodeAndSemesters = ({ + coursecode, yearcode +}) => { + const route = '/v2/populationstatistics/coursesbycoursecode' + const prefix = 'GET_POPULATION_COURSES_' + const query = { + coursecode, yearcode + } + const body = { + coursecode, yearcode + } + return callController(route, prefix, body, 'post', query) +} + export const clearPopulationCourses = () => ({ type: 'CLEAR_POPULATIONS_COURSES' }) From e7db17d7665eb97550ae7b9fb5e0bd9980250f50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A4kinen=20Sasu=20S?= Date: Mon, 8 Jul 2019 15:43:28 +0300 Subject: [PATCH 03/15] update students on demand supporrtt --- .../src/routes/population.js | 6 +- services/backend/updater_writer/index.js | 16 ++--- services/updater_api/index.js | 72 +++++++++---------- services/updater_scheduler/index.js | 3 +- services/updater_scheduler/src/api.js | 6 +- .../src/schedule_students.js | 24 +++++-- .../src/student_list_updater.js | 11 ++- 7 files changed, 72 insertions(+), 66 deletions(-) diff --git a/services/backend/oodikone2-backend/src/routes/population.js b/services/backend/oodikone2-backend/src/routes/population.js index 307adb2f13..6e7a93d500 100644 --- a/services/backend/oodikone2-backend/src/routes/population.js +++ b/services/backend/oodikone2-backend/src/routes/population.js @@ -1,7 +1,7 @@ const router = require('express').Router() const Population = require('../services/populations') const Filters = require('../services/filters') -const { updateStudents } = require('../services/doo_api_database_updater/database_updater') +const { updateStudents } = require('../services/updaterService') const StudyrightService = require('../services/studyrights') // POST instead of GET because of too long params and "sensitive" data @@ -121,8 +121,8 @@ router.post('/updatedatabase', async (req, res) => { const studentnumbers = req.body console.log(studentnumbers) if (studentnumbers) { - await updateStudents(studentnumbers, 128) - res.status(200).json('Updated') + await updateStudents(studentnumbers) + res.status(200).json('Scheduled') } else { res.status(400).end() } diff --git a/services/backend/updater_writer/index.js b/services/backend/updater_writer/index.js index 5109a9a2e6..ef114cbfea 100644 --- a/services/backend/updater_writer/index.js +++ b/services/backend/updater_writer/index.js @@ -14,7 +14,7 @@ stan.on('connect', function () { const sub = stan.subscribe('UpdateWrite', 'updater.workers', opts) const attSub = stan.subscribe('UpdateAttainmentDates', opts) - const dumpSub = stan.subscribe('DumpDatabase') + const prioSub = stan.subscribe('PriorityWrite', 'updater.workers', opts) sub.on('message', async (msg) => { const data = JSON.parse(msg.getData()) @@ -30,14 +30,10 @@ stan.on('connect', function () { await updateAttainmentMeta() msg.ack() }) - dumpSub.on('message', async (_) => { - await dumpDatabase() - stan.publish('ScheduleAll', null, (err, guid) => { - if (err) { - console.log(err) - } else { - console.log('calling re-scheduling database') - } - }) + prioSub.on('message', async (msg) => { + const data = JSON.parse(msg.getData()) + await updateStudent(data, stan) + msg.ack() + stan.publish('status', `${data.studentInfo.studentnumber}:DONE`, (err) => { if (err) console.log(err) }) }) }) \ No newline at end of file diff --git a/services/updater_api/index.js b/services/updater_api/index.js index a47dd8c864..ee41eea35c 100644 --- a/services/updater_api/index.js +++ b/services/updater_api/index.js @@ -9,45 +9,43 @@ opts.setAckWait(15 * 60 * 1000); // 15min opts.setDeliverAllAvailable() opts.setDurableName('durable') opts.setMaxInFlight(3) + +const handleMessage = async (priority) => async (msg) => { // :d + let data = '' + const message = msg.getData() + if (!message) { + console.log('undefined message') + msg.ack() + return + } + if (message === 'meta') { + data = await getMeta() + stan.publish('UpdateWrite', JSON.stringify(data)) + msg.ack() + stan.publish('status', `${message}:FETCHED`, (err) => { if (err) console.log( 'STATUS PUBLISH FAILED', err) }) + } else { + // TODO: check that its a valid studentnumber and just ack it if its not + data = await getStudent(message) + try { + // TODO: check that data is properly structured(?) + stan.publish(priority ? 'UpdateWrite' : 'PriorityWrite', JSON.stringify(data), (err, guid) => { + if (err) { + return err + } else { + msg.ack() + stan.publish('status', `${message}:FETCHED`, (err) => { if (err) console.log('STATUS PUBLISH FAIELD', err) }) + } + }) + } catch (e) { + console.log(e) + } + } +} stan.on('connect', function () { const sub = stan.subscribe('UpdateApi', 'updater.workers', opts) - const prioSub = stan.subscribe('Priority_Update', 'updater.workers', opts) + const prioSub = stan.subscribe('PriorityApi', 'updater.workers', opts) - sub.on('message', async (msg) => { - let data = '' - const message = msg.getData() - if (!message) { - console.log('undefined message') - msg.ack() - return - } - if (message === 'meta') { - data = await getMeta() - stan.publish('UpdateWrite', JSON.stringify(data)) - msg.ack() - stan.publish('status', `${message}:FETCHED`, (err) => { if (err) console.log( 'STATUS PUBLISH FAILED', err) }) - } else { - // TODO: check that its a valid studentnumber and just ack it if its not - data = await getStudent(message) - try { - // TODO: check that data is properly structured(?) - stan.publish('UpdateWrite', JSON.stringify(data), (err, guid) => { - if (err) { - return err - } else { - msg.ack() - stan.publish('status', `${message}:FETCHED`, (err) => { if (err) console.log('STATUS PUBLISH FAIELD', err) }) - } - }) - } catch (e) { - console.log(e) - } - } - }) - - prioSub.on('message', async (msg) => { - await updateStudent(msg.getData()) - msg.ack() - }); + sub.on('message', handleMessage(false)) + prioSub.on('message', handleMessage(true)) }) \ No newline at end of file diff --git a/services/updater_scheduler/index.js b/services/updater_scheduler/index.js index e25499637b..ab487cf366 100644 --- a/services/updater_scheduler/index.js +++ b/services/updater_scheduler/index.js @@ -6,8 +6,7 @@ const logger = require('./logger') const { updateStudentNumberList } = require('./src/student_list_updater') const { scheduleActiveStudents, scheduleAllStudentsAndMeta } = require('./src/schedule_students') const { getOldestTasks, getCurrentStatus } = require('./src/SchedulingStatistics') -const { app } = require('./src/api') -console.log(app) +require('./src/api') let updatedCount = 0 let scheduledCount = 0 diff --git a/services/updater_scheduler/src/api.js b/services/updater_scheduler/src/api.js index f2a154cae9..b2fe7272e9 100644 --- a/services/updater_scheduler/src/api.js +++ b/services/updater_scheduler/src/api.js @@ -2,11 +2,15 @@ const express = require('express') const app = express() const port = process.env.PORT const bodyParser = require('body-parser') - +const { scheduleStudentsByArray } = require('./schedule_students') app.use(bodyParser.json()) app.get('/ping', (req, res) => res.json({ message: 'pong '})) +app.post('/update', async (req, res) => { + const msg = await scheduleStudentsByArray(req.body) + res.json({ message: msg }) +}) app.listen(port, () => console.log(`listening on port ${port}!`)) module.exports = { app } \ No newline at end of file diff --git a/services/updater_scheduler/src/schedule_students.js b/services/updater_scheduler/src/schedule_students.js index 90e22f51f8..7e3f889cfd 100644 --- a/services/updater_scheduler/src/schedule_students.js +++ b/services/updater_scheduler/src/schedule_students.js @@ -2,13 +2,13 @@ const { stan } = require('./nats_connection') const Schedule = require('../models') const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)) -const publish = async (tasks) => { - let rampup = 300 +const publish = async (tasks, priority = false) => { + let rampup = 300 for (const task of tasks) { if (rampup > 1) { rampup = rampup - 1 } - stan.publish('UpdateApi', task.task, (err, guid) => { + stan.publish(priority ? 'UpdateApi' : 'PriorityApi', task.task, (err, guid) => { console.log(guid) if (err) { console.log('publish failed', err) @@ -24,8 +24,8 @@ const publish = async (tasks) => { } const shuffleArray = (array) => { for (let i = array.length - 1; i > 0; i--) { - const j = Math.floor(Math.random() * (i + 1)); - [array[i], array[j]] = [array[j], array[i]]; + const j = Math.floor(Math.random() * (i + 1)); + [array[i], array[j]] = [array[j], array[i]]; } return array } @@ -36,8 +36,18 @@ const scheduleActiveStudents = async () => { } const scheduleAllStudentsAndMeta = async () => { - const tasks = [{ task:'meta', type: 'other', active: 'false' }, ...shuffleArray(await Schedule.find({ type: 'student' }))] + const tasks = [{ task: 'meta', type: 'other', active: 'false' }, ...shuffleArray(await Schedule.find({ type: 'student' }))] console.log(tasks.length, 'tasks to schedule') publish(tasks) } -module.exports = { scheduleActiveStudents, scheduleAllStudentsAndMeta } \ No newline at end of file + +const scheduleStudentsByArray = async (studentNumbers) => { + try { + const tasks = await Schedule.find({ type: 'student', task: { $in: studentNumbers } }) + publish(tasks, true) + } catch (e) { + return e + } + return 'scheduled' +} +module.exports = { scheduleActiveStudents, scheduleAllStudentsAndMeta, scheduleStudentsByArray } \ No newline at end of file diff --git a/services/updater_scheduler/src/student_list_updater.js b/services/updater_scheduler/src/student_list_updater.js index 564a63d37c..907c7f0f5f 100644 --- a/services/updater_scheduler/src/student_list_updater.js +++ b/services/updater_scheduler/src/student_list_updater.js @@ -7,6 +7,7 @@ const moment = require('moment') async function updateStudentNumberList() { const { KEY_PATH, CERT_PATH, TOKEN, NODE_ENV, OODI_ADDR, STUDENT_NUMBERS } = process.env + console.log(TOKEN) const agent = KEY_PATH && CERT_PATH ? new https.Agent({ cert: fs.readFileSync(CERT_PATH, 'utf8'), @@ -22,25 +23,23 @@ async function updateStudentNumberList() { }) instance.defaults.httpsAgent = agent - console.log(NODE_ENV) if (NODE_ENV === 'development') { - axios.defaults.params = { + instance.defaults.params['token'] = { token: TOKEN } - } - + const getStudentNumberChecksum = studentNumber => { const studentNumberString = String(studentNumber) let checksumNumbers = [7, 3, 1] let checksum = 0 - + for (let i = 0; i < studentNumberString.length; i++) { // go from end t start let currentNumber = studentNumberString[studentNumberString.length - (i + 1)] checksum += currentNumber * (checksumNumbers[i % checksumNumbers.length]) } - + return (10 - (checksum % 10)) % 10 } const requestStudent = async (studentNumber) => { From 2c06cd529539aa3f1198977dacfea7818d05ccdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A4kinen=20Sasu=20S?= Date: Mon, 8 Jul 2019 16:34:05 +0300 Subject: [PATCH 04/15] fix travis typo --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 31cd499f7f..6ac2be956e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,7 @@ jobs: - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.production.yml - docker ps - cat nginx/log - - docker-compose -f docker-compose.yml -f ./docker/ocker-compose.lateste2e.production.yml logs -f & + - docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.production.yml logs -f & - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done - time npm run test_services - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record From 2598f54dea9655fe313b8de45d55d89fe956a564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A4kinen=20Sasu=20S?= Date: Mon, 8 Jul 2019 16:55:45 +0300 Subject: [PATCH 05/15] fix subscription functions to await --- services/updater_api/index.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/updater_api/index.js b/services/updater_api/index.js index ee41eea35c..ba68b9ea9f 100644 --- a/services/updater_api/index.js +++ b/services/updater_api/index.js @@ -41,11 +41,11 @@ const handleMessage = async (priority) => async (msg) => { // :d } } } -stan.on('connect', function () { +stan.on('connect', async () => { const sub = stan.subscribe('UpdateApi', 'updater.workers', opts) const prioSub = stan.subscribe('PriorityApi', 'updater.workers', opts) - - sub.on('message', handleMessage(false)) - prioSub.on('message', handleMessage(true)) + + sub.on('message', await handleMessage(false)) + prioSub.on('message', await handleMessage(true)) }) \ No newline at end of file From dafa5561a45a1327e535fab2625fdd494bb9de21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A4kinen=20Sasu=20S?= Date: Mon, 8 Jul 2019 17:25:29 +0300 Subject: [PATCH 06/15] rm volumes from default backend dco conf --- docker-compose.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4eb4566d93..7ecadb67ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,11 +99,6 @@ services: ANALYTICS_URL: http://analytics:4568 NATS_URI: nats://nats:4222 UPDATER_URL: http://updater_scheduler:3678 - volumes: - - ./services/backend/oodikone2-backend/:/usr/src/app - - ./services/backend/shared/models:/usr/src/app/src/models - - ./services/backend/shared/migrations:/usr/src/app/src/database/migrations - - ./services/backend/shared/migrations_kone:/usr/src/app/src/database/migrations_kone container_name: backend depends_on: From 3b41db701e0747342895863b410943cf1705b3cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A4kinen=20Sasu=20S?= Date: Tue, 9 Jul 2019 09:45:56 +0300 Subject: [PATCH 07/15] add transaction lock to updating attainment meta so it wont crash on deadlock --- .../updater/update_attainment_dates.js | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/services/backend/updater_writer/updater/update_attainment_dates.js b/services/backend/updater_writer/updater/update_attainment_dates.js index 70af4e03c6..a89969da22 100644 --- a/services/backend/updater_writer/updater/update_attainment_dates.js +++ b/services/backend/updater_writer/updater/update_attainment_dates.js @@ -2,20 +2,26 @@ const { sequelize } = require('../database/connection') const updateAttainmentDates = async () => { const { schema } = sequelize.options - await sequelize.query( - `UPDATE ${schema}.course + const transaction = await sequelize.transaction() + try { + await sequelize.query( + `UPDATE ${schema}.course SET max_attainment_date = cr.max FROM (select course_code, max(attainment_date) from ${schema}.credit group by course_code) cr WHERE course.code=cr.course_code`, - { type: sequelize.QueryTypes.UPDATE } - ) - await sequelize.query( - `UPDATE ${schema}.course + { type: sequelize.QueryTypes.UPDATE, transaction, lock: transaction.LOCK.UPDATE } + ) + await sequelize.query( + `UPDATE ${schema}.course SET min_attainment_date = cr.min FROM (select course_code, min(attainment_date) from ${schema}.credit group by course_code) cr WHERE course.code=cr.course_code`, - { type: sequelize.QueryTypes.UPDATE } - ) + { type: sequelize.QueryTypes.UPDATE, transaction, lock: transaction.LOCK.UPDATE } + ) + transaction.commit() + } catch (e) { + transaction.rollback() + } } module.exports = { From 3f8b4746c29eff5d323303f4ff3c907da567b27e Mon Sep 17 00:00:00 2001 From: Warro Date: Tue, 9 Jul 2019 09:48:51 +0300 Subject: [PATCH 08/15] Refactored Travis so that it only does one build --- .travis.yml | 88 +++++++++---------- .../docker-compose.lateste2e.production.yml | 62 ------------- docker/docker-compose.lateste2e.yml | 18 ++-- nginx.production.conf => nginx.conf | 0 nginx.staging.conf | 19 ---- package.json | 2 +- run.sh | 2 +- scripts.sh | 6 +- 8 files changed, 58 insertions(+), 139 deletions(-) delete mode 100644 docker/docker-compose.lateste2e.production.yml rename nginx.production.conf => nginx.conf (100%) delete mode 100644 nginx.staging.conf diff --git a/.travis.yml b/.travis.yml index 31cd499f7f..42c03c17dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,50 +18,50 @@ before_script: - docker-compose --version - export AUTHOR_NAME="$(git log -1 ${TRAVIS_COMMIT} --pretty="%aN")" -jobs: - include: - - stage: build & test & push & deploy - if: NOT tag IS present - env: - - IMAGE=STAGING - script: - - docker login -u $DOCKER_USER -p $DOCKER_PASS - - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml - - docker ps - - cat nginx/log - - docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f & - - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done - - time npm run test_services - - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record - - docker ps -a - - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi - deploy: # deploy is always skipped for PR - provider: script - skip_cleanup: true - script: "docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}" - on: - branch: master - - # in parallel - if: type = pull_request OR NOT branch = master - env: - - IMAGE=PRODUCTION - script: - - docker login -u $DOCKER_USER -p $DOCKER_PASS - - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.production.yml - - docker ps - - cat nginx/log - - docker-compose -f docker-compose.yml -f ./docker/ocker-compose.lateste2e.production.yml logs -f & - - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done - - time npm run test_services - - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record - - docker ps -a - - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi - deploy: # deploy is always skipped for PR - provider: script - skip_cleanup: true - script: "docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.production.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}" - on: - tags: true +stages: + - name: login to docker + script: + - docker login -u $DOCKER_USER -p $DOCKER_PASS + - name: build staging + if: NOT tag IS present + env: + - IMAGE=STAGING + - TAG=staging + script: + - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG} + - name: build latest + if: tag IS present + env: + - IMAGE=PRODUCTION + - TAG=latest + script: + - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG} + - name: run tests + script: + - docker ps + - cat nginx/log + - TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f & + - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done + - time npm run test_services + - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record + - docker ps -a + - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi + - name: deploy to staging + if: NOT tag IS present + deploy: + provider: script + skip_cleanup: true + script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}" + on: + branch: master + - name: deploy to production + if: tag IS present + deploy: + provider: script + skip_cleanup: true + script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}" + on: + tags: true after_failure: - "curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${IMAGE} buildi paskana. STOP AND FIX tai tulee kenkää! *${AUTHOR_NAME}* ${TRAVIS_BUILD_WEB_URL}\\\"}\" ${SLACKBOT_URL}" diff --git a/docker/docker-compose.lateste2e.production.yml b/docker/docker-compose.lateste2e.production.yml deleted file mode 100644 index fc5eea51ec..0000000000 --- a/docker/docker-compose.lateste2e.production.yml +++ /dev/null @@ -1,62 +0,0 @@ -version: '3' - -services: - user_db: - environment: - - POSTGRES_DB=user_db - - db: - environment: - - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test - - backend: - image: toska/oodikone2-backend:latest - ports: - - "9229:9229" - environment: - FRONT_URL: http://localhost:1337 - ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres - - frontend: - image: toska/oodikone2-frontend:latest - build: - args: - BASE_PATH: / - ports: - - "5000:5000" - - userservice: - image: toska/oodikone2-userservice:latest - - usageservice: - image: toska/oodikone2-usageservice:latest - - updater_api: - image: toska/updater_api:latest - - updater_writer: - image: toska/updater_writer:latest - - updater_scheduler: - image: toska/updater_scheduler:latest - command: ["echo", "mluukkai on orjuuttaja"] - - analytics: - image: toska/oodikone2-analytics:latest - - nginx: - image: nginx:latest - volumes: - - ./nginx.production.conf:/etc/nginx/nginx.conf - - ./nginx/error.log:/etc/nginx/error_log.log - - ./nginx/log:/var/log/nginx/localhost.error_log - - ./nginx/cache/:/etc/nginx/cache - - /etc/letsencrypt/:/etc/letsencrypt/ - ports: - - 1337:80 - depends_on: - - frontend - - backend - - userservice - - analytics - container_name: nginx diff --git a/docker/docker-compose.lateste2e.yml b/docker/docker-compose.lateste2e.yml index e8553d34bb..8afa33b1a7 100644 --- a/docker/docker-compose.lateste2e.yml +++ b/docker/docker-compose.lateste2e.yml @@ -10,7 +10,7 @@ services: - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test backend: - image: toska/oodikone2-backend:staging + image: toska/oodikone2-backend:${TAG} ports: - "9229:9229" environment: @@ -18,7 +18,7 @@ services: ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres frontend: - image: toska/oodikone2-frontend:staging + image: toska/oodikone2-frontend:${TAG} build: args: BASE_PATH: / @@ -26,28 +26,28 @@ services: - "5000:5000" userservice: - image: toska/oodikone2-userservice:staging + image: toska/oodikone2-userservice:${TAG} usageservice: - image: toska/oodikone2-usageservice:staging + image: toska/oodikone2-usageservice:${TAG} updater_api: - image: toska/updater_api:staging + image: toska/updater_api:${TAG} updater_writer: - image: toska/updater_writer:staging + image: toska/updater_writer:${TAG} updater_scheduler: - image: toska/updater_scheduler:staging + image: toska/updater_scheduler:${TAG} command: ["echo", "mluukkai on orjuuttaja"] analytics: - image: toska/oodikone2-analytics:staging + image: toska/oodikone2-analytics:${TAG} nginx: image: nginx:latest volumes: - - ./nginx.staging.conf:/etc/nginx/nginx.conf + - ./nginx.conf:/etc/nginx/nginx.conf - ./nginx/error.log:/etc/nginx/error_log.log - ./nginx/log:/var/log/nginx/localhost.error_log - ./nginx/cache/:/etc/nginx/cache diff --git a/nginx.production.conf b/nginx.conf similarity index 100% rename from nginx.production.conf rename to nginx.conf diff --git a/nginx.staging.conf b/nginx.staging.conf deleted file mode 100644 index e780f2d73b..0000000000 --- a/nginx.staging.conf +++ /dev/null @@ -1,19 +0,0 @@ -events { worker_connections 1024; } - -http { - - rewrite_log on; - error_log /var/log/nginx/localhost.error_log notice; - - server { - listen 80; - - location / { - proxy_pass http://frontend:5000/; - } - - location /api/ { - proxy_pass http://backend:8080/; - } - } -} diff --git a/package.json b/package.json index 4ef320e3bd..793d2400cb 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "docker:logs:backend": "npm run docker -- logs -f backend", "docker:restart": "npm run docker -- restart", "docker:restart:backend": "npm run docker -- restart backend", - "docker:obliterate": "docker stop $(docker ps -aq) && docker system prune", + "docker:obliterate": "docker stop $(docker ps -aq) ; docker rmi -f $(docker images -a -q) ; docker system prune", "start": "npm run docker:up && npm run docker:logs" }, "husky": { diff --git a/run.sh b/run.sh index 47d42cac2f..134f5fd2d1 100755 --- a/run.sh +++ b/run.sh @@ -33,7 +33,7 @@ do case $opt in "e2e setup") mopo - run_e2e_setup ./docker/docker-compose.lateste2e.yml + run_e2e_setup ./docker/docker-compose.lateste2e.yml staging ;; "Anon setup") mopo diff --git a/scripts.sh b/scripts.sh index fccdd5c8b3..afa0d9352a 100644 --- a/scripts.sh +++ b/scripts.sh @@ -164,10 +164,10 @@ run_e2e_setup () { echo "Getting anon backups from the private repository. " get_anon_oodikone echo "Building images" - docker-compose -f docker-compose.yml -f $1 build + TAG=$2 docker-compose -f docker-compose.yml -f $1 build echo "Setup oodikone db from dump." - docker-compose -f docker-compose.yml -f $1 up -d db user_db db_kone + TAG=$2 docker-compose -f docker-compose.yml -f $1 up -d db user_db db_kone db_anon_setup_full echo "Starting services." - docker-compose -f docker-compose.yml -f $1 up -d + TAG=$2 docker-compose -f docker-compose.yml -f $1 up -d } From 67bcce5fd49c015c57fabfee1e968e295ad7b3ed Mon Sep 17 00:00:00 2001 From: esakemp Date: Tue, 9 Jul 2019 10:17:19 +0300 Subject: [PATCH 09/15] bck bttn wrk --- .travis.yml | 88 ++-- README.md | 14 +- docker-compose.lateste2e.production.yml | 249 ----------- docker-compose.lateste2e.yml | 251 ----------- docker-compose.yml | 153 +------ .../docker-compose.dev.real.yml | 0 docker/docker-compose.dev.yml | 170 ++++++++ docker/docker-compose.lateste2e.yml | 62 +++ nginx.production.conf => nginx.conf | 0 nginx.staging.conf | 19 - package.json | 12 +- run.sh | 2 +- scripts.sh | 34 +- .../oodikone2-backend/src/conf-backend.js | 4 +- .../src/routes/population.js | 79 +++- .../services/course_yearly_stats_counter.js | 4 +- .../oodikone2-backend/src/services/courses.js | 4 +- .../src/services/populations.js | 65 +-- .../src/services/students.js | 47 ++- .../src/services/updaterService.js | 21 + services/backend/updater_writer/index.js | 16 +- .../updater/update_attainment_dates.js | 22 +- .../src/components/CourseStatistics/index.jsx | 4 +- .../PopulationCourseStats/index.jsx | 16 +- .../components/PopulationSearchForm/index.jsx | 10 +- .../components/PopulationStatistics/index.jsx | 15 +- .../src/components/StudentDetails/index.jsx | 3 +- .../src/redux/populationCourses.js | 14 + services/updater_api/index.js | 74 ++-- services/updater_scheduler/index.js | 1 + services/updater_scheduler/package-lock.json | 388 ++++++++++++++++++ services/updater_scheduler/package.json | 1 + services/updater_scheduler/src/api.js | 16 + .../src/schedule_students.js | 24 +- .../src/student_list_updater.js | 11 +- 35 files changed, 1034 insertions(+), 859 deletions(-) delete mode 100644 docker-compose.lateste2e.production.yml delete mode 100644 docker-compose.lateste2e.yml rename docker-compose.real.yml => docker/docker-compose.dev.real.yml (100%) create mode 100644 docker/docker-compose.dev.yml create mode 100644 docker/docker-compose.lateste2e.yml rename nginx.production.conf => nginx.conf (100%) delete mode 100644 nginx.staging.conf create mode 100644 services/backend/oodikone2-backend/src/services/updaterService.js create mode 100644 services/updater_scheduler/src/api.js diff --git a/.travis.yml b/.travis.yml index 4cb784ee4a..42c03c17dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,50 +18,50 @@ before_script: - docker-compose --version - export AUTHOR_NAME="$(git log -1 ${TRAVIS_COMMIT} --pretty="%aN")" -jobs: - include: - - stage: build & test & push & deploy - if: NOT tag IS present - env: - - IMAGE=STAGING - script: - - docker login -u $DOCKER_USER -p $DOCKER_PASS - - source scripts.sh && time run_e2e_setup docker-compose.lateste2e.yml - - docker ps - - cat nginx/log - - docker-compose logs -f & - - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done - - time npm run test_services - - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record - - docker ps -a - - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi - deploy: # deploy is always skipped for PR - provider: script - skip_cleanup: true - script: "docker-compose -f docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}" - on: - branch: master - - # in parallel - if: type = pull_request OR NOT branch = master - env: - - IMAGE=PRODUCTION - script: - - docker login -u $DOCKER_USER -p $DOCKER_PASS - - source scripts.sh && time run_e2e_setup docker-compose.lateste2e.production.yml - - docker ps - - cat nginx/log - - docker-compose logs -f & - - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done - - time npm run test_services - - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record - - docker ps -a - - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi - deploy: # deploy is always skipped for PR - provider: script - skip_cleanup: true - script: "docker-compose -f docker-compose.lateste2e.production.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}" - on: - tags: true +stages: + - name: login to docker + script: + - docker login -u $DOCKER_USER -p $DOCKER_PASS + - name: build staging + if: NOT tag IS present + env: + - IMAGE=STAGING + - TAG=staging + script: + - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG} + - name: build latest + if: tag IS present + env: + - IMAGE=PRODUCTION + - TAG=latest + script: + - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG} + - name: run tests + script: + - docker ps + - cat nginx/log + - TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f & + - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done + - time npm run test_services + - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record + - docker ps -a + - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi + - name: deploy to staging + if: NOT tag IS present + deploy: + provider: script + skip_cleanup: true + script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}" + on: + branch: master + - name: deploy to production + if: tag IS present + deploy: + provider: script + skip_cleanup: true + script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}" + on: + tags: true after_failure: - "curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${IMAGE} buildi paskana. STOP AND FIX tai tulee kenkää! *${AUTHOR_NAME}* ${TRAVIS_BUILD_WEB_URL}\\\"}\" ${SLACKBOT_URL}" diff --git a/README.md b/README.md index e3c3a2bbd7..5f4080ddf3 100644 --- a/README.md +++ b/README.md @@ -58,27 +58,27 @@ The mapping of ports and environment variables are also defined in the docker-co #### With anonymized data: ``` -docker-compose up -d +npm run docker:up ``` #### With real data: ``` -docker-compose -f docker-compose.yml -f docker-compose.real.yml up -d +docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml -f ./docker/docker-compose.dev.real.yml up -d ``` ### Stop the development environment ``` -docker-compose down +npm run docker:down ``` ### Restart container(s) ``` -docker-compose restart # all +npm run docker:restart # all -docker-compose restart backend # just backend +npm run docker:restart:backend # just backend ``` ### View the containers in the running environment @@ -90,9 +90,9 @@ docker-compose ps ``` -docker-compose logs -f # all +npm run docker:logs # all -docker-compose logs -f backend # just backend +npm run docker:logs:backend # just backend ``` diff --git a/docker-compose.lateste2e.production.yml b/docker-compose.lateste2e.production.yml deleted file mode 100644 index fd1e60b903..0000000000 --- a/docker-compose.lateste2e.production.yml +++ /dev/null @@ -1,249 +0,0 @@ -version: '3' - -services: - nats: - image: nats-streaming - command: -cid updaterNATS - expose: - - "4222" - container_name: nats - - analytics_db: - image: postgres:9.6.3 - ports: - - "5423:5432" - volumes: - - ./analytics_pgdata:/var/lib/postgresql/data - container_name: oodi_analytics_db - environment: - - POSTGRES_USER=postgres - - POSTGRES_DB=analytics_db - - user_db: - image: postgres:9.6.3 - ports: - - "5422:5432" - volumes: - - ./user_pgdata:/var/lib/postgresql/data - container_name: oodi_user_db - environment: - - POSTGRES_USER=postgres - - POSTGRES_DB=user_db - - db: - image: postgres:9.6.3 - ports: - - "5421:5432" - volumes: - - ./pgdata:/var/lib/postgresql/data - - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro - container_name: oodi_db - environment: - - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test - - POSTGRES_USER=postgres - - db_kone: - image: postgres:9.6.3 - ports: - - "5425:5432" - volumes: - - ./pgdata_kone:/var/lib/postgresql/data - - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro - container_name: db_kone - environment: - - POSTGRES_MULTIPLE_DATABASES=db_kone - - POSTGRES_USER=postgres - - redis: - image: redis - command: ["redis-server", "--appendonly", "yes"] - ports: - - "6379:6379" - volumes: - - ./redis-data:/data - container_name: redis - - analytics_redis: - image: redis - command: ["redis-server", "--appendonly", "yes"] - ports: - - "6380:6379" - volumes: - - ./analytics-redis-data:/data - container_name: analytics_redis - - backend: - image: toska/oodikone2-backend:latest - build: - context: services/backend - dockerfile: oodikone2-backend/Dockerfile - ports: - - "8080:8080" - - "9229:9229" - environment: - WAIT_HOSTS: redis:6379,db:5432,db_kone:5432,nats:4222 - REDIS: redis - DB_URL: postgres://postgres@db:5432/tkt_oodi - DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone - TEST_DB: postgres://postgres@db:5432/tkt_oodi_test - TEST_DB_KONE: postgres://postgres@db_kone:5432/db_kone_test - FRONT_URL: http://localhost:1337 - TOKEN_SECRET: IWannaBeTheVeryBest - OODILEARN_URL: http://oodilearn:5000 - USERSERVICE_URL: http://userservice:4567 - USAGESERVICE_URL: http://usageservice:545 - USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 - USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 - ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres - ANALYTICS_URL: http://analytics:4568 - NATS_URI: nats://nats:4222 - container_name: backend - depends_on: - - db - - redis - - nats - - frontend: - image: toska/oodikone2-frontend:latest - build: - context: services/oodikone2-frontend - args: - BASE_PATH: / - ports: - - "5000:5000" - environment: - WAIT_HOSTS: user_db:5432 - BACKEND_ADDR: backend - ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres - container_name: frontend - - userservice: - image: toska/oodikone2-userservice:latest - build: - context: services/oodikone2-userservice - ports: - - 4567:4567 - environment: - - WAIT_HOSTS=user_db:5432 - - DB_URL=postgres://postgres@user_db:5432/user_db - - TEST_DB=postgres://postgres@user_db:5432/user_db_test - - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 - - TOKEN_SECRET=IWannaBeTheVeryBest - depends_on: - - user_db - container_name: userservice - - usageservice: - image: toska/oodikone2-usageservice:latest - build: - context: services/oodikone2-usageservice - ports: - - 545:545 - environment: - - WAIT_HOSTS=usage_db:5432 - - PORT=545 - - DB_URL=postgres://postgres@usage_db:5432/usage_db - - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 - depends_on: - - usage_db - container_name: usageservice - - usage_db: - image: postgres:9.6.3 - ports: - - "5424:5432" - volumes: - - ./usage_pgdata:/var/lib/postgresql/data - container_name: oodi_usage_db - environment: - - POSTGRES_USER=postgres - - POSTGRES_DB=usage_db - - analytics: - image: toska/oodikone2-analytics:latest - build: - context: services/oodikone2-analytics - ports: - - 4568:4568 - environment: - - WAIT_HOSTS=analytics_db:5432,analytics_redis:6379 - - REDIS=analytics_redis - - DB_URL=postgres://postgres@analytics_db:5432/analytics_db - - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test - depends_on: - - analytics_db - - analytics_redis - container_name: analytics - - updater_api: - image: toska/updater_api:latest - build: - context: services/updater_api - depends_on: - - nats - environment: - - WAIT_HOSTS=nats:4222 - - NATS_URI=nats://nats:4222 - - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi - - TOKEN=toskab1gs8cret5 - container_name: updater_api - - updater_writer: - image: toska/updater_writer:latest - build: - context: services/backend - dockerfile: updater_writer/Dockerfile - depends_on: - - nats - - db - environment: - WAIT_HOSTS: nats:4222,db:5432 - NATS_URI: nats://nats:4222 - DB_URL: postgres://postgres@db:5432/tkt_oodi - TEST_DB: postgres://postgres@db:5432/tkt_oodi_test - TEST_DB_SCHEMA: updater_test - container_name: updater_writer - - scheduler_mongo: - image: mongo - volumes: - - ./mongodata:/data/db - expose: - - "27017" - container_name: scheduler_db - - updater_scheduler: - image: toska/updater_scheduler:latest - build: - context: services/updater_scheduler - command: ["echo", "mluukkai on orjuuttaja"] - depends_on: - - nats - - scheduler_mongo - environment: - - WAIT_HOSTS=nats:4222,scheduler_mongo:27017 - - NATS_URI=nats://nats:4222 - - MONGO_URI=mongodb://scheduler_mongo:27017 - - STUDENT_NUMBERS=./all_student_numbers.txt - - ACTIVE_STUDENTS=./active_student_numbers.txt - - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi - - TOKEN=toskab1gs8cret5 - container_name: updater_scheduler - - nginx: - image: nginx:latest - volumes: - - ./nginx.production.conf:/etc/nginx/nginx.conf - - ./nginx/error.log:/etc/nginx/error_log.log - - ./nginx/log:/var/log/nginx/localhost.error_log - - ./nginx/cache/:/etc/nginx/cache - - /etc/letsencrypt/:/etc/letsencrypt/ - ports: - - 1337:80 - depends_on: - - frontend - - backend - - userservice - - analytics - - container_name: nginx diff --git a/docker-compose.lateste2e.yml b/docker-compose.lateste2e.yml deleted file mode 100644 index 3dbb38f905..0000000000 --- a/docker-compose.lateste2e.yml +++ /dev/null @@ -1,251 +0,0 @@ -version: '3' - -services: - nats: - image: nats-streaming - command: -cid updaterNATS --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore - expose: - - "4222" - volumes: - - ./datastore:/datastore - container_name: nats - - analytics_db: - image: postgres:9.6.3 - ports: - - "5423:5432" - volumes: - - ./analytics_pgdata:/var/lib/postgresql/data - container_name: oodi_analytics_db - environment: - - POSTGRES_USER=postgres - - POSTGRES_DB=analytics_db - - user_db: - image: postgres:9.6.3 - ports: - - "5422:5432" - volumes: - - ./user_pgdata:/var/lib/postgresql/data - container_name: oodi_user_db - environment: - - POSTGRES_USER=postgres - - POSTGRES_DB=user_db - - db: - image: postgres:9.6.3 - ports: - - "5421:5432" - volumes: - - ./pgdata:/var/lib/postgresql/data - - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro - container_name: oodi_db - environment: - - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test - - POSTGRES_USER=postgres - - db_kone: - image: postgres:9.6.3 - ports: - - "5425:5432" - volumes: - - ./pgdata_kone:/var/lib/postgresql/data - - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro - container_name: db_kone - environment: - - POSTGRES_MULTIPLE_DATABASES=db_kone - - POSTGRES_USER=postgres - - redis: - image: redis - command: ["redis-server", "--appendonly", "yes"] - ports: - - "6379:6379" - volumes: - - ./redis-data:/data - container_name: redis - - analytics_redis: - image: redis - command: ["redis-server", "--appendonly", "yes"] - ports: - - "6380:6379" - volumes: - - ./analytics-redis-data:/data - container_name: analytics_redis - - backend: - image: toska/oodikone2-backend:staging - build: - context: services/backend - dockerfile: oodikone2-backend/Dockerfile - ports: - - "8080:8080" - - "9229:9229" - environment: - WAIT_HOSTS: redis:6379,db:5432,db_kone:5432,nats:4222 - REDIS: redis - DB_URL: postgres://postgres@db:5432/tkt_oodi - DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone - TEST_DB: postgres://postgres@db:5432/tkt_oodi_test - TEST_DB_KONE: postgres://postgres@db_kone:5432/db_kone_test - FRONT_URL: http://localhost:1337 - TOKEN_SECRET: IWannaBeTheVeryBest - OODILEARN_URL: http://oodilearn:5000 - USERSERVICE_URL: http://userservice:4567 - USAGESERVICE_URL: http://usageservice:545 - USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 - USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 - ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres - ANALYTICS_URL: http://analytics:4568 - NATS_URI: nats://nats:4222 - container_name: backend - depends_on: - - db - - redis - - nats - - frontend: - image: toska/oodikone2-frontend:staging - build: - context: services/oodikone2-frontend - args: - BASE_PATH: / - ports: - - "5000:5000" - environment: - WAIT_HOSTS: user_db:5432 - BACKEND_ADDR: backend - ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres - container_name: frontend - - userservice: - image: toska/oodikone2-userservice:staging - build: - context: services/oodikone2-userservice - ports: - - 4567:4567 - environment: - - WAIT_HOSTS=user_db:5432 - - DB_URL=postgres://postgres@user_db:5432/user_db - - TEST_DB=postgres://postgres@user_db:5432/user_db_test - - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 - - TOKEN_SECRET=IWannaBeTheVeryBest - depends_on: - - user_db - container_name: userservice - - usageservice: - image: toska/oodikone2-usageservice:staging - build: - context: services/oodikone2-usageservice - ports: - - 545:545 - environment: - - WAIT_HOSTS=usage_db:5432 - - PORT=545 - - DB_URL=postgres://postgres@usage_db:5432/usage_db - - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 - depends_on: - - usage_db - container_name: usageservice - - usage_db: - image: postgres:9.6.3 - ports: - - "5424:5432" - volumes: - - ./usage_pgdata:/var/lib/postgresql/data - container_name: oodi_usage_db - environment: - - POSTGRES_USER=postgres - - POSTGRES_DB=usage_db - - analytics: - image: toska/oodikone2-analytics:staging - build: - context: services/oodikone2-analytics - ports: - - 4568:4568 - environment: - - WAIT_HOSTS=analytics_db:5432,analytics_redis:6379 - - REDIS=analytics_redis - - DB_URL=postgres://postgres@analytics_db:5432/analytics_db - - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test - depends_on: - - analytics_db - - analytics_redis - container_name: analytics - - updater_api: - image: toska/updater_api:staging - build: - context: services/updater_api - depends_on: - - nats - environment: - - WAIT_HOSTS=nats:4222 - - NATS_URI=nats://nats:4222 - - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi - - TOKEN=toskab1gs8cret5 - container_name: updater_api - - updater_writer: - image: toska/updater_writer:staging - build: - context: services/backend - dockerfile: updater_writer/Dockerfile - depends_on: - - nats - - db - environment: - WAIT_HOSTS: nats:4222,db:5432 - NATS_URI: nats://nats:4222 - DB_URL: postgres://postgres@db:5432/tkt_oodi - TEST_DB: postgres://postgres@db:5432/tkt_oodi_test - TEST_DB_SCHEMA: updater_test - container_name: updater_writer - - scheduler_mongo: - image: mongo - volumes: - - ./mongodata:/data/db - expose: - - "27017" - container_name: scheduler_db - - updater_scheduler: - image: toska/updater_scheduler:staging - build: - context: services/updater_scheduler - command: ["echo", "mluukkai on orjuuttaja"] - depends_on: - - nats - - scheduler_mongo - environment: - - WAIT_HOSTS=nats:4222,scheduler_mongo:27017 - - NATS_URI=nats://nats:4222 - - MONGO_URI=mongodb://scheduler_mongo:27017 - - STUDENT_NUMBERS=./all_student_numbers.txt - - ACTIVE_STUDENTS=./active_student_numbers.txt - - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi - - TOKEN=toskab1gs8cret5 - container_name: updater_scheduler - - nginx: - image: nginx:latest - volumes: - - ./nginx.staging.conf:/etc/nginx/nginx.conf - - ./nginx/error.log:/etc/nginx/error_log.log - - ./nginx/log:/var/log/nginx/localhost.error_log - - ./nginx/cache/:/etc/nginx/cache - - /etc/letsencrypt/:/etc/letsencrypt/ - ports: - - 1337:80 - depends_on: - - frontend - - backend - - userservice - - analytics - - container_name: nginx diff --git a/docker-compose.yml b/docker-compose.yml index d2a82df6e1..7ecadb67ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,43 +1,17 @@ version: '3' services: - # zookeeper: - # image: wurstmeister/zookeeper - # ports: - # - "2181:2181" - # environment: - # - KAFKA_BROKER_ID=1 - # container_name: zookeeper - - # kafka: - # image: wurstmeister/kafka - # container_name: kafka - # depends_on: - # - zookeeper - # ports: - # - "9092:9092" - # environment: - # - HOSTNAME_COMMAND=route -n | awk '/UG[ \t]/{print $$2}' - # - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - # - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true - # - KAFKA_DELETE_TOPIC_ENABLE=true - # - KAFKA_CREATE_TOPICS=studentnumbers:1:1,priority_studentnumbers:1:1 - # - KAFKA_LOG_DIRS=/kafka/kafka-logs-1 - # volumes: - # - ./kafka/docker.sock:/var/run/docker.sock - # - ./kafka/logs:/kafka/kafka-logs-1 - nats: image: nats-streaming - command: -cid updaterNATS --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore + command: -cid updaterNATS --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore expose: - "4222" - volumes: - - ./datastore:/datastore ports: - "8222:8222" - "4222:4222" container_name: nats + volumes: + - ./datastore:/datastore analytics_db: image: postgres:9.6.3 @@ -59,7 +33,6 @@ services: container_name: oodi_user_db environment: - POSTGRES_USER=postgres - - POSTGRES_MULTIPLE_DATABASES=user_db,user_db_real usage_db: image: postgres:9.6.3 @@ -81,7 +54,6 @@ services: - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro container_name: oodi_db environment: - - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test,tkt_oodi_real - POSTGRES_USER=postgres db_kone: @@ -105,24 +77,10 @@ services: - ./redis-data:/data container_name: redis - # mongo_db: - # image: mongo - # restart: unless-stopped - # volumes: - # - ./backups/mongo/oodilearn:/dump - # - ./mongodata:/data/db - # ports: - # - "27017:27017" - # container_name: mongo_db - backend: build: context: services/backend dockerfile: oodikone2-backend/Dockerfile - args: - NODE_ENV: 'development' - command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] - restart: unless-stopped ports: - "8080:8080" environment: @@ -132,7 +90,6 @@ services: DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone TEST_DB: postgres://postgres@db:5432/tkt_oodi_test TEST_DB_KONE: postgres://postgres@db_kone:5432/db_kone_test - FRONT_URL: http://localhost:8000 TOKEN_SECRET: IWannaBeTheVeryBest OODILEARN_URL: http://oodilearn:5000 USERSERVICE_URL: http://userservice:4567 @@ -141,11 +98,7 @@ services: USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 ANALYTICS_URL: http://analytics:4568 NATS_URI: nats://nats:4222 - volumes: - - ./services/backend/oodikone2-backend/:/usr/src/app - - ./services/backend/shared/models:/usr/src/app/src/models - - ./services/backend/shared/migrations:/usr/src/app/src/database/migrations - - ./services/backend/shared/migrations_kone:/usr/src/app/src/database/migrations_kone + UPDATER_URL: http://updater_scheduler:3678 container_name: backend depends_on: @@ -156,49 +109,14 @@ services: frontend: build: context: services/oodikone2-frontend - args: - NODE_ENV: 'development' - command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run docker'] - restart: unless-stopped - ports: - - "8081:8081" - - "7777:7777" environment: BACKEND_ADDR: backend ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres - USER_ADMINER_URL: http://localhost:5050/?pgsql=user_db&username=postgres - ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres - USAGE_ADMINER_URL: http://localhost:5050/?pgsql=usage_db&username=postgres - KONE_ADMINER_URL: http://localhost:5050/?pgsql=db_kone&username=postgres - volumes: - - ./services/oodikone2-frontend/:/usr/src/app container_name: frontend - # oodilearn: - # build: - # context: services/oodilearn/server - # dockerfile: Dockerfile - # restart: unless-stopped - # volumes: - # - ./services/oodilearn/data:/data - # - ./services/oodilearn/server:/src - # - ./services/oodilearn/models:/models - # ports: - # - "5000:5000" - # environment: - # - ENV=development - # - MONGO_URI=mongodb://mongo_db:27017/oodilearn - # container_name: oodilearn - userservice: build: context: services/oodikone2-userservice - args: - NODE_ENV: 'development' - command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] - restart: unless-stopped - volumes: - - ./services/oodikone2-userservice/:/usr/src/app ports: - 4567:4567 environment: @@ -214,10 +132,6 @@ services: usageservice: build: context: services/oodikone2-usageservice - args: - NODE_ENV: 'development' - command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] - restart: unless-stopped ports: - 545:545 environment: @@ -225,8 +139,6 @@ services: PORT: '545' DB_URL: postgres://postgres@usage_db:5432/usage_db SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019 - volumes: - - ./services/oodikone2-usageservice/:/usr/src/app depends_on: - usage_db container_name: usageservice @@ -234,12 +146,6 @@ services: updater_api: build: context: services/updater_api - args: - NODE_ENV: 'development' - command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] - restart: unless-stopped - volumes: - - ./services/updater_api/:/usr/src/app depends_on: - nats environment: @@ -253,16 +159,6 @@ services: build: context: services/backend dockerfile: updater_writer/Dockerfile - args: - NODE_ENV: 'development' - command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] - restart: unless-stopped - volumes: - - ./services/backend/updater_writer/:/usr/src/app - - ./services/backend/shared/models:/usr/src/app/models - - ./services/backend/shared/migrations:/usr/src/app/database/migrations - - ./services/backend/shared/migrations_kone:/usr/src/app/database/migrations_kone - depends_on: - nats - db @@ -272,28 +168,19 @@ services: DB_URL: postgres://postgres@db:5432/tkt_oodi TEST_DB: postgres://postgres@db:5432/tkt_oodi_test TEST_DB_SCHEMA: updater_test - DB_SCHEMA_KONE: kone_data - - + container_name: updater_writer scheduler_mongo: image: mongo - restart: unless-stopped volumes: - ./mongodata:/data/db expose: - "27017" - container_name: scheduler_db + container_name: scheduler_mongo updater_scheduler: build: context: services/updater_scheduler - args: - NODE_ENV: 'development' - command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] - restart: unless-stopped - volumes: - - ./services/updater_scheduler/:/usr/src/app depends_on: - nats - scheduler_mongo @@ -308,26 +195,19 @@ services: - LOG_HOST=toska.cs.helsinki.fi - LOG_PORT=9501 - LOG_HOSTNAME=updater_scheduler - + - PORT=3678 container_name: updater_scheduler - analytics: build: context: services/oodikone2-analytics - args: - NODE_ENV: 'development' - command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] - restart: unless-stopped - volumes: - - ./services/oodikone2-analytics/:/usr/src/app ports: - 4568:4568 environment: - WAIT_HOSTS: analytics_db:5432,analytics_redis:6379 - REDIS: analytics_redis - DB_URL: postgres://postgres@analytics_db:5432/analytics_db - TEST_DB: postgres://postgres@analytics_db:5432/analytics_db_test + - WAIT_HOSTS=analytics_db:5432,analytics_redis:6379 + - REDIS=analytics_redis + - DB_URL=postgres://postgres@analytics_db:5432/analytics_db + - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test depends_on: - analytics_db - analytics_redis @@ -341,14 +221,3 @@ services: volumes: - ./analytics-redis-data:/data container_name: analytics_redis - - adminer: - image: adminer:4.6.2 - environment: - - ADMINER_DESIGN=pepa-linha - volumes: - - ./adminer-plugins:/var/www/html/plugins-enabled - ports: - - 5050:8080 - restart: always - container_name: adminer diff --git a/docker-compose.real.yml b/docker/docker-compose.dev.real.yml similarity index 100% rename from docker-compose.real.yml rename to docker/docker-compose.dev.real.yml diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml new file mode 100644 index 0000000000..e13691d297 --- /dev/null +++ b/docker/docker-compose.dev.yml @@ -0,0 +1,170 @@ +version: '3' + +services: + user_db: + environment: + - POSTGRES_MULTIPLE_DATABASES=user_db,user_db_real + + db: + environment: + - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test,tkt_oodi_real + + backend: + build: + args: + NODE_ENV: 'development' + command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] + restart: unless-stopped + environment: + FRONT_URL: http://localhost:8000 + volumes: + - ./services/backend/oodikone2-backend/:/usr/src/app + - ./services/backend/shared/models:/usr/src/app/src/models + - ./services/backend/shared/migrations:/usr/src/app/src/database/migrations + - ./services/backend/shared/migrations_kone:/usr/src/app/src/database/migrations_kone + + frontend: + build: + args: + NODE_ENV: 'development' + command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run docker'] + restart: unless-stopped + ports: + - "8081:8081" + - "7777:7777" + environment: + USER_ADMINER_URL: http://localhost:5050/?pgsql=user_db&username=postgres + ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres + USAGE_ADMINER_URL: http://localhost:5050/?pgsql=usage_db&username=postgres + KONE_ADMINER_URL: http://localhost:5050/?pgsql=db_kone&username=postgres + volumes: + - ./services/oodikone2-frontend/:/usr/src/app + + userservice: + build: + args: + NODE_ENV: 'development' + command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] + restart: unless-stopped + volumes: + - ./services/oodikone2-userservice/:/usr/src/app + + usageservice: + build: + args: + NODE_ENV: 'development' + command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] + restart: unless-stopped + volumes: + - ./services/oodikone2-usageservice/:/usr/src/app + + updater_api: + build: + args: + NODE_ENV: 'development' + command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] + restart: unless-stopped + volumes: + - ./services/updater_api/:/usr/src/app + + updater_writer: + build: + args: + NODE_ENV: 'development' + command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] + restart: unless-stopped + volumes: + - ./services/backend/updater_writer/:/usr/src/app + - ./services/backend/shared/models:/usr/src/app/models + - ./services/backend/shared/migrations:/usr/src/app/database/migrations + - ./services/backend/shared/migrations_kone:/usr/src/app/database/migrations_kone + environment: + DB_SCHEMA_KONE: kone_data + + scheduler_mongo: + restart: unless-stopped + + updater_scheduler: + build: + args: + NODE_ENV: 'development' + command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] + restart: unless-stopped + volumes: + - ./services/updater_scheduler/:/usr/src/app + environment: + - LOG_HOST=toska.cs.helsinki.fi + - LOG_PORT=9501 + - LOG_HOSTNAME=updater_scheduler + + analytics: + build: + args: + NODE_ENV: 'development' + command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev'] + restart: unless-stopped + volumes: + - ./services/oodikone2-analytics/:/usr/src/app + + adminer: + image: adminer:4.6.2 + environment: + - ADMINER_DESIGN=pepa-linha + volumes: + - ./adminer-plugins:/var/www/html/plugins-enabled + ports: + - 5050:8080 + restart: always + container_name: adminer + + # zookeeper: + # image: wurstmeister/zookeeper + # ports: + # - "2181:2181" + # environment: + # - KAFKA_BROKER_ID=1 + # container_name: zookeeper + + # kafka: + # image: wurstmeister/kafka + # container_name: kafka + # depends_on: + # - zookeeper + # ports: + # - "9092:9092" + # environment: + # - HOSTNAME_COMMAND=route -n | awk '/UG[ \t]/{print $$2}' + # - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + # - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true + # - KAFKA_DELETE_TOPIC_ENABLE=true + # - KAFKA_CREATE_TOPICS=studentnumbers:1:1,priority_studentnumbers:1:1 + # - KAFKA_LOG_DIRS=/kafka/kafka-logs-1 + # volumes: + # - ./kafka/docker.sock:/var/run/docker.sock + # - ./kafka/logs:/kafka/kafka-logs-1 + + # mongo_db: + # image: mongo + # restart: unless-stopped + # volumes: + # - ./backups/mongo/oodilearn:/dump + # - ./mongodata:/data/db + # ports: + # - "27017:27017" + # container_name: mongo_db + + # oodilearn: + # build: + # context: services/oodilearn/server + # dockerfile: Dockerfile + # restart: unless-stopped + # volumes: + # - ./services/oodilearn/data:/data + # - ./services/oodilearn/server:/src + # - ./services/oodilearn/models:/models + # ports: + # - "5000:5000" + # environment: + # - ENV=development + # - MONGO_URI=mongodb://mongo_db:27017/oodilearn + # container_name: oodilearn diff --git a/docker/docker-compose.lateste2e.yml b/docker/docker-compose.lateste2e.yml new file mode 100644 index 0000000000..8afa33b1a7 --- /dev/null +++ b/docker/docker-compose.lateste2e.yml @@ -0,0 +1,62 @@ +version: '3' + +services: + user_db: + environment: + - POSTGRES_DB=user_db + + db: + environment: + - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test + + backend: + image: toska/oodikone2-backend:${TAG} + ports: + - "9229:9229" + environment: + FRONT_URL: http://localhost:1337 + ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres + + frontend: + image: toska/oodikone2-frontend:${TAG} + build: + args: + BASE_PATH: / + ports: + - "5000:5000" + + userservice: + image: toska/oodikone2-userservice:${TAG} + + usageservice: + image: toska/oodikone2-usageservice:${TAG} + + updater_api: + image: toska/updater_api:${TAG} + + updater_writer: + image: toska/updater_writer:${TAG} + + updater_scheduler: + image: toska/updater_scheduler:${TAG} + command: ["echo", "mluukkai on orjuuttaja"] + + analytics: + image: toska/oodikone2-analytics:${TAG} + + nginx: + image: nginx:latest + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf + - ./nginx/error.log:/etc/nginx/error_log.log + - ./nginx/log:/var/log/nginx/localhost.error_log + - ./nginx/cache/:/etc/nginx/cache + - /etc/letsencrypt/:/etc/letsencrypt/ + ports: + - 1337:80 + depends_on: + - frontend + - backend + - userservice + - analytics + container_name: nginx diff --git a/nginx.production.conf b/nginx.conf similarity index 100% rename from nginx.production.conf rename to nginx.conf diff --git a/nginx.staging.conf b/nginx.staging.conf deleted file mode 100644 index e780f2d73b..0000000000 --- a/nginx.staging.conf +++ /dev/null @@ -1,19 +0,0 @@ -events { worker_connections 1024; } - -http { - - rewrite_log on; - error_log /var/log/nginx/localhost.error_log notice; - - server { - listen 80; - - location / { - proxy_pass http://frontend:5000/; - } - - location /api/ { - proxy_pass http://backend:8080/; - } - } -} diff --git a/package.json b/package.json index 0ac6ceecb9..793d2400cb 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,17 @@ "cypress:record": "cypress run --config videoUploadOnPasses=false -P ./", "concurrently": "concurrently", "test": "./run_all_tests.sh", - "test_services": "./run_service_tests.sh" + "test_services": "./run_service_tests.sh", + "docker": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml", + "docker:build": "npm run docker -- build", + "docker:up": "npm run docker -- up -d", + "docker:down": "npm run docker -- down", + "docker:logs": "npm run docker -- logs -f", + "docker:logs:backend": "npm run docker -- logs -f backend", + "docker:restart": "npm run docker -- restart", + "docker:restart:backend": "npm run docker -- restart backend", + "docker:obliterate": "docker stop $(docker ps -aq) ; docker rmi -f $(docker images -a -q) ; docker system prune", + "start": "npm run docker:up && npm run docker:logs" }, "husky": { "hooks": { diff --git a/run.sh b/run.sh index 6a20f8addd..134f5fd2d1 100755 --- a/run.sh +++ b/run.sh @@ -33,7 +33,7 @@ do case $opt in "e2e setup") mopo - run_e2e_setup docker-compose.lateste2e.yml + run_e2e_setup ./docker/docker-compose.lateste2e.yml staging ;; "Anon setup") mopo diff --git a/scripts.sh b/scripts.sh index 8bca1a0a42..afa0d9352a 100644 --- a/scripts.sh +++ b/scripts.sh @@ -10,6 +10,10 @@ PSQL_REAL_DB_BACKUP="$BACKUP_DIR/latest-pg.sqz" KONE_REAL_DB_BACKUP="$BACKUP_DIR/latest-kone-pg.sqz" USER_REAL_DB_BACKUP="$BACKUP_DIR/latest-user-pg.sqz" +docker-compose-dev () { + docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml "$@" +} + retry () { for i in {1..60} do @@ -96,17 +100,17 @@ db_anon_setup_full () { } reset_real_db () { - docker-compose down - docker-compose up -d db user_db db_kone + docker-compose-dev down + docker-compose-dev up -d db user_db db_kone db_setup_full - docker-compose down + docker-compose-dev down } reset_db () { - docker-compose down - docker-compose up -d db user_db db_kone + docker-compose-dev down + docker-compose-dev up -d db user_db db_kone db_anon_setup_full - docker-compose down + docker-compose-dev down } install_cli_npm_packages () { @@ -127,12 +131,12 @@ run_full_setup () { echo "Getting anon backups from the private repository. " get_anon_oodikone echo "Building images" - docker-compose build + docker-compose-dev build echo "Setup oodikone db from dump." - docker-compose up -d db user_db db_kone + docker-compose-dev up -d db user_db db_kone db_setup_full db_anon_setup_full - docker-compose down + docker-compose-dev down show_instructions } @@ -144,11 +148,11 @@ run_anon_full_setup () { echo "Getting anon backups from the private repository. " get_anon_oodikone echo "Building images" - docker-compose build + docker-compose-dev build echo "Setup oodikone db from dump." - docker-compose up -d db user_db db_kone + docker-compose-dev up -d db user_db db_kone db_anon_setup_full - docker-compose down + docker-compose-dev down show_instructions } @@ -160,10 +164,10 @@ run_e2e_setup () { echo "Getting anon backups from the private repository. " get_anon_oodikone echo "Building images" - docker-compose -f $1 build + TAG=$2 docker-compose -f docker-compose.yml -f $1 build echo "Setup oodikone db from dump." - docker-compose -f $1 up -d db user_db db_kone + TAG=$2 docker-compose -f docker-compose.yml -f $1 up -d db user_db db_kone db_anon_setup_full echo "Starting services." - docker-compose -f $1 up -d + TAG=$2 docker-compose -f docker-compose.yml -f $1 up -d } diff --git a/services/backend/oodikone2-backend/src/conf-backend.js b/services/backend/oodikone2-backend/src/conf-backend.js index d18da0a3ab..c6bb7dd8ed 100644 --- a/services/backend/oodikone2-backend/src/conf-backend.js +++ b/services/backend/oodikone2-backend/src/conf-backend.js @@ -21,6 +21,7 @@ const CERT_PATH = process.env.CERT_PATH // production/staging only const KEY_PATH = process.env.KEY_PATH // production/staging only const OODILEARN_URL = process.env.OODILEARN_URL const USERSERVICE_URL = process.env.USERSERVICE_URL +const UPDATER_URL = process.env.UPDATER_URL const USAGESERVICE_URL = process.env.USAGESERVICE_URL const ANALYTICS_URL = process.env.ANALYTICS_URL const PORT = isTest ? 8079 : 8080 @@ -81,5 +82,6 @@ module.exports = { requiredGroup, OODI_SECRET, OODI_SECRET_HEADER_KEY, - isTest + isTest, + UPDATER_URL } \ No newline at end of file diff --git a/services/backend/oodikone2-backend/src/routes/population.js b/services/backend/oodikone2-backend/src/routes/population.js index 307adb2f13..978ae3f72b 100644 --- a/services/backend/oodikone2-backend/src/routes/population.js +++ b/services/backend/oodikone2-backend/src/routes/population.js @@ -1,8 +1,11 @@ const router = require('express').Router() const Population = require('../services/populations') const Filters = require('../services/filters') -const { updateStudents } = require('../services/doo_api_database_updater/database_updater') +const { updateStudents } = require('../services/updaterService') + +const Student = require('../services/students') const StudyrightService = require('../services/studyrights') +const UserService = require('../services/userService') // POST instead of GET because of too long params and "sensitive" data router.post('/v2/populationstatistics/courses', async (req, res) => { @@ -31,6 +34,37 @@ router.post('/v2/populationstatistics/courses', async (req, res) => { } }) +router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) => { + try { + if (!req.body.yearcode || !req.body.coursecode) { + res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' }) + return + } + const { coursecode, yearcode } = req.body + let studentnumberlist + const studentnumbers = await Student.findByCourseAndSemesters(coursecode, yearcode) + + const { roles, userId } = req.decodedToken + if (roles && roles.map(r => r.group_code).includes('admin')) { + studentnumberlist = studentnumbers + } else { + const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId) + const codes = unitsUserCanAccess.map(unit => unit.id) + studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes) + } + const result = await Population.bottlenecksOf(req.body, studentnumberlist) + + if (result.error) { + res.status(400).json(result) + return + } + + res.json(result) + } catch (e) { + res.status(400).json({ error: e }) + } +}) + router.get('/v3/populationstatistics', async (req, res) => { const { startYear, semesters, studyRights: studyRightsJSON } = req.query try { @@ -73,6 +107,45 @@ router.get('/v3/populationstatistics', async (req, res) => { } }) +router.get('/v3/populationstatisticsbycourse', async (req, res) => { + const { coursecode, yearcode } = req.query + const semesters = ['FALL', 'SPRING'] + let studentnumberlist + console.log(coursecode, yearcode) + const studentnumbers = await Student.findByCourseAndSemesters(coursecode, yearcode) + console.log(studentnumbers) + const { roles, userId } = req.decodedToken + if (roles && roles.map(r => r.group_code).includes('admin')) { + studentnumberlist = studentnumbers + } else { + const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId) + const codes = unitsUserCanAccess.map(unit => unit.id) + studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes) + } + try { + const result = await Population.optimizedStatisticsOf({ + startYear: 1900, + endYear: 2200, + studyRights: [], + semesters, + months: 1000 + }, studentnumberlist) + + if (result.error) { + console.log(result.error) + res.status(400).end() + return + } + + console.log(`request completed ${new Date()}`) + res.json(result) + } catch (e) { + console.log(e) + res.status(400).json({ error: e }) + } +}) + + router.get('/v2/populationstatistics/filters', async (req, res) => { let results = [] @@ -121,8 +194,8 @@ router.post('/updatedatabase', async (req, res) => { const studentnumbers = req.body console.log(studentnumbers) if (studentnumbers) { - await updateStudents(studentnumbers, 128) - res.status(200).json('Updated') + await updateStudents(studentnumbers) + res.status(200).json('Scheduled') } else { res.status(400).end() } diff --git a/services/backend/oodikone2-backend/src/services/course_yearly_stats_counter.js b/services/backend/oodikone2-backend/src/services/course_yearly_stats_counter.js index cec78b98c9..09407b99c7 100644 --- a/services/backend/oodikone2-backend/src/services/course_yearly_stats_counter.js +++ b/services/backend/oodikone2-backend/src/services/course_yearly_stats_counter.js @@ -127,12 +127,14 @@ class CourseYearlyStatsCounter { formatStudentStatistics(students) { const grades = {} const classes = {} + const studentnumbers = [] Object.entries(students).forEach(([studentnumber, stat]) => { const { grade, category } = stat grades[grade] = grades[grade] ? grades[grade].concat(studentnumber) : [studentnumber] classes[category] = classes[category] ? classes[category].concat(studentnumber) : [studentnumber] + studentnumbers.includes(studentnumber) ? null : studentnumbers.push(studentnumber) }) - return { grades, classes } + return { grades, classes, studentnumbers } } formatGroupStatistics() { diff --git a/services/backend/oodikone2-backend/src/services/courses.js b/services/backend/oodikone2-backend/src/services/courses.js index 5488b736dd..d6ed736640 100644 --- a/services/backend/oodikone2-backend/src/services/courses.js +++ b/services/backend/oodikone2-backend/src/services/courses.js @@ -196,6 +196,7 @@ const creditsOf = async (codes) => { fail: credits.filter(Credit.failed).length, pass: credits.filter(Credit.passed).length, students: credits.length, + studentnumbers: credits.map(cr => cr.student_studentnumber) } } @@ -238,7 +239,7 @@ const oneYearStats = (instances, year, separate, allInstancesUntilYear) => { return { studentsThatPassedThisYear, studentsThatFailedThisYear, allStudentsThatFailedEver, passedStudentsThatFailedBefore, passedStudentsOnFirstTry, failedStudentsThatFailedBefore, - failedStudentsOnFirstTry, gradeDistribution + failedStudentsOnFirstTry, gradeDistribution, studentnumbers: thisSemester.studentnumbers } } const stats = [] @@ -345,6 +346,7 @@ const yearlyStatsOf = async (code, year, separate, language) => { if (yearInst) { for (let year = start; year < end; year++) { stats = oneYearStats(yearInst, year, separate, allInstancesUntilYear) + console.logs(stats) if (stats.length > 0) { resultStats.push(...stats) resultProgrammes = getProgrammesFromStats(stats) diff --git a/services/backend/oodikone2-backend/src/services/populations.js b/services/backend/oodikone2-backend/src/services/populations.js index 5ff7a0515f..52676e6b2a 100644 --- a/services/backend/oodikone2-backend/src/services/populations.js +++ b/services/backend/oodikone2-backend/src/services/populations.js @@ -143,7 +143,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa creditsOfStudentLaakis : creditsOfStudentOther const students = await Student.findAll({ - attributes: ['firstnames', 'lastname', 'studentnumber', + attributes: ['firstnames', 'lastname', 'studentnumber', 'home_country_en', 'dateofuniversityenrollment', 'creditcount', 'matriculationexamination', 'abbreviatedname', 'email', 'updatedAt', 'gender_code', 'gender_fi', 'gender_sv', 'gender_en'], include: [ @@ -423,7 +423,7 @@ const formatStudentsForApi = async (students, startDate, endDate, { studyRights } } -const optimizedStatisticsOf = async (query) => { +const optimizedStatisticsOf = async (query, studentnumberlist) => { if (!query.semesters.map(semester => semester === 'FALL' || semester === 'SPRING').every(e => e === true)) { return { error: 'Semester should be either SPRING OR FALL' } } @@ -438,7 +438,8 @@ const optimizedStatisticsOf = async (query) => { studyRights, startDate, months, endDate, exchangeStudents, cancelledStudents, nondegreeStudents } = parseQueryParams(query) - const studentnumbers = + const studentnumbers = studentnumberlist ? + studentnumberlist : await studentnumbersWithAllStudyrightElements( studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents ) @@ -509,26 +510,39 @@ const parseCreditInfo = credit => ({ date: credit.attainment_date }) -const bottlenecksOf = async (query) => { - if (!query.semesters.map(semester => semester === 'FALL' || semester === 'SPRING').every(e => e === true)) { - return { error: 'Semester should be either SPRING OR FALL' } - } - if (query.studentStatuses && - !query.studentStatuses.map( - status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE' - ).every(e => e === true)) { - return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE' } - } - const { studyRights, startDate, endDate, months, exchangeStudents, cancelledStudents } = parseQueryParams(query) - - if (query.selectedStudents) { - const allStudents = - await studentnumbersWithAllStudyrightElements( - studyRights, startDate, endDate, exchangeStudents, cancelledStudents - ) - const disallowedRequest = - checkThatSelectedStudentsAreUnderRequestedStudyright(query.selectedStudents, allStudents) - if (disallowedRequest) return { error: 'Trying to request unauthorized students data' } +const bottlenecksOf = async (query, studentnumberlist) => { + let allstudents + let courses + if (!studentnumberlist) { + if (!query.semesters.map(semester => semester === 'FALL' || semester === 'SPRING').every(e => e === true)) { + return { error: 'Semester should be either SPRING OR FALL' } + } + if (query.studentStatuses && + !query.studentStatuses.map( + status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE' + ).every(e => e === true)) { + return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE' } + } + const { studyRights, startDate, endDate, months, exchangeStudents, cancelledStudents } = parseQueryParams(query) + + if (query.selectedStudents) { + const allStudents = + await studentnumbersWithAllStudyrightElements( + studyRights, startDate, endDate, exchangeStudents, cancelledStudents + ) + const disallowedRequest = + checkThatSelectedStudentsAreUnderRequestedStudyright(query.selectedStudents, allStudents) + if (disallowedRequest) return { error: 'Trying to request unauthorized students data' } + } + + const studentnumbers = query.selectedStudents || + await studentnumbersWithAllStudyrightElements(studyRights, startDate, endDate, exchangeStudents, cancelledStudents) + allstudents = studentnumbers.reduce((numbers, num) => ({ ...numbers, [num]: true }), {}) + courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months)) + + } else { + allstudents = studentnumberlist + courses = await findCourses(allstudents, new Date()) } const bottlenecks = { @@ -538,11 +552,6 @@ const bottlenecksOf = async (query) => { const codeduplicates = await getMainCodesMap() - const studentnumbers = query.selectedStudents || - await studentnumbersWithAllStudyrightElements(studyRights, startDate, endDate, exchangeStudents, cancelledStudents) - const allstudents = studentnumbers.reduce((numbers, num) => ({ ...numbers, [num]: true }), {}) - const courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months)) - const allcoursestatistics = await courses.reduce(async (coursestatistics, course) => { const stats = await coursestatistics let { code, name, disciplines, course_type } = course diff --git a/services/backend/oodikone2-backend/src/services/students.js b/services/backend/oodikone2-backend/src/services/students.js index f4b1a923e2..49f304504d 100644 --- a/services/backend/oodikone2-backend/src/services/students.js +++ b/services/backend/oodikone2-backend/src/services/students.js @@ -1,4 +1,5 @@ const Sequelize = require('sequelize') +const { sequelize } = require('../database/connection') const moment = require('moment') const { Student, Credit, Course, Studyright, StudyrightElement, ElementDetails } = require('../models') const Op = Sequelize.Op @@ -41,6 +42,24 @@ const byId = async (id) => Student.findByPk(id, { ] }) + +const findByCourseAndSemesters = async (coursecode, yearcode) => + sequelize.query(` + SELECT + studentnumber, credit.course_code, attainment_date + FROM student + INNER JOIN credit ON + student.studentnumber=credit.student_studentnumber + WHERE + course_code=:coursecode AND + attainment_date + BETWEEN + (select startdate FROM semesters where yearcode=:yearcode ORDER BY semestercode LIMIT 1) AND + (select enddate FROM semesters where yearcode=:yearcode ORDER BY semestercode DESC LIMIT 1); + `, + { replacements: { coursecode, yearcode }, type: sequelize.QueryTypes.SELECT }) + .map(st => st.studentnumber) + const byAbreviatedNameOrStudentNumber = (searchTerm) => { return Student.findAll({ where: { @@ -280,10 +299,36 @@ const bySearchTermAndElementsNew = async (searchterm, codes) => { return matches.map(formatStudent) } +const filterStudentnumbersByAccessrights = async (studentnumbers, codes) => { + const students = await Student.findAll({ + include: { + model: StudyrightElement, + required: true, + where: { + code: { + [Op.in]: codes + } + } + }, + where: { + studentnumber: { + [Op.in]: studentnumbers + } + } + }) + return students.map(student => student.studentnumber) +} + const NEW_SEARCH = true const bySearchTerm = NEW_SEARCH ? bySearchTermNew : bySearchTermOld const bySearchTermAndElements = NEW_SEARCH ? bySearchTermAndElementsNew : bySearchTermAndElementsOld module.exports = { - withId, bySearchTerm, createStudent, updateStudent, bySearchTermAndElements + withId, + bySearchTerm, + createStudent, + updateStudent, + bySearchTermAndElements, + filterStudentnumbersByAccessrights, + findByCourseAndSemesters } \ No newline at end of file diff --git a/services/backend/oodikone2-backend/src/services/updaterService.js b/services/backend/oodikone2-backend/src/services/updaterService.js new file mode 100644 index 0000000000..1fdeb76f28 --- /dev/null +++ b/services/backend/oodikone2-backend/src/services/updaterService.js @@ -0,0 +1,21 @@ +const axios = require('axios') +const { UPDATER_URL } = require('../conf-backend') + +const client = axios.create({ baseURL: UPDATER_URL }) + +const ping = async () => { + const url = '/ping' + const response = await axios.get(url) + return response.data +} + +const updateStudents = async (studentNumbers) => { + const response = await client.post('/update', studentNumbers) + return response.data +} + + +module.exports = { + ping, + updateStudents +} \ No newline at end of file diff --git a/services/backend/updater_writer/index.js b/services/backend/updater_writer/index.js index 5109a9a2e6..ef114cbfea 100644 --- a/services/backend/updater_writer/index.js +++ b/services/backend/updater_writer/index.js @@ -14,7 +14,7 @@ stan.on('connect', function () { const sub = stan.subscribe('UpdateWrite', 'updater.workers', opts) const attSub = stan.subscribe('UpdateAttainmentDates', opts) - const dumpSub = stan.subscribe('DumpDatabase') + const prioSub = stan.subscribe('PriorityWrite', 'updater.workers', opts) sub.on('message', async (msg) => { const data = JSON.parse(msg.getData()) @@ -30,14 +30,10 @@ stan.on('connect', function () { await updateAttainmentMeta() msg.ack() }) - dumpSub.on('message', async (_) => { - await dumpDatabase() - stan.publish('ScheduleAll', null, (err, guid) => { - if (err) { - console.log(err) - } else { - console.log('calling re-scheduling database') - } - }) + prioSub.on('message', async (msg) => { + const data = JSON.parse(msg.getData()) + await updateStudent(data, stan) + msg.ack() + stan.publish('status', `${data.studentInfo.studentnumber}:DONE`, (err) => { if (err) console.log(err) }) }) }) \ No newline at end of file diff --git a/services/backend/updater_writer/updater/update_attainment_dates.js b/services/backend/updater_writer/updater/update_attainment_dates.js index 70af4e03c6..a89969da22 100644 --- a/services/backend/updater_writer/updater/update_attainment_dates.js +++ b/services/backend/updater_writer/updater/update_attainment_dates.js @@ -2,20 +2,26 @@ const { sequelize } = require('../database/connection') const updateAttainmentDates = async () => { const { schema } = sequelize.options - await sequelize.query( - `UPDATE ${schema}.course + const transaction = await sequelize.transaction() + try { + await sequelize.query( + `UPDATE ${schema}.course SET max_attainment_date = cr.max FROM (select course_code, max(attainment_date) from ${schema}.credit group by course_code) cr WHERE course.code=cr.course_code`, - { type: sequelize.QueryTypes.UPDATE } - ) - await sequelize.query( - `UPDATE ${schema}.course + { type: sequelize.QueryTypes.UPDATE, transaction, lock: transaction.LOCK.UPDATE } + ) + await sequelize.query( + `UPDATE ${schema}.course SET min_attainment_date = cr.min FROM (select course_code, min(attainment_date) from ${schema}.credit group by course_code) cr WHERE course.code=cr.course_code`, - { type: sequelize.QueryTypes.UPDATE } - ) + { type: sequelize.QueryTypes.UPDATE, transaction, lock: transaction.LOCK.UPDATE } + ) + transaction.commit() + } catch (e) { + transaction.rollback() + } } module.exports = { diff --git a/services/oodikone2-frontend/src/components/CourseStatistics/index.jsx b/services/oodikone2-frontend/src/components/CourseStatistics/index.jsx index c1e80b04b7..a827a59540 100644 --- a/services/oodikone2-frontend/src/components/CourseStatistics/index.jsx +++ b/services/oodikone2-frontend/src/components/CourseStatistics/index.jsx @@ -69,7 +69,7 @@ class CourseStatistics extends Component { } render() { - const { statsIsEmpty } = this.props + const { statsIsEmpty, history } = this.props const panes = this.getPanes() return (
@@ -77,7 +77,7 @@ class CourseStatistics extends Component { Course Statistics - {statsIsEmpty ? : ( + {statsIsEmpty || history.location.search === '' ? : ( { @@ -515,7 +510,6 @@ export default connect( { setPopulationFilter, removePopulationFilterOfCourse, - getCourseStats, clearCourseStats } )(withRouter(PopulationCourseStats)) diff --git a/services/oodikone2-frontend/src/components/PopulationSearchForm/index.jsx b/services/oodikone2-frontend/src/components/PopulationSearchForm/index.jsx index 19bc8c7f55..88bf40508c 100644 --- a/services/oodikone2-frontend/src/components/PopulationSearchForm/index.jsx +++ b/services/oodikone2-frontend/src/components/PopulationSearchForm/index.jsx @@ -142,14 +142,15 @@ class PopulationSearchForm extends Component { this.pushQueryToUrl(query) } - fetchPopulation = (query, tag) => { + fetchPopulation = (query) => { + const { selectedTag } = this.state const queryCodes = Object.values(query.studyRights).filter(e => e != null) const uuid = uuidv4() const request = { ...query, studyRights: queryCodes, uuid } this.setState({ isLoading: true }) this.props.setLoading() Promise.all([ - this.props.getPopulationStatistics({ ...query, uuid, tag }), + this.props.getPopulationStatistics({ ...query, uuid, tag: selectedTag }), this.props.getPopulationCourses(request), this.props.getPopulationFilters(request), this.props.getMandatoryCourses(query.studyRights.programme) @@ -653,7 +654,7 @@ class PopulationSearchForm extends Component { /> @@ -668,7 +669,8 @@ class PopulationSearchForm extends Component { } render() { - if (!this.shouldRenderSearchForm()) { + const { location } = this.props + if (!this.shouldRenderSearchForm() && location.search !== '') { return null } const { translate } = this.props diff --git a/services/oodikone2-frontend/src/components/PopulationStatistics/index.jsx b/services/oodikone2-frontend/src/components/PopulationStatistics/index.jsx index e8546014ba..562a2a5992 100644 --- a/services/oodikone2-frontend/src/components/PopulationStatistics/index.jsx +++ b/services/oodikone2-frontend/src/components/PopulationStatistics/index.jsx @@ -1,7 +1,7 @@ import React, { PureComponent } from 'react' import { connect } from 'react-redux' import { getActiveLanguage, getTranslate } from 'react-localize-redux' -import { func, bool } from 'prop-types' +import { func, bool, shape } from 'prop-types' import { Header, Segment, Divider } from 'semantic-ui-react' import PopulationSearchForm from '../PopulationSearchForm' @@ -16,11 +16,12 @@ class PopulationStatistics extends PureComponent { static propTypes = { translate: func.isRequired, populationFound: bool.isRequired, - loading: bool.isRequired + loading: bool.isRequired, + location: shape({}).isRequired } renderPopulationSearch = () => { - const { translate, populationFound, loading } = this.props + const { translate, populationFound, loading, location } = this.props const { Main } = infoTooltips.PopulationStatistics const title = populationFound ? translate('populationStatistics.foundTitle') : @@ -33,20 +34,20 @@ class PopulationStatistics extends PureComponent { - + {location.search !== '' ? () : null} ) } render() { - const { translate } = this.props + const { translate, location } = this.props return (
{translate('populationStatistics.header')}
- { this.renderPopulationSearch() } - + {this.renderPopulationSearch()} + {location.search !== '' ? () : null}
) diff --git a/services/oodikone2-frontend/src/components/StudentDetails/index.jsx b/services/oodikone2-frontend/src/components/StudentDetails/index.jsx index 8dfeb2487a..9ac88d9b13 100644 --- a/services/oodikone2-frontend/src/components/StudentDetails/index.jsx +++ b/services/oodikone2-frontend/src/components/StudentDetails/index.jsx @@ -35,8 +35,7 @@ class StudentDetails extends Component { const queryObject = { ...rest, courseCodes: JSON.stringify(courseCodes) } const searchString = qs.stringify(queryObject) this.props.clearCourseStats() - history.push('/coursestatistics/') - history.push({ search: searchString }) + history.push(`/coursestatistics?${searchString}`) } renderCreditsGraph = () => { diff --git a/services/oodikone2-frontend/src/redux/populationCourses.js b/services/oodikone2-frontend/src/redux/populationCourses.js index 719c28b8a1..002cd844ea 100644 --- a/services/oodikone2-frontend/src/redux/populationCourses.js +++ b/services/oodikone2-frontend/src/redux/populationCourses.js @@ -20,6 +20,20 @@ export const getPopulationCourses = ({ return callController(route, prefix, body, 'post', query) } +export const getPopulationCoursesByCodeAndSemesters = ({ + coursecode, yearcode +}) => { + const route = '/v2/populationstatistics/coursesbycoursecode' + const prefix = 'GET_POPULATION_COURSES_' + const query = { + coursecode, yearcode + } + const body = { + coursecode, yearcode + } + return callController(route, prefix, body, 'post', query) +} + export const clearPopulationCourses = () => ({ type: 'CLEAR_POPULATIONS_COURSES' }) diff --git a/services/updater_api/index.js b/services/updater_api/index.js index a47dd8c864..ba68b9ea9f 100644 --- a/services/updater_api/index.js +++ b/services/updater_api/index.js @@ -9,45 +9,43 @@ opts.setAckWait(15 * 60 * 1000); // 15min opts.setDeliverAllAvailable() opts.setDurableName('durable') opts.setMaxInFlight(3) -stan.on('connect', function () { - const sub = stan.subscribe('UpdateApi', 'updater.workers', opts) - const prioSub = stan.subscribe('Priority_Update', 'updater.workers', opts) - - sub.on('message', async (msg) => { - let data = '' - const message = msg.getData() - if (!message) { - console.log('undefined message') - msg.ack() - return - } - if (message === 'meta') { - data = await getMeta() - stan.publish('UpdateWrite', JSON.stringify(data)) - msg.ack() - stan.publish('status', `${message}:FETCHED`, (err) => { if (err) console.log( 'STATUS PUBLISH FAILED', err) }) - } else { - // TODO: check that its a valid studentnumber and just ack it if its not - data = await getStudent(message) - try { - // TODO: check that data is properly structured(?) - stan.publish('UpdateWrite', JSON.stringify(data), (err, guid) => { - if (err) { - return err - } else { - msg.ack() - stan.publish('status', `${message}:FETCHED`, (err) => { if (err) console.log('STATUS PUBLISH FAIELD', err) }) - } - }) - } catch (e) { - console.log(e) - } +const handleMessage = async (priority) => async (msg) => { // :d + let data = '' + const message = msg.getData() + if (!message) { + console.log('undefined message') + msg.ack() + return + } + if (message === 'meta') { + data = await getMeta() + stan.publish('UpdateWrite', JSON.stringify(data)) + msg.ack() + stan.publish('status', `${message}:FETCHED`, (err) => { if (err) console.log( 'STATUS PUBLISH FAILED', err) }) + } else { + // TODO: check that its a valid studentnumber and just ack it if its not + data = await getStudent(message) + try { + // TODO: check that data is properly structured(?) + stan.publish(priority ? 'UpdateWrite' : 'PriorityWrite', JSON.stringify(data), (err, guid) => { + if (err) { + return err + } else { + msg.ack() + stan.publish('status', `${message}:FETCHED`, (err) => { if (err) console.log('STATUS PUBLISH FAIELD', err) }) + } + }) + } catch (e) { + console.log(e) } - }) + } +} +stan.on('connect', async () => { - prioSub.on('message', async (msg) => { - await updateStudent(msg.getData()) - msg.ack() - }); + const sub = stan.subscribe('UpdateApi', 'updater.workers', opts) + const prioSub = stan.subscribe('PriorityApi', 'updater.workers', opts) + + sub.on('message', await handleMessage(false)) + prioSub.on('message', await handleMessage(true)) }) \ No newline at end of file diff --git a/services/updater_scheduler/index.js b/services/updater_scheduler/index.js index 7cce2984f5..ab487cf366 100644 --- a/services/updater_scheduler/index.js +++ b/services/updater_scheduler/index.js @@ -6,6 +6,7 @@ const logger = require('./logger') const { updateStudentNumberList } = require('./src/student_list_updater') const { scheduleActiveStudents, scheduleAllStudentsAndMeta } = require('./src/schedule_students') const { getOldestTasks, getCurrentStatus } = require('./src/SchedulingStatistics') +require('./src/api') let updatedCount = 0 let scheduledCount = 0 diff --git a/services/updater_scheduler/package-lock.json b/services/updater_scheduler/package-lock.json index 6dacfba8aa..f367346e49 100644 --- a/services/updater_scheduler/package-lock.json +++ b/services/updater_scheduler/package-lock.json @@ -9,6 +9,15 @@ "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, + "accepts": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", + "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", + "requires": { + "mime-types": "~2.1.24", + "negotiator": "0.6.2" + } + }, "ansi-align": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-2.0.0.tgz", @@ -64,6 +73,11 @@ "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", "integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=" }, + "array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + }, "array-unique": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz", @@ -173,6 +187,33 @@ "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz", "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA==" }, + "body-parser": { + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", + "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", + "requires": { + "bytes": "3.1.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", + "on-finished": "~2.3.0", + "qs": "6.7.0", + "raw-body": "2.4.0", + "type-is": "~1.6.17" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + } + } + }, "boxen": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/boxen/-/boxen-1.3.0.tgz", @@ -228,6 +269,11 @@ "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.1.tgz", "integrity": "sha512-jCGVYLoYMHDkOsbwJZBCqwMHyH4c+wzgI9hG7Z6SZJRXWr+x58pdIbm2i9a/jFGCkRJqRUr8eoI7lDWa0hTkxg==" }, + "bytes": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", + "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" + }, "cache-base": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", @@ -396,6 +442,29 @@ "xdg-basedir": "^3.0.0" } }, + "content-disposition": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", + "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", + "requires": { + "safe-buffer": "5.1.2" + } + }, + "content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + }, + "cookie": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", + "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==" + }, + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, "copy-descriptor": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz", @@ -491,6 +560,16 @@ } } }, + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + }, + "destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + }, "diagnostics": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/diagnostics/-/diagnostics-1.1.1.tgz", @@ -514,6 +593,11 @@ "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=" }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, "enabled": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/enabled/-/enabled-1.0.2.tgz", @@ -522,16 +606,31 @@ "env-variable": "0.0.x" } }, + "encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + }, "env-variable": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.5.tgz", "integrity": "sha512-zoB603vQReOFvTg5xMl9I1P2PnHsHQQKTEowsKKD7nseUfJq6UWzK+4YtlWUO1nhiQUxe6XMkk+JleSZD1NZFA==" }, + "escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, + "etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + }, "execa": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", @@ -586,6 +685,53 @@ } } }, + "express": { + "version": "4.17.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", + "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", + "requires": { + "accepts": "~1.3.7", + "array-flatten": "1.1.1", + "body-parser": "1.19.0", + "content-disposition": "0.5.3", + "content-type": "~1.0.4", + "cookie": "0.4.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.1.2", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.5", + "qs": "6.7.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.1.2", + "send": "0.17.1", + "serve-static": "1.14.1", + "setprototypeof": "1.1.1", + "statuses": "~1.5.0", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + } + } + }, "extend-shallow": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz", @@ -695,6 +841,30 @@ } } }, + "finalhandler": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", + "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "requires": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", + "unpipe": "~1.0.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + } + } + }, "follow-redirects": { "version": "1.5.10", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", @@ -718,6 +888,11 @@ "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", "integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=" }, + "forwarded": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", + "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" + }, "fragment-cache": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", @@ -726,6 +901,11 @@ "map-cache": "^0.2.2" } }, + "fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + }, "fsevents": { "version": "1.2.9", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.9.tgz", @@ -1287,6 +1467,26 @@ } } }, + "http-errors": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", + "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.1", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.0" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, "ignore-by-default": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", @@ -1312,6 +1512,11 @@ "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" }, + "ipaddr.js": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.0.tgz", + "integrity": "sha512-M4Sjn6N/+O6/IXSJseKqHoFc+5FdGJ22sXqnjTpdZweHK64MzEPAyQZyEU3R/KRv2GLoa7nNtg/C2Ev6m7z+eA==" + }, "is-accessor-descriptor": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", @@ -1587,12 +1792,27 @@ "object-visit": "^1.0.0" } }, + "media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + }, "memory-pager": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", "optional": true }, + "merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + }, + "methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + }, "micromatch": { "version": "3.1.10", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz", @@ -1613,6 +1833,24 @@ "to-regex": "^3.0.2" } }, + "mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" + }, + "mime-db": { + "version": "1.40.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz", + "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==" + }, + "mime-types": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz", + "integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==", + "requires": { + "mime-db": "1.40.0" + } + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -1767,6 +2005,11 @@ "ts-nkeys": "^1.0.8" } }, + "negotiator": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", + "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" + }, "node-cron": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-2.0.3.tgz", @@ -1873,6 +2116,14 @@ "isobject": "^3.0.1" } }, + "on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "requires": { + "ee-first": "1.1.1" + } + }, "one-time": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/one-time/-/one-time-0.0.4.tgz", @@ -1899,6 +2150,11 @@ "semver": "^5.1.0" } }, + "parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" + }, "pascalcase": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz", @@ -1924,6 +2180,11 @@ "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=" }, + "path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, "pify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", @@ -1944,6 +2205,15 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" }, + "proxy-addr": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz", + "integrity": "sha512-t/7RxHXPH6cJtP0pRG6smSr9QJidhB+3kXu0KgXnbGYMgzEnUxRQ4/LDdfOwZEMyIh3/xHb8PX3t+lfL9z+YVQ==", + "requires": { + "forwarded": "~0.1.2", + "ipaddr.js": "1.9.0" + } + }, "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -1954,6 +2224,27 @@ "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.6.tgz", "integrity": "sha512-NdF35+QsqD7EgNEI5mkI/X+UwaxVEbQaz9f4IooEmMUv6ZPmlTQYGjBPJGgrlzNdjSvIy4MWMg6Q6vCgBO2K+w==" }, + "qs": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", + "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" + }, + "range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" + }, + "raw-body": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", + "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", + "requires": { + "bytes": "3.1.0", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, "rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", @@ -2072,6 +2363,11 @@ "ret": "~0.1.10" } }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, "saslprep": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", @@ -2094,6 +2390,59 @@ "semver": "^5.0.3" } }, + "send": { + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", + "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", + "requires": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "~1.7.2", + "mime": "1.6.0", + "ms": "2.1.1", + "on-finished": "~2.3.0", + "range-parser": "~1.2.1", + "statuses": "~1.5.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } + } + }, + "ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + } + } + }, + "serve-static": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", + "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", + "requires": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.17.1" + } + }, "set-value": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz", @@ -2115,6 +2464,11 @@ } } }, + "setprototypeof": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", + "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" + }, "shebang-command": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", @@ -2319,6 +2673,11 @@ } } }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" + }, "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", @@ -2418,6 +2777,11 @@ "repeat-string": "^1.6.1" } }, + "toidentifier": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", + "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" + }, "touch": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz", @@ -2444,6 +2808,15 @@ "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-1.0.1.tgz", "integrity": "sha512-kcoMoKTPYnoeS50tzoqjPY3Uv9axeuuFAZY9M/9zFnhoVvRfxz9K29IMPD7jGmt2c8SW7i3gT9WqDl2+nV7p4A==" }, + "type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + } + }, "tz-offset": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/tz-offset/-/tz-offset-0.0.1.tgz", @@ -2507,6 +2880,11 @@ "crypto-random-string": "^1.0.0" } }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + }, "unset-value": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz", @@ -2593,6 +2971,16 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, + "utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + }, + "vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", diff --git a/services/updater_scheduler/package.json b/services/updater_scheduler/package.json index 66560b9594..9246a33587 100644 --- a/services/updater_scheduler/package.json +++ b/services/updater_scheduler/package.json @@ -16,6 +16,7 @@ "license": "ISC", "dependencies": { "axios": "^0.19.0", + "express": "^4.17.1", "moment": "^2.24.0", "mongoose": "^5.5.8", "node-cron": "^2.0.3", diff --git a/services/updater_scheduler/src/api.js b/services/updater_scheduler/src/api.js new file mode 100644 index 0000000000..b2fe7272e9 --- /dev/null +++ b/services/updater_scheduler/src/api.js @@ -0,0 +1,16 @@ +const express = require('express') +const app = express() +const port = process.env.PORT +const bodyParser = require('body-parser') +const { scheduleStudentsByArray } = require('./schedule_students') +app.use(bodyParser.json()) + + +app.get('/ping', (req, res) => res.json({ message: 'pong '})) + +app.post('/update', async (req, res) => { + const msg = await scheduleStudentsByArray(req.body) + res.json({ message: msg }) +}) +app.listen(port, () => console.log(`listening on port ${port}!`)) +module.exports = { app } \ No newline at end of file diff --git a/services/updater_scheduler/src/schedule_students.js b/services/updater_scheduler/src/schedule_students.js index 90e22f51f8..7e3f889cfd 100644 --- a/services/updater_scheduler/src/schedule_students.js +++ b/services/updater_scheduler/src/schedule_students.js @@ -2,13 +2,13 @@ const { stan } = require('./nats_connection') const Schedule = require('../models') const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)) -const publish = async (tasks) => { - let rampup = 300 +const publish = async (tasks, priority = false) => { + let rampup = 300 for (const task of tasks) { if (rampup > 1) { rampup = rampup - 1 } - stan.publish('UpdateApi', task.task, (err, guid) => { + stan.publish(priority ? 'UpdateApi' : 'PriorityApi', task.task, (err, guid) => { console.log(guid) if (err) { console.log('publish failed', err) @@ -24,8 +24,8 @@ const publish = async (tasks) => { } const shuffleArray = (array) => { for (let i = array.length - 1; i > 0; i--) { - const j = Math.floor(Math.random() * (i + 1)); - [array[i], array[j]] = [array[j], array[i]]; + const j = Math.floor(Math.random() * (i + 1)); + [array[i], array[j]] = [array[j], array[i]]; } return array } @@ -36,8 +36,18 @@ const scheduleActiveStudents = async () => { } const scheduleAllStudentsAndMeta = async () => { - const tasks = [{ task:'meta', type: 'other', active: 'false' }, ...shuffleArray(await Schedule.find({ type: 'student' }))] + const tasks = [{ task: 'meta', type: 'other', active: 'false' }, ...shuffleArray(await Schedule.find({ type: 'student' }))] console.log(tasks.length, 'tasks to schedule') publish(tasks) } -module.exports = { scheduleActiveStudents, scheduleAllStudentsAndMeta } \ No newline at end of file + +const scheduleStudentsByArray = async (studentNumbers) => { + try { + const tasks = await Schedule.find({ type: 'student', task: { $in: studentNumbers } }) + publish(tasks, true) + } catch (e) { + return e + } + return 'scheduled' +} +module.exports = { scheduleActiveStudents, scheduleAllStudentsAndMeta, scheduleStudentsByArray } \ No newline at end of file diff --git a/services/updater_scheduler/src/student_list_updater.js b/services/updater_scheduler/src/student_list_updater.js index 564a63d37c..907c7f0f5f 100644 --- a/services/updater_scheduler/src/student_list_updater.js +++ b/services/updater_scheduler/src/student_list_updater.js @@ -7,6 +7,7 @@ const moment = require('moment') async function updateStudentNumberList() { const { KEY_PATH, CERT_PATH, TOKEN, NODE_ENV, OODI_ADDR, STUDENT_NUMBERS } = process.env + console.log(TOKEN) const agent = KEY_PATH && CERT_PATH ? new https.Agent({ cert: fs.readFileSync(CERT_PATH, 'utf8'), @@ -22,25 +23,23 @@ async function updateStudentNumberList() { }) instance.defaults.httpsAgent = agent - console.log(NODE_ENV) if (NODE_ENV === 'development') { - axios.defaults.params = { + instance.defaults.params['token'] = { token: TOKEN } - } - + const getStudentNumberChecksum = studentNumber => { const studentNumberString = String(studentNumber) let checksumNumbers = [7, 3, 1] let checksum = 0 - + for (let i = 0; i < studentNumberString.length; i++) { // go from end t start let currentNumber = studentNumberString[studentNumberString.length - (i + 1)] checksum += currentNumber * (checksumNumbers[i % checksumNumbers.length]) } - + return (10 - (checksum % 10)) % 10 } const requestStudent = async (studentNumber) => { From 54308c7dce9ab38a5aea6bed0df8efdfa2461b68 Mon Sep 17 00:00:00 2001 From: Warro Date: Tue, 9 Jul 2019 10:22:56 +0300 Subject: [PATCH 10/15] Added stages to travis --- .travis.yml | 80 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 34 deletions(-) diff --git a/.travis.yml b/.travis.yml index 42c03c17dc..4c95479506 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,50 +18,62 @@ before_script: - docker-compose --version - export AUTHOR_NAME="$(git log -1 ${TRAVIS_COMMIT} --pretty="%aN")" +jobs: + include: + - stage: login to docker + script: + - docker login -u $DOCKER_USER -p $DOCKER_PASS + - stage: build staging + if: NOT tag IS present + env: + - IMAGE=STAGING + - TAG=staging + script: + - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG} + - stage: build latest + if: tag IS present + env: + - IMAGE=PRODUCTION + - TAG=latest + script: + - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG} + - stage: run tests + script: + - docker ps + - cat nginx/log + - TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f & + - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done + - time npm run test_services + - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record + - docker ps -a + - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi + - stage: deploy to staging + if: NOT tag IS present + deploy: + provider: script + skip_cleanup: true + script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}" + on: + branch: master + - stage: deploy to production + if: tag IS present + deploy: + provider: script + skip_cleanup: true + script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}" + on: + tags: true stages: - name: login to docker - script: - - docker login -u $DOCKER_USER -p $DOCKER_PASS - name: build staging if: NOT tag IS present - env: - - IMAGE=STAGING - - TAG=staging - script: - - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG} - name: build latest if: tag IS present - env: - - IMAGE=PRODUCTION - - TAG=latest - script: - - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG} - name: run tests - script: - - docker ps - - cat nginx/log - - TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f & - - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done - - time npm run test_services - - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record - - docker ps -a - - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi - name: deploy to staging - if: NOT tag IS present - deploy: - provider: script - skip_cleanup: true - script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}" - on: - branch: master + if: NOT tag IS present AND branch = master - name: deploy to production if: tag IS present - deploy: - provider: script - skip_cleanup: true - script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}" - on: - tags: true after_failure: - "curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${IMAGE} buildi paskana. STOP AND FIX tai tulee kenkää! *${AUTHOR_NAME}* ${TRAVIS_BUILD_WEB_URL}\\\"}\" ${SLACKBOT_URL}" From 97b48da7cf35adec0f3b253f6edefb075f060132 Mon Sep 17 00:00:00 2001 From: Warro Date: Tue, 9 Jul 2019 10:41:13 +0300 Subject: [PATCH 11/15] Combined stages so that data can be shared in different deploy stages --- .travis.yml | 57 +++++++++++++++++++++++------------------------------ 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4c95479506..4685b2f981 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,25 +20,13 @@ before_script: jobs: include: - - stage: login to docker - script: - - docker login -u $DOCKER_USER -p $DOCKER_PASS - - stage: build staging - if: NOT tag IS present + - stage: build & test & push & deploy staging env: - IMAGE=STAGING - TAG=staging script: + - docker login -u $DOCKER_USER -p $DOCKER_PASS - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG} - - stage: build latest - if: tag IS present - env: - - IMAGE=PRODUCTION - - TAG=latest - script: - - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG} - - stage: run tests - script: - docker ps - cat nginx/log - TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f & @@ -47,32 +35,37 @@ jobs: - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record - docker ps -a - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi - - stage: deploy to staging - if: NOT tag IS present deploy: provider: script skip_cleanup: true script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}" on: branch: master - - stage: deploy to production - if: tag IS present - deploy: - provider: script - skip_cleanup: true - script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}" - on: - tags: true + - stage: build & test & push & deploy production + env: + - IMAGE=PRODUCTION + - TAG=latest + script: + - docker login -u $DOCKER_USER -p $DOCKER_PASS + - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG} + - docker ps + - cat nginx/log + - TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f & + - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done + - time npm run test_services + - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record + - docker ps -a + - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi + deploy: + provider: script + skip_cleanup: true + script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}" + on: + tags: true stages: - - name: login to docker - - name: build staging + - name: build & test & push & deploy staging if: NOT tag IS present - - name: build latest - if: tag IS present - - name: run tests - - name: deploy to staging - if: NOT tag IS present AND branch = master - - name: deploy to production + - name: build & test & push & deploy latest if: tag IS present after_failure: From aa6fdd34e935cc0592146dd269ae36903443e7a6 Mon Sep 17 00:00:00 2001 From: Warro Date: Tue, 9 Jul 2019 10:41:47 +0300 Subject: [PATCH 12/15] fixed typo --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4685b2f981..b7288facf8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,7 +65,7 @@ jobs: stages: - name: build & test & push & deploy staging if: NOT tag IS present - - name: build & test & push & deploy latest + - name: build & test & push & deploy production if: tag IS present after_failure: From 0e2134874fe9b865e1d58c0b6c0fb7b94705b65d Mon Sep 17 00:00:00 2001 From: Warro Date: Tue, 9 Jul 2019 10:45:01 +0300 Subject: [PATCH 13/15] changed prod to latest --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b7288facf8..d102d9de42 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,7 +41,7 @@ jobs: script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}" on: branch: master - - stage: build & test & push & deploy production + - stage: build & test & push & deploy latest env: - IMAGE=PRODUCTION - TAG=latest @@ -65,7 +65,7 @@ jobs: stages: - name: build & test & push & deploy staging if: NOT tag IS present - - name: build & test & push & deploy production + - name: build & test & push & deploy latest if: tag IS present after_failure: From 80740e53e7feeebea6e61189e17a4348c0b1b8b1 Mon Sep 17 00:00:00 2001 From: Warro Date: Tue, 9 Jul 2019 10:48:24 +0300 Subject: [PATCH 14/15] fixed syntax error in .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d102d9de42..c4d5ae1f9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,7 +41,7 @@ jobs: script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}" on: branch: master - - stage: build & test & push & deploy latest + - stage: build & test & push & deploy latest env: - IMAGE=PRODUCTION - TAG=latest From 464733fcc0db0896a322523de97eb4decd7c2623 Mon Sep 17 00:00:00 2001 From: Warro Date: Tue, 9 Jul 2019 10:51:10 +0300 Subject: [PATCH 15/15] fixed another syntax error in .travis.yml --- .travis.yml | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.travis.yml b/.travis.yml index c4d5ae1f9d..0a464bcd1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,27 +41,27 @@ jobs: script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}" on: branch: master - - stage: build & test & push & deploy latest - env: - - IMAGE=PRODUCTION - - TAG=latest - script: - - docker login -u $DOCKER_USER -p $DOCKER_PASS - - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG} - - docker ps - - cat nginx/log - - TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f & - - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done - - time npm run test_services - - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record - - docker ps -a - - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi - deploy: - provider: script - skip_cleanup: true - script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}" - on: - tags: true + - stage: build & test & push & deploy latest + env: + - IMAGE=PRODUCTION + - TAG=latest + script: + - docker login -u $DOCKER_USER -p $DOCKER_PASS + - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG} + - docker ps + - cat nginx/log + - TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f & + - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done + - time npm run test_services + - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record + - docker ps -a + - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi + deploy: + provider: script + skip_cleanup: true + script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}" + on: + tags: true stages: - name: build & test & push & deploy staging if: NOT tag IS present