From 362166e55817c10f49e4c8d17a2e287965900ebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A4kinen=20Sasu=20S?= Date: Fri, 24 May 2019 10:19:41 +0300 Subject: [PATCH 1/2] winston to api --- .gitignore | 3 +- services/updater_api/package-lock.json | 183 +++++++++++++++++++++++++ services/updater_api/package.json | 4 +- 3 files changed, 188 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index d346f829bc..78a0e5b642 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,5 @@ services/updater_scheduler/all_student_numbers.txt services/updater_scheduler/debug.log services/updater_scheduler/active_student_numbers.txt services/oodikone2-backend/debug.log -datastore \ No newline at end of file +datastore +services/updater_api/debug.log \ No newline at end of file diff --git a/services/updater_api/package-lock.json b/services/updater_api/package-lock.json index 2eb00935ff..ba4e18a848 100644 --- a/services/updater_api/package-lock.json +++ b/services/updater_api/package-lock.json @@ -74,6 +74,14 @@ "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c=" }, + "async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.2.tgz", + "integrity": "sha512-H1qVYh1MYhEEFLsP97cVKqCGo7KfCyTt6uEWqsTBr9SO84oK9Uwbyd/yCW+6rKJLHksBNUVWZDAjfS+Ccx0Bbg==", + "requires": { + "lodash": "^4.17.11" + } + }, "async-each": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.3.tgz", @@ -298,6 +306,15 @@ "object-visit": "^1.0.0" } }, + "color": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz", + "integrity": "sha512-jCpd5+s0s0t7p3pHQKpnJ0TpQKKdleP71LWcA0aqiljpiuAkOSUFN/dyH8ZwF0hRmFlrIuRhufds1QyEP9EB+w==", + "requires": { + "color-convert": "^1.9.1", + "color-string": "^1.5.2" + } + }, "color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -311,6 +328,34 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" }, + "color-string": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.3.tgz", + "integrity": "sha512-dC2C5qeWoYkxki5UAXapdjqO672AM4vZuPGRQfO8b5HKuKGBbKWpITyDYN7TOFKvRW7kOgAn3746clDBMDJyQw==", + "requires": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "colornames": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/colornames/-/colornames-1.1.1.tgz", + "integrity": "sha1-+IiQMGhcfE/54qVZ9Qd+t2qBb5Y=" + }, + "colors": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.3.3.tgz", + "integrity": "sha512-mmGt/1pZqYRjMxB1axhTo16/snVZ5krrKkcmMeVKxzECMMXoCgnvTPp10QgHfcbQZw8Dq2jMNG6je4JlWU0gWg==" + }, + "colorspace": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.2.tgz", + "integrity": "sha512-vt+OoIP2d76xLhjwbBaucYlNSpPsrJWPlBTtwCpQKIu6/CSMutyzX93O/Do0qzpH3YoHEes8YEFXyZ797rEhzQ==", + "requires": { + "color": "3.0.x", + "text-hex": "1.0.x" + } + }, "component-emitter": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", @@ -429,6 +474,16 @@ } } }, + "diagnostics": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/diagnostics/-/diagnostics-1.1.1.tgz", + "integrity": "sha512-8wn1PmdunLJ9Tqbx+Fx/ZEuHfJf4NKSN2ZBj7SJC/OWRWha843+WsTjqMe1B5E3p28jqBlp+mJ2fPVxPyNgYKQ==", + "requires": { + "colorspace": "1.1.x", + "enabled": "1.0.x", + "kuler": "1.0.x" + } + }, "dot-prop": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.0.tgz", @@ -447,6 +502,19 @@ "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=" }, + "enabled": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-1.0.2.tgz", + "integrity": "sha1-ll9lE9LC0cX0ZStkouM5ZGf8L5M=", + "requires": { + "env-variable": "0.0.x" + } + }, + "env-variable": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.5.tgz", + "integrity": "sha512-zoB603vQReOFvTg5xMl9I1P2PnHsHQQKTEowsKKD7nseUfJq6UWzK+4YtlWUO1nhiQUxe6XMkk+JleSZD1NZFA==" + }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -584,6 +652,16 @@ } } }, + "fast-safe-stringify": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.6.tgz", + "integrity": "sha512-q8BZ89jjc+mz08rSxROs8VsrBBcn1SIw1kq9NjolL509tkABRk9io01RAjSaEv1Xb2uFLt8VtRiZbGp5H8iDtg==" + }, + "fecha": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-2.3.3.tgz", + "integrity": "sha512-lUGBnIamTAwk4znq5BcqsDaxSmZ9nDVJaij6NvRt/Tg4R69gERA+otPKbS86ROw9nxVMw2/mp1fnaiWqbs6Sdg==" + }, "fill-range": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", @@ -1230,6 +1308,11 @@ } } }, + "is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + }, "is-binary-path": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-1.0.1.tgz", @@ -1402,6 +1485,14 @@ "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==" }, + "kuler": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-1.0.1.tgz", + "integrity": "sha512-J9nVUucG1p/skKul6DU3PUZrhs0LPulNaeUOox0IyXDi8S4CztTHs1gQphhuZmzXG7VOQSf6NJfKuzteQLv9gQ==", + "requires": { + "colornames": "^1.1.1" + } + }, "latest-version": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-3.1.0.tgz", @@ -1415,6 +1506,25 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" }, + "logform": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.1.2.tgz", + "integrity": "sha512-+lZh4OpERDBLqjiwDLpAWNQu6KMjnlXH2ByZwCuSqVPJletw0kTWJf5CgSNAUKn1KUkv3m2cUz/LK8zyEy7wzQ==", + "requires": { + "colors": "^1.2.1", + "fast-safe-stringify": "^2.0.4", + "fecha": "^2.3.3", + "ms": "^2.1.1", + "triple-beam": "^1.3.0" + }, + "dependencies": { + "ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + } + } + }, "lowercase-keys": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", @@ -1642,6 +1752,11 @@ "isobject": "^3.0.1" } }, + "one-time": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-0.0.4.tgz", + "integrity": "sha1-+M33eISCb+Tf+T46nMN7HkSAdC4=" + }, "p-finally": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", @@ -1864,6 +1979,14 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" }, + "simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=", + "requires": { + "is-arrayish": "^0.3.1" + } + }, "snapdragon": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", @@ -1999,6 +2122,11 @@ "extend-shallow": "^3.0.0" } }, + "stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" + }, "static-extend": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz", @@ -2069,6 +2197,11 @@ "execa": "^0.7.0" } }, + "text-hex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" + }, "timed-out": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", @@ -2120,6 +2253,11 @@ "nopt": "~1.0.10" } }, + "triple-beam": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", + "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" + }, "ts-nkeys": { "version": "1.0.12", "resolved": "https://registry.npmjs.org/ts-nkeys/-/ts-nkeys-1.0.12.tgz", @@ -2293,6 +2431,51 @@ "string-width": "^2.1.1" } }, + "winston": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.2.1.tgz", + "integrity": "sha512-zU6vgnS9dAWCEKg/QYigd6cgMVVNwyTzKs81XZtTFuRwJOcDdBg7AU0mXVyNbs7O5RH2zdv+BdNZUlx7mXPuOw==", + "requires": { + "async": "^2.6.1", + "diagnostics": "^1.1.1", + "is-stream": "^1.1.0", + "logform": "^2.1.1", + "one-time": "0.0.4", + "readable-stream": "^3.1.1", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.3.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.3.0.tgz", + "integrity": "sha512-EsI+s3k3XsW+fU8fQACLN59ky34AZ14LoeVZpYwmZvldCFo0r0gnelwF2TcMjLor/BTL5aDJVBMkss0dthToPw==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, + "winston-log2gelf": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/winston-log2gelf/-/winston-log2gelf-2.0.4.tgz", + "integrity": "sha512-HfZWFPD0MCeBgceVKHueRrVm4xOuJ+u52BcgctKFHHORj7feWszqvAezEwY0E9wM6QRZ2uQoWXRIRV/m2RNGng==", + "requires": { + "winston-transport": "^4.2.0" + } + }, + "winston-transport": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.3.0.tgz", + "integrity": "sha512-B2wPuwUi3vhzn/51Uukcao4dIduEiPOcOt9HJ3QeaXgkJ5Z7UwpBzxS4ZGNHtrxrUvTwemsQiSys0ihOf8Mp1A==", + "requires": { + "readable-stream": "^2.3.6", + "triple-beam": "^1.2.0" + } + }, "write-file-atomic": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.2.tgz", diff --git a/services/updater_api/package.json b/services/updater_api/package.json index c359e87d5b..2d5abd9fb2 100644 --- a/services/updater_api/package.json +++ b/services/updater_api/package.json @@ -16,6 +16,8 @@ "lodash": "^4.17.11", "moment": "^2.24.0", "node-nats-streaming": "^0.2.2", - "nodemon": "^1.18.10" + "nodemon": "^1.18.10", + "winston": "^3.2.1", + "winston-log2gelf": "^2.0.4" } } From bcd20539c1e6be5ad5bbd80dd80e5d84a6ffb0b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A4kinen=20Sasu=20S?= Date: Fri, 24 May 2019 12:00:50 +0300 Subject: [PATCH 2/2] modify logging --- .../backend/updater_writer/updater/database_updater.js | 7 ++----- .../updater_api/doo_api_database_updater/oodi_interface.js | 4 ++-- services/updater_api/logger.js | 2 -- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/services/backend/updater_writer/updater/database_updater.js b/services/backend/updater_writer/updater/database_updater.js index bd5cc94b72..f3bf12300a 100644 --- a/services/backend/updater_writer/updater/database_updater.js +++ b/services/backend/updater_writer/updater/database_updater.js @@ -38,7 +38,7 @@ const updateAttainments = (studyAttainments, transaction) => studyAttainments.ma creditTeachers.length > 0 && Promise.all(creditTeachers.map(cT => CreditTeacher.upsert(cT, { transaction }))) ]) } catch (e) { - console.log(e) + console.log('attainment update failed', e) } }) @@ -61,12 +61,9 @@ const updateStudent = async (student) => { await deleteStudentStudyrights(studentInfo.studentnumber, transaction) // this needs to be done because Oodi just deletes deprecated studyrights from students ( big yikes ) await Student.upsert(studentInfo, { transaction }) - try { await Promise.all(semesterEnrollments.map(SE => SemesterEnrollment.upsert(SE, { transaction }))) - } catch(e) { - console.log(e) - } + if (studyAttainments) await Promise.all(updateAttainments(studyAttainments, transaction)) if (studyRights) await Promise.all(updateStudyRights(studyRights, transaction)) diff --git a/services/updater_api/doo_api_database_updater/oodi_interface.js b/services/updater_api/doo_api_database_updater/oodi_interface.js index a52e4d966e..bd253119ea 100644 --- a/services/updater_api/doo_api_database_updater/oodi_interface.js +++ b/services/updater_api/doo_api_database_updater/oodi_interface.js @@ -36,7 +36,7 @@ const attemptGetFor = async (url, attempts = 5) => { attempt += 1 try { response = await getUrl(url) - logger.info('requested url', { url, status: response.status }) + logger.info('requested url', { url, success: response.status === 200 }) return response } catch (error) { if (attempt === attempts) { @@ -63,7 +63,7 @@ console.log(process.env.NODE_ENV) const getStudent = async studentNumber => { const url = `${base_url}/students/${studentNumber}/info` try { - const response = await getUrl(url) + const response = await attemptGetFor(url) const data = response.data.data return data } catch (e) { diff --git a/services/updater_api/logger.js b/services/updater_api/logger.js index caa3b72faf..09394083e5 100644 --- a/services/updater_api/logger.js +++ b/services/updater_api/logger.js @@ -16,8 +16,6 @@ if (process.env.NODE_ENV !== 'test') { transports.push(new winston.transports.File({ filename: 'debug.log' })) } -transports.push(new winston.transports.Console({ level: 'debug' })) - const logger = winston.createLogger({ transports })