Merge pull request #1083 from UniversityOfHelsinkiCS/trunk

Refactored Travis to only do one build per push and other mayhem

.travis.yml

@@ -20,48 +20,53 @@ before_script:
 
 jobs:
   include:
-    - stage: build & test & push & deploy
-      if: NOT tag IS present
+    - stage: build & test & push & deploy staging
       env:
         - IMAGE=STAGING
+        - TAG=staging
       script:
         - docker login -u $DOCKER_USER -p $DOCKER_PASS
-        - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml
+        - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG}
         - docker ps
         - cat nginx/log
-        - docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f &
+        - TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f &
         - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done
         - time npm run test_services
         - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record
         - docker ps -a
         - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi
-      deploy: # deploy is always skipped for PR
+      deploy:
         provider: script
         skip_cleanup: true
-        script: "docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}"
+        script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}"
         on:
           branch: master
-    - # in parallel
-      if: type = pull_request OR NOT branch = master
+    - stage: build & test & push & deploy latest
       env:
         - IMAGE=PRODUCTION
+        - TAG=latest
       script:
         - docker login -u $DOCKER_USER -p $DOCKER_PASS
-        - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.production.yml
+        - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG}
         - docker ps
         - cat nginx/log
-        - docker-compose -f docker-compose.yml -f ./docker/ocker-compose.lateste2e.production.yml logs -f &
+        - TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f &
         - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done
         - time npm run test_services
         - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record
         - docker ps -a
         - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi
-      deploy: # deploy is always skipped for PR
+      deploy:
         provider: script
         skip_cleanup: true
-        script: "docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.production.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}"
+        script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}"
         on:
           tags: true
+stages:
+  - name: build & test & push & deploy staging
+    if: NOT tag IS present
+  - name: build & test & push & deploy latest
+    if: tag IS present
 
 after_failure:
   - "curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${IMAGE} buildi paskana. STOP AND FIX tai tulee kenkää! *${AUTHOR_NAME}* ${TRAVIS_BUILD_WEB_URL}\\\"}\" ${SLACKBOT_URL}"

docker-compose.yml

@@ -98,6 +98,8 @@ services:
       USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       ANALYTICS_URL: http://analytics:4568
       NATS_URI: nats://nats:4222
+      UPDATER_URL: http://updater_scheduler:3678
+
     container_name: backend
     depends_on:
       - db
@@ -190,6 +192,10 @@ services:
       - ACTIVE_STUDENTS=./active_student_numbers.txt
       - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
       - TOKEN=toskab1gs8cret5
+      - LOG_HOST=toska.cs.helsinki.fi
+      - LOG_PORT=9501
+      - LOG_HOSTNAME=updater_scheduler
+      - PORT=3678
     container_name: updater_scheduler
 
   analytics:

docker/docker-compose.lateste2e.yml

@@ -10,44 +10,44 @@ services:
       - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test
 
   backend:
-    image: toska/oodikone2-backend:staging
+    image: toska/oodikone2-backend:${TAG}
     ports:
       - "9229:9229"
     environment:
       FRONT_URL: http://localhost:1337
       ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
 
   frontend:
-    image: toska/oodikone2-frontend:staging
+    image: toska/oodikone2-frontend:${TAG}
     build:
       args:
         BASE_PATH: /
     ports:
       - "5000:5000"
 
   userservice:
-    image: toska/oodikone2-userservice:staging
+    image: toska/oodikone2-userservice:${TAG}
 
   usageservice:
-    image: toska/oodikone2-usageservice:staging
+    image: toska/oodikone2-usageservice:${TAG}
 
   updater_api:
-    image: toska/updater_api:staging
+    image: toska/updater_api:${TAG}
 
   updater_writer:
-    image: toska/updater_writer:staging
+    image: toska/updater_writer:${TAG}
 
   updater_scheduler:
-    image: toska/updater_scheduler:staging
+    image: toska/updater_scheduler:${TAG}
     command: ["echo", "mluukkai on orjuuttaja"]
 
   analytics:
-    image: toska/oodikone2-analytics:staging
+    image: toska/oodikone2-analytics:${TAG}
 
   nginx:
     image: nginx:latest
     volumes:
-      - ./nginx.staging.conf:/etc/nginx/nginx.conf
+      - ./nginx.conf:/etc/nginx/nginx.conf
       - ./nginx/error.log:/etc/nginx/error_log.log
       - ./nginx/log:/var/log/nginx/localhost.error_log
       - ./nginx/cache/:/etc/nginx/cache

package.json

@@ -18,7 +18,7 @@
     "docker:logs:backend": "npm run docker -- logs -f backend",
     "docker:restart": "npm run docker -- restart",
     "docker:restart:backend": "npm run docker -- restart backend",
-    "docker:obliterate": "docker stop $(docker ps -aq) && docker system prune",
+    "docker:obliterate": "docker stop $(docker ps -aq) ; docker rmi -f $(docker images -a -q) ; docker system prune",
     "start": "npm run docker:up && npm run docker:logs"
   },
   "husky": {

run.sh

@@ -33,7 +33,7 @@ do
     case $opt in
         "e2e setup")
             mopo
-            run_e2e_setup ./docker/docker-compose.lateste2e.yml
+            run_e2e_setup ./docker/docker-compose.lateste2e.yml staging
             ;;
         "Anon setup")
             mopo

scripts.sh

@@ -164,10 +164,10 @@ run_e2e_setup () {
     echo "Getting anon backups from the private repository. "
     get_anon_oodikone
     echo "Building images"
-    docker-compose -f docker-compose.yml -f $1 build
+    TAG=$2 docker-compose -f docker-compose.yml -f $1 build
     echo "Setup oodikone db from dump."
-    docker-compose -f docker-compose.yml -f $1 up -d db user_db db_kone
+    TAG=$2 docker-compose -f docker-compose.yml -f $1 up -d db user_db db_kone
     db_anon_setup_full
     echo "Starting services."
-    docker-compose -f docker-compose.yml -f $1 up -d
+    TAG=$2 docker-compose -f docker-compose.yml -f $1 up -d
 }

services/backend/oodikone2-backend/src/conf-backend.js

@@ -21,6 +21,7 @@ const CERT_PATH = process.env.CERT_PATH // production/staging only
 const KEY_PATH = process.env.KEY_PATH // production/staging only
 const OODILEARN_URL = process.env.OODILEARN_URL
 const USERSERVICE_URL = process.env.USERSERVICE_URL
+const UPDATER_URL = process.env.UPDATER_URL
 const USAGESERVICE_URL = process.env.USAGESERVICE_URL
 const ANALYTICS_URL = process.env.ANALYTICS_URL
 const PORT = isTest ? 8079 : 8080
@@ -81,5 +82,6 @@ module.exports = {
   requiredGroup,
   OODI_SECRET,
   OODI_SECRET_HEADER_KEY,
-  isTest
+  isTest,
+  UPDATER_URL
 }

services/backend/oodikone2-backend/src/routes/population.js

@@ -1,8 +1,11 @@
 const router = require('express').Router()
 const Population = require('../services/populations')
 const Filters = require('../services/filters')
-const { updateStudents } = require('../services/doo_api_database_updater/database_updater')
+const { updateStudents }  = require('../services/updaterService')
+
+const Student = require('../services/students')
 const StudyrightService = require('../services/studyrights')
+const UserService = require('../services/userService')
 
 // POST instead of GET because of too long params and "sensitive" data
 router.post('/v2/populationstatistics/courses', async (req, res) => {
@@ -31,6 +34,37 @@ router.post('/v2/populationstatistics/courses', async (req, res) => {
   }
 })
 
+router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) => {
+  try {
+    if (!req.body.yearcode || !req.body.coursecode) {
+      res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
+      return
+    }
+    const { coursecode, yearcode } = req.body
+    let studentnumberlist
+    const studentnumbers = await Student.findByCourseAndSemesters(coursecode, yearcode)
+
+    const { roles, userId } = req.decodedToken
+    if (roles && roles.map(r => r.group_code).includes('admin')) {
+      studentnumberlist = studentnumbers
+    } else {
+      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
+      const codes = unitsUserCanAccess.map(unit => unit.id)
+      studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
+    }
+    const result = await Population.bottlenecksOf(req.body, studentnumberlist)
+
+    if (result.error) {
+      res.status(400).json(result)
+      return
+    }
+
+    res.json(result)
+  } catch (e) {
+    res.status(400).json({ error: e })
+  }
+})
+
 router.get('/v3/populationstatistics', async (req, res) => {
   const { startYear, semesters, studyRights: studyRightsJSON } = req.query
   try {
@@ -73,6 +107,45 @@ router.get('/v3/populationstatistics', async (req, res) => {
   }
 })
 
+router.get('/v3/populationstatisticsbycourse', async (req, res) => {
+  const { coursecode, yearcode } = req.query
+  const semesters = ['FALL', 'SPRING']
+  let studentnumberlist
+  console.log(coursecode, yearcode)
+  const studentnumbers = await Student.findByCourseAndSemesters(coursecode, yearcode)
+  console.log(studentnumbers)
+  const { roles, userId } = req.decodedToken
+  if (roles && roles.map(r => r.group_code).includes('admin')) {
+    studentnumberlist = studentnumbers
+  } else {
+    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
+    const codes = unitsUserCanAccess.map(unit => unit.id)
+    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
+  }
+  try {
+    const result = await Population.optimizedStatisticsOf({
+      startYear: 1900,
+      endYear: 2200,
+      studyRights: [],
+      semesters, 
+      months: 1000
+    }, studentnumberlist)
+
+    if (result.error) {
+      console.log(result.error)
+      res.status(400).end()
+      return
+    }
+
+    console.log(`request completed ${new Date()}`)
+    res.json(result)
+  } catch (e) {
+    console.log(e)
+    res.status(400).json({ error: e })
+  }
+})
+
+
 router.get('/v2/populationstatistics/filters', async (req, res) => {
 
   let results = []
@@ -121,8 +194,8 @@ router.post('/updatedatabase', async (req, res) => {
   const studentnumbers = req.body
   console.log(studentnumbers)
   if (studentnumbers) {
-    await updateStudents(studentnumbers, 128)
-    res.status(200).json('Updated')
+    await updateStudents(studentnumbers)
+    res.status(200).json('Scheduled')
   } else {
     res.status(400).end()
   }

services/backend/oodikone2-backend/src/services/course_yearly_stats_counter.js

@@ -127,12 +127,14 @@ class CourseYearlyStatsCounter {
   formatStudentStatistics(students) {
     const grades = {}
     const classes = {}
+    const studentnumbers = []
     Object.entries(students).forEach(([studentnumber, stat]) => {
       const { grade, category } = stat
       grades[grade] = grades[grade] ? grades[grade].concat(studentnumber) : [studentnumber]
       classes[category] = classes[category] ? classes[category].concat(studentnumber) : [studentnumber]
+      studentnumbers.includes(studentnumber) ? null : studentnumbers.push(studentnumber)
     })
-    return { grades, classes }
+    return { grades, classes, studentnumbers }
   }
 
   formatGroupStatistics() {