Merge pull request #966 from UniversityOfHelsinkiCS/trunk

Add sorting to overview and students listing/search

services/backend/oodikone2-backend/scripts/force_sync_database.js

@@ -1,13 +1,10 @@
-const { sequelize, sequelizeKone } = require('../src/models/index')
-const { seedMigrations } = require('../src/database/seed_migrations')
-
-const options = { force: true }
+const { seedAllMigrations } = require('../src/database/seed_migrations')
+const { forceSyncDatabase } = require('../src/database/connection')
 
 const sync = async () => {
   try {
-    await sequelize.sync(options)
-    await sequelizeKone.sync(options)
-    await seedMigrations()
+    await forceSyncDatabase()
+    await seedAllMigrations()
     console.log('Force sync succeeded. ')
     process.exit(0)
   } catch (e) {

services/backend/oodikone2-backend/scripts/seed_migrations.js

@@ -1,8 +1,8 @@
-const { seedMigrations } = require('../src/database/seed_migrations')
+const { seedAllMigrations } = require('../src/database/seed_migrations')
 
 const run = async () => {
   try {
-    await seedMigrations()
+    await seedAllMigrations()
     process.exit(0)
   } catch (e) {
     console.log(`Seeding migrations failed: ${e}`)

services/backend/oodikone2-backend/src/database/connection.js

@@ -76,8 +76,7 @@ const runMigrationsKone = async () => {
   }
 }
 
-const migrationPromise = !conf.isTest && conf.DB_SCHEMA === 'public' ? runMigrations().then(() => runMigrationsKone())
-  : Promise.resolve()
+const migrationPromise = !conf.isTest ? runMigrations().then(() => runMigrationsKone()) : Promise.resolve()
 
 const forceSyncDatabase = async () => {
   try {
@@ -90,8 +89,8 @@ const forceSyncDatabase = async () => {
   } catch (e) {
     // console.log(e)
   }
-  await sequelize.sync({ force: true, schema: conf.DB_SCHEMA, searchPath: conf.DB_SCHEMA })
-  await sequelizeKone.sync({ force: true, schema: conf.DB_SCHEMA_KONE, searchPath: conf.DB_SCHEMA_KONE })
+  await sequelize.sync({ force: true })
+  await sequelizeKone.sync({ force: true })
 }
 
 module.exports = {

services/backend/oodikone2-backend/src/database/seed_migrations.js

@@ -17,12 +17,11 @@ const seedMigrationsKone = async (migrationfilepath=DEFAULT_PATH_KONE) => {
   await saveMigrationsToDatabase(MigrationKone)(filenames)
 }
 
-const run = async () => {
+const seedAllMigrations = async () => {
   await seedMigrations()
   await seedMigrationsKone()
 }
 
 module.exports = {
-  run,
-  seedMigrations
+  seedAllMigrations
 }

services/backend/oodikone2-backend/src/routes/teachers.js

@@ -20,6 +20,12 @@ router.get('/top', async (req, res) => {
   res.json(result)
 })
 
+router.post('/top', async (req, res) => {
+  const { startyearcode, endyearcode } = req.body
+  res.status(200).end()
+  await topteachers.findAndSaveTeachers(startyearcode, endyearcode)
+})
+
 router.get('/top/categories', async (req, res) => {
   const result = await topteachers.getCategoriesAndYears()
   res.json(result)

services/backend/oodikone2-backend/src/services/semesters.js

@@ -1,5 +1,6 @@
-const { Op } = require('sequelize')
+const sequelize = require('sequelize')
 const { Semester } = require('../models/index')
+const { Op } = sequelize
 
 const getSemestersAndYears = async before => {
   const semesters = await Semester.findAll({
@@ -24,6 +25,15 @@ const getSemestersAndYears = async before => {
   return result
 }
 
+const getMaxYearcode = async () => {
+  const aa = await Semester.findAll({
+    attributes: [[sequelize.fn('max', sequelize.col('yearcode')), 'maxYearCode']],
+    raw: true
+  })
+  return aa[0].maxYearCode
+}
+
 module.exports = {
-  getSemestersAndYears
+  getSemestersAndYears,
+  getMaxYearcode
 }

services/backend/oodikone2-backend/src/services/studytrack.js

@@ -8,8 +8,6 @@ const { studentnumbersWithAllStudyrightElements } = require('./populations')
 const { semesterStart, semesterEnd } = require('../util/semester')
 const isNumber = str => !Number.isNaN(Number(str))
 
-const FIVE_YEARS_IN_MONTHS = 60
-
 const studytrackToProviderCode = code => {
   const [left, right] = code.split('_')
   const prefix = [...left].filter(isNumber).join('')
@@ -66,7 +64,7 @@ const productivityStatsForProvider = async (providercode, since) => {
 
 const formatGraduatedStudyright = ({ studyrightid, enddate, studystartdate }) => {
   const year = enddate && enddate.getFullYear()
-  const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'days')
+  const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
   return { studyrightid, year, timeToGraduation }
 }
 
@@ -94,10 +92,32 @@ const findGraduated = (studytrack, since) => Studyright.findAll({
 
 const graduatedStatsFromStudyrights = studyrights => {
   const stats = {}
-  studyrights.forEach(({ year }) => {
-    const graduated = stats[year] || 0
-    stats[year] = graduated + 1
+  let graduationTimes = []
+  studyrights.forEach(({ year, timeToGraduation }) => {
+    const graduated = stats[year] ? stats[year].graduated : 0
+    stats[year] = {
+      graduated: graduated + 1,
+      timesToGraduation: stats[year] ?
+        [...stats[year].timesToGraduation, timeToGraduation || 0] : [ timeToGraduation || 0]
+    }
+    graduationTimes = [...graduationTimes, timeToGraduation || 0]
+  })
+  const median = (values) => {
+    if (values.length === 0) return 0
+
+    values.sort((a, b) => a - b)
+
+    var half = Math.floor(values.length / 2)
+
+    if (values.length % 2)
+      return values[half]
+
+    return (values[half - 1] + values[half]) / 2.0
+  }
+  Object.keys(stats).forEach(year => {
+    stats[year].medianGraduationTime = median(stats[year].timesToGraduation)
   })
+  stats['medianGraduationTime'] = median(graduationTimes)
   return stats
 }
 
@@ -153,10 +173,10 @@ const thesisProductivityForStudytrack = async code => {
 
 const combineStatistics = (creditStats, studyrightStats, thesisStats) => {
   const stats = { ...creditStats }
-  console.log(studyrightStats)
   Object.keys(stats).forEach(year => {
     const thesis = thesisStats[year] || {}
-    stats[year].graduated = studyrightStats[year] || 0
+    stats[year].graduated = studyrightStats[year] ? studyrightStats[year].graduated : 0
+    stats[year].medianGraduationTime = studyrightStats[year] ? studyrightStats[year].medianGraduationTime : 0
     stats[year].bThesis = thesis.bThesis || 0
     stats[year].mThesis = thesis.mThesis || 0
   })
@@ -327,7 +347,8 @@ const productivityStats = async (studentnumbers, startDate, studytrack, endDate)
     thesesFromClass(studentnumbers, startDate, studytrack),
     gendersFromClass(studentnumbers),
     countriesFromClass(studentnumbers),
-    tranferredToStudyprogram(studentnumbers, startDate, studytrack, endDate)])
+    tranferredToStudyprogram(studentnumbers, startDate, studytrack, endDate)
+  ])
 }
 
 const getYears = (since) => {
@@ -357,7 +378,8 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
     transferred: 0
   }
   const years = getYears(since)
-  const graduationTimeLimit = studytrack[0] === 'K' ? 36 : 24 // studyprogramme starts with K if bachelors and M if masters
+  // studyprogramme starts with K if bachelors and M if masters
+  const graduationTimeLimit = studytrack[0] === 'K' ? 36 : 24
 
   const arr = await Promise.all(years.map(async year => {
     const startDate = `${year}-${semesterStart['FALL']}`
@@ -367,7 +389,7 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
       await productivityStats(studentnumbers, startDate, studytrack, endDate)
     delete genders[null]
     delete countries[null]
-    
+
     const creditValues = credits.reduce((acc, curr) => {
       acc.mte30 = curr >= 30 ? acc.mte30 + 1 : acc.mte30
       acc.mte60 = curr >= 60 ? acc.mte60 + 1 : acc.mte60
@@ -402,7 +424,7 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
       year: `${year}-${year + 1}`,
       credits: credits.map(cr => cr === null ? 0 : cr),
       graduated: graduated.length,
-      inTargetTime, 
+      inTargetTime,
       thesisM: theses.MASTER || 0,
       thesisB: theses.BACHELOR || 0,
       genders,

services/backend/oodikone2-backend/src/services/studytrack.test.js

@@ -125,8 +125,8 @@ test('graduatedStatsFromStudyrights calculates stats correctly', () => {
   ]
   const stats = graduatedStatsFromStudyrights(studyrights)
   expect(stats).toMatchObject({
-    2015: 2,
-    2014: 1
+    2015: { graduated: 2., medianGraduationTime: 0, timesToGraduation: [ 0, 0 ]},
+    2014: { graduated: 1., medianGraduationTime: 0, timesToGraduation: [ 0 ]}
   })
 })
 
@@ -137,8 +137,8 @@ test('combineStatistics returns correctly formatted array', () => {
     2014: { year: 2014, credits: 20 }
   }
   const studyrightStats = {
-    2015: 2,
-    2016: 1
+    2015: { graduated: 2, medianGraduationTime: 1.5, timesToGraduation: [1, 2] },
+    2016: { graduated: 1, medianGraduationTime: 0, timesToGraduation: [ 0 ] }
   }
   const thesisStats = {
     2014: { mThesis: 1 },
@@ -150,21 +150,24 @@ test('combineStatistics returns correctly formatted array', () => {
     mThesis: 2,
     bThesis: 1,
     credits: 40,
+    medianGraduationTime: 1.5,
     graduated: 2
   })
   expect(stats).toContainEqual({
     year: 2014,
     mThesis: 1,
     bThesis: 0,
     credits: 20,
-    graduated: 0
+    graduated: 0,
+    medianGraduationTime: 0
   })
   expect(stats).toContainEqual({
     year: 2016,
     mThesis: 0,
     bThesis: 0,
     credits: 5,
-    graduated: 1
+    graduated: 1,
+    medianGraduationTime: 0
   })
 })
 
@@ -175,14 +178,16 @@ test('productivityStatsForStudytrack integrates', async () => {
     graduated: 0,
     bThesis: 0,
     mThesis: 1,
-    credits: 40
+    credits: 40,
+    medianGraduationTime: 0
   })
   expect(stats.data).toContainEqual({
     year: 2016,
     graduated: 1,
     mThesis: 0,
     bThesis: 0,
-    credits: 5
+    credits: 5,
+    medianGraduationTime: 0
   })
 })
 

services/backend/oodikone2-backend/src/services/topteachers.js

@@ -1,5 +1,5 @@
 const { redisClient } = require('./redis')
-const { getSemestersAndYears } = require('./semesters')
+const { getSemestersAndYears, getMaxYearcode } = require('./semesters')
 const { Teacher, Semester, Credit, Course } = require('../models/index')
 const { Op } = require('sequelize')
 
@@ -126,6 +126,13 @@ const findTopTeachers = async (yearcode) => {
   }
 }
 
+const findAndSaveTeachers = async (startcode = 50, to) => {
+  const endcode = to ? to : await getMaxYearcode()
+  for (let code = startcode; code <= endcode; code++) {
+    await findAndSaveTopTeachers(code)
+  }
+}
+
 const findAndSaveTopTeachers = async yearcode => {
   const { all, openuni } = await findTopTeachers(yearcode)
   await setTeacherStats(ID.OPENUNI, yearcode, openuni)
@@ -138,6 +145,7 @@ module.exports = {
   setTeacherStats,
   getCategoriesAndYears,
   findTopTeachers,
+  findAndSaveTeachers,
   findAndSaveTopTeachers,
   ID
 }

services/backend/oodikone2-backend/test/jest/globals.js

@@ -1,11 +1,11 @@
 const { sequelize } = require('../../src/models/index')
 const { redisClient } = require('../../src/services/redis')
 const { forceSyncDatabase } = require('../../src/database/connection')
-const { seedMigrations } = require('../../src/database/seed_migrations')
+const { seedAllMigrations } = require('../../src/database/seed_migrations')
 
 beforeAll(async () => {
   await forceSyncDatabase()
-  await seedMigrations()
+  await seedAllMigrations()
 })
 
 afterAll(async () => {

services/backend/shared/models/index.js

@@ -1,5 +1,5 @@
 const Sequelize = require('sequelize')
-const { sequelize, migrationPromise } = require('../database/connection')
+const { sequelize, sequelizeKone, migrationPromise } = require('../database/connection')
 const conf = require('../conf-backend')
 
 const ThesisTypeEnums = {
@@ -135,13 +135,13 @@ const Tag = sequelize.define('tag',
   }
 )
 
-const MigrationKone = sequelize.define('migrations', {
+const MigrationKone = sequelize.define('migrationsKone', {
   name: {
     type: Sequelize.STRING,
     primaryKey: true
   }
 }, {
-  tablename: 'migrations',
+  tableName: 'migrations',
   timestamps: false,
   schema: conf.DB_SCHEMA_KONE
 })
@@ -303,7 +303,7 @@ const StudyrightElement = sequelize.define('studyright_elements',
         fields: ['startdate']
       }
     ],
-    tablename: 'studyright_elements'
+    tableName: 'studyright_elements'
   }
 )
 
@@ -317,7 +317,7 @@ const ElementDetails = sequelize.define('element_details',
     type: { type: Sequelize.INTEGER }
   },
   {
-    tablename: 'element_details'
+    tableName: 'element_details'
   }
 )
 
@@ -549,7 +549,7 @@ const Migration = sequelize.define('migrations', {
     primaryKey: true
   }
 }, {
-  tablename: 'migrations',
+  tableName: 'migrations',
   timestamps: false
 })
 
@@ -695,6 +695,7 @@ module.exports = {
   Tag,
   Teacher,
   sequelize,
+  sequelizeKone,
   migrationPromise,
   Organisation,
   StudentList,