From 40bf94555340f3700ff2af8711cb89e1d5d04476 Mon Sep 17 00:00:00 2001 From: Valtteri Kantanen Date: Tue, 20 Aug 2024 16:27:05 +0300 Subject: [PATCH] [University] Fix university view crashing --- services/backend/src/routes.js | 2 +- services/backend/src/routes/university.js | 202 ---------------- services/backend/src/routes/university.ts | 227 ++++++++++++++++++ .../faculty/facultyGraduationTimes.ts | 2 +- 4 files changed, 229 insertions(+), 204 deletions(-) delete mode 100644 services/backend/src/routes/university.js create mode 100644 services/backend/src/routes/university.ts diff --git a/services/backend/src/routes.js b/services/backend/src/routes.js index 91b083af21..87c5c82365 100644 --- a/services/backend/src/routes.js +++ b/services/backend/src/routes.js @@ -32,7 +32,7 @@ const studyProgrammeCriteria = require('./routes/studyProgrammeCriteria').defaul const studyProgrammePins = require('./routes/studyProgrammePins').default const tags = require('./routes/tags').default const teachers = require('./routes/teachers').default -const university = require('./routes/university') +const university = require('./routes/university').default const updater = require('./routes/updater').default const usersToska = require('./routes/users').default const usersFd = require('./routes/usersFd').default diff --git a/services/backend/src/routes/university.js b/services/backend/src/routes/university.js deleted file mode 100644 index 7b390536a6..0000000000 --- a/services/backend/src/routes/university.js +++ /dev/null @@ -1,202 +0,0 @@ -const router = require('express').Router() - -const { serviceProvider } = require('../config') -const { magicFacultyCode } = require('../config/organizationConstants') -const { getSortedFaculties } = require('../services/faculty/facultyHelpers') -const { getFacultyProgressStats, getGraduationStats } = require('../services/faculty/facultyService') -const { getMedian } = require('../services/studyProgramme/studyProgrammeHelpers') - -const degreeNames = ['bachelor', 'bachelorMaster', 'master', 'licentiate', 'doctor'] - -const getProgrammeNames = faculties => { - return faculties.reduce((obj, faculty) => { - const { name, ...rest } = faculty.dataValues - obj[faculty.code] = { ...rest, ...name } - return obj - }, {}) -} - -router.get('/allprogressstats', async (req, res) => { - const specialGroups = req.query?.specialsIncluded === 'true' ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED' - const graduated = req.query?.graduated - const allFaculties = await getSortedFaculties() - const facultyCodes = allFaculties.map(faculty => faculty.code) - const codeToData = {} - - for (const facultyCode of facultyCodes) { - const data = await getFacultyProgressStats(facultyCode, specialGroups, graduated) - if (!data) { - return res.status(404).send('Data missing from server: Refreshing faculty data required') - } - codeToData[facultyCode] = data - } - - const universityData = { - years: codeToData[magicFacultyCode].years, - yearlyBachelorTitles: codeToData[magicFacultyCode].yearlyBachelorTitles, - yearlyBcMsTitles: codeToData[magicFacultyCode].yearlyBcMsTitles, - yearlyMasterTitles: codeToData[magicFacultyCode].yearlyMasterTitles, - yearlyLicentiateTitles: codeToData[magicFacultyCode].yearlyLicentiateTitles, - programmeNames: getProgrammeNames(allFaculties), - bachelorsProgStats: {}, - creditCounts: { - bachelor: {}, - }, - } - - const unifyProgressStats = progStats => { - return progStats.reduce( - (all, prog) => { - prog.forEach((yearStats, yearIndex) => - yearStats.forEach((category, categoryIndex) => { - all[yearIndex][categoryIndex] += prog[yearIndex][categoryIndex] - }) - ) - return all - }, - // eslint-disable-next-line no-unused-vars - progStats[0].map(year => year.map(_num => 0)) - ) - } - - for (const facultyCode of facultyCodes) { - const facultyData = codeToData[facultyCode] - for (const year of universityData.years.slice(1).reverse()) { - for (const fieldName of degreeNames) { - if (!facultyData.creditCounts[fieldName] || Object.keys(facultyData.creditCounts[fieldName]).length === 0) - continue - if (!universityData.creditCounts[fieldName]) universityData.creditCounts[fieldName] = {} - if (!universityData.creditCounts[fieldName][year]) universityData.creditCounts[fieldName][year] = [] - universityData.creditCounts[fieldName][year].push(...facultyData.creditCounts[fieldName][year]) - } - for (const fieldName of [ - 'bachelorsProgStats', - 'bcMsProgStats', - 'licentiateProgStats', - 'mastersProgStats', - 'doctoralProgStats', - ]) { - if (!universityData[fieldName]) universityData[fieldName] = {} - if (Object.keys(facultyData[fieldName]).length === 0) continue - universityData[fieldName][facultyCode] = unifyProgressStats(Object.values(facultyData[fieldName])) - } - } - } - - // Remove ELL bachelor+master progresstats for now, because it is problematic due to different credit limits - // when expanding the table rows to show 'programme'(faculty) specific progress bars - delete universityData.bcMsProgStats.H90 - - return res.status(200).json(universityData) -}) - -router.get('/allgraduationstats', async (_req, res) => { - const degreeNames = ['bachelor', 'bcMsCombo', 'master', 'licentiate', 'doctor'] - const allFaculties = await getSortedFaculties() - const facultyCodes = allFaculties.map(faculty => faculty.code) - const facultyData = {} - const timesArrays = [] // keep book of these to null them in the end, large lists not used in frontend - const programmeFilter = serviceProvider === 'toska' ? 'NEW_STUDY_PROGRAMMES' : 'ALL_PROGRAMMES' - for (const facultyCode of facultyCodes) { - const data = await getGraduationStats(facultyCode, programmeFilter, true) - if (!data) return res.status(500).json({ message: `Did not find data for ${facultyCode}` }) - facultyData[facultyCode] = data - } - - const unifyTotals = (facultyData, universityData, isLast) => { - for (const degree of degreeNames) { - if (!universityData[degree]) universityData[degree] = [] - if (!facultyData[degree]) continue - for (const yearStats of facultyData[degree]) { - const universityStats = universityData[degree] - const universityYearStats = universityData[degree].find(stats => stats.name === yearStats.name) - if (!universityYearStats) { - universityStats.push(yearStats) - } else { - universityYearStats.times.push(...yearStats.times) - timesArrays.push(universityYearStats.times) - universityYearStats.amount += yearStats.amount - universityYearStats.statistics.onTime += yearStats.statistics.onTime - universityYearStats.statistics.yearOver += yearStats.statistics.yearOver - universityYearStats.statistics.wayOver += yearStats.statistics.wayOver - if (isLast) { - universityYearStats.y = getMedian(universityYearStats.times) - } - } - } - } - } - - const unifyProgrammeStats = (universityData, facultyData, facultyCode) => { - for (const degree of degreeNames) { - if (!facultyData[degree]) continue - if (!universityData[degree]) universityData[degree] = {} - for (const yearData of facultyData[degree]) { - if (!universityData[degree][yearData.name]) { - universityData[degree][yearData.name] = { programmes: [], data: [] } - } - const uniYearStats = universityData[degree][yearData.name] - if (!uniYearStats.programmes.find(prog => prog === facultyCode)) { - uniYearStats.programmes.push(facultyCode) - } - const uniYearFacultyStats = uniYearStats.data.find(item => item.code === facultyCode) - const yearDataClone = { ...yearData, times: null, statistics: { ...yearData.statistics } } - if (!uniYearFacultyStats) { - uniYearStats.data.push({ ...yearDataClone, name: facultyCode, code: facultyCode }) - } else { - uniYearFacultyStats.y = yearData.y - uniYearFacultyStats.amount += yearData.amount - uniYearFacultyStats.statistics.onTime += yearData.statistics.onTime - uniYearFacultyStats.statistics.yearOver += yearData.statistics.yearOver - uniYearFacultyStats.statistics.wayOver += yearData.statistics.wayOver - } - } - } - - return universityData - } - - const universityData = { - goals: { - bachelor: 36, - bcMsCombo: 60, - master: 24, - doctor: 48, - licentiate: 78, - }, - programmeNames: getProgrammeNames(allFaculties), - byGradYear: { medians: {}, programmes: { medians: {} } }, - classSizes: { programmes: {} }, - } - - for (let i = 0; i < facultyCodes.length; i++) { - const facultyCode = facultyCodes[i] - const data = facultyData[facultyCode] - unifyTotals(data.byGradYear.medians, universityData.byGradYear.medians, i === facultyCodes.length - 1) - unifyProgrammeStats(universityData.byGradYear.programmes.medians, data.byGradYear.medians, facultyCode) - for (const degree of degreeNames) { - if (!data.classSizes[degree]) { - continue - } - if (!universityData.classSizes[degree]) { - universityData.classSizes[degree] = data.classSizes[degree] - } else { - Object.entries(data.classSizes[degree]).forEach(([key, value]) => { - universityData.classSizes[degree][key] += value - }) - } - } - - const { programmes, ...rest } = data.classSizes - universityData.classSizes.programmes[facultyCode] = rest - } - - // Empty "times" arrays because that's not needed anymore. - timesArrays.forEach(arr => { - arr.length = 0 - }) - - return res.status(200).json(universityData) -}) - -module.exports = router diff --git a/services/backend/src/routes/university.ts b/services/backend/src/routes/university.ts new file mode 100644 index 0000000000..fda1d53adb --- /dev/null +++ b/services/backend/src/routes/university.ts @@ -0,0 +1,227 @@ +// const router = require('express').Router() +import { Request, Response, Router } from 'express' +import { cloneDeep } from 'lodash' + +import { serviceProvider } from '../config' +import { magicFacultyCode } from '../config/organizationConstants' +import { Organization } from '../models' +import { getDegreeProgrammesOfFaculty } from '../services/faculty/faculty' +import { countGraduationTimes, LevelGraduationStats } from '../services/faculty/facultyGraduationTimes' +import { getSortedFaculties } from '../services/faculty/facultyHelpers' +import { + getFacultyProgressStats, + getGraduationStats, + setGraduationStats, + setFacultyProgressStats, +} from '../services/faculty/facultyService' +import { combineFacultyStudentProgress, FacultyProgressData } from '../services/faculty/facultyStudentProgress' +import { getMedian } from '../services/studyProgramme/studyProgrammeHelpers' +import { Graduated } from '../types' +import logger from '../util/logger' + +const router = Router() + +const degreeNames = ['bachelor', 'bachelorMaster', 'master', 'doctor'] as const + +const getProgrammeNames = (faculties: Organization[]) => { + return faculties.reduce>((obj, faculty) => { + const { name, code } = faculty.dataValues + obj[faculty.code] = { code, ...name } + return obj + }, {}) +} + +interface GetProgressStatsRequest extends Request { + query: { + specialsIncluded: 'true' | undefined + graduated: Graduated + } +} + +router.get('/allprogressstats', async (req: GetProgressStatsRequest, res: Response) => { + const specialGroups = req.query?.specialsIncluded === 'true' ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED' + const graduated = req.query?.graduated + const allFaculties = await getSortedFaculties() + const facultyCodes = allFaculties.map(faculty => faculty.code) + const codeToData: Record = {} + + for (const facultyCode of facultyCodes) { + let data: any = await getFacultyProgressStats(facultyCode, specialGroups, graduated) + if (!data) { + logger.info(`Data missing from server: Refreshing progress faculty data for faculty ${facultyCode}`) + const programmes = await getDegreeProgrammesOfFaculty(facultyCode, true) + data = await combineFacultyStudentProgress(facultyCode, programmes, specialGroups, graduated) + await setFacultyProgressStats(data, specialGroups, graduated) + } + codeToData[facultyCode] = data + } + + const universityData = { + years: codeToData[magicFacultyCode].years, + yearlyBachelorTitles: codeToData[magicFacultyCode].yearlyBachelorTitles, + yearlyBcMsTitles: codeToData[magicFacultyCode].yearlyBcMsTitles, + yearlyMasterTitles: codeToData[magicFacultyCode].yearlyMasterTitles, + programmeNames: getProgrammeNames(allFaculties), + bachelorsProgStats: {} as Record, + bcMsProgStats: {} as Record, + mastersProgStats: {} as Record, + doctoralProgStats: {} as Record, + creditCounts: { + bachelor: {} as Record, + bachelorMaster: {} as Record, + master: {} as Record, + doctor: {} as Record, + }, + } + + const unifyProgressStats = (progStats: number[][][]) => { + return progStats.reduce( + (all, prog) => { + prog.forEach((yearStats, yearIndex) => + yearStats.forEach((category, categoryIndex) => { + all[yearIndex][categoryIndex] += prog[yearIndex][categoryIndex] + }) + ) + return all + }, + progStats[0].map(year => year.map(() => 0)) + ) + } + + for (const facultyCode of facultyCodes) { + const facultyData = codeToData[facultyCode] + for (const year of universityData.years.slice(1).reverse()) { + for (const fieldName of degreeNames) { + if (!facultyData.creditCounts[fieldName] || Object.keys(facultyData.creditCounts[fieldName]).length === 0) + continue + if (!universityData.creditCounts[fieldName][year]) universityData.creditCounts[fieldName][year] = [] + universityData.creditCounts[fieldName][year].push(...facultyData.creditCounts[fieldName][year]) + } + const progStats = ['bachelorsProgStats', 'bcMsProgStats', 'mastersProgStats', 'doctoralProgStats'] as const + for (const fieldName of progStats) { + if (Object.keys(facultyData[fieldName] || {}).length === 0) continue + universityData[fieldName][facultyCode] = unifyProgressStats(Object.values(facultyData[fieldName])) + } + } + } + + res.status(200).json(universityData) +}) + +router.get('/allgraduationstats', async (_req: Request, res: Response) => { + const degreeNames = ['bachelor', 'bcMsCombo', 'master', 'doctor'] as const + const allFaculties = await getSortedFaculties() + const facultyCodes = allFaculties.map(faculty => faculty.code) + const facultyData: Record>> = {} + const programmeFilter = serviceProvider === 'toska' ? 'NEW_STUDY_PROGRAMMES' : 'ALL_PROGRAMMES' + for (const facultyCode of facultyCodes) { + let data: any = await getGraduationStats(facultyCode, programmeFilter, true) + if (!data) { + logger.info(`Data missing from server: Refreshing graduation faculty data for faculty ${facultyCode}`) + const programmes = await getDegreeProgrammesOfFaculty(facultyCode, true) + data = await countGraduationTimes(facultyCode, programmes) + await setGraduationStats(data, programmeFilter) + } + facultyData[facultyCode] = data + } + + const universityData = { + goals: { bachelor: 36, bcMsCombo: 60, master: 24, doctor: 48 }, + programmeNames: getProgrammeNames(allFaculties), + byGradYear: { + medians: {} as Record, + programmes: { + medians: {} as Record< + string, + Record }> + >, + }, + }, + classSizes: { + bachelor: null as Record | null, + bcMsCombo: null as Record | null, + master: null as Record | null, + doctor: null as Record | null, + programmes: {} as Record>>, + }, + } + + const unifyTotals = (facultyData: Record, isLast: boolean) => { + const mediansForUniversity = universityData.byGradYear.medians + for (const degree of degreeNames) { + if (!mediansForUniversity[degree]) mediansForUniversity[degree] = [] + if (!facultyData[degree]) continue + for (const yearStats of facultyData[degree]) { + const universityStats = mediansForUniversity[degree] + const universityYearStats = mediansForUniversity[degree].find(stats => stats.name === yearStats.name) + if (!universityYearStats) { + universityStats.push(yearStats) + } else { + universityYearStats.times.push(...yearStats.times) + universityYearStats.amount += yearStats.amount + universityYearStats.statistics.onTime += yearStats.statistics.onTime + universityYearStats.statistics.yearOver += yearStats.statistics.yearOver + universityYearStats.statistics.wayOver += yearStats.statistics.wayOver + if (isLast) { + universityYearStats.median = getMedian(universityYearStats.times) + } + } + } + } + } + + const unifyProgrammeStats = (facultyData: Record, facultyCode: string) => { + const mediansForUniversity = universityData.byGradYear.programmes.medians + for (const degree of degreeNames) { + if (!facultyData[degree]) continue + if (!mediansForUniversity[degree]) mediansForUniversity[degree] = {} + for (const yearData of facultyData[degree]) { + if (!mediansForUniversity[degree][yearData.name]) { + mediansForUniversity[degree][yearData.name] = { programmes: [], data: [] } + } + const uniYearStats = mediansForUniversity[degree][yearData.name] + if (!uniYearStats.programmes.find(prog => prog === facultyCode)) { + uniYearStats.programmes.push(facultyCode) + } + const uniYearFacultyStats = uniYearStats.data.find(item => item.code === facultyCode) + if (!uniYearFacultyStats) { + uniYearStats.data.push({ ...cloneDeep(yearData), name: facultyCode, code: facultyCode }) + } else { + uniYearFacultyStats.times.push(...uniYearFacultyStats.times) + uniYearFacultyStats.median = getMedian(uniYearFacultyStats.times) + uniYearFacultyStats.amount += yearData.amount + uniYearFacultyStats.statistics.onTime += yearData.statistics.onTime + uniYearFacultyStats.statistics.yearOver += yearData.statistics.yearOver + uniYearFacultyStats.statistics.wayOver += yearData.statistics.wayOver + } + } + } + } + + for (let i = 0; i < facultyCodes.length; i++) { + const facultyCode = facultyCodes[i] + const data = facultyData[facultyCode] + unifyTotals(data.byGradYear.medians, i === facultyCodes.length - 1) + unifyProgrammeStats(data.byGradYear.medians, facultyCode) + for (const degree of degreeNames) { + if (!data.classSizes[degree]) { + continue + } + const facultyClassSizes = data.classSizes[degree] as Record + if (!universityData.classSizes[degree]) { + universityData.classSizes[degree] = facultyClassSizes + } else { + Object.entries(facultyClassSizes).forEach(([key, value]) => { + universityData.classSizes[degree]![key] += value + }) + } + } + + const { programmes, ...rest } = data.classSizes + universityData.classSizes.programmes[facultyCode] = rest as Record> + } + + res.status(200).json(universityData) +}) + +export default router diff --git a/services/backend/src/services/faculty/facultyGraduationTimes.ts b/services/backend/src/services/faculty/facultyGraduationTimes.ts index bca98baca2..dd4787cedc 100644 --- a/services/backend/src/services/faculty/facultyGraduationTimes.ts +++ b/services/backend/src/services/faculty/facultyGraduationTimes.ts @@ -11,7 +11,7 @@ import { } from '../studyProgramme/studyTrackStats' import type { ProgrammesOfOrganization } from './faculty' -type LevelGraduationStats = Omit & { median: number } +export type LevelGraduationStats = Omit & { median: number } type ProgrammeStats = { data: Array & { code: string }>