Merge pull request #1899 from UniversityOfHelsinkiCS/trunk

Implement new admin sparkline area chart proto, bug fixing, sis migrations

services/oodikone2-frontend/src/components/PopulationSearchForm/index.jsx

@@ -13,7 +13,7 @@ import moment from 'moment'
 import { getPopulationStatistics, clearPopulations } from '../../redux/populations'
 import { getPopulationCourses } from '../../redux/populationCourses'
 import { getPopulationSelectedStudentCourses, clearSelected } from '../../redux/populationSelectedStudentCourses'
-import { getPopulationFilters, setPopulationFilter, clearPopulationFilters } from '../../redux/populationFilters'
+import { getPopulationFilters, setPopulationFilter, resetPopulationFilters } from '../../redux/populationFilters'
 import { getMandatoryCourses } from '../../redux/populationMandatoryCourses'
 import { getSemesters } from '../../redux/semesters'
 import { transferTo } from '../../populationFilters'
@@ -109,7 +109,7 @@ const PopulationSearchForm = props => {
     setState({ isLoading: true })
     props.setLoading()
     props.clearSelected()
-    props.clearPopulationFilters()
+    props.resetPopulationFilters()
     fetchPopulationPromises.current = cancelablePromise(
       Promise.all([
         props.getPopulationStatistics({ ...formattedQueryParams, uuid, onProgress }),
@@ -164,7 +164,7 @@ const PopulationSearchForm = props => {
     setDidMount(true)
     return () => {
       if (fetchPopulationPromises.current) fetchPopulationPromises.current.cancel()
-      props.clearPopulationFilters()
+      props.resetPopulationFilters()
     }
   }, [location.search])
 
@@ -634,7 +634,7 @@ PopulationSearchForm.propTypes = {
   getMandatoryCourses: func.isRequired,
   getPopulationFilters: func.isRequired,
   setPopulationFilter: func.isRequired,
-  clearPopulationFilters: func.isRequired,
+  resetPopulationFilters: func.isRequired,
   queries: shape({}).isRequired,
   studyProgrammes: shape({}), //eslint-disable-line
   degrees: arrayOf(dropdownType), //eslint-disable-line
@@ -676,7 +676,7 @@ export default withRouter(
       getPopulationFilters,
       getMandatoryCourses,
       setPopulationFilter,
-      clearPopulationFilters,
+      resetPopulationFilters,
       getDegreesAndProgrammes,
       setLoading,
       getSemesters,

services/oodikone2-frontend/src/components/StudyProgramme/ThesisCourses/SearchResult.jsx

@@ -50,6 +50,14 @@ SearchResult.defaultProps = {
   courses: []
 }
 
-const mapStateToProps = state => ({ courses: getCourseSearchResults(state).courses })
-
+const mapStateToProps = state => {
+  const { thesisCourses } = state
+  const { data } = thesisCourses || null
+  const thesisCourseCodes = data.map(c => c.courseCode)
+  const searchResults = getCourseSearchResults(state).courses
+  const filteredResults = searchResults.filter(c => !thesisCourseCodes.includes(c.code))
+  return {
+    courses: filteredResults
+  }
+}
 export default connect(mapStateToProps)(SearchResult)

services/oodikone2-frontend/src/redux/populationFilters.js

@@ -7,6 +7,10 @@ export const clearPopulationFilters = () => ({
   type: 'CLEAR_POPULATION_FILTERS'
 })
 
+export const resetPopulationFilters = () => ({
+  type: 'RESET_POPULATION_FILTERS'
+})
+
 export const setPopulationFilter = filter => ({
   type: 'ADD_POPULATION_FILTER',
   filter
@@ -107,6 +111,13 @@ const reducer = (state = initial, action) => {
         filters: [],
         refreshNeeded: true
       }
+
+    case 'RESET_POPULATION_FILTERS':
+      return {
+        ...state,
+        filters: []
+      }
+
     case 'ALTER_POPULATION_COURSE_FILTER': {
       const toAlter = state.filters.find(f => f.id === action.id)
       const { course } = toAlter.params

services/oodikone2-frontend/src/redux/populationSelectedStudentCourses.js

@@ -71,9 +71,7 @@ const reducer = (state = defaultState, action) => {
         query: action.query
       }
     case 'CLEAR_SELECTED':
-      return {
-        state: defaultState
-      }
+      return defaultState
     default:
       return state
   }

services/sis-updater-scheduler/src/scheduler.js

@@ -97,7 +97,7 @@ const scheduleStudents = async () => {
     table: 'persons',
     whereNotNull: 'student_number',
     pluck: 'id',
-    limit: isDev ? 10000 : null
+    limit: isDev ? 100 : null
   })
 
   await redisSet(REDIS_TOTAL_STUDENTS_KEY, totalStudents)

services/sis-updater-worker/src/db/migrations/20200226_00_course_index_and_course_id_to_credit.js

@@ -0,0 +1,16 @@
+const { STRING } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.removeConstraint('credit', 'credit_course_code_fkey')
+    await queryInterface.addColumn('credit', 'course_id', {
+      type: STRING,
+      references: {
+        model: 'course',
+        key: 'id'
+      }
+    })
+    await queryInterface.addIndex('course', ['code'])
+  },
+  down: async () => {}
+}

services/sis-updater-worker/src/db/models/course.js

@@ -15,10 +15,10 @@ Course.init(
     name: {
       type: JSONB
     },
-    latestInstanceDate: {
+    latest_instance_date: {
       type: DATE
     },
-    isStudyModule: {
+    is_study_module: {
       type: BOOLEAN
     },
     coursetypecode: {
@@ -30,21 +30,23 @@ Course.init(
     enddate: {
       type: DATE
     },
-    maxAttainmentDate: {
+    max_attainment_date: {
       type: DATE
     },
-    minAttainmentDate: {
+    min_attainment_date: {
       type: DATE
     },
     createdAt: {
+      field: 'created_at',
       type: DATE
     },
     updatedAt: {
+      field: 'updated_at',
       type: DATE
     }
   },
   {
-    underscored: true,
+    underscored: false,
     sequelize: dbConnections.sequelize,
     modelName: 'course',
     tableName: 'course'

services/sis-updater-worker/src/db/models/credit.js

@@ -40,6 +40,9 @@ Credit.init(
       type: DATE
     },
     course_code: {
+      type: STRING
+    },
+    course_id: {
       type: STRING,
       references: {
         model: 'course',

services/sis-updater-worker/src/db/models/index.js

@@ -45,8 +45,8 @@ Credit.improved = credit => credit.credittypecode === CREDIT_TYPE_CODES.IMPROVED
 Credit.belongsTo(Student, { foreignKey: 'student_studentnumber', targetKey: 'studentnumber' })
 Student.hasMany(Credit, { foreignKey: 'student_studentnumber', sourceKey: 'studentnumber' })
 
-Credit.belongsTo(Course, { foreignKey: 'course_code' })
-Course.hasMany(Credit, { foreignKey: 'course_code' })
+Credit.belongsTo(Course, { foreignKey: 'course_id' })
+Course.hasMany(Credit, { foreignKey: 'course_id' })
 
 Credit.belongsTo(CreditType, { foreignKey: 'credittypecode', targetKey: 'credittypecode' })
 Credit.belongsToMany(Teacher, { through: CreditTeacher, foreignKey: 'credit_id' })

services/sis-updater-worker/src/index.js

@@ -24,7 +24,11 @@ const msgParser = f => async msg => {
   } catch (e) {
     console.log('Updating failed', e)
   } finally {
-    msg.ack()
+    try {
+      msg.ack()
+    } catch (e) {
+      console.log('Failed acking message...')
+    }
   }
 }
 

services/sis-updater-worker/src/updater/index.js

@@ -1,3 +1,4 @@
+const { Op } = require('sequelize')
 const { groupBy, flatten, flattenDeep, sortBy, mapValues, uniqBy } = require('lodash')
 const {
   Organization,
@@ -43,11 +44,11 @@ const initDaysToSemesters = async () => {
   }, {})
 }
 
-const getCreditTypeCodeFromAttainment = attainment => {
+const getCreditTypeCodeFromAttainment = (attainment, passed) => {
   const { primary, state } = attainment
+  if (!passed || state === 'FAILED') return 10
   if (!primary) return 7
   if (state === 'ATTAINED') return 4
-  if (state === 'FAILED') return 10
   return 9
 }
 
@@ -171,9 +172,9 @@ const updateCourses = async (courseIdToAttainments, groupIdToCourse) => {
       name,
       code,
       coursetypecode,
-      minAttainmentDate: min_attainment_date,
-      maxAttainmentDate: max_attainment_date,
-      latestInstanceDate: max_attainment_date,
+      min_attainment_date,
+      max_attainment_date,
+      latest_instance_date: max_attainment_date,
       startdate,
       enddate,
       isStudyModule: false
@@ -516,9 +517,14 @@ const updateAttainments = async attainments => {
       const {
         localId,
         numericCorrespondence,
+        passed,
         abbreviation: { fi }
       } = curr
-      if (!res[localId]) res[localId] = numericCorrespondence || fi
+      if (!res[localId])
+        res[localId] = {
+          value: numericCorrespondence || fi,
+          passed
+        }
       return res
     }, {})
     return res
@@ -534,6 +540,19 @@ const updateAttainments = async attainments => {
     return res
   }, {})
 
+  const courseGroupIdToCourseCode = (
+    await Course.findAll({
+      where: {
+        id: {
+          [Op.in]: Object.values(courseUnitIdToCourseGroupId)
+        }
+      }
+    })
+  ).reduce((res, curr) => {
+    res[curr.id] = curr.code
+    return res
+  })
+
   const organisations = await selectFromSnapshotsByIds(
     'organisations',
     flatten(attainments.map(a => a.organisations.map(o => o.organisationId)))
@@ -567,14 +586,21 @@ const updateAttainments = async attainments => {
 
       return {
         id: a.id,
-        grade: gradeScaleIdToGradeIdsToGrades[grade_scale_id][grade_id],
+        grade: gradeScaleIdToGradeIdsToGrades[grade_scale_id][grade_id].value,
         student_studentnumber: personIdToStudentNumber[a.person_id],
         credits: a.credits,
         createdate: a.registration_date,
-        credittypecode: getCreditTypeCodeFromAttainment(a),
+        credittypecode: getCreditTypeCodeFromAttainment(
+          a,
+          gradeScaleIdToGradeIdsToGrades[grade_scale_id][grade_id].passed
+        ),
         attainment_date: a.attainment_date,
-        course_code:
+        course_id:
           a.type === 'CourseUnitAttainment' ? courseUnitIdToCourseGroupId[a.course_unit_id] : a.module_group_id,
+        course_code:
+          a.type === 'CourseUnitAttainment'
+            ? courseGroupIdToCourseCode[courseUnitIdToCourseGroupId[a.course_unit_id]]
+            : courseGroupIdToCourseCode[a.module_group_id],
         semestercode: targetSemester.semestercode,
         semester_composite: targetSemester.composite,
         isStudyModule: a.type === 'ModuleAttainment',