Merge pull request #920 from UniversityOfHelsinkiCS/trunk

Tests and file persisting nats

.gitignore

@@ -23,4 +23,5 @@ services/updater_writer/updater/test_assets/meta.json
 services/updater_scheduler/all_student_numbers.txt
 services/updater_scheduler/debug.log
 services/updater_scheduler/active_student_numbers.txt
-services/oodikone2-backend/debug.log
+services/oodikone2-backend/debug.log
+datastore

cypress/integration/Course_statistics.js

@@ -0,0 +1,83 @@
+
+describe('Course Statistics tests', () => {
+  beforeEach(() => {
+    cy.server({
+      onAnyRequest: function (route, proxy) {
+        if (Cypress.config().baseUrl.includes("http://localhost:1337/")) {
+          proxy.xhr.setRequestHeader('uid', 'tktl')
+          proxy.xhr.setRequestHeader('shib-session-id', 'mock-shibboleth')
+          proxy.xhr.setRequestHeader('hygroupcn', 'grp-oodikone-users')
+          proxy.xhr.setRequestHeader('edupersonaffiliation', 'asdasd')
+        }
+      }
+    })
+    console.log(Cypress.config().baseUrl)
+    cy.visit(Cypress.config().baseUrl)
+    cy.contains("Course statistics").click()
+    cy.contains("Search for courses")
+  })
+
+  it('Searching single course having duplicate mappings shows course statistics', () => {
+    cy.contains("Fetch statistics").should('be.disabled')
+    cy.url().should('include', '/coursestatistics')
+    cy.contains("Search for courses")
+    cy.get("input[placeholder='Search by entering a course code']").type('TKT20003')
+    cy.contains("tr", "TKT20003").within(($row) => {
+      cy.get('button').click()
+    })
+    cy.contains("Fetch statistics").should('be.enabled').click()
+    cy.contains("Search for courses").should('not.exist')
+
+    cy.contains("Käyttöjärjestelmät")
+    cy.contains("TKT20003")
+    cy.contains("582640") // old mapped code
+
+    cy.contains(".tabular.menu a", "Table").click()
+    cy.contains("All")
+    cy.contains(".modeSelectorRow a", "Cumulative").click()
+    cy.contains(".modeSelectorRow a", "Student").click()
+    cy.contains(".modeSelectorRow a", "Grades").click()
+
+    cy.contains(".tabular.menu a", "Pass rate chart").click()
+    cy.get("div.modeSelectorContainer").click()
+    cy.contains("svg", "Pass rate chart")
+
+    cy.contains(".tabular.menu a", "Grade distribution chart").click()
+    cy.get("div.modeSelectorContainer").click()
+    cy.contains("svg", "Grades")
+
+    cy.contains("a", "New query").click()
+    cy.contains("Search for courses")
+  })
+
+  it('Searching multiple courses having duplicate mappings shows course statistics', () => {
+    cy.contains("Fetch statistics").should('be.disabled')
+    cy.url().should('include', '/coursestatistics')
+    cy.contains("Search for courses")
+    cy.get("input[placeholder='Search by entering a course code']").type('TKT')
+    cy.contains("tr", "TKT20003").within(($row) => {
+      cy.get('button').click()
+    })
+    cy.contains("tr", "TKT10002").within(($row) => {
+      cy.get('button').click()
+    })
+    cy.contains("Fetch statistics").should('be.enabled').click()
+    cy.contains("Search for courses").should('not.exist')
+
+    cy.contains('.courseNameCell', "Käyttöjärjestelmät").contains("TKT20003").click()
+    cy.contains('.courseNameCell', "Ohjelmoinnin perusteet").should('not.exist');
+    cy.contains("TKT20003")
+    cy.contains("582640") // old mapped code
+    cy.contains("Summary").click()
+
+    cy.contains('.courseNameCell', "Ohjelmoinnin perusteet").contains("TKT10002").click()
+    cy.contains('.courseNameCell', "Käyttöjärjestelmät").should('not.exist');
+    cy.contains("TKT10002")
+    cy.contains("581325") // old mapped code
+    cy.contains("Summary").click()
+
+    cy.contains("a", "New query").click()
+    cy.contains("Search for courses")
+  })
+
+})

cypress/integration/Population_statistics.js

@@ -102,7 +102,7 @@ describe('Population Statistics tests', () => {
     cy.contains("DIGI-000A")
     cy.go("back")
 
-   
+
 
     cy.contains("Courses of Population").parentsUntil(".ui.segment").parent().within(() => {
       cy.contains("number at least").siblings().within(() => cy.get("input").clear().type("0"))
@@ -252,6 +252,6 @@ describe('Population Statistics tests', () => {
       })
     })
     cy.get("button").contains("Delete for good").click({ force: true })
-    
+
   })
 })

cypress/integration/Studyprogramme_overview.js

@@ -15,6 +15,30 @@ describe('Studyprogramme overview', () => {
     cy.contains("Study Programme", { timeout: 100000 })
   })
 
+  it('can search for course mappings', () => {
+    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click()
+    cy.contains('Code Mapper').click()
+    cy.contains('tr', 'TKT20003 Käyttöjärjestelmät').get('input').type('582219')
+    cy.contains('tr', 'TKT20003 Käyttöjärjestelmät').get('.results').contains("Käyttöjärjestelmät (582219)")
+    cy.contains('tr', 'TKT20003 Käyttöjärjestelmät').contains('button', "Add")
+  })
+
+  it('can view course groups', () => {
+    cy.contains("Kasvatustieteiden kandiohjelma").click()
+    cy.contains('Course Groups').click()
+
+    cy.contains('tr', 'Test course group').get('i.edit').click()
+    cy.contains("Edit group")
+    cy.get('.prompt').type("Professori Pekka")
+    cy.contains("Add teacher").parent().contains("9000960")
+    cy.contains("Teachers in group").parent().contains("9000960")
+    cy.get("i.reply.link.icon").click()
+
+    cy.contains('tr a', 'Test course group').click()
+    cy.contains("Total teachers")
+    cy.get("i.reply.icon").click()
+  })
+
   it('renders progress and productivity tables', () => {
     cy.contains("Tietojenkäsittelytieteen kandiohjelma").click()
     cy.get('table').should('have.length', 2)
@@ -40,5 +64,5 @@ describe('Studyprogramme overview', () => {
     cy.contains('Thesis Courses').click()
     cy.contains('Add thesis course').click()
     cy.contains('No results')
-  })  
+  })
 })

docker-compose.lateste2e.yml

@@ -3,9 +3,11 @@ version: '3'
 services:
   nats:
     image: nats-streaming
-    command: -cid updaterNATS
+    command: -cid updaterNATS  --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore 
     expose:
       - "4222"
+    volumes:
+      - ./datastore:/datastore
     container_name: nats
 
   analytics_db:

docker-compose.yml

@@ -29,9 +29,11 @@ services:
 
   nats:
     image: nats-streaming
-    command: -cid updaterNATS
+    command: -cid updaterNATS  --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore 
     expose:
       - "4222"
+    volumes:
+      - ./datastore:/datastore
     ports:
       - "8222:8222"
       - "4222:4222"

services/backend/oodikone2-backend/src/routes/population.js

@@ -5,7 +5,7 @@ const { updateStudents } = require('../services/doo_api_database_updater/databas
 const StudyrightService = require('../services/studyrights')
 
 // POST instead of GET because of too long params and "sensitive" data
-router.post('/v2/populationstatistics/courses', async (req, res) => { 
+router.post('/v2/populationstatistics/courses', async (req, res) => {
   try {
     if (!req.body.year || !req.body.semesters || !req.body.studyRights) {
       res.status(400).json({ error: 'The body should have a year, semester and study rights defined' })
@@ -32,30 +32,33 @@ router.post('/v2/populationstatistics/courses', async (req, res) => {
 })
 
 router.get('/v3/populationstatistics', async (req, res) => {
-  console.log(req.query.year, req.query.semesters, req.query.studyRights)
+  const { year, semesters, studyRights: studyRightsJSON } = req.query
   try {
-    if (!req.query.year || !req.query.semesters || !req.query.studyRights) {
-      res.status(400).json({ error: 'The query should have a year, semester and study rights defined' })
+    if (!year || !semesters || !studyRightsJSON) {
+      res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })
       return
     }
-    if (!Array.isArray(req.query.studyRights)) { // studyRights should always be an array
-      req.query.studyRights = [req.query.studyRights]
-    }
-    req.query.studyRights = req.query.studyRights.filter(sr => sr !== 'undefined')
-    const roles = req.decodedToken.roles
-    if (!roles || !roles.map(r => r.group_code).includes('admin')) {
-      const elements = new Set(req.decodedToken.rights)
-      if (req.query.studyRights.some(code => !elements.has(code))) {
-        res.status(403).json([])
-        return
+    let studyRights = null
+    try {
+      studyRights = JSON.parse(studyRightsJSON)
+      const { roles, rights } = req.decodedToken
+      if (!roles || !roles.map(r => r.group_code).includes('admin')) {
+        if (!rights.includes(studyRights.programme)) {
+          res.status(403).json([])
+          return
+        }
       }
+    } catch(e) {
+      console.error(e)
+      res.status(400).json({ error: 'The query had invalid studyRights' })
+      return
     }
-    
+
     if (req.query.months == null) {
       req.query.months = 12
     }
 
-    const result = await Population.optimizedStatisticsOf(req.query)
+    const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights})
 
     if (result.error) {
       console.log(result.error)

services/backend/oodikone2-backend/src/services/populations.js

@@ -302,7 +302,7 @@ const parseQueryParams = query => {
     exchangeStudents,
     cancelledStudents,
     nondegreeStudents,
-    studyRights,
+    studyRights: Array.isArray(studyRights) ? studyRights : Object.values(studyRights),
     months,
     startDate,
     endDate

services/backend/oodikone2-backend/src/services/studyrights.js

@@ -333,18 +333,21 @@ const getFilteredAssociations = async (codes) => {
   console.log(codes)
   const associations = await getAssociations()
   associations.programmes = _.pick(associations.programmes, codes)
+
+  const degrees = []
+  const studyTracks = []
   Object.keys(associations.programmes).forEach(k => {
     Object.keys(associations.programmes[k].enrollmentStartYears).forEach(year => {
       const yearData = associations.programmes[k].enrollmentStartYears[year]
-      yearData.degrees = _.pick(yearData.degrees, codes)
-      yearData.studyTracks = _.pick(yearData.studyTracks, codes)
+      degrees.push(...Object.keys(yearData.degrees))
+      studyTracks.push(...Object.keys(yearData.studyTracks))
     })
   })
-  associations.degrees = _.pick(associations.degrees, codes)
+  associations.degrees = _.pick(associations.degrees, degrees)
   Object.keys(associations.degrees).forEach(k => {
     associations.degrees[k].programmes = _.pick(associations.degrees[k].programmes, codes)
   })
-  associations.studyTracks = _.pick(associations.studyTracks, codes)
+  associations.studyTracks = _.pick(associations.studyTracks, studyTracks)
   Object.keys(associations.studyTracks).forEach(k => {
     associations.studyTracks[k].programmes = _.pick(associations.studyTracks[k].programmes, codes)
   })

services/oodikone2-frontend/src/components/PopulationFilters/CanceledStudyright.jsx

@@ -1,7 +1,7 @@
 import React, { Component } from 'react'
 import { connect } from 'react-redux'
 import { Dropdown, Icon, Form, Segment, Button } from 'semantic-ui-react'
-import { shape, func, arrayOf, string } from 'prop-types'
+import { shape, func, string } from 'prop-types'
 import InfoBox from '../InfoBox'
 import infoTooltips from '../../common/InfoToolTips'
 import { canceledStudyright } from '../../populationFilters'
@@ -12,7 +12,7 @@ class CanceledStudyright extends Component {
     filter: shape({}).isRequired,
     removePopulationFilter: func.isRequired,
     setPopulationFilter: func.isRequired,
-    studyrights: arrayOf(string).isRequired
+    studyrights: shape({ programme: string, degree: string, studyTrack: string }).isRequired
   }
 
   state = {

services/oodikone2-frontend/src/components/PopulationFilters/SimpleExtentGraduated.jsx

@@ -86,7 +86,7 @@ const SimpleExtentGraduated = (props) => {
 
 
 const mapStateToProps = ({ settings, populations, populationDegreesAndProgrammes }) => {
-  const code = populations.query.studyRights[0]
+  const code = populations.query.studyRights.programme
   const studyrightName = populationDegreesAndProgrammes.data.programmes[code].name
   return { language: settings.language, programme: studyrightName, code }
 }

services/oodikone2-frontend/src/components/PopulationFilters/TransferToStudyrightFilter.jsx

@@ -91,7 +91,7 @@ class TransferToStudyrightFilter extends Component {
 }
 
 const mapStateToProps = (state) => {
-  const code = state.populations.query.studyRights[0]
+  const code = state.populations.query.studyRights.programme
   const studyrightName = state.populationDegreesAndProgrammes.data.programmes[code].name
   return ({
     language: state.settings.language,

services/oodikone2-frontend/src/components/PopulationFilters/index.jsx

@@ -1,7 +1,7 @@
 import React, { Component } from 'react'
 import { connect } from 'react-redux'
 import { Segment, Header, Button, Form, Radio, Modal, Icon, TextArea, Input } from 'semantic-ui-react'
-import { object, func, arrayOf, bool, string } from 'prop-types'
+import { object, func, arrayOf, bool, shape, string } from 'prop-types'
 import _ from 'lodash'
 import uuidv4 from 'uuid/v4'
 
@@ -71,7 +71,7 @@ class PopulationFilters extends Component {
     setComplementFilter: func.isRequired,
     savePopulationFilters: func.isRequired,
     setPopulationFilter: func.isRequired,
-    studyRights: arrayOf(string).isRequired,
+    studyRights: shape({ programme: string, degree: string, studyTrack: string }).isRequired,
     populationFilters: object.isRequired, //eslint-disable-line
     populationCourses: object.isRequired //eslint-disable-line
   }
@@ -138,7 +138,7 @@ class PopulationFilters extends Component {
       id: uuidv4(),
       name: this.state.presetName,
       description: this.state.presetDescription,
-      population: this.props.studyRights,
+      population: Object.values(this.props.studyRights),
       filters: this.props.filters
     }
     this.setState({ presetName: '', presetDescription: '' })

services/oodikone2-frontend/src/components/PopulationQueryCard/index.jsx

@@ -111,7 +111,7 @@ PopulationQueryCard.propTypes = {
   query: shape({
     year: oneOfType([string, number]),
     semester: string,
-    studyRights: arrayOf(string),
+    studyRights: shape({ programme: string, degree: string, studyTrack: string }),
     uuid: string
   }).isRequired,
   removeSampleFn: func.isRequired,

services/oodikone2-frontend/src/components/PopulationSearchForm/index.jsx

@@ -133,15 +133,14 @@ class PopulationSearchForm extends Component {
   }
 
   fetchPopulation = (query) => {
-    let queryCodes = []
-    queryCodes = [...Object.values(query.studyRights).filter(e => e != null)]
-    const backendQuery = { ...query, studyRights: queryCodes }
+    const queryCodes = Object.values(query.studyRights).filter(e => e != null)
+
     const uuid = uuidv4()
-    const request = { ...backendQuery, uuid }
+    const request = { ...query, studyRights: queryCodes, uuid }
     this.setState({ isLoading: true })
     this.props.setLoading()
     Promise.all([
-      this.props.getPopulationStatistics(request),
+      this.props.getPopulationStatistics({ ...query, uuid }),
       this.props.getPopulationCourses(request),
       this.props.getPopulationFilters(request),
       this.props.getMandatoryCourses(query.studyRights.programme)

services/oodikone2-frontend/src/components/PopulationSearchHistory/index.jsx

@@ -43,13 +43,13 @@ class PopulationSearchHistory extends Component {
           population={populations.data}
           query={populations.query}
           queryId={0}
-          unit={units.data.programmes[populations.query.studyRights[0]]} // Possibly deprecated
+          unit={units.data.programmes[populations.query.studyRights.programme]} // Possibly deprecated
           units={
             ([
               ...Object.values(units.data.programmes),
               ...Object.values(units.data.degrees),
               ...Object.values(units.data.studyTracks)
-            ]).filter(u => populations.query.studyRights.includes(u.code))
+            ]).filter(u => Object.values(populations.query.studyRights).includes(u.code))
           }
           removeSampleFn={this.removePopulation}
           updateStudentsFn={() => this.props.updatePopulationStudents(studentNumberList)}

services/oodikone2-frontend/src/components/PopulationStudents/index.jsx

@@ -447,7 +447,7 @@ const mapStateToProps = ({ settings, populations, populationCourses, populationM
     showNames: settings.namesVisible,
     showList: settings.studentlistVisible,
     language: settings.language,
-    queryStudyrights: populations.query.studyRights,
+    queryStudyrights: Object.values(populations.query.studyRights),
     mandatoryCourses: populationMandatoryCourses.data,
     mandatoryPassed
   }