Engaging Crowds: switch to Datasette for subject browse & search (#2132)

* Add subjectSet.isIndexed

* use Datasette to get subjects

* use Datasette query params

* Remove old demo values

* Add tests

* Encode URL query params
Encode URL query params and add a test case with reserved characters.

packages/app-project/src/helpers/fetchSubjectSets/fetchSubjectSets.js

@@ -13,6 +13,7 @@ async function fetchSubjectSetData(subjectSetIDs, env) {
     const response = await panoptes.get('/subject_sets', query)
     subject_sets = response.body.subject_sets
     await Promise.allSettled(subject_sets.map(subjectSet => fetchPreviewImage(subjectSet, env)))
+    await Promise.allSettled(subject_sets.map(subjectSet => hasIndexedSubjects(subjectSet)))
   } catch (error) {
     console.error(error)
     logToSentry(error)
@@ -53,6 +54,12 @@ async function fetchPreviewImage (subjectSet, env) {
   }
 }
 
+async function hasIndexedSubjects(subjectSet) {
+  const response = await fetch(`https://subject-set-search-api.zooniverse.org/subjects/${subjectSet.id}.json`)
+  const data = await response.json()
+  subjectSet.isIndexed = !!data.rows
+}
+
 export default async function fetchSubjectSets(workflow, env) {
   const subjectSetCounts = await fetchWorkflowCellectStatus(workflow)
   let subjectSetIDs = Object.keys(subjectSetCounts)

packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.spec.js

@@ -48,6 +48,7 @@ describe('Helpers > fetchWorkflowsHelper', function () {
     return {
       id,
       display_name: `test set ${id}`,
+      isIndexed: false,
       set_member_subjects_count: 10,
       subjects: mockSetSubjects
     }

packages/app-project/src/helpers/getDefaultPageProps/getDefaultPageProps.spec.js

@@ -65,6 +65,7 @@ describe('Helpers > getDefaultPageProps', function () {
     return {
       id,
       display_name: `test set ${id}`,
+      isIndexed: false,
       set_member_subjects_count: 10,
       subjects: mockSetSubjects
     }

packages/app-project/src/helpers/getStaticPageProps/getStaticPageProps.spec.js

@@ -66,6 +66,7 @@ describe('Helpers > getStaticPageProps', function () {
     return {
       id,
       display_name: `test set ${id}`,
+      isIndexed: false,
       set_member_subjects_count: 10,
       subjects: mockSetSubjects
     }

packages/app-project/src/screens/ClassifyPage/components/WorkflowMenu/WorkflowMenu.js

@@ -22,8 +22,7 @@ export default function WorkflowMenu({
   const [ activeWorkflow, setActiveWorkflow ] = useState(workflowFromUrl)
 
   function onSelectSubjectSet(event, subjectSet) {
-    const useSubjectSelection = activeWorkflow.id === '16106'
-    if (useSubjectSelection) {
+    if (subjectSet.isIndexed) {
       event.preventDefault()
       setActiveSubjectSet(subjectSet)
       return false
@@ -42,6 +41,7 @@ export default function WorkflowMenu({
 
   function onClose() {
     setActiveWorkflow(null)
+    setActiveSubjectSet(null)
   }
 
   let modalContent = (

packages/app-project/src/screens/ProjectHomePage/components/Hero/components/WorkflowMenu/WorkflowMenu.js

@@ -27,8 +27,7 @@ export default function WorkflowMenu({ workflows }) {
   }
 
   function onSelectSubjectSet(event, subjectSet) {
-    const useSubjectSelection = activeWorkflow.id === '16106'
-    if (useSubjectSelection) {
+    if (subjectSet.isIndexed) {
       event.preventDefault()
       setActiveSubjectSet(subjectSet)
       return false
@@ -38,6 +37,7 @@ export default function WorkflowMenu({ workflows }) {
 
   function onClose() {
     setActiveWorkflow(null)
+    setActiveSubjectSet(null)
   }
 
   let baseUrl = `/projects/${owner}/${project}/classify`

packages/app-project/src/shared/components/SubjectPicker/SubjectPicker.js

@@ -18,14 +18,14 @@ counterpart.registerTranslations('en', en)
   https://matthiasott.com/notes/the-thing-with-leading-in-css
 */
 
-const StyledBox = styled(Box)`
+export const StyledBox = styled(Box)`
   min-height: 30px;
 `
-const StyledHeading = styled(Heading)`
+export const StyledHeading = styled(Heading)`
   line-height: 100%;
 `
 
-const SubjectDataTable = styled(DataTable)`
+export const SubjectDataTable = styled(DataTable)`
   button {
     padding: 0;
   }
@@ -44,13 +44,13 @@ const PAGE_SIZE = 100
 export default function SubjectPicker({ baseUrl, subjectSet, workflow }) {
   const [ rows, setRows ] = useState([])
   const [ query, setQuery ] = useState('')
-  const [ sortField, setSortField ] = useState('subject_id')
+  const [ sortField, setSortField ] = useState('priority')
   const [ sortOrder, setSortOrder ] = useState('asc')
   const { indexFields } = subjectSet.metadata
-  const customHeaders = indexFields ? indexFields.split(',') : ['date', 'title', 'creators']
+  const customHeaders = indexFields.split(',')
 
   async function fetchSubjectData() {
-    const subjects = await fetchSubjects('15582', query, sortField, sortOrder, PAGE_SIZE)
+    const subjects = await fetchSubjects(subjectSet.id, query, sortField, sortOrder, PAGE_SIZE)
     const rows = await fetchRows(subjects, workflow, PAGE_SIZE)
     setRows(rows)
   }

packages/app-project/src/shared/components/SubjectPicker/SubjectPicker.spec.js

@@ -0,0 +1,94 @@
+import { mount, shallow } from 'enzyme'
+import { Grommet } from 'grommet'
+import nock from 'nock'
+import zooTheme from '@zooniverse/grommet-theme'
+
+import SubjectPicker, { StyledBox, SubjectDataTable } from './SubjectPicker'
+
+describe('Components > Subject Picker', function () {
+  let wrapper
+  const workflow = {
+    id: '123345'
+  }
+  const subjectSet = {
+    id: '4567',
+    display_name: 'Test subject set',
+    metadata: {
+      indexFields: 'Date,Page'
+    }
+  }
+
+  before(function () {
+    wrapper = shallow(
+      <SubjectPicker
+        baseUrl="/workflow/12345/subject-set/4567"
+        subjectSet={subjectSet}
+        workflow={workflow}
+      />
+    )
+  })
+
+  it('should render', function () {
+    expect(wrapper).to.be.ok()
+  })
+
+  it('should show the subject set name', function () {
+    const displayName = wrapper.find(StyledBox)
+    expect(displayName).to.be.ok()
+  })
+
+  describe('subject data table', function () {
+    let wrapper
+
+    before(async function () {
+      const columns = [
+        'subject_id',
+        'Page',
+        'Date'
+      ]
+      const rows = [
+        ['1', '43', '23 January 1916'],
+        ['2', '44', '24 January 1916'],
+        ['3', '45', '25 January 1916']
+      ]
+      nock('https://subject-set-search-api.zooniverse.org/subjects')
+      .get('/4567.json')
+      .query(true)
+      .reply(200, {
+        columns,
+        rows
+      })
+      nock('https://panoptes-staging.zooniverse.org/api')
+      .get('/subject_workflow_statuses')
+      .query(true)
+      .reply(200, {
+        subject_workflow_statuses: [
+          { classifications_count: 0, retired_at: null, links: { subject: '1' }},
+          { classifications_count: 3, retired_at: null, links: { subject: '2' }},
+          { classifications_count: 5, retired_at: "2018-01-30T21:09:49.396Z", links: { subject: '3' }}
+        ]
+      })
+      wrapper = mount(
+        <SubjectPicker
+          subjectSet={subjectSet}
+          workflow={workflow}
+        />,
+        { wrappingComponent: Grommet, wrappingComponentProps: { theme: zooTheme } }
+      )
+      // workaround to wait for the mock API calls to resolve
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    })
+
+    it('should have column headings, including indexed subject fields', function () {
+      const dataTable = wrapper.find(SubjectDataTable)
+      const headers = dataTable.prop('columns').map(column => column.header)
+      expect(headers).to.deep.equal(['subject_id', 'Date', 'Page', 'status'])
+    })
+
+    it('should have a row for each subject', function () {
+      wrapper.update()
+      const dataTable = wrapper.find(SubjectDataTable)
+      expect(dataTable.prop('data').length).to.equal(3)
+    })
+  })
+})

packages/app-project/src/shared/components/SubjectPicker/helpers/checkRetiredStatus.js

@@ -2,7 +2,7 @@ import { env, panoptes } from '@zooniverse/panoptes-js'
 import { logToSentry } from '@helpers/logger'
 
 export default async function checkRetiredStatus(subject_ids, workflow, page_size=20) {
-  const workflow_id = '5329'
+  const workflow_id = workflow.id
   const retirementStatuses = {}
   try {
     const response = await panoptes

packages/app-project/src/shared/components/SubjectPicker/helpers/checkRetiredStatus.spec.js

@@ -0,0 +1,36 @@
+import nock from 'nock'
+import checkRetiredStatus from './checkRetiredStatus'
+
+describe('Components > Subject Picker > helpers > checkRetiredStatus', function () {
+  let retirementStatuses
+
+  before(async function () {
+    const subject_ids = ['1', '2', '3']
+    const workflow = {
+      id: '1'
+    }
+    const panoptes = nock('https://panoptes-staging.zooniverse.org/api')
+    .get('/subject_workflow_statuses')
+    .query(true)
+    .reply(200, {
+      subject_workflow_statuses: [
+        { classifications_count: 0, retired_at: null, links: { subject: '1' }},
+        { classifications_count: 3, retired_at: null, links: { subject: '2' }},
+        { classifications_count: 5, retired_at: "2018-01-30T21:09:49.396Z", links: { subject: '3' }}
+      ]
+    })
+    retirementStatuses = await checkRetiredStatus(subject_ids, workflow)
+  })
+
+  it('should set the status of unclassified subjects', function () {
+    expect(retirementStatuses['1']).to.equal('Unclassified')
+  })
+
+  it('should set the status of classified subjects', function () {
+    expect(retirementStatuses['2']).to.equal('In progress')
+  })
+
+  it('should set the status of retired subjects', function () {
+    expect(retirementStatuses['3']).to.equal('Retired')
+  })
+})

packages/app-project/src/shared/components/SubjectPicker/helpers/fetchRows.js

@@ -1,15 +1,17 @@
 import { checkRetiredStatus } from './'
 
 export default async function fetchRows(subjects, workflow, page_size) {
-  const subject_ids = subjects.map(subject => subject.subject_id).join(',')
+  const { columns, rows } = subjects
+  const IDColumn = columns.indexOf('subject_id')
+  const subject_ids = rows.map(row => row[IDColumn]).join(',')
   const retirementStatuses = await checkRetiredStatus(subject_ids, workflow, page_size)
-  const rows = subjects.map(subject => {
-    const { id, subject_id, ...fields } = subject
-    return {
-      subject_id,
-      status: retirementStatuses[subject_id],
-      ...fields
-    }
+  const data = rows.map(row => {
+    const subject = {}
+    columns.forEach((column, index) => {
+      subject[column] = row[index]
+    })
+    subject.status = retirementStatuses[subject.subject_id]
+    return subject
   })
-  return rows
+  return data
 }

packages/app-project/src/shared/components/SubjectPicker/helpers/fetchRows.spec.js

@@ -0,0 +1,42 @@
+import nock from 'nock'
+import fetchRows from './fetchRows'
+
+describe('Components > Subject Picker > helpers > fetchRows', function () {
+  let data
+  const expectedData = [
+    { subject_id: '1', Page: '43', Date: '23 January 1916', status: 'Unclassified' },
+    { subject_id: '2', Page: '44', Date: '24 January 1916', status: 'In progress' },
+    { subject_id: '3', Page: '45', Date: '25 January 1916', status: 'Retired' },
+  ]
+
+  before(async function () {
+    const columns = [
+      'subject_id',
+      'Page',
+      'Date'
+    ]
+    const rows = [
+      ['1', '43', '23 January 1916'],
+      ['2', '44', '24 January 1916'],
+      ['3', '45', '25 January 1916']
+    ]
+    const workflow = {
+      id: '1'
+    }
+    const panoptes = nock('https://panoptes-staging.zooniverse.org/api')
+    .get('/subject_workflow_statuses')
+    .query(true)
+    .reply(200, {
+      subject_workflow_statuses: [
+        { classifications_count: 0, retired_at: null, links: { subject: '1' }},
+        { classifications_count: 3, retired_at: null, links: { subject: '2' }},
+        { classifications_count: 5, retired_at: "2018-01-30T21:09:49.396Z", links: { subject: '3' }}
+      ]
+    })
+    data = await fetchRows({ columns, rows }, workflow)
+  })
+
+  it('should generate subject data table rows with classification statuses', function () {
+    expect(data).to.deep.equal(expectedData)
+  })
+})

packages/app-project/src/shared/components/SubjectPicker/helpers/fetchSubjects.js

@@ -1,15 +1,14 @@
-const API_HOST = 'https://redsea.zooniverse.org/search'
+const API_HOST = 'https://subject-set-search-api.zooniverse.org/subjects'
 
 export default async function fetchSubjects(
   subjectSetID,
   query='',
-  sortField='subject_id',
-  sortOrder='asc',
+  sortField='priority',
   page_size=20
 ) {
-  const url = `${API_HOST}/${subjectSetID}?${query}&limit=${page_size}&sort_field=${sortField}&sort_order=${sortOrder}`
+  const url = `${API_HOST}/${subjectSetID}.json?${query}&_sort=${sortField}`
   const mode = 'cors'
   const response = await fetch(url, { mode })
-  const results = await response.json()
-  return results
+  const data = await response.json()
+  return data
 }

packages/app-project/src/shared/components/SubjectPicker/helpers/searchParams.js

@@ -1,10 +1,10 @@
 export default function searchParams(data) {
-  let query = ''
+  let query = []
   Object.entries(data).forEach(([key, value]) => {
     if (value !== '') {
-      query += `@${key}:${value}*`
+      query.push(`${key}__contains=${encodeURIComponent(value)}`)
     }
   })
-  const urlParams = (query !== '') ? `filter_field=${query}` : ''
+  const urlParams = query.join('&')
   return urlParams
 }

packages/app-project/src/shared/components/SubjectPicker/helpers/searchParams.spec.js

@@ -2,14 +2,14 @@ import searchParams from './searchParams'
 
 describe('Components > Subject Picker > helpers > searchParams', function () {
   describe('with data', function () {
-    it('should generate a filter_field query param', function () {
+    it('should generate Datasette query params', function () {
       const data = {
-        creators: 'Smith',
+        creators: 'Smith & Jones',
         date: '',
         title: 'boston'
       }
       const query = searchParams(data)
-      expect(query).to.equal('filter_field=@creators:Smith*@title:boston*')
+      expect(query).to.equal('creators__contains=Smith%20%26%20Jones&title__contains=boston')
     })
   })