Engaging Crowds: Subject set completeness (#2016)

Adds a query which gets available subject numbers, for a given workflow, from the Cellect API.
Removes included subject sets from the workflows API query (see #678.) Subject sets are fetched separately, by ID, from the Panoptes API. This increases the number of API requests required to build each page, but allows us to limit subject set requests to grouped workflows.
Updates the API mocks, in data-fetching tests, to include grouped/non-grouped cases and to add the /subject_sets endpoint.
Refactors data-fetching into `fetchWorkflowsHelper` and `fetchSubjectSets`.
Wraps the data-fetching requests in try/catch, so that we can log API errors to Sentry.

Co-authored-by: Shaun A. Noordin <[email protected]>

packages/app-project/package.json

@@ -51,6 +51,7 @@
     "morgan": "^1.10.0",
     "newrelic": "~7.1.0",
     "next": "~9.5.5",
+    "node-fetch": "~2.6.1",
     "panoptes-client": "~3.2.1",
     "path-match": "~1.2.4",
     "polished": "~3.6.4",

packages/app-project/src/helpers/fetchSubjectSets/fetchSubjectSets.js

@@ -0,0 +1,64 @@
+import { panoptes } from '@zooniverse/panoptes-js'
+import fetch from 'node-fetch'
+
+import { logToSentry } from '@helpers/logger'
+
+async function fetchSubjectSetData(subjectSetIDs, env) {
+  let subject_sets = []
+  try {
+    const query = {
+      env,
+      id: subjectSetIDs.join(',')
+    }
+    const response = await panoptes.get('/subject_sets', query)
+    subject_sets = response.body.subject_sets
+    await Promise.allSettled(subject_sets.map(subjectSet => fetchPreviewImage(subjectSet, env)))
+  } catch (error) {
+    console.error(error)
+    logToSentry(error)
+  }
+  return subject_sets
+}
+
+async function fetchWorkflowCellectStatus(workflow) {
+  let groups = {}
+  if (workflow.grouped) {
+    try {
+      const workflowURL = `https://cellect.zooniverse.org/workflows/${workflow.id}/status`
+      const response = await fetch(workflowURL)
+      const body = await response.json()
+      groups = body.groups ?? {}
+    } catch (error) {
+      console.error(error)
+      logToSentry(error)
+    }
+  }
+  return groups
+}
+
+async function fetchPreviewImage (subjectSet, env) {
+  try {
+    const response = await panoptes
+      .get('/set_member_subjects', {
+        env,
+        subject_set_id: subjectSet.id,
+        include: 'subject',
+        page_size: 1
+      })
+    const { linked } = response.body
+    subjectSet.subjects = linked.subjects
+  } catch (error) {
+    console.error(error)
+    logToSentry(error)
+  }
+}
+
+export default async function fetchSubjectSets(workflow, env) {
+  const subjectSetCounts = await fetchWorkflowCellectStatus(workflow)
+  const subjectSetIDs = Object.keys(subjectSetCounts)
+  const subjectSets = await fetchSubjectSetData(subjectSetIDs, env)
+  subjectSets.forEach(subjectSet => {
+    subjectSet.availableSubjects = subjectSetCounts[subjectSet.id]
+  })
+  return subjectSets
+}

packages/app-project/src/helpers/fetchSubjectSets/index.js

@@ -0,0 +1 @@
+export { default } from './fetchSubjectSets'

packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js

@@ -1,67 +1,72 @@
 import { panoptes } from '@zooniverse/panoptes-js'
+import fetch from 'node-fetch'
+
+import { logToSentry } from '@helpers/logger'
+import fetchSubjectSets from '@helpers/fetchSubjectSets'
 
 async function fetchWorkflowData (activeWorkflows, env) {
-  const response = await panoptes
-    .get('/workflows', {
+  try {
+    const query = {
       complete: false,
       env,
-      fields: 'completeness,grouped',
-      id: activeWorkflows.join(','),
-      include: 'subject_sets'
-    })
-  const { workflows, linked } = response.body
-  const subjectSets = linked ? linked.subject_sets : []
-  await Promise.allSettled(subjectSets.map(subjectSet => fetchPreviewImage(subjectSet, env)))
-  return { subjectSets, workflows }
+      fields: 'completeness,display_name,grouped',
+      id: activeWorkflows.join(',')
+    }
+    const response = await panoptes.get('/workflows', query)
+    return response.body.workflows
+  } catch (error) {
+    logToSentry(error)
+    throw error
+  }
 }
-
-function fetchDisplayNames (language, activeWorkflows, env) {
-  return panoptes
-    .get('/translations', {
+async function fetchDisplayNames (language, activeWorkflows, env) {
+  let displayNames = {}
+  try {
+    const response = await panoptes.get('/translations', {
       env,
       fields: 'strings,translated_id',
       language,
       'translated_id': activeWorkflows.join(','),
       'translated_type': 'workflow'
     })
-    .then(response => response.body.translations)
-    .then(createDisplayNamesMap)
+    const { translations } = response.body
+    displayNames = createDisplayNamesMap(translations)
+  } catch (error) {
+    logToSentry(error)
+    throw error
+  }
+  return displayNames
 }
 
-async function fetchPreviewImage (subjectSet, env) {
-  const response = await panoptes
-    .get('/set_member_subjects', {
-      env,
-      subject_set_id: subjectSet.id,
-      include: 'subject',
-      page_size: 1
-    })
-  const { linked } = response.body
-  subjectSet.subjects = linked.subjects
+async function buildWorkflow(workflow, displayName, isDefault, env) {
+  const workflowData = {
+    completeness: workflow.completeness || 0,
+    default: isDefault,
+    displayName,
+    grouped: workflow.grouped,
+    id: workflow.id,
+    subjectSets: []
+  }
+  if (workflow.grouped) {
+    workflowData.subjectSets = await fetchSubjectSets(workflow, env)
+  }
+
+  return workflowData
 }
 
 async function fetchWorkflowsHelper (language = 'en', activeWorkflows, defaultWorkflow, env) {
-  const { subjectSets, workflows } = await fetchWorkflowData(activeWorkflows, env)
+  const workflows = await fetchWorkflowData(activeWorkflows, env)
   const workflowIds = workflows.map(workflow => workflow.id)
   const displayNames = await fetchDisplayNames(language, workflowIds, env)
 
-  return workflows.map(workflow => {
+  const awaitWorkflows = workflows.map(workflow => {
     const isDefault = workflows.length === 1 || workflow.id === defaultWorkflow
-    const workflowSubjectSets = workflow.links.subject_sets
-      .map(subjectSetID => {
-        return subjectSets.find(subjectSet => subjectSet.id === subjectSetID)
-      })
-      .filter(Boolean)
-
-    return {
-      completeness: workflow.completeness || 0,
-      default: isDefault,
-      displayName: displayNames[workflow.id],
-      grouped: workflow.grouped,
-      id: workflow.id,
-      subjectSets: workflowSubjectSets
-    }
+    const displayName = displayNames[workflow.id] || workflow.display_name
+    return buildWorkflow(workflow, displayName, isDefault, env)
   })
+  const workflowStatuses = await Promise.allSettled(awaitWorkflows)
+  const workflowsWithSubjectSets = workflowStatuses.map(result => result.value || result.reason)
+  return workflowsWithSubjectSets
 }
 
 function createDisplayNamesMap (translations) {