cleanup

SatcherInstitute · Dec 27, 2024 · 5a2d5ee · 5a2d5ee
1 parent 1b32f67
commit 5a2d5ee
Show file tree

Hide file tree

Showing 10 changed files with 141 additions and 196 deletions.
diff --git a/frontend/playwright-tests/externalUrls.spec.ts b/frontend/playwright-tests/externalUrls.spec.ts
@@ -40,15 +40,15 @@ test(`Fetch First 100 Blog Posts`, async ({ page }) => {
     'https://hetblog.dreamhosters.com/wp-json/wp/v2/posts?_embed&per_page=100'
   const response = await page.goto(url, { waitUntil: 'domcontentloaded' })
   if (response?.status() !== 200)
-    console.log('\n🙀', url, response?.status(), '\n')
+    console.error('\n🙀', url, response?.status(), '\n')
 })
 
 for (const url of Object.values(urlMap)) {
   if (!url || knownFlakyUrls.includes(url)) continue
   test(`${url}`, async ({ page }) => {
     const response = await page.goto(url, { waitUntil: 'domcontentloaded' })
     if (response?.status() !== 200)
-      console.log('\n🙀', url, response?.status(), '\n')
+      console.error('\n🙀', url, response?.status(), '\n')
   })
 }
 
@@ -59,7 +59,7 @@ for (const url of RESOURCES.resources
   test(`Resource Page: ${url}`, async ({ page }) => {
     const response = await page.goto(url)
     if (response?.status() !== 200)
-      console.log('\n🙀', url, response?.status(), '\n')
+      console.error('\n🙀', url, response?.status(), '\n')
   })
 }
 
@@ -70,7 +70,7 @@ for (const url of PDOH_RESOURCES.resources
   test(`PDOH_RESOURCES Page: ${url}`, async ({ page }) => {
     const response = await page.goto(url)
     if (response?.status() !== 200)
-      console.log('\n🙀', url, response?.status(), '\n')
+      console.error('\n🙀', url, response?.status(), '\n')
   })
 }
 
@@ -81,7 +81,7 @@ for (const url of EQUITY_INDEX_RESOURCES.resources
   test(`EQUITY_INDEX_RESOURCES Page: ${url}`, async ({ page }) => {
     const response = await page.goto(url)
     if (response?.status() !== 200)
-      console.log('\n🙀', url, response?.status(), '\n')
+      console.error('\n🙀', url, response?.status(), '\n')
   })
 }
 
@@ -92,7 +92,7 @@ for (const url of ECONOMIC_EQUITY_RESOURCES.resources
   test(`ECONOMIC_EQUITY_RESOURCES Page: ${url}`, async ({ page }) => {
     const response = await page.goto(url)
     if (response?.status() !== 200)
-      console.log('\n🙀', url, response?.status(), '\n')
+      console.error('\n🙀', url, response?.status(), '\n')
   })
 }
 
@@ -103,7 +103,7 @@ for (const url of AIAN_RESOURCES.resources
   test(`AIAN_RESOURCES Page: ${url}`, async ({ page }) => {
     const response = await page.goto(url)
     if (response?.status() !== 200)
-      console.log('\n🙀', url, response?.status(), '\n')
+      console.error('\n🙀', url, response?.status(), '\n')
   })
 }
 
@@ -114,7 +114,7 @@ for (const url of API_RESOURCES.resources
   test(`API_RESOURCES Page: ${url}`, async ({ page }) => {
     const response = await page.goto(url)
     if (response?.status() !== 200)
-      console.log('\n🙀', url, response?.status(), '\n')
+      console.error('\n🙀', url, response?.status(), '\n')
   })
 }
 
@@ -125,7 +125,7 @@ for (const url of HISP_RESOURCES.resources
   test(`HISP_RESOURCES Page: ${url}`, async ({ page }) => {
     const response = await page.goto(url)
     if (response?.status() !== 200)
-      console.log('\n🙀', url, response?.status(), '\n')
+      console.error('\n🙀', url, response?.status(), '\n')
   })
 }
 
@@ -136,7 +136,7 @@ for (const url of MENTAL_HEALTH_RESOURCES.resources
   test(`MENTAL_HEALTH_RESOURCES Page: ${url}`, async ({ page }) => {
     const response = await page.goto(url)
     if (response?.status() !== 200)
-      console.log('\n🙀', url, response?.status(), '\n')
+      console.error('\n🙀', url, response?.status(), '\n')
   })
 }
 for (const url of COVID_RESOURCES.resources
@@ -146,7 +146,7 @@ for (const url of COVID_RESOURCES.resources
   test(`COVID_RESOURCES Page: ${url}`, async ({ page }) => {
     const response = await page.goto(url)
     if (response?.status() !== 200)
-      console.log('\n🙀', url, response?.status(), '\n')
+      console.error('\n🙀', url, response?.status(), '\n')
   })
 }
 for (const url of COVID_VACCINATION_RESOURCES.resources
@@ -156,6 +156,6 @@ for (const url of COVID_VACCINATION_RESOURCES.resources
   test(`COVID_VACCINATION_RESOURCES Page: ${url}`, async ({ page }) => {
     const response = await page.goto(url)
     if (response?.status() !== 200)
-      console.log('\n🙀', url, response?.status(), '\n')
+      console.error('\n🙀', url, response?.status(), '\n')
   })
 }
diff --git a/frontend/playwright-tests/landing_page.ci.spec.ts b/frontend/playwright-tests/landing_page.ci.spec.ts
@@ -1,6 +1,6 @@
 import { test } from '@playwright/test'
 
 test(`Site Loads`, async ({ page, baseURL }) => {
-  console.log(`Running tests against: ${baseURL}`)
+  console.info(`Running tests against: ${baseURL}`)
   await page.goto('/', { waitUntil: 'commit' })
 })
diff --git a/frontend/src/ErrorBoundaryDropParams.jsx b/frontend/src/ErrorBoundaryDropParams.jsx
@@ -12,7 +12,7 @@ export default class ErrorBoundary extends React.Component {
   }
 
   componentDidCatch(error, info) {
-    console.log(error, info.componentStack)
+    console.error(error, info.componentStack)
   }
 
   render() {

diff --git a/frontend/src/data/config/DatasetMetadata.ts b/frontend/src/data/config/DatasetMetadata.ts
@@ -191,6 +191,8 @@ export type DatasetId =
   | 'census_pop_estimates-race_and_ethnicity'
   | 'chr_data-race_and_ethnicity_county_current'
   | 'chr_data-race_and_ethnicity_county_historical'
+  | 'chr_data-alls_county_current'
+  | 'chr_data-alls_county_historical'
   | 'covid_tracking_project-cases_by_race_state'
   | 'covid_tracking_project-deaths_by_race_state'
   | 'covid_tracking_project-hospitalizations_by_race_state'
@@ -916,10 +918,22 @@ export const DatasetMetadataMap: Record<DatasetId, DatasetMetadata> = {
   },
   'chr_data-race_and_ethnicity_county_historical': {
     name: 'Prevalence of multiple chronic disease, behavioral health, and social determinants of health by county, with race/ethnicity breakdowns for some topics.',
+    original_data_sourced: '2016-2021',
+    contains_nh: true,
+    source_id: 'chr',
+  },
+  'chr_data-alls_county_current': {
+    name: 'Prevalence of multiple chronic disease, behavioral health, and social determinants of health by county.',
     original_data_sourced: '2021',
     contains_nh: true,
     source_id: 'chr',
   },
+  'chr_data-alls_county_historical': {
+    name: 'Prevalence of multiple chronic disease, behavioral health, and social determinants of health by county.',
+    original_data_sourced: '2016-2021',
+    contains_nh: true,
+    source_id: 'chr',
+  },
   the_unitedstates_project: {
     name: '@unitedstates is a shared commons of data and tools for the United States. Made by the public, used by the public. Featuring work from people with the Sunlight Foundation, GovTrack.us, the New York Times, the Electronic Frontier Foundation, and the internet.',
     original_data_sourced: '1915-2023',

diff --git a/frontend/src/data/config/MetadataMap.test.ts b/frontend/src/data/config/MetadataMap.test.ts
@@ -7,7 +7,7 @@ describe('Test Data Source URLs', () => {
   test('Links all use HTTPS', () => {
     Object.values(dataSourceMetadataMap).forEach((metadata) => {
       const testUrl = metadata.data_source_link
-      console.log(testUrl, '--')
+      console.info(testUrl, '--')
 
       expect(testUrl.slice(0, 8)).toEqual('https://')
     })

diff --git a/frontend/src/data/providers/AhrProvider.test.ts b/frontend/src/data/providers/AhrProvider.test.ts
@@ -5,6 +5,7 @@ import {
   getDataFetcher,
   resetCacheDebug,
 } from '../../utils/globals'
+import type { ScrollableHashId } from '../../utils/hooks/useStepObserver'
 import {
   type DatasetId,
   type DatasetIdWithStateFIPSCode,
@@ -21,10 +22,10 @@ async function ensureCorrectDatasetsDownloaded(
   ahrDatasetId: DatasetId,
   baseBreakdown: Breakdowns,
   demographicType: DemographicType,
+  cardId?: ScrollableHashId,
 ) {
   const ahrProvider = new AhrProvider()
   const specificId = appendFipsIfNeeded(ahrDatasetId, baseBreakdown)
-
   dataFetcher.setFakeDatasetLoaded(specificId, [])
 
   // Evaluate the response with requesting "All" field
@@ -34,9 +35,9 @@ async function ensureCorrectDatasetsDownloaded(
       baseBreakdown.addBreakdown(demographicType),
       'suicide',
       'current',
+      cardId,
     ),
   )
-
   expect(dataFetcher.getNumLoadDatasetCalls()).toBe(1)
 
   const consumedDatasetIds: Array<DatasetId | DatasetIdWithStateFIPSCode> = [
@@ -115,9 +116,10 @@ describe('AhrProvider', () => {
 
   test('County and Sex Breakdown (should just get the ALLs)', async () => {
     await ensureCorrectDatasetsDownloaded(
-      'chr_data-race_and_ethnicity_county_current',
+      'chr_data-alls_county_current',
       Breakdowns.forFips(new Fips('01001')),
       SEX,
+      'rates-over-time', // need to mock a call from a card that should get fallbacks
     )
   })
 })
diff --git a/frontend/src/data/providers/AhrProvider.ts b/frontend/src/data/providers/AhrProvider.ts
@@ -1,15 +1,14 @@
 import { getParentDropdownFromDataTypeId } from '../../utils/MadLibs'
 import { getDataManager } from '../../utils/globals'
-import type { DatasetId } from '../config/DatasetMetadata'
 import type { DropdownVarId } from '../config/DropDownIds'
 import { BEHAVIORAL_HEALTH_CATEGORY_DROPDOWNIDS } from '../config/MetricConfigBehavioralHealth'
 import type { DataTypeId, MetricId } from '../config/MetricConfigTypes'
-import type {
-  Breakdowns,
-  DemographicBreakdownKey,
-  TimeView,
-} from '../query/Breakdowns'
-import { type MetricQuery, MetricQueryResponse } from '../query/MetricQuery'
+import type { Breakdowns } from '../query/Breakdowns'
+import {
+  type MetricQuery,
+  MetricQueryResponse,
+  resolveDatasetId,
+} from '../query/MetricQuery'
 import { appendFipsIfNeeded } from '../utils/datasetutils'
 import VariableProvider from './VariableProvider'
 
@@ -139,102 +138,15 @@ class AhrProvider extends VariableProvider {
     ])
   }
 
-  getDatasetId(
-    breakdowns: Breakdowns,
-    dataTypeId?: DataTypeId,
-    timeView?: TimeView,
-  ): DatasetId | undefined {
-    const currentDropdown: DropdownVarId | undefined =
-      dataTypeId && getParentDropdownFromDataTypeId(dataTypeId)
-    const isBehavioralHealth =
-      currentDropdown &&
-      BEHAVIORAL_HEALTH_CATEGORY_DROPDOWNIDS.includes(currentDropdown as any)
-
-    if (breakdowns.geography === 'national') {
-      if (timeView === 'current') {
-        if (breakdowns.hasOnlyRace())
-          return isBehavioralHealth
-            ? 'graphql_ahr_data-behavioral_health_race_and_ethnicity_national_current'
-            : 'graphql_ahr_data-non-behavioral_health_race_and_ethnicity_national_current'
-        if (breakdowns.hasOnlySex())
-          return isBehavioralHealth
-            ? 'graphql_ahr_data-behavioral_health_sex_national_current'
-            : 'graphql_ahr_data-non-behavioral_health_sex_national_current'
-        if (breakdowns.hasOnlyAge())
-          return isBehavioralHealth
-            ? 'graphql_ahr_data-behavioral_health_age_national_current'
-            : 'graphql_ahr_data-non-behavioral_health_age_national_current'
-      }
-
-      if (timeView === 'historical') {
-        if (breakdowns.hasOnlyRace())
-          return isBehavioralHealth
-            ? 'graphql_ahr_data-behavioral_health_race_and_ethnicity_national_historical'
-            : 'graphql_ahr_data-non-behavioral_health_race_and_ethnicity_national_historical'
-        if (breakdowns.hasOnlySex())
-          return isBehavioralHealth
-            ? 'graphql_ahr_data-behavioral_health_sex_national_historical'
-            : 'graphql_ahr_data-non-behavioral_health_sex_national_historical'
-        if (breakdowns.hasOnlyAge())
-          return isBehavioralHealth
-            ? 'graphql_ahr_data-behavioral_health_age_national_historical'
-            : 'graphql_ahr_data-non-behavioral_health_age_national_historical'
-      }
-    }
-    if (breakdowns.geography === 'state') {
-      if (timeView === 'current') {
-        if (breakdowns.hasOnlyRace())
-          return isBehavioralHealth
-            ? 'graphql_ahr_data-behavioral_health_race_and_ethnicity_state_current'
-            : 'graphql_ahr_data-non-behavioral_health_race_and_ethnicity_state_current'
-        if (breakdowns.hasOnlySex())
-          return isBehavioralHealth
-            ? 'graphql_ahr_data-behavioral_health_sex_state_current'
-            : 'graphql_ahr_data-non-behavioral_health_sex_state_current'
-        if (breakdowns.hasOnlyAge())
-          return isBehavioralHealth
-            ? 'graphql_ahr_data-behavioral_health_age_state_current'
-            : 'graphql_ahr_data-non-behavioral_health_age_state_current'
-      }
-
-      if (timeView === 'historical') {
-        if (breakdowns.hasOnlyRace())
-          return isBehavioralHealth
-            ? 'graphql_ahr_data-behavioral_health_race_and_ethnicity_state_historical'
-            : 'graphql_ahr_data-non-behavioral_health_race_and_ethnicity_state_historical'
-        if (breakdowns.hasOnlySex())
-          return isBehavioralHealth
-            ? 'graphql_ahr_data-behavioral_health_sex_state_historical'
-            : 'graphql_ahr_data-non-behavioral_health_sex_state_historical'
-        if (breakdowns.hasOnlyAge())
-          return isBehavioralHealth
-            ? 'graphql_ahr_data-behavioral_health_age_state_historical'
-            : 'graphql_ahr_data-non-behavioral_health_age_state_historical'
-      }
-    }
-    // some county data is available via CHR
-    if (
-      breakdowns.geography === 'county' &&
-      dataTypeId &&
-      CHR_DATATYPE_IDS.includes(dataTypeId)
-    ) {
-      if (breakdowns.hasExactlyOneDemographic()) {
-        if (timeView === 'current') {
-          return 'chr_data-race_and_ethnicity_county_current'
-
-        }
-        if (timeView === 'historical') {
-          return 'chr_data-race_and_ethnicity_county_historical'
-        }
-      }
-    }
-  }
-
   async getDataInternal(
     metricQuery: MetricQuery,
   ): Promise<MetricQueryResponse> {
-    const { breakdowns, dataTypeId, timeView } = metricQuery
-    const datasetId = this.getDatasetId(breakdowns, dataTypeId, timeView)
+    const { isChr, categoryPrefix } = getDatasetDetails(metricQuery)
+    const { datasetId, isFallbackId, breakdowns } = resolveDatasetId(
+      isChr ? 'chr_data' : 'graphql_ahr_data',
+      isChr ? '' : categoryPrefix,
+      metricQuery,
+    )
 
     if (!datasetId) {
       return new MetricQueryResponse([], [])
@@ -248,25 +160,13 @@ class AhrProvider extends VariableProvider {
     df = this.filterByGeo(df, breakdowns)
     df = this.renameGeoColumns(df, breakdowns)
 
-    if (
-      breakdowns.geography === 'county' &&
-      dataTypeId &&
-      CHR_DATATYPE_IDS.includes(dataTypeId) &&
-      !breakdowns.hasOnlyRace()
-    ) {
-      let requestedDemographic: DemographicBreakdownKey = 'race_and_ethnicity'
-      // CHR: treat the "All" rows from by race as "All" for sex/age
-      df = df.filter((row) => row['race_and_ethnicity'] === 'All')
-      if (breakdowns.hasOnlySex()) requestedDemographic = 'sex'
-      if (breakdowns.hasOnlyAge()) requestedDemographic = 'age'
-      df = df.renameSeries({
-        race_and_ethnicity: requestedDemographic,
-      })
+    if (isFallbackId) {
+      df = this.castAllsAsRequestedDemographicBreakdown(df, breakdowns)
+    } else {
+      df = this.applyDemographicBreakdownFilters(df, breakdowns)
+      df = this.removeUnrequestedColumns(df, metricQuery)
     }
 
-    df = this.applyDemographicBreakdownFilters(df, breakdowns)
-    df = this.removeUnrequestedColumns(df, metricQuery)
-
     return new MetricQueryResponse(df.toArray(), consumedDatasetIds)
   }
 
@@ -288,3 +188,26 @@ class AhrProvider extends VariableProvider {
 }
 
 export default AhrProvider
+
+function getDatasetDetails(metricQuery: MetricQuery) {
+  const { dataTypeId, breakdowns } = metricQuery
+  if (
+    dataTypeId &&
+    CHR_DATATYPE_IDS.includes(dataTypeId) &&
+    breakdowns.geography === 'county'
+  )
+    return { isChr: true, categoryPrefix: '' }
+
+  const currentDropdown: DropdownVarId | undefined =
+    dataTypeId && getParentDropdownFromDataTypeId(dataTypeId)
+
+  const isBehavioralHealth =
+    currentDropdown &&
+    BEHAVIORAL_HEALTH_CATEGORY_DROPDOWNIDS.includes(currentDropdown as any)
+
+  const categoryPrefix = isBehavioralHealth
+    ? 'behavioral_health_'
+    : 'non-behavioral_health_'
+
+  return { isChr: false, categoryPrefix }
+}