partial update of providers. not ready to merge; backend needs lots o…

…f updates first

frontend/src/data/providers/AcsConditionProvider.ts

@@ -1,10 +1,13 @@
 import { getDataManager } from '../../utils/globals'
-import type { TimeView, Breakdowns } from '../query/Breakdowns'
-import { type MetricQuery, MetricQueryResponse } from '../query/MetricQuery'
-import VariableProvider from './VariableProvider'
-import { appendFipsIfNeeded } from '../utils/datasetutils'
 import type { DataTypeId, MetricId } from '../config/MetricConfigTypes'
-import type { DatasetId } from '../config/DatasetMetadata'
+import type { Breakdowns } from '../query/Breakdowns'
+import {
+  type MetricQuery,
+  MetricQueryResponse,
+  resolveDatasetId,
+} from '../query/MetricQuery'
+import { appendFipsIfNeeded } from '../utils/datasetutils'
+import VariableProvider from './VariableProvider'
 
 export const ACS_CONDITION_DATATYPES: DataTypeId[] = [
   'health_insurance',
@@ -31,87 +34,30 @@ class AcsConditionProvider extends VariableProvider {
     super('acs_condition_provider', ACS_CONDITION_METRICS)
   }
 
-  getDatasetId(
-    breakdowns: Breakdowns,
-    _dataTypeId?: DataTypeId,
-    timeView?: TimeView,
-  ): DatasetId | undefined {
-    if (timeView === 'historical') {
-      if (breakdowns.geography === 'national') {
-        if (breakdowns.hasOnlyRace())
-          return 'acs_condition-by_race_national_historical'
-        if (breakdowns.hasOnlyAge())
-          return 'acs_condition-by_age_national_historical'
-        if (breakdowns.hasOnlySex())
-          return 'acs_condition-by_sex_national_historical'
-      }
-      if (breakdowns.geography === 'state') {
-        if (breakdowns.hasOnlyRace())
-          return 'acs_condition-by_race_state_historical'
-        if (breakdowns.hasOnlyAge())
-          return 'acs_condition-by_age_state_historical'
-        if (breakdowns.hasOnlySex())
-          return 'acs_condition-by_sex_state_historical'
-      }
-
-      if (breakdowns.geography === 'county') {
-        if (breakdowns.hasOnlyRace())
-          return 'acs_condition-by_race_county_historical'
-        if (breakdowns.hasOnlyAge())
-          return 'acs_condition-by_age_county_historical'
-        if (breakdowns.hasOnlySex())
-          return 'acs_condition-by_sex_county_historical'
-      }
-    }
-
-    if (timeView === 'current') {
-      if (breakdowns.geography === 'national') {
-        if (breakdowns.hasOnlyRace())
-          return 'acs_condition-by_race_national_current'
-        if (breakdowns.hasOnlyAge())
-          return 'acs_condition-by_age_national_current'
-        if (breakdowns.hasOnlySex())
-          return 'acs_condition-by_sex_national_current'
-      }
-      if (breakdowns.geography === 'state') {
-        if (breakdowns.hasOnlyRace())
-          return 'acs_condition-by_race_state_current'
-        if (breakdowns.hasOnlyAge()) return 'acs_condition-by_age_state_current'
-        if (breakdowns.hasOnlySex()) return 'acs_condition-by_sex_state_current'
-      }
-
-      if (breakdowns.geography === 'county') {
-        if (breakdowns.hasOnlyRace())
-          return 'acs_condition-by_race_county_current'
-        if (breakdowns.hasOnlyAge())
-          return 'acs_condition-by_age_county_current'
-        if (breakdowns.hasOnlySex())
-          return 'acs_condition-by_sex_county_current'
-      }
-    }
-  }
-
   async getDataInternal(
     metricQuery: MetricQuery,
   ): Promise<MetricQueryResponse> {
-    const breakdowns = metricQuery.breakdowns
-    const timeView = metricQuery.timeView
-    const datasetId = this.getDatasetId(breakdowns, undefined, timeView)
+    const { breakdowns, datasetId, isFallbackId } = resolveDatasetId(
+      'acs_condition',
+      'by_',
+      metricQuery,
+    )
     if (!datasetId) {
       return new MetricQueryResponse([], [])
     }
     const specificDatasetId = appendFipsIfNeeded(datasetId, breakdowns)
     const acsDataset = await getDataManager().loadDataset(specificDatasetId)
 
     let df = acsDataset.toDataFrame()
-
-    // If requested, filter geography by state or county level
-    // We apply the geo filter right away to reduce subsequent calculation times
     df = this.filterByGeo(df, breakdowns)
     df = this.renameGeoColumns(df, breakdowns)
 
-    df = this.applyDemographicBreakdownFilters(df, breakdowns)
-    df = this.removeUnrequestedColumns(df, metricQuery)
+    if (isFallbackId) {
+      df = this.castAllsAsRequestedDemographicBreakdown(df, breakdowns)
+    } else {
+      df = this.applyDemographicBreakdownFilters(df, breakdowns)
+      df = this.removeUnrequestedColumns(df, metricQuery)
+    }
 
     return new MetricQueryResponse(df.toArray(), [datasetId])
   }

frontend/src/data/providers/AcsPopulationProvider.ts

@@ -1,10 +1,10 @@
 import type { IDataFrame } from 'data-forge'
 import { getDataManager } from '../../utils/globals'
+import type { DatasetId } from '../config/DatasetMetadata'
 import type { Breakdowns } from '../query/Breakdowns'
 import { type MetricQuery, MetricQueryResponse } from '../query/MetricQuery'
 import { appendFipsIfNeeded } from '../utils/datasetutils'
 import VariableProvider from './VariableProvider'
-import type { DatasetId } from '../config/DatasetMetadata'
 
 export function GetAcsDatasetId(breakdowns: Breakdowns): DatasetId | undefined {
   if (breakdowns.geography === 'national') {
@@ -44,7 +44,6 @@ class AcsPopulationProvider extends VariableProvider {
 
     df = this.applyDemographicBreakdownFilters(df, breakdowns)
     df = this.removeUnrequestedColumns(df, metricQuery)
-
     const datasetId = this.getDatasetId(breakdowns)
     let consumedDatasetIds
     if (datasetId) consumedDatasetIds = [datasetId]

frontend/src/data/providers/AhrProvider.ts

@@ -1,17 +1,17 @@
 import { getDataManager } from '../../utils/globals'
 import { getParentDropdownFromDataTypeId } from '../../utils/MadLibs'
 import type { DatasetId } from '../config/DatasetMetadata'
-import type { DataTypeId, MetricId } from '../config/MetricConfigTypes'
+import type { DropdownVarId } from '../config/DropDownIds'
 import { BEHAVIORAL_HEALTH_CATEGORY_DROPDOWNIDS } from '../config/MetricConfigBehavioralHealth'
+import type { DataTypeId, MetricId } from '../config/MetricConfigTypes'
 import type {
-  DemographicBreakdownKey,
   Breakdowns,
+  DemographicBreakdownKey,
   TimeView,
 } from '../query/Breakdowns'
 import { type MetricQuery, MetricQueryResponse } from '../query/MetricQuery'
 import { appendFipsIfNeeded } from '../utils/datasetutils'
 import VariableProvider from './VariableProvider'
-import type { DropdownVarId } from '../config/DropDownIds'
 
 const CHR_DATATYPE_IDS_ONLY_ALLS: DataTypeId[] = [
   'diabetes',
@@ -257,8 +257,12 @@ class AhrProvider extends VariableProvider {
       })
     }
 
-    df = this.applyDemographicBreakdownFilters(df, breakdowns)
-    df = this.removeUnrequestedColumns(df, metricQuery)
+    if (isFallbackId) {
+      df = this.castAllsAsRequestedDemographicBreakdown(df, breakdowns)
+    } else {
+      df = this.applyDemographicBreakdownFilters(df, breakdowns)
+      df = this.removeUnrequestedColumns(df, metricQuery)
+    }
 
     return new MetricQueryResponse(df.toArray(), consumedDatasetIds)
   }

frontend/src/data/providers/CawpProvider.ts

@@ -1,24 +1,27 @@
 import { getDataManager } from '../../utils/globals'
-import type { MetricId, DataTypeId } from '../config/MetricConfigTypes'
+import type { DataTypeId, MetricId } from '../config/MetricConfigTypes'
 
-import type { TimeView, Breakdowns } from '../query/Breakdowns'
-import { type MetricQuery, MetricQueryResponse } from '../query/MetricQuery'
-import { GetAcsDatasetId } from './AcsPopulationProvider'
-import VariableProvider from './VariableProvider'
+import type { Breakdowns } from '../query/Breakdowns'
 import {
-  UNKNOWN_RACE,
+  type MetricQuery,
+  MetricQueryResponse,
+  resolveDatasetId,
+} from '../query/MetricQuery'
+import {
+  type RaceAndEthnicityGroup,
+  HISP_W,
   HISPANIC,
   MULTI,
-  UNREPRESENTED,
-  type RaceAndEthnicityGroup,
-  OTHER_W,
   MULTI_W,
   OTHER_STANDARD,
+  OTHER_W,
+  UNKNOWN_RACE,
   UNKNOWN_W,
-  HISP_W,
+  UNREPRESENTED,
 } from '../utils/Constants'
-import type { DatasetId } from '../config/DatasetMetadata'
 import { appendFipsIfNeeded } from '../utils/datasetutils'
+import { GetAcsDatasetId } from './AcsPopulationProvider'
+import VariableProvider from './VariableProvider'
 
 export const CAWP_CONGRESS_COUNTS: MetricId[] = [
   'women_this_race_us_congress_count',
@@ -76,41 +79,21 @@ class CawpProvider extends VariableProvider {
     super('cawp_provider', CAWP_METRICS)
   }
 
-  getDatasetId(
-    breakdowns: Breakdowns,
-    dataTypeId?: DataTypeId,
-    timeView?: TimeView,
-  ): DatasetId | undefined {
-    if (timeView === 'current') {
-      if (breakdowns.geography === 'national' && breakdowns.hasOnlyRace())
-        return 'cawp_time_data-race_and_ethnicity_national_current'
-      if (breakdowns.geography === 'state' && breakdowns.hasOnlyRace())
-        return 'cawp_time_data-race_and_ethnicity_state_current'
-    }
-    if (timeView === 'historical') {
-      if (breakdowns.geography === 'national' && breakdowns.hasOnlyRace())
-        return 'cawp_time_data-race_and_ethnicity_national_historical'
-      if (breakdowns.geography === 'state' && breakdowns.hasOnlyRace())
-        return 'cawp_time_data-race_and_ethnicity_state_historical'
-    }
-  }
-
   async getDataInternal(
     metricQuery: MetricQuery,
   ): Promise<MetricQueryResponse> {
-    const breakdowns = metricQuery.breakdowns
-    const timeView = metricQuery.timeView
-    const datasetId = this.getDatasetId(breakdowns, undefined, timeView)
+    const { breakdowns, datasetId, isFallbackId, timeView } = resolveDatasetId(
+      'cawp_time_data',
+      '',
+      metricQuery,
+    )
     if (!datasetId) {
       return new MetricQueryResponse([], [])
     }
     const specificDatasetId = appendFipsIfNeeded(datasetId, breakdowns)
     const cawp = await getDataManager().loadDataset(specificDatasetId)
     let df = cawp.toDataFrame()
 
-    df = this.filterByGeo(df, breakdowns)
-    df = this.renameGeoColumns(df, breakdowns)
-
     const consumedDatasetIds = [datasetId]
 
     // no population numbers used for rates, only comparison pop. and pct_rel_inequity
@@ -143,8 +126,15 @@ class CawpProvider extends VariableProvider {
       consumedDatasetIds.push('the_unitedstates_project')
     }
 
-    df = this.applyDemographicBreakdownFilters(df, breakdowns)
-    df = this.removeUnrequestedColumns(df, metricQuery)
+    df = this.filterByGeo(df, breakdowns)
+    df = this.renameGeoColumns(df, breakdowns)
+
+    if (isFallbackId) {
+      df = this.castAllsAsRequestedDemographicBreakdown(df, breakdowns)
+    } else {
+      df = this.applyDemographicBreakdownFilters(df, breakdowns)
+      df = this.removeUnrequestedColumns(df, metricQuery)
+    }
 
     return new MetricQueryResponse(df.toArray(), consumedDatasetIds)
   }

frontend/src/data/providers/CdcCovidProvider.ts

@@ -1,16 +1,16 @@
+import type { IDataFrame } from 'data-forge'
 import { getDataManager } from '../../utils/globals'
-import type { Breakdowns, TimeView } from '../query/Breakdowns'
-import { type MetricQuery, MetricQueryResponse } from '../query/MetricQuery'
-import { GetAcsDatasetId } from './AcsPopulationProvider'
-import VariableProvider from './VariableProvider'
-import { appendFipsIfNeeded } from '../utils/datasetutils'
-import type { DatasetId } from '../config/DatasetMetadata'
-import type { DataTypeId } from '../config/MetricConfigTypes'
 import {
   AGE_ADJUST_COVID_DEATHS_US_SETTING,
   AGE_ADJUST_COVID_HOSP_US_SETTING,
 } from '../../utils/internalRoutes'
-import type { IDataFrame } from 'data-forge'
+import type { DatasetId } from '../config/DatasetMetadata'
+import type { DataTypeId } from '../config/MetricConfigTypes'
+import type { Breakdowns, TimeView } from '../query/Breakdowns'
+import { type MetricQuery, MetricQueryResponse } from '../query/MetricQuery'
+import { appendFipsIfNeeded } from '../utils/datasetutils'
+import { GetAcsDatasetId } from './AcsPopulationProvider'
+import VariableProvider from './VariableProvider'
 
 // when alternate data types are available, provide a link to the national level, by race report for that data type
 export const dataTypeLinkMap: Partial<Record<DataTypeId, string>> = {

frontend/src/data/providers/IncarcerationProvider.tsx

@@ -1,12 +1,16 @@
 import { getDataManager } from '../../utils/globals'
+import type { DataTypeId, MetricId } from '../config/MetricConfigTypes'
 import type { Breakdowns, TimeView } from '../query/Breakdowns'
-import { type MetricQuery, MetricQueryResponse } from '../query/MetricQuery'
-import type { MetricId, DataTypeId } from '../config/MetricConfigTypes'
+import {
+  type MetricQuery,
+  MetricQueryResponse,
+  resolveDatasetId,
+} from '../query/MetricQuery'
 
-import VariableProvider from './VariableProvider'
-import { GetAcsDatasetId } from './AcsPopulationProvider'
-import { appendFipsIfNeeded } from '../utils/datasetutils'
 import type { DatasetId } from '../config/DatasetMetadata'
+import { appendFipsIfNeeded } from '../utils/datasetutils'
+import { GetAcsDatasetId } from './AcsPopulationProvider'
+import VariableProvider from './VariableProvider'
 
 // states with combined prison and jail systems
 export const COMBINED_INCARCERATION_STATES_LIST = [
@@ -88,9 +92,19 @@ class IncarcerationProvider extends VariableProvider {
   async getDataInternal(
     metricQuery: MetricQuery,
   ): Promise<MetricQueryResponse> {
-    const breakdowns = metricQuery.breakdowns
-    const timeView = metricQuery.timeView
-    const datasetId = this.getDatasetId(breakdowns, undefined, timeView)
+    const isVera = metricQuery.breakdowns.geography === 'county'
+
+    const datasetName = isVera
+      ? 'vera_incarceration_county'
+      : 'bjs_incarceration_data'
+    const prefix = isVera ? 'by_' : ''
+
+    const { breakdowns, datasetId, isFallbackId, timeView } = resolveDatasetId(
+      datasetName,
+      prefix,
+      metricQuery,
+    )
+
     if (!datasetId) {
       return new MetricQueryResponse([], [])
     }
@@ -132,8 +146,12 @@ class IncarcerationProvider extends VariableProvider {
       }
     }
 
-    df = this.applyDemographicBreakdownFilters(df, breakdowns)
-    df = this.removeUnrequestedColumns(df, metricQuery)
+    if (isFallbackId) {
+      df = this.castAllsAsRequestedDemographicBreakdown(df, breakdowns)
+    } else {
+      df = this.applyDemographicBreakdownFilters(df, breakdowns)
+      df = this.removeUnrequestedColumns(df, metricQuery)
+    }
 
     return new MetricQueryResponse(df.toArray(), consumedDatasetIds)
   }