From 5248671d20d8d85514802ff60294a0afb5ac455c Mon Sep 17 00:00:00 2001 From: Ben Hammond Date: Fri, 22 Nov 2024 14:53:17 -0700 Subject: [PATCH] partial update of providers. not ready to merge; backend needs lots of updates first --- .../data/providers/AcsConditionProvider.ts | 92 ++++--------------- .../data/providers/AcsPopulationProvider.ts | 3 +- frontend/src/data/providers/AhrProvider.ts | 14 ++- frontend/src/data/providers/CawpProvider.ts | 66 ++++++------- .../src/data/providers/CdcCovidProvider.ts | 16 ++-- .../data/providers/IncarcerationProvider.tsx | 38 ++++++-- .../src/data/providers/PhrmaBrfssProvider.ts | 22 ++++- frontend/src/data/providers/PhrmaProvider.ts | 8 +- .../src/data/providers/VaccineProvider.ts | 2 - .../src/data/providers/VariableProvider.ts | 19 +--- frontend/src/data/query/MetricQuery.ts | 6 +- 11 files changed, 123 insertions(+), 163 deletions(-) diff --git a/frontend/src/data/providers/AcsConditionProvider.ts b/frontend/src/data/providers/AcsConditionProvider.ts index eac1def3b8..1e07165ea3 100644 --- a/frontend/src/data/providers/AcsConditionProvider.ts +++ b/frontend/src/data/providers/AcsConditionProvider.ts @@ -1,10 +1,13 @@ import { getDataManager } from '../../utils/globals' -import type { TimeView, Breakdowns } from '../query/Breakdowns' -import { type MetricQuery, MetricQueryResponse } from '../query/MetricQuery' -import VariableProvider from './VariableProvider' -import { appendFipsIfNeeded } from '../utils/datasetutils' import type { DataTypeId, MetricId } from '../config/MetricConfigTypes' -import type { DatasetId } from '../config/DatasetMetadata' +import type { Breakdowns } from '../query/Breakdowns' +import { + type MetricQuery, + MetricQueryResponse, + resolveDatasetId, +} from '../query/MetricQuery' +import { appendFipsIfNeeded } from '../utils/datasetutils' +import VariableProvider from './VariableProvider' export const ACS_CONDITION_DATATYPES: DataTypeId[] = [ 'health_insurance', @@ -31,72 +34,14 @@ class AcsConditionProvider extends VariableProvider { super('acs_condition_provider', ACS_CONDITION_METRICS) } - getDatasetId( - breakdowns: Breakdowns, - _dataTypeId?: DataTypeId, - timeView?: TimeView, - ): DatasetId | undefined { - if (timeView === 'historical') { - if (breakdowns.geography === 'national') { - if (breakdowns.hasOnlyRace()) - return 'acs_condition-by_race_national_historical' - if (breakdowns.hasOnlyAge()) - return 'acs_condition-by_age_national_historical' - if (breakdowns.hasOnlySex()) - return 'acs_condition-by_sex_national_historical' - } - if (breakdowns.geography === 'state') { - if (breakdowns.hasOnlyRace()) - return 'acs_condition-by_race_state_historical' - if (breakdowns.hasOnlyAge()) - return 'acs_condition-by_age_state_historical' - if (breakdowns.hasOnlySex()) - return 'acs_condition-by_sex_state_historical' - } - - if (breakdowns.geography === 'county') { - if (breakdowns.hasOnlyRace()) - return 'acs_condition-by_race_county_historical' - if (breakdowns.hasOnlyAge()) - return 'acs_condition-by_age_county_historical' - if (breakdowns.hasOnlySex()) - return 'acs_condition-by_sex_county_historical' - } - } - - if (timeView === 'current') { - if (breakdowns.geography === 'national') { - if (breakdowns.hasOnlyRace()) - return 'acs_condition-by_race_national_current' - if (breakdowns.hasOnlyAge()) - return 'acs_condition-by_age_national_current' - if (breakdowns.hasOnlySex()) - return 'acs_condition-by_sex_national_current' - } - if (breakdowns.geography === 'state') { - if (breakdowns.hasOnlyRace()) - return 'acs_condition-by_race_state_current' - if (breakdowns.hasOnlyAge()) return 'acs_condition-by_age_state_current' - if (breakdowns.hasOnlySex()) return 'acs_condition-by_sex_state_current' - } - - if (breakdowns.geography === 'county') { - if (breakdowns.hasOnlyRace()) - return 'acs_condition-by_race_county_current' - if (breakdowns.hasOnlyAge()) - return 'acs_condition-by_age_county_current' - if (breakdowns.hasOnlySex()) - return 'acs_condition-by_sex_county_current' - } - } - } - async getDataInternal( metricQuery: MetricQuery, ): Promise { - const breakdowns = metricQuery.breakdowns - const timeView = metricQuery.timeView - const datasetId = this.getDatasetId(breakdowns, undefined, timeView) + const { breakdowns, datasetId, isFallbackId } = resolveDatasetId( + 'acs_condition', + 'by_', + metricQuery, + ) if (!datasetId) { return new MetricQueryResponse([], []) } @@ -104,14 +49,15 @@ class AcsConditionProvider extends VariableProvider { const acsDataset = await getDataManager().loadDataset(specificDatasetId) let df = acsDataset.toDataFrame() - - // If requested, filter geography by state or county level - // We apply the geo filter right away to reduce subsequent calculation times df = this.filterByGeo(df, breakdowns) df = this.renameGeoColumns(df, breakdowns) - df = this.applyDemographicBreakdownFilters(df, breakdowns) - df = this.removeUnrequestedColumns(df, metricQuery) + if (isFallbackId) { + df = this.castAllsAsRequestedDemographicBreakdown(df, breakdowns) + } else { + df = this.applyDemographicBreakdownFilters(df, breakdowns) + df = this.removeUnrequestedColumns(df, metricQuery) + } return new MetricQueryResponse(df.toArray(), [datasetId]) } diff --git a/frontend/src/data/providers/AcsPopulationProvider.ts b/frontend/src/data/providers/AcsPopulationProvider.ts index e82f227796..1208678b29 100644 --- a/frontend/src/data/providers/AcsPopulationProvider.ts +++ b/frontend/src/data/providers/AcsPopulationProvider.ts @@ -1,10 +1,10 @@ import type { IDataFrame } from 'data-forge' import { getDataManager } from '../../utils/globals' +import type { DatasetId } from '../config/DatasetMetadata' import type { Breakdowns } from '../query/Breakdowns' import { type MetricQuery, MetricQueryResponse } from '../query/MetricQuery' import { appendFipsIfNeeded } from '../utils/datasetutils' import VariableProvider from './VariableProvider' -import type { DatasetId } from '../config/DatasetMetadata' export function GetAcsDatasetId(breakdowns: Breakdowns): DatasetId | undefined { if (breakdowns.geography === 'national') { @@ -44,7 +44,6 @@ class AcsPopulationProvider extends VariableProvider { df = this.applyDemographicBreakdownFilters(df, breakdowns) df = this.removeUnrequestedColumns(df, metricQuery) - const datasetId = this.getDatasetId(breakdowns) let consumedDatasetIds if (datasetId) consumedDatasetIds = [datasetId] diff --git a/frontend/src/data/providers/AhrProvider.ts b/frontend/src/data/providers/AhrProvider.ts index 8c00a2f4ef..1baca0a2c3 100644 --- a/frontend/src/data/providers/AhrProvider.ts +++ b/frontend/src/data/providers/AhrProvider.ts @@ -1,17 +1,17 @@ import { getDataManager } from '../../utils/globals' import { getParentDropdownFromDataTypeId } from '../../utils/MadLibs' import type { DatasetId } from '../config/DatasetMetadata' -import type { DataTypeId, MetricId } from '../config/MetricConfigTypes' +import type { DropdownVarId } from '../config/DropDownIds' import { BEHAVIORAL_HEALTH_CATEGORY_DROPDOWNIDS } from '../config/MetricConfigBehavioralHealth' +import type { DataTypeId, MetricId } from '../config/MetricConfigTypes' import type { - DemographicBreakdownKey, Breakdowns, + DemographicBreakdownKey, TimeView, } from '../query/Breakdowns' import { type MetricQuery, MetricQueryResponse } from '../query/MetricQuery' import { appendFipsIfNeeded } from '../utils/datasetutils' import VariableProvider from './VariableProvider' -import type { DropdownVarId } from '../config/DropDownIds' const CHR_DATATYPE_IDS_ONLY_ALLS: DataTypeId[] = [ 'diabetes', @@ -257,8 +257,12 @@ class AhrProvider extends VariableProvider { }) } - df = this.applyDemographicBreakdownFilters(df, breakdowns) - df = this.removeUnrequestedColumns(df, metricQuery) + if (isFallbackId) { + df = this.castAllsAsRequestedDemographicBreakdown(df, breakdowns) + } else { + df = this.applyDemographicBreakdownFilters(df, breakdowns) + df = this.removeUnrequestedColumns(df, metricQuery) + } return new MetricQueryResponse(df.toArray(), consumedDatasetIds) } diff --git a/frontend/src/data/providers/CawpProvider.ts b/frontend/src/data/providers/CawpProvider.ts index 766e9c4ebc..c77945a35e 100644 --- a/frontend/src/data/providers/CawpProvider.ts +++ b/frontend/src/data/providers/CawpProvider.ts @@ -1,24 +1,27 @@ import { getDataManager } from '../../utils/globals' -import type { MetricId, DataTypeId } from '../config/MetricConfigTypes' +import type { DataTypeId, MetricId } from '../config/MetricConfigTypes' -import type { TimeView, Breakdowns } from '../query/Breakdowns' -import { type MetricQuery, MetricQueryResponse } from '../query/MetricQuery' -import { GetAcsDatasetId } from './AcsPopulationProvider' -import VariableProvider from './VariableProvider' +import type { Breakdowns } from '../query/Breakdowns' import { - UNKNOWN_RACE, + type MetricQuery, + MetricQueryResponse, + resolveDatasetId, +} from '../query/MetricQuery' +import { + type RaceAndEthnicityGroup, + HISP_W, HISPANIC, MULTI, - UNREPRESENTED, - type RaceAndEthnicityGroup, - OTHER_W, MULTI_W, OTHER_STANDARD, + OTHER_W, + UNKNOWN_RACE, UNKNOWN_W, - HISP_W, + UNREPRESENTED, } from '../utils/Constants' -import type { DatasetId } from '../config/DatasetMetadata' import { appendFipsIfNeeded } from '../utils/datasetutils' +import { GetAcsDatasetId } from './AcsPopulationProvider' +import VariableProvider from './VariableProvider' export const CAWP_CONGRESS_COUNTS: MetricId[] = [ 'women_this_race_us_congress_count', @@ -76,31 +79,14 @@ class CawpProvider extends VariableProvider { super('cawp_provider', CAWP_METRICS) } - getDatasetId( - breakdowns: Breakdowns, - dataTypeId?: DataTypeId, - timeView?: TimeView, - ): DatasetId | undefined { - if (timeView === 'current') { - if (breakdowns.geography === 'national' && breakdowns.hasOnlyRace()) - return 'cawp_time_data-race_and_ethnicity_national_current' - if (breakdowns.geography === 'state' && breakdowns.hasOnlyRace()) - return 'cawp_time_data-race_and_ethnicity_state_current' - } - if (timeView === 'historical') { - if (breakdowns.geography === 'national' && breakdowns.hasOnlyRace()) - return 'cawp_time_data-race_and_ethnicity_national_historical' - if (breakdowns.geography === 'state' && breakdowns.hasOnlyRace()) - return 'cawp_time_data-race_and_ethnicity_state_historical' - } - } - async getDataInternal( metricQuery: MetricQuery, ): Promise { - const breakdowns = metricQuery.breakdowns - const timeView = metricQuery.timeView - const datasetId = this.getDatasetId(breakdowns, undefined, timeView) + const { breakdowns, datasetId, isFallbackId, timeView } = resolveDatasetId( + 'cawp_time_data', + '', + metricQuery, + ) if (!datasetId) { return new MetricQueryResponse([], []) } @@ -108,9 +94,6 @@ class CawpProvider extends VariableProvider { const cawp = await getDataManager().loadDataset(specificDatasetId) let df = cawp.toDataFrame() - df = this.filterByGeo(df, breakdowns) - df = this.renameGeoColumns(df, breakdowns) - const consumedDatasetIds = [datasetId] // no population numbers used for rates, only comparison pop. and pct_rel_inequity @@ -143,8 +126,15 @@ class CawpProvider extends VariableProvider { consumedDatasetIds.push('the_unitedstates_project') } - df = this.applyDemographicBreakdownFilters(df, breakdowns) - df = this.removeUnrequestedColumns(df, metricQuery) + df = this.filterByGeo(df, breakdowns) + df = this.renameGeoColumns(df, breakdowns) + + if (isFallbackId) { + df = this.castAllsAsRequestedDemographicBreakdown(df, breakdowns) + } else { + df = this.applyDemographicBreakdownFilters(df, breakdowns) + df = this.removeUnrequestedColumns(df, metricQuery) + } return new MetricQueryResponse(df.toArray(), consumedDatasetIds) } diff --git a/frontend/src/data/providers/CdcCovidProvider.ts b/frontend/src/data/providers/CdcCovidProvider.ts index 811664b8ff..c0eabfb65d 100644 --- a/frontend/src/data/providers/CdcCovidProvider.ts +++ b/frontend/src/data/providers/CdcCovidProvider.ts @@ -1,16 +1,16 @@ +import type { IDataFrame } from 'data-forge' import { getDataManager } from '../../utils/globals' -import type { Breakdowns, TimeView } from '../query/Breakdowns' -import { type MetricQuery, MetricQueryResponse } from '../query/MetricQuery' -import { GetAcsDatasetId } from './AcsPopulationProvider' -import VariableProvider from './VariableProvider' -import { appendFipsIfNeeded } from '../utils/datasetutils' -import type { DatasetId } from '../config/DatasetMetadata' -import type { DataTypeId } from '../config/MetricConfigTypes' import { AGE_ADJUST_COVID_DEATHS_US_SETTING, AGE_ADJUST_COVID_HOSP_US_SETTING, } from '../../utils/internalRoutes' -import type { IDataFrame } from 'data-forge' +import type { DatasetId } from '../config/DatasetMetadata' +import type { DataTypeId } from '../config/MetricConfigTypes' +import type { Breakdowns, TimeView } from '../query/Breakdowns' +import { type MetricQuery, MetricQueryResponse } from '../query/MetricQuery' +import { appendFipsIfNeeded } from '../utils/datasetutils' +import { GetAcsDatasetId } from './AcsPopulationProvider' +import VariableProvider from './VariableProvider' // when alternate data types are available, provide a link to the national level, by race report for that data type export const dataTypeLinkMap: Partial> = { diff --git a/frontend/src/data/providers/IncarcerationProvider.tsx b/frontend/src/data/providers/IncarcerationProvider.tsx index e40a861332..ee103c7c62 100644 --- a/frontend/src/data/providers/IncarcerationProvider.tsx +++ b/frontend/src/data/providers/IncarcerationProvider.tsx @@ -1,12 +1,16 @@ import { getDataManager } from '../../utils/globals' +import type { DataTypeId, MetricId } from '../config/MetricConfigTypes' import type { Breakdowns, TimeView } from '../query/Breakdowns' -import { type MetricQuery, MetricQueryResponse } from '../query/MetricQuery' -import type { MetricId, DataTypeId } from '../config/MetricConfigTypes' +import { + type MetricQuery, + MetricQueryResponse, + resolveDatasetId, +} from '../query/MetricQuery' -import VariableProvider from './VariableProvider' -import { GetAcsDatasetId } from './AcsPopulationProvider' -import { appendFipsIfNeeded } from '../utils/datasetutils' import type { DatasetId } from '../config/DatasetMetadata' +import { appendFipsIfNeeded } from '../utils/datasetutils' +import { GetAcsDatasetId } from './AcsPopulationProvider' +import VariableProvider from './VariableProvider' // states with combined prison and jail systems export const COMBINED_INCARCERATION_STATES_LIST = [ @@ -88,9 +92,19 @@ class IncarcerationProvider extends VariableProvider { async getDataInternal( metricQuery: MetricQuery, ): Promise { - const breakdowns = metricQuery.breakdowns - const timeView = metricQuery.timeView - const datasetId = this.getDatasetId(breakdowns, undefined, timeView) + const isVera = metricQuery.breakdowns.geography === 'county' + + const datasetName = isVera + ? 'vera_incarceration_county' + : 'bjs_incarceration_data' + const prefix = isVera ? 'by_' : '' + + const { breakdowns, datasetId, isFallbackId, timeView } = resolveDatasetId( + datasetName, + prefix, + metricQuery, + ) + if (!datasetId) { return new MetricQueryResponse([], []) } @@ -132,8 +146,12 @@ class IncarcerationProvider extends VariableProvider { } } - df = this.applyDemographicBreakdownFilters(df, breakdowns) - df = this.removeUnrequestedColumns(df, metricQuery) + if (isFallbackId) { + df = this.castAllsAsRequestedDemographicBreakdown(df, breakdowns) + } else { + df = this.applyDemographicBreakdownFilters(df, breakdowns) + df = this.removeUnrequestedColumns(df, metricQuery) + } return new MetricQueryResponse(df.toArray(), consumedDatasetIds) } diff --git a/frontend/src/data/providers/PhrmaBrfssProvider.ts b/frontend/src/data/providers/PhrmaBrfssProvider.ts index 9186fcef9b..abebfb2887 100644 --- a/frontend/src/data/providers/PhrmaBrfssProvider.ts +++ b/frontend/src/data/providers/PhrmaBrfssProvider.ts @@ -3,7 +3,11 @@ import type { DatasetId } from '../config/DatasetMetadata' import type { DropdownVarId } from '../config/DropDownIds' import type { DataTypeId, MetricId } from '../config/MetricConfigTypes' import type { Breakdowns } from '../query/Breakdowns' -import { type MetricQuery, MetricQueryResponse } from '../query/MetricQuery' +import { + type MetricQuery, + MetricQueryResponse, + resolveDatasetId, +} from '../query/MetricQuery' import { appendFipsIfNeeded } from '../utils/datasetutils' import VariableProvider from './VariableProvider' @@ -115,8 +119,12 @@ class PhrmaBrfssProvider extends VariableProvider { async getDataInternal( metricQuery: MetricQuery, ): Promise { - const breakdowns = metricQuery.breakdowns - const datasetId = this.getDatasetId(breakdowns) + const { breakdowns, datasetId, isFallbackId } = resolveDatasetId( + 'phrma_brfss_data', + '', + metricQuery, + ) + if (!datasetId) { return new MetricQueryResponse([], []) } @@ -128,8 +136,12 @@ class PhrmaBrfssProvider extends VariableProvider { df = this.renameGeoColumns(df, breakdowns) const consumedDatasetIds = [datasetId] - df = this.applyDemographicBreakdownFilters(df, breakdowns) - df = this.removeUnrequestedColumns(df, metricQuery) + if (isFallbackId) { + df = this.castAllsAsRequestedDemographicBreakdown(df, breakdowns) + } else { + df = this.applyDemographicBreakdownFilters(df, breakdowns) + df = this.removeUnrequestedColumns(df, metricQuery) + } const data = df.toArray() diff --git a/frontend/src/data/providers/PhrmaProvider.ts b/frontend/src/data/providers/PhrmaProvider.ts index 00635bd8aa..011b56464c 100644 --- a/frontend/src/data/providers/PhrmaProvider.ts +++ b/frontend/src/data/providers/PhrmaProvider.ts @@ -148,8 +148,12 @@ class PhrmaProvider extends VariableProvider { df = this.renameGeoColumns(df, breakdowns) const consumedDatasetIds = [datasetId] - df = this.applyDemographicBreakdownFilters(df, breakdowns) - df = this.removeUnrequestedColumns(df, metricQuery) + if (isFallbackId) { + df = this.castAllsAsRequestedDemographicBreakdown(df, breakdowns) + } else { + df = this.applyDemographicBreakdownFilters(df, breakdowns) + df = this.removeUnrequestedColumns(df, metricQuery) + } return new MetricQueryResponse(df.toArray(), consumedDatasetIds) } diff --git a/frontend/src/data/providers/VaccineProvider.ts b/frontend/src/data/providers/VaccineProvider.ts index 599f93b51e..ef832761d0 100644 --- a/frontend/src/data/providers/VaccineProvider.ts +++ b/frontend/src/data/providers/VaccineProvider.ts @@ -81,8 +81,6 @@ class VaccineProvider extends VariableProvider { } } if (breakdowns.geography === 'county') { - // We merge this in on the backend, no need to redownload it here - // but we want to provide the proper citation consumedDatasetIds.push('acs_population-by_race_county') } diff --git a/frontend/src/data/providers/VariableProvider.ts b/frontend/src/data/providers/VariableProvider.ts index 579c047159..7294a7666f 100644 --- a/frontend/src/data/providers/VariableProvider.ts +++ b/frontend/src/data/providers/VariableProvider.ts @@ -1,8 +1,7 @@ import type { IDataFrame } from 'data-forge' -import type { DatasetId } from '../config/DatasetMetadata' -import type { DataTypeId, MetricId } from '../config/MetricConfigTypes' +import type { MetricId } from '../config/MetricConfigTypes' import type { ProviderId } from '../loading/VariableProviderMap' -import type { Breakdowns, TimeView } from '../query/Breakdowns' +import type { Breakdowns } from '../query/Breakdowns' import { createMissingDataResponse, type MetricQuery, @@ -134,20 +133,6 @@ abstract class VariableProvider { breakdowns: Breakdowns, metricIds?: MetricId[], ): boolean - - // TODO: remove getDatasetId and getFallbackAllsDatasetId once all providers have migrated in favor of resolveDatasetId in datasetutils - - getDatasetId?( - breakdown: Breakdowns, - dataTypeId?: DataTypeId, - timeView?: TimeView, - ): DatasetId | undefined - - getFallbackAllsDatasetId?( - breakdown: Breakdowns, - dataTypeId?: DataTypeId, - timeView?: TimeView, - ): DatasetId | undefined } export default VariableProvider diff --git a/frontend/src/data/query/MetricQuery.ts b/frontend/src/data/query/MetricQuery.ts index 09263681d0..eda3ba29f1 100644 --- a/frontend/src/data/query/MetricQuery.ts +++ b/frontend/src/data/query/MetricQuery.ts @@ -170,6 +170,7 @@ export function resolveDatasetId( breakdowns: Breakdowns datasetId?: DatasetId isFallbackId?: boolean + timeView?: TimeView } { const { breakdowns, timeView } = metricQuery const requestedDemographic: DemographicType = @@ -182,6 +183,7 @@ export function resolveDatasetId( return { breakdowns, datasetId: requestedDatasetId as DatasetId, + timeView, } } @@ -193,6 +195,7 @@ export function resolveDatasetId( return { breakdowns, datasetId: requestedRaceDatasetId as DatasetId, + timeView, } } } @@ -210,6 +213,7 @@ export function resolveDatasetId( ? (fallbackAllsDatasetId as DatasetId) : undefined, isFallbackId: isFallbackEligible, + timeView, } } @@ -217,5 +221,5 @@ export function resolveDatasetId( console.warn( `Invalid datasetId requests:\n${requestedDatasetId}${requestedRaceDatasetId ? '\n' + requestedRaceDatasetId : ''}\n${fallbackAllsDatasetId}\nNone of those known datasetIds. Did you update DatasetId type?`, ) - return { breakdowns } + return { breakdowns, timeView } }