Transform eia861 short form

docs/release_notes.rst

@@ -65,6 +65,8 @@ EIA AEO
   <https://www.eia.gov/outlooks/aeo/tables_ref.php>`__, which include future
   projections related to electric power and renewable energy through the year
   2050, across a variety of scenarios. See :issue:`3368` and :pr:`3538`.
+* Added new :ref:`core_eia861__yearly_short_form` table from EIA861 which contains
+  the shorter version of EIA861. See issues :issue:`3540` and PR :pr:`3565`.
 * Added new tables from EIA AEO table 54:
 
   * :ref:`core_eiaaeo__yearly_projected_generation_in_electric_sector_by_technology`

migrations/versions/827e88186933_add_eia861_short_form_table_and_update_.py

@@ -0,0 +1,80 @@
+"""Add EIA861 short_form table and update column names in 860
+
+Revision ID: 827e88186933
+Revises: 6dcb5e3e07dd
+Create Date: 2024-05-23 15:33:28.522903
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '827e88186933'
+down_revision = '6dcb5e3e07dd'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('core_eia861__yearly_short_form',
+    sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'),
+    sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'),
+    sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'),
+    sa.Column('entity_type', sa.Text(), nullable=True, comment='Entity type of principal owner.'),
+    sa.Column('state', sa.Text(), nullable=False, comment='Two letter US state abbreviation.'),
+    sa.Column('balancing_authority_code_eia', sa.Text(), nullable=False, comment='EIA short code identifying a balancing authority. May include Canadian and Mexican BAs.'),
+    sa.Column('sales_revenue', sa.Float(), nullable=True, comment='Revenue from electricity sold.'),
+    sa.Column('sales_mwh', sa.Float(), nullable=True, comment='Quantity of electricity sold in MWh.'),
+    sa.Column('customers', sa.Float(), nullable=True, comment='Number of customers.'),
+    sa.Column('num_water_heaters', sa.Integer(), nullable=True, comment="The number of grid-enabled water heaters added to the respondent's program this year - if the respondent has DSM program for grid-enabled water heaters (as defined by DOE’s Office of Energy Efficiency and Renewable Energy)."),
+    sa.Column('has_net_metering', sa.Boolean(), nullable=True, comment='Whether the plant has a net metering agreement in effect during the reporting year.  (Only displayed for facilities that report the sun or wind as an energy source). This field was only reported up until 2015'),
+    sa.Column('has_demand_side_management', sa.Boolean(), nullable=True, comment='Whether there were strategies or measures used to control electricity demand by customers'),
+    sa.Column('has_time_responsive_programs', sa.Boolean(), nullable=True, comment='Whether the respondent operates any time-based rate programs (e.g., real-time pricing, critical peak pricing, variable peak pricing and time-of-use rates administered through a tariff).'),
+    sa.Column('has_green_pricing', sa.Boolean(), nullable=True, comment='Whether a green pricing program was associated with this utility during the reporting year.'),
+    sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'),
+    sa.ForeignKeyConstraint(['balancing_authority_code_eia'], ['core_eia__codes_balancing_authorities.code'], name=op.f('fk_core_eia861__yearly_short_form_balancing_authority_code_eia_core_eia__codes_balancing_authorities')),
+    sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_short_form_data_maturity_core_pudl__codes_data_maturities')),
+    sa.ForeignKeyConstraint(['utility_id_eia'], ['core_eia__entity_utilities.utility_id_eia'], name=op.f('fk_core_eia861__yearly_short_form_utility_id_eia_core_eia__entity_utilities')),
+    sa.PrimaryKeyConstraint('utility_id_eia', 'state', 'report_date', 'balancing_authority_code_eia', name=op.f('pk_core_eia861__yearly_short_form'))
+    )
+    with op.batch_alter_table('core_eia860__scd_plants', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('has_net_metering', sa.Boolean(), nullable=True, comment='Whether the plant has a net metering agreement in effect during the reporting year.  (Only displayed for facilities that report the sun or wind as an energy source). This field was only reported up until 2015'))
+        batch_op.drop_column('net_metering')
+
+    with op.batch_alter_table('core_eia861__yearly_demand_response_water_heater', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('num_water_heaters', sa.Integer(), nullable=True, comment="The number of grid-enabled water heaters added to the respondent's program this year - if the respondent has DSM program for grid-enabled water heaters (as defined by DOE’s Office of Energy Efficiency and Renewable Energy)."))
+        batch_op.drop_column('water_heater')
+
+    with op.batch_alter_table('core_eia861__yearly_demand_side_management_misc', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('has_time_responsive_programs', sa.Boolean(), nullable=True, comment='Whether the respondent operates any time-based rate programs (e.g., real-time pricing, critical peak pricing, variable peak pricing and time-of-use rates administered through a tariff).'))
+        batch_op.drop_column('time_responsive_programs')
+
+    with op.batch_alter_table('out_eia__yearly_plants', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('has_net_metering', sa.Boolean(), nullable=True, comment='Whether the plant has a net metering agreement in effect during the reporting year.  (Only displayed for facilities that report the sun or wind as an energy source). This field was only reported up until 2015'))
+        batch_op.drop_column('net_metering')
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('out_eia__yearly_plants', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('net_metering', sa.BOOLEAN(), nullable=True))
+        batch_op.drop_column('has_net_metering')
+
+    with op.batch_alter_table('core_eia861__yearly_demand_side_management_misc', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('time_responsive_programs', sa.BOOLEAN(), nullable=True))
+        batch_op.drop_column('has_time_responsive_programs')
+
+    with op.batch_alter_table('core_eia861__yearly_demand_response_water_heater', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('water_heater', sa.INTEGER(), nullable=True))
+        batch_op.drop_column('num_water_heaters')
+
+    with op.batch_alter_table('core_eia860__scd_plants', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('net_metering', sa.BOOLEAN(), nullable=True))
+        batch_op.drop_column('has_net_metering')
+
+    op.drop_table('core_eia861__yearly_short_form')
+    # ### end Alembic commands ###

src/pudl/metadata/fields.py

@@ -989,6 +989,10 @@
         "description": "Annual demand per km2 of a given service territory.",
         "unit": "MWh/km2",
     },
+    "has_demand_side_management": {
+        "type": "boolean",
+        "description": "Whether there were strategies or measures used to control electricity demand by customers",
+    },
     "depreciation_type": {
         "type": "string",
         "description": (
@@ -1827,6 +1831,10 @@
         "type": "string",
         "description": "Category of geographic aggregation in EIA bulk electricity data.",
     },
+    "has_green_pricing": {
+        "type": "boolean",
+        "description": "Whether a green pricing program was associated with this utility during the reporting year.",
+    },
     "green_pricing_revenue": {
         "type": "number",
         "description": (
@@ -2463,9 +2471,9 @@
         "description": "Net output for load (net generation - energy used for pumping) in megawatt-hours.",
         "unit": "MWh",
     },
-    "net_metering": {
+    "has_net_metering": {
         "type": "boolean",
-        "description": "Did this plant have a net metering agreement in effect during the reporting year?  (Only displayed for facilities that report the sun or wind as an energy source). This field was only reported up until 2015",
+        "description": "Whether the plant has a net metering agreement in effect during the reporting year.  (Only displayed for facilities that report the sun or wind as an energy source). This field was only reported up until 2015",
         # TODO: Is this really boolean? Or do we have non-null strings that mean False?
     },
     "net_output_penalty": {
@@ -4076,7 +4084,7 @@
             "and predefined, based on season, day of week, and time of day."
         ),
     },
-    "time_responsive_programs": {
+    "has_time_responsive_programs": {
         "type": "boolean",
         "description": (
             "Whether the respondent operates any time-based rate programs (e.g., "
@@ -4384,7 +4392,7 @@
         "unit": "MMBtu_per_hour",
         "description": "Design waste-heat input rate at maximum continuous steam flow where a waste-heat boiler is a boiler that receives all or a substantial portion of its energy input from the noncumbustible exhaust gases of a separate fuel-burning process (MMBtu per hour).",
     },
-    "water_heater": {
+    "num_water_heaters": {
         "type": "integer",
         "description": (
             "The number of grid-enabled water heaters added to the respondent's "

src/pudl/metadata/resources/__init__.py

@@ -56,7 +56,7 @@
             "natural_gas_pipeline_name_2",
             "natural_gas_pipeline_name_3",
             "nerc_region",
-            "net_metering",
+            "has_net_metering",
             "pipeline_notes",
             "primary_purpose_id_naics",
             "regulatory_status_code",

src/pudl/metadata/resources/eia.py

@@ -851,7 +851,7 @@
                 "natural_gas_pipeline_name_2",
                 "natural_gas_pipeline_name_3",
                 "nerc_region",
-                "net_metering",
+                "has_net_metering",
                 "pipeline_notes",
                 "primary_purpose_id_naics",
                 "regulatory_status_code",

src/pudl/metadata/resources/eia860.py

@@ -351,7 +351,7 @@
                 "natural_gas_pipeline_name_2",
                 "natural_gas_pipeline_name_3",
                 "nerc_region",
-                "net_metering",
+                "has_net_metering",
                 "pipeline_notes",
                 "primary_purpose_id_naics",
                 "regulatory_status_code",
@@ -493,6 +493,7 @@
                     "core_eia861__yearly_operational_data_revenue",
                     "core_eia861__yearly_reliability",
                     "core_eia861__yearly_sales",
+                    "core_eia861__yearly_short_form",
                     "core_eia861__yearly_service_territory",
                     "out_ferc714__summarized_demand",
                     "core_eia861__assn_utility",

src/pudl/metadata/resources/eia861.py

@@ -124,7 +124,7 @@
                 "report_date",
                 "state",
                 "utility_id_eia",
-                "water_heater",
+                "num_water_heaters",
                 "data_maturity",
             ],
             "primary_key": [
@@ -213,7 +213,7 @@
                 "reported_as_another_company",
                 "short_form",
                 "state",
-                "time_responsive_programs",
+                "has_time_responsive_programs",
                 "utility_id_eia",
                 "utility_name_eia",
                 "data_maturity",
@@ -694,6 +694,52 @@
         "sources": ["eia861"],
         "etl_group": "eia861",
     },
+    "core_eia861__yearly_short_form": {
+        "description": (
+            """This is data extracted from Form EIA-861S, a shorter version of Form EIA-861.
+The data started being reported in 2012. However, the 2019 data is not available.
+They are expected to submit the completed Form EIA-861S to EIA by April 30th, following the end of the prior calendar year.
+Utilities report on Form EIA-861S if they:
+
+- Report less than 200,000 megawatthours on the last previous Form EIA-861.
+
+- Provide only bundled service (generation and distribution).
+
+- Are not needed to ensure acceptable quality of statistical estimates.
+
+- Are not part of the aggregate TVA or WPPI.
+
+- Do not report on Form EIA-861M."""
+        ),
+        "schema": {
+            "fields": [
+                "report_date",
+                "utility_id_eia",
+                "utility_name_eia",
+                "entity_type",
+                "state",
+                "balancing_authority_code_eia",
+                "sales_revenue",
+                "sales_mwh",
+                "customers",
+                "num_water_heaters",
+                "has_net_metering",
+                "has_demand_side_management",
+                "has_time_responsive_programs",
+                "has_green_pricing",
+                "data_maturity",
+            ],
+            "primary_key": [
+                "utility_id_eia",
+                "state",
+                "report_date",
+                "balancing_authority_code_eia",
+            ],
+        },
+        "field_namespace": "eia",
+        "sources": ["eia861"],
+        "etl_group": "eia861",
+    },
     "core_eia861__yearly_service_territory": {
         "description": "County FIPS codes for counties composing utility service territories.",
         "schema": {

src/pudl/output/pudltabl.py

@@ -181,6 +181,7 @@ def _register_output_methods(self: Self):
             # eia861 (clean)
             "core_eia861__yearly_service_territory": "service_territory_eia861",
             "core_eia861__yearly_sales": "sales_eia861",
+            "core_eia861__yearly_short_form": "short_form_eia861",
             "core_eia861__yearly_advanced_metering_infrastructure": "advanced_metering_infrastructure_eia861",
             "core_eia861__yearly_demand_response": "demand_response_eia861",
             "core_eia861__yearly_demand_response_water_heater": "demand_response_water_heater_eia861",

src/pudl/package_data/eia860/column_maps/plant.csv

@@ -20,7 +20,7 @@ sector_name_eia,,,,,,,,,sector_name,sector_name,sector_name,sector_name,sector_n
 sector_id_eia,,,,,,,,,sector_number,sector_number,sector,sector,sector,sector,sector,sector,sector,sector,sector,sector,sector,sector
 ferc_cogen_status,,,,,,,ferc_cogen,ferc_cogen,ferc_cogen,ferc_cogen,ferc_cogen,ferc_cogeneration_status,ferc_cogeneration_status,ferc_cogeneration_status,ferc_cogeneration_status,ferc_cogeneration_status,ferc_cogeneration_status,ferc_cogeneration_status,ferc_cogeneration_status,ferc_cogeneration_status,ferc_cogeneration_status,ferc_cogeneration_status
 ferc_cogen_docket_no,,,,,,,ferc_cogen_docket,ferc_cogen_docket,ferc_cogen_docket,ferc_cogen_docket,ferc_cogen_docket,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number
-net_metering,,,,,,,,,,,,,net_metering_for_facilities_with_solar_or_wind_generation,net_metering_for_facilities_with_solar_or_wind_generation,net_metering_for_facilities_with_solar_or_wind_generation,,,,,,,
+has_net_metering,,,,,,,,,,,,,net_metering_for_facilities_with_solar_or_wind_generation,net_metering_for_facilities_with_solar_or_wind_generation,net_metering_for_facilities_with_solar_or_wind_generation,,,,,,,
 ferc_small_power_producer,,,,,,,ferc_small_power,ferc_small_power,ferc_small_power,ferc_small_power,ferc_small_power,ferc_small_power_producer_status,ferc_small_power_producer_status,ferc_small_power_producer_status,ferc_small_power_producer_status,ferc_small_power_producer_status,ferc_small_power_producer_status,ferc_small_power_producer_status,ferc_small_power_producer_status,ferc_small_power_producer_status,ferc_small_power_producer_status,ferc_small_power_producer_status
 ferc_small_power_producer_docket_no,,,,,,,ferc_small_power_docket,ferc_small_power_docket,ferc_small_power_docket,ferc_small_power_docket,ferc_small_power_docket,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number
 ferc_exempt_wholesale_generator,,,,,,,ferc_exempt_wholesale,ferc_exempt_wholesale,ferc_exempt_wholesale,ferc_exempt_wholesale,ferc_exempt_wholesale,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status

src/pudl/package_data/eia861/column_maps/demand_response_eia861.csv

@@ -35,5 +35,5 @@ commercial_other_costs,30,30,30,30,31,31,32,31,31,31
 industrial_other_costs,31,31,31,31,32,32,33,32,32,32
 transportation_other_costs,32,32,32,32,33,33,34,33,33,33
 total_other_costs,33,33,33,33,34,34,35,34,34,34
-water_heater,34,34,34,34,35,35,36,35,35,35
+num_water_heaters,34,34,34,34,35,35,36,35,35,35
 early_release,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1

src/pudl/package_data/eia861/column_maps/demand_side_management_eia861.csv

@@ -74,7 +74,7 @@ total_load_management_annual_cost,45,49,49,49,49,49,49,49,49,49,65,65,70
 total_load_management_annual_incentive_payment,-1,50,50,50,50,50,50,50,50,50,75,75,75
 total_annual_indirect_program_cost,46,51,51,51,51,51,51,51,51,51,80,80,80
 total_annual_total_cost,47,52,52,52,52,52,52,52,52,52,85,85,85
-demand_side_management,-1,-1,-1,-1,-1,-1,-1,53,53,53,86,86,-1
+has_demand_side_management,-1,-1,-1,-1,-1,-1,-1,53,53,53,86,86,-1
 reported_as_another_company,-1,-1,-1,-1,-1,-1,-1,54,54,54,87,87,-1
 energy_savings_independently_verified,-1,-1,-1,-1,-1,-1,-1,55,55,55,88,88,-1
 energy_savings_estimates_independently_verified,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,89,89,-1
@@ -85,7 +85,7 @@ commercial_price_responsiveness_customers,-1,-1,-1,-1,-1,-1,-1,59,59,59,93,93,87
 industrial_price_responsiveness_customers,-1,-1,-1,-1,-1,-1,-1,60,60,60,94,94,88
 transportation_price_responsiveness_customers,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,95,95,89
 other_price_responsiveness_customers,-1,-1,-1,-1,-1,-1,-1,61,61,61,-1,-1,-1
-time_responsive_programs,-1,-1,-1,-1,-1,-1,-1,62,62,62,96,96,-1
+has_time_responsive_programs,-1,-1,-1,-1,-1,-1,-1,62,62,62,96,96,-1
 residential_time_responsiveness_customers,-1,-1,-1,-1,-1,-1,-1,63,63,63,97,97,90
 commercial_time_responsiveness_customers,-1,-1,-1,-1,-1,-1,-1,64,64,64,98,98,91
 industrial_time_responsiveness_customers,-1,-1,-1,-1,-1,-1,-1,65,65,65,99,99,92

src/pudl/package_data/eia861/column_maps/frame_eia861.csv

@@ -13,7 +13,7 @@ distribution_systems,8,9,9,9,10,10,10
 dynamic_pricing,9,10,10,10,11,11,11
 energy_efficiency,10,11,11,11,12,12,12
 mergers,11,12,12,12,13,13,13
-net_metering,12,13,13,13,14,14,14
+has_net_metering,12,13,13,13,14,14,14
 non_net_metering_distributed,13,14,14,14,15,15,15
 operational_data,14,15,15,15,16,16,16
 reliability,15,16,16,16,17,17,17