Merge pull request #2162 from catalyst-cooperative/transform_f1_elc_o…

…p_mnt_expn

Transform `f1_elc_op_mnt_expn`

docs/release_notes.rst

@@ -22,9 +22,11 @@ Data Coverage
   * :ref:`transmission_ferc1`, see issue :issue:`1822` & PR :pr:`2103`
   * :ref:`utility_plant_summary_ferc1`, see issue :issue:`1806` & PR :pr:`2105`.
   * :ref:`balance_sheet_assets_ferc1`, see issue :issue:`1805` & PRs :pr:`2112,2127`.
+  * :ref:`balance_sheet_liabilities_ferc1`, see issue :issue:`1810` & PR :pr:`2134`.
   * :ref:`depreciation_amortization_summary_ferc1`, see issue :issue:`1816` & PR
     :pr:`2143`.
   * :ref:`income_statement_ferc1`, see issue :issue:`1813` & PR :pr:`2147`.
+  * :ref:`electric_opex_ferc1`, see issue :issue:`1817` & PR :pr:`2162`.
   * :ref:`retained_earnings_ferc1`, see issue :issue:`1811` & PR :pr:`2155`.
 
 .. _release-v2022.11.30:

src/pudl/extract/ferc1.py

@@ -141,10 +141,6 @@
         "dbf": "f1_purchased_pwr",
         "xbrl": "purchased_power_326",
     },
-    "electric_oandm_ferc1": {
-        "dbf": "f1_324_elc_expns",
-        "xbrl": "electric_operations_and_maintenance_expenses_320",
-    },
     "electric_energy_sources_ferc1": {
         "dbf": "f1_elctrc_erg_acct",
         "xbrl": "electric_energy_account_401a",
@@ -161,6 +157,10 @@
         "dbf": "f1_xmssn_line",
         "xbrl": "transmission_line_statistics_422",
     },
+    "electric_opex_ferc1": {
+        "dbf": "f1_elc_op_mnt_expn",
+        "xbrl": "electric_operations_and_maintenance_expenses_320",
+    },
     "balance_sheet_liabilities_ferc1": {
         "dbf": "f1_bal_sheet_cr",
         "xbrl": "comparative_balance_sheet_liabilities_and_other_credits_110",

src/pudl/metadata/fields.py

@@ -588,6 +588,12 @@
     },
     "exchange_energy_delivered_mwh": {"type": "number", "unit": "MWh"},
     "exchange_energy_received_mwh": {"type": "number", "unit": "MWh"},
+    "expense": {
+        "type": "number",
+        "unit": "USD",
+        "description": "The amount of a given expense in USD.",
+    },
+    "expense_type": {"type": "string", "description": "The type of expense."},
     "ferc_account": {
         "type": "string",
         "description": "Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting.",

src/pudl/metadata/resources/ferc1.py

@@ -135,6 +135,24 @@
         "etl_group": "ferc1",
         "field_namespace": "ferc1",
     },
+    "electric_opex_ferc1": {
+        "description": "Operating and maintenance costs associated with producing electricty.",
+        "schema": {
+            "fields": [
+                "utility_id_ferc1",
+                "report_year",
+                "expense",
+                "expense_type",
+                "record_id",
+                "ferc_account",
+                "row_type_xbrl",
+            ],
+            "primary_key": ["utility_id_ferc1", "report_year", "expense_type"],
+        },
+        "sources": ["ferc1"],
+        "etl_group": "ferc1",
+        "field_namespace": "ferc1",
+    },
     "ferc_accounts": {
         "description": "Account numbers from the FERC Uniform System of Accounts for Electric Plant, which is defined in Code of Federal Regulations (CFR) Title 18, Chapter I, Subchapter C, Part 101. (See e.g. https://www.law.cornell.edu/cfr/text/18/part-101).",
         "schema": {

src/pudl/package_data/settings/etl_fast.yml

@@ -20,6 +20,7 @@ ferc_to_sqlite_settings:
       - f1_elctrc_erg_acct
       - f1_utltyplnt_smmry
       - f1_xmssn_line
+      - f1_elc_op_mnt_expn
       - f1_bal_sheet_cr
       - f1_comp_balance_db
       - f1_income_stmnt
@@ -62,6 +63,8 @@ ferc_to_sqlite_settings:
       - statement_of_income_114_instant
       - summary_of_depreciation_and_amortization_charges_section_a_336_duration
       - summary_of_depreciation_and_amortization_charges_section_a_336_instant
+      - electric_operations_and_maintenance_expenses_320_duration
+      - electric_operations_and_maintenance_expenses_320_instant
       - retained_earnings_118_duration
       - retained_earnings_118_instant
 
@@ -96,6 +99,7 @@ datasets:
       - transmission_ferc1
       - electric_energy_sources_ferc1
       - electric_energy_dispositions_ferc1
+      - electric_opex_ferc1
       - utility_plant_summary_ferc1
       - balance_sheet_liabilities_ferc1
       - depreciation_amortization_summary_ferc1

src/pudl/package_data/settings/etl_full.yml

@@ -561,6 +561,7 @@ datasets:
       - electric_energy_sources_ferc1
       - electric_energy_dispositions_ferc1
       - utility_plant_summary_ferc1
+      - electric_opex_ferc1
       - balance_sheet_liabilities_ferc1
       - depreciation_amortization_summary_ferc1
       - balance_sheet_assets_ferc1

src/pudl/transform/ferc1.py

@@ -73,6 +73,7 @@ class TableIdFerc1(enum.Enum):
     ELECTRIC_ENERGY_SOURCES_FERC1 = "electric_energy_sources_ferc1"
     ELECTRIC_ENERGY_DISPOSITIONS_FERC1 = "electric_energy_dispositions_ferc1"
     UTILITY_PLANT_SUMMARY_FERC1 = "utility_plant_summary_ferc1"
+    ELECTRIC_OPEX_FERC1 = "electric_opex_ferc1"
     BALANCE_SHEET_LIABILITIES = "balance_sheet_liabilities_ferc1"
     DEPRECIATION_AMORTIZATION_SUMMARY_FERC1 = "depreciation_amortization_summary_ferc1"
     BALANCE_SHEET_ASSETS_FERC1 = "balance_sheet_assets_ferc1"
@@ -926,7 +927,7 @@ def merge_xbrl_metadata(
         if not params:
             params = self.params.merge_xbrl_metadata
         if params.on:
-            logger.info(f"{self.table_id.value}: merging metadata")
+            logger.info(f"{self.table_id.value}: Merging metadata")
             df = merge_xbrl_metadata(df, self.xbrl_metadata, params)
         return df
 
@@ -948,13 +949,17 @@ def align_row_numbers_dbf(
     def drop_duplicate_rows_dbf(
         self, df: pd.DataFrame, params: DropDuplicateRowsDbf | None = None
     ) -> pd.DataFrame:
-        """Drop the duplicate DBF rows when the PKs and data columns are the same.
+        """Drop the DBF rows where the PKs and data columns are duplicated.
 
         Wrapper function for :func:`drop_duplicate_rows_dbf`.
         """
         if params is None:
             params = self.params.drop_duplicate_rows_dbf
         if params.table_name:
+            logger.info(
+                f"{self.table_id.value}: Dropping rows where primary key and data "
+                "columns are duplicated."
+            )
             df = drop_duplicate_rows_dbf(df, params=params)
         return df
 
@@ -3236,6 +3241,34 @@ def merge_xbrl_metadata(
         return df
 
 
+class ElectricOpexFerc1TableTransformer(Ferc1AbstractTableTransformer):
+    """Transformer class for :ref:`electric_opex_ferc1` table."""
+
+    table_id: TableIdFerc1 = TableIdFerc1.ELECTRIC_OPEX_FERC1
+    has_unique_record_ids: bool = False
+
+    def targeted_drop_duplicates_dbf(self, raw_df: pd.DataFrame) -> pd.DataFrame:
+        """Drop incorrect duplicate from 2002.
+
+        In 2002, utility_id_ferc1_dbf 96 reported two values for
+        administrative_and_general_operation_expense. I found the correct value by
+        looking at the prev_yr_amt value in 2003. This removes the incorrect row.
+        """
+        start_len = len(raw_df)
+        raw_df = raw_df[
+            ~((raw_df["report_year"] == 2002) & (raw_df["crnt_yr_amt"] == 35990321))
+        ]
+        if (dropped := start_len - len(raw_df)) > 1:
+            raise AssertionError(f"More rows dropped than expected: {dropped}")
+        logger.info("Heyyyy dropping that one row")
+        return raw_df
+
+    @cache_df(key="dbf")
+    def process_dbf(self, raw_dbf: pd.DataFrame) -> pd.DataFrame:
+        """Process DBF but drop a bad row that is flagged by drop_duplicates."""
+        return super().process_dbf(self.targeted_drop_duplicates_dbf(raw_dbf))
+
+
 def transform(
     ferc1_dbf_raw_dfs: dict[str, pd.DataFrame],
     ferc1_xbrl_raw_dfs: dict[str, dict[str, pd.DataFrame]],
@@ -3270,6 +3303,7 @@ def transform(
         "electric_energy_sources_ferc1": ElectricEnergySourcesFerc1TableTransformer,
         "electric_energy_dispositions_ferc1": ElectricEnergyDispositionsFerc1TableTransformer,
         "utility_plant_summary_ferc1": UtilityPlantSummaryFerc1TableTransformer,
+        "electric_opex_ferc1": ElectricOpexFerc1TableTransformer,
         "balance_sheet_liabilities_ferc1": BalanceSheetLiabilitiesFerc1TableTransformer,
         "depreciation_amortization_summary_ferc1": DepreciationAmortizationSummaryFerc1TableTransformer,
         "balance_sheet_assets_ferc1": BalanceSheetAssetsFerc1TableTransformer,