Consistently use baseline expanded_income to fuzz reform results in dropq tables

RELEASES.md

@@ -4,13 +4,15 @@ Go
 [here](https://github.com/open-source-economics/Tax-Calculator/pulls?q=is%3Apr+is%3Aclosed)
 for a complete commit history.
 
-Release 0.Y.Z on 2017-??-??
+Release 0.11.0 on 2017-??-??
 ----------------------------
 (last merged pull request is
 [#xxxx](https://github.com/open-source-economics/Tax-Calculator/pull/xxxx))
 
 **API Changes**
-- None
+- Revise dropq distribution and difference tables used by TaxBrain
+  [[#1537](https://github.com/open-source-economics/Tax-Calculator/pull/1537)
+  by Anderson Frailey and Martin Holmer]
 
 **New Features**
 - None

taxcalc/dropq/dropq.py

@@ -1,6 +1,7 @@
 """
 The dropq functions are used by TaxBrain to call Tax-Calculator in order
-to maintain the privacy of the micro data being used by TaxBrain.
+to maintain the privacy of the micro data being used by TaxBrain.  This
+is done by adding random "fuzz" to the results in each table cell.
 """
 # CODING-STYLE CHECKS:
 # pep8 --ignore=E402 dropq.py
@@ -20,7 +21,7 @@
 # specify constants
 PLAN_COLUMN_TYPES = [float] * len(TABLE_LABELS)
 
-DIFF_COLUMN_TYPES = [int, int, int, float, float, str, str, str]
+DIFF_COLUMN_TYPES = [int, int, int, float, float, str, str, str, str]
 
 DECILE_ROW_NAMES = ['perc0-10', 'perc10-20', 'perc20-30', 'perc30-40',
                     'perc40-50', 'perc50-60', 'perc60-70', 'perc70-80',
@@ -102,98 +103,111 @@ def run_nth_year_tax_calc_model(year_n, start_year,
     np.random.seed(seed)  # pylint: disable=no-member
 
     # construct dropq summary results from raw results
-    (m2_dec, m1_dec, df_dec, pdf_dec, cdf_dec,
-     m2_bin, m1_bin, df_bin, pdf_bin, cdf_bin,
-     itax_sumd, ptax_sumd, comb_sumd,
-     itax_sum1, ptax_sum1, comb_sum1,
-     itax_sum2, ptax_sum2, comb_sum2) = dropq_summary(rawres1, rawres2, mask)
-
-    # construct DataFrames containing selected summary results
-    totsd = [itax_sumd, ptax_sumd, comb_sumd]
-    fiscal_tots_diff = pd.DataFrame(data=totsd, index=TOTAL_ROW_NAMES)
-
-    tots1 = [itax_sum1, ptax_sum1, comb_sum1]
-    fiscal_tots_baseline = pd.DataFrame(data=tots1, index=TOTAL_ROW_NAMES)
-
-    tots2 = [itax_sum2, ptax_sum2, comb_sum2]
-    fiscal_tots_reform = pd.DataFrame(data=tots2, index=TOTAL_ROW_NAMES)
-
-    # remove negative incomes from selected summary results
-    df_bin.drop(df_bin.index[0], inplace=True)
-    pdf_bin.drop(pdf_bin.index[0], inplace=True)
-    cdf_bin.drop(cdf_bin.index[0], inplace=True)
-    m2_bin.drop(m2_bin.index[0], inplace=True)
-    m1_bin.drop(m1_bin.index[0], inplace=True)
+    (dist1_dec, dist2_dec,
+     diff_itax_dec, diff_ptax_dec, diff_comb_dec,
+     dist1_bin, dist2_bin,
+     diff_itax_bin, diff_ptax_bin, diff_comb_bin,
+     aggr_itax_d, aggr_ptax_d, aggr_comb_d,
+     aggr_itax_1, aggr_ptax_1, aggr_comb_1,
+     aggr_itax_2, aggr_ptax_2, aggr_comb_2) = dropq_summary(rawres1,
+                                                            rawres2,
+                                                            mask)
+
+    # construct DataFrames containing aggregate tax totals
+    # ... for reform-minus-baseline difference
+    aggrd = [aggr_itax_d, aggr_ptax_d, aggr_comb_d]
+    aggr_d = pd.DataFrame(data=aggrd, index=TOTAL_ROW_NAMES)
+    # ... for baseline
+    aggr1 = [aggr_itax_1, aggr_ptax_1, aggr_comb_1]
+    aggr_1 = pd.DataFrame(data=aggr1, index=TOTAL_ROW_NAMES)
+    # ... for reform
+    aggr2 = [aggr_itax_2, aggr_ptax_2, aggr_comb_2]
+    aggr_2 = pd.DataFrame(data=aggr2, index=TOTAL_ROW_NAMES)
 
     elapsed_time = time.time() - start_time
     print('elapsed time for this run: ', elapsed_time)
 
-    def append_year(tdf):
+    def append_year(pdf):
         """
-        append_year embedded function
+        append_year embedded function revises all column names in pdf
         """
-        tdf.columns = [str(col) + '_{}'.format(year_n) for col in tdf.columns]
-        return tdf
+        pdf.columns = [str(col) + '_{}'.format(year_n) for col in pdf.columns]
+        return pdf
 
     # optionally return non-JSON results
     if not return_json:
-        return (append_year(m2_dec), append_year(m1_dec), append_year(df_dec),
-                append_year(pdf_dec), append_year(cdf_dec),
-                append_year(m2_bin), append_year(m1_bin), append_year(df_bin),
-                append_year(pdf_bin), append_year(cdf_bin),
-                append_year(fiscal_tots_diff),
-                append_year(fiscal_tots_baseline),
-                append_year(fiscal_tots_reform))
-
-    # optionally construct JSON results
-    decile_row_names_i = [x + '_' + str(year_n) for x in DECILE_ROW_NAMES]
-    m2_dec_table_i = create_json_table(m2_dec,
-                                       row_names=decile_row_names_i,
-                                       column_types=PLAN_COLUMN_TYPES)
-    m1_dec_table_i = create_json_table(m1_dec,
-                                       row_names=decile_row_names_i,
-                                       column_types=PLAN_COLUMN_TYPES)
-    df_dec_table_i = create_json_table(df_dec,
-                                       row_names=decile_row_names_i,
-                                       column_types=DIFF_COLUMN_TYPES)
-    pdf_dec_table_i = create_json_table(pdf_dec,
-                                        row_names=decile_row_names_i,
-                                        column_types=DIFF_COLUMN_TYPES)
-    cdf_dec_table_i = create_json_table(cdf_dec,
-                                        row_names=decile_row_names_i,
-                                        column_types=DIFF_COLUMN_TYPES)
-    bin_row_names_i = [x + '_' + str(year_n) for x in BIN_ROW_NAMES]
-    m2_bin_table_i = create_json_table(m2_bin,
-                                       row_names=bin_row_names_i,
-                                       column_types=PLAN_COLUMN_TYPES)
-    m1_bin_table_i = create_json_table(m1_bin,
-                                       row_names=bin_row_names_i,
-                                       column_types=PLAN_COLUMN_TYPES)
-    df_bin_table_i = create_json_table(df_bin,
-                                       row_names=bin_row_names_i,
-                                       column_types=DIFF_COLUMN_TYPES)
-    pdf_bin_table_i = create_json_table(pdf_bin,
-                                        row_names=bin_row_names_i,
-                                        column_types=DIFF_COLUMN_TYPES)
-    cdf_bin_table_i = create_json_table(cdf_bin,
-                                        row_names=bin_row_names_i,
-                                        column_types=DIFF_COLUMN_TYPES)
-    total_row_names_i = [x + '_' + str(year_n) for x in TOTAL_ROW_NAMES]
-    fiscal_yr_total_df = create_json_table(fiscal_tots_diff,
-                                           row_names=total_row_names_i)
-    fiscal_yr_total_df = dict((k, v[0]) for k, v in fiscal_yr_total_df.items())
-    fiscal_yr_total_bl = create_json_table(fiscal_tots_baseline,
-                                           row_names=total_row_names_i)
-    fiscal_yr_total_bl = dict((k, v[0]) for k, v in fiscal_yr_total_bl.items())
-    fiscal_yr_total_rf = create_json_table(fiscal_tots_reform,
-                                           row_names=total_row_names_i)
-    fiscal_yr_total_rf = dict((k, v[0]) for k, v in fiscal_yr_total_rf.items())
+        return (append_year(dist2_dec),
+                append_year(dist1_dec),
+                append_year(diff_itax_dec),
+                append_year(diff_ptax_dec),
+                append_year(diff_comb_dec),
+                append_year(dist2_bin),
+                append_year(dist1_bin),
+                append_year(diff_itax_bin),
+                append_year(diff_ptax_bin),
+                append_year(diff_comb_bin),
+                append_year(aggr_d),
+                append_year(aggr_1),
+                append_year(aggr_2))
+
+    # optionally construct JSON results tables for year n
+    dec_row_names_n = [x + '_' + str(year_n) for x in DECILE_ROW_NAMES]
+    dist2_dec_table_n = create_json_table(dist2_dec,
+                                          row_names=dec_row_names_n,
+                                          column_types=PLAN_COLUMN_TYPES)
+    dist1_dec_table_n = create_json_table(dist1_dec,
+                                          row_names=dec_row_names_n,
+                                          column_types=PLAN_COLUMN_TYPES)
+    diff_itax_dec_table_n = create_json_table(diff_itax_dec,
+                                              row_names=dec_row_names_n,
+                                              column_types=DIFF_COLUMN_TYPES)
+    diff_ptax_dec_table_n = create_json_table(diff_ptax_dec,
+                                              row_names=dec_row_names_n,
+                                              column_types=DIFF_COLUMN_TYPES)
+    diff_comb_dec_table_n = create_json_table(diff_comb_dec,
+                                              row_names=dec_row_names_n,
+                                              column_types=DIFF_COLUMN_TYPES)
+    bin_row_names_n = [x + '_' + str(year_n) for x in BIN_ROW_NAMES]
+    dist2_bin_table_n = create_json_table(dist2_bin,
+                                          row_names=bin_row_names_n,
+                                          column_types=PLAN_COLUMN_TYPES)
+    dist1_bin_table_n = create_json_table(dist1_bin,
+                                          row_names=bin_row_names_n,
+                                          column_types=PLAN_COLUMN_TYPES)
+    diff_itax_bin_table_n = create_json_table(diff_itax_bin,
+                                              row_names=bin_row_names_n,
+                                              column_types=DIFF_COLUMN_TYPES)
+    diff_ptax_bin_table_n = create_json_table(diff_ptax_bin,
+                                              row_names=bin_row_names_n,
+                                              column_types=DIFF_COLUMN_TYPES)
+    diff_comb_bin_table_n = create_json_table(diff_comb_bin,
+                                              row_names=bin_row_names_n,
+                                              column_types=DIFF_COLUMN_TYPES)
+    total_row_names_n = [x + '_' + str(year_n) for x in TOTAL_ROW_NAMES]
+    aggr_d_table_n = create_json_table(aggr_d,
+                                       row_names=total_row_names_n)
+    aggr_d_table_n = dict((k, v[0]) for k, v in aggr_d_table_n.items())
+    aggr_1_table_n = create_json_table(aggr_1,
+                                       row_names=total_row_names_n)
+    aggr_1_table_n = dict((k, v[0]) for k, v in aggr_1_table_n.items())
+    aggr_2_table_n = create_json_table(aggr_2,
+                                       row_names=total_row_names_n)
+    aggr_2_table_n = dict((k, v[0]) for k, v in aggr_2_table_n.items())
 
     # return JSON results
-    return (m2_dec_table_i, m1_dec_table_i, df_dec_table_i, pdf_dec_table_i,
-            cdf_dec_table_i, m2_bin_table_i, m1_bin_table_i, df_bin_table_i,
-            pdf_bin_table_i, cdf_bin_table_i, fiscal_yr_total_df,
-            fiscal_yr_total_bl, fiscal_yr_total_rf)
+    return (dist2_dec_table_n,
+            dist1_dec_table_n,
+            diff_itax_dec_table_n,
+            diff_ptax_dec_table_n,
+            diff_comb_dec_table_n,
+            dist2_bin_table_n,
+            dist1_bin_table_n,
+            diff_itax_bin_table_n,
+            diff_ptax_bin_table_n,
+            diff_comb_bin_table_n,
+            aggr_d_table_n,
+            aggr_1_table_n,
+            aggr_2_table_n)
 
 
 def run_nth_year_gdp_elast_model(year_n, start_year,
@@ -218,10 +232,10 @@ def run_nth_year_gdp_elast_model(year_n, start_year,
     # return gdp_effect results
     if return_json:
         gdp_df = pd.DataFrame(data=[gdp_effect], columns=['col0'])
-        gdp_elast_names_i = [x + '_' + str(year_n)
+        gdp_elast_names_n = [x + '_' + str(year_n)
                              for x in GDP_ELAST_ROW_NAMES]
         gdp_elast_total = create_json_table(gdp_df,
-                                            row_names=gdp_elast_names_i,
+                                            row_names=gdp_elast_names_n,
                                             num_decimals=5)
         gdp_elast_total = dict((k, v[0]) for k, v in gdp_elast_total.items())
         return gdp_elast_total