From 9e8f4f405f57bbfe4970f1f12f62bbd06f3dbd6e Mon Sep 17 00:00:00 2001 From: Jordan Day Date: Thu, 29 Aug 2024 14:28:35 +0100 Subject: [PATCH 1/8] quick and easy fix for period and refernce being index --- mbs_results/data_cleaning.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mbs_results/data_cleaning.py b/mbs_results/data_cleaning.py index 82244950..c71c4fab 100755 --- a/mbs_results/data_cleaning.py +++ b/mbs_results/data_cleaning.py @@ -214,6 +214,9 @@ def join_manual_constructions( pd.DataFrame dataframe with correctly formatted column datatypes. """ + if period not in df.columns or reference not in df.columns: + df = df.reset_index() + if not is_same_dtype(df, manual_constructions, period): manual_constructions[period] = convert_column_to_datetime( manual_constructions[period] From 5d96b220c3698dc0b845662717da600f3b45d0e0 Mon Sep 17 00:00:00 2001 From: Jordan Day Date: Thu, 29 Aug 2024 16:05:17 +0100 Subject: [PATCH 2/8] quick fix to filter on question_no, set default value for column for quicker testing --- mbs_results/data_cleaning.py | 67 +++++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 16 deletions(-) diff --git a/mbs_results/data_cleaning.py b/mbs_results/data_cleaning.py index c71c4fab..9ca6515a 100755 --- a/mbs_results/data_cleaning.py +++ b/mbs_results/data_cleaning.py @@ -188,6 +188,7 @@ def join_manual_constructions( manual_constructions: pd.DataFrame, reference: str, period: str, + question_no: str = "question_no", **config ): """ @@ -206,6 +207,8 @@ def join_manual_constructions( the name of the reference column period: str the name of the period column + period: str + the name of the question number column **config: Dict main pipeline configuration. Can be used to input the entire config dictionary @@ -214,27 +217,42 @@ def join_manual_constructions( pd.DataFrame dataframe with correctly formatted column datatypes. """ - if period not in df.columns or reference not in df.columns: - df = df.reset_index() - if not is_same_dtype(df, manual_constructions, period): - manual_constructions[period] = convert_column_to_datetime( - manual_constructions[period] - ) + question_no_from_df = df[question_no].unique().tolist() + manual_constructions_filter = manual_constructions.loc[ + manual_constructions[question_no].isin(question_no_from_df) + ] + + if manual_constructions_filter.empty: + # return original df as nothing present to use + # as manual construction + return df + else: + # manual_constructions = manual_constructions_filter + if period not in df.columns or reference not in df.columns: + df = df.reset_index() + + if not is_same_dtype(df, manual_constructions, period): + manual_constructions[period] = convert_column_to_datetime( + manual_constructions[period] + ) - if not is_same_dtype(df, manual_constructions, reference): - manual_constructions[reference] = manual_constructions[reference].astype( - df[reference].dtype - ) + if not is_same_dtype(df, manual_constructions, reference): + manual_constructions[reference] = manual_constructions[reference].astype( + df[reference].dtype + ) - manual_constructions.set_index([reference, period], inplace=True) - df.set_index([reference, period], inplace=True) + manual_constructions.set_index([reference, period], inplace=True) + df.set_index([reference, period], inplace=True) - validate_manual_constructions(df, manual_constructions) + validate_manual_constructions(df, manual_constructions) - return df.merge( - manual_constructions, on=[reference, period], how="left", suffixes=("", "_man") - ).reset_index() + return df.merge( + manual_constructions, + on=[reference, period, question_no], + how="left", + suffixes=("", "_man"), + ).reset_index() def is_same_dtype(df: pd.DataFrame, df2: pd.DataFrame, col_name: str) -> bool: @@ -355,3 +373,20 @@ def create_imputation_class( ) return df + + +if __name__ == "__main__": + df = pd.DataFrame( + np.array([[202401, 2, 3, 10], [202402, 5, 6, 30], [202403, 8, 9, 30]]), + columns=["period", "reference", "c","question_no"], + ) + df2 = pd.DataFrame( + np.array([[202401, 2, 3, 10], [202402, 5, 6, 30], [202403, 8, 9, 20]]), + columns=["period", "reference", "c","question_no"], + ) + # df2 = df2.loc[df2["period"] == 1] + # print(df2.empty) + df["period"] = convert_column_to_datetime(df["period"]) + df.set_index(['period'],inplace=True) + output = join_manual_constructions(df, df2, period="period", reference="reference") + print(output) From 0bcaaae891a99b2a0b7aaec579c66801d027c11f Mon Sep 17 00:00:00 2001 From: Jordan Day Date: Thu, 29 Aug 2024 16:11:42 +0100 Subject: [PATCH 3/8] remove name main section --- mbs_results/data_cleaning.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/mbs_results/data_cleaning.py b/mbs_results/data_cleaning.py index 9ca6515a..f92fd95f 100755 --- a/mbs_results/data_cleaning.py +++ b/mbs_results/data_cleaning.py @@ -373,20 +373,3 @@ def create_imputation_class( ) return df - - -if __name__ == "__main__": - df = pd.DataFrame( - np.array([[202401, 2, 3, 10], [202402, 5, 6, 30], [202403, 8, 9, 30]]), - columns=["period", "reference", "c","question_no"], - ) - df2 = pd.DataFrame( - np.array([[202401, 2, 3, 10], [202402, 5, 6, 30], [202403, 8, 9, 20]]), - columns=["period", "reference", "c","question_no"], - ) - # df2 = df2.loc[df2["period"] == 1] - # print(df2.empty) - df["period"] = convert_column_to_datetime(df["period"]) - df.set_index(['period'],inplace=True) - output = join_manual_constructions(df, df2, period="period", reference="reference") - print(output) From bc497582238275e199a90bb54b6e363fd35dd593 Mon Sep 17 00:00:00 2001 From: Jordan Day Date: Thu, 29 Aug 2024 16:13:55 +0100 Subject: [PATCH 4/8] adding question_no into index setting --- mbs_results/data_cleaning.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mbs_results/data_cleaning.py b/mbs_results/data_cleaning.py index f92fd95f..6193a0fd 100755 --- a/mbs_results/data_cleaning.py +++ b/mbs_results/data_cleaning.py @@ -242,8 +242,8 @@ def join_manual_constructions( df[reference].dtype ) - manual_constructions.set_index([reference, period], inplace=True) - df.set_index([reference, period], inplace=True) + manual_constructions.set_index([reference, period, question_no], inplace=True) + df.set_index([reference, period, question_no], inplace=True) validate_manual_constructions(df, manual_constructions) From b0c280a3057c64f0996b3891b79fadbf6a4da42a Mon Sep 17 00:00:00 2001 From: Jordan Day Date: Thu, 29 Aug 2024 16:42:23 +0100 Subject: [PATCH 5/8] refactoring to drop question col after filtering --- mbs_results/data_cleaning.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/mbs_results/data_cleaning.py b/mbs_results/data_cleaning.py index 6193a0fd..0214e9df 100755 --- a/mbs_results/data_cleaning.py +++ b/mbs_results/data_cleaning.py @@ -228,28 +228,28 @@ def join_manual_constructions( # as manual construction return df else: - # manual_constructions = manual_constructions_filter + manual_constructions_filter.drop(columns = [question_no],inplace=True) if period not in df.columns or reference not in df.columns: df = df.reset_index() - if not is_same_dtype(df, manual_constructions, period): - manual_constructions[period] = convert_column_to_datetime( - manual_constructions[period] + if not is_same_dtype(df, manual_constructions_filter, period): + manual_constructions_filter[period] = convert_column_to_datetime( + manual_constructions_filter[period] ) - if not is_same_dtype(df, manual_constructions, reference): - manual_constructions[reference] = manual_constructions[reference].astype( + if not is_same_dtype(df, manual_constructions_filter, reference): + manual_constructions_filter[reference] = manual_constructions_filter[reference].astype( df[reference].dtype ) - manual_constructions.set_index([reference, period, question_no], inplace=True) - df.set_index([reference, period, question_no], inplace=True) + manual_constructions_filter.set_index([reference, period], inplace=True) + df.set_index([reference, period], inplace=True) - validate_manual_constructions(df, manual_constructions) + validate_manual_constructions(df, manual_constructions_filter) return df.merge( - manual_constructions, - on=[reference, period, question_no], + manual_constructions_filter, + on=[reference, period], how="left", suffixes=("", "_man"), ).reset_index() From 3da3841093693ee59adc3492f57af12beb0963a2 Mon Sep 17 00:00:00 2001 From: Jordan Day Date: Wed, 4 Sep 2024 13:35:28 +0100 Subject: [PATCH 6/8] Updating unit tests to include question_no column --- .../rom_test_data_case_mc_10_input.csv | 26 ++++++------ .../rom_test_data_case_mc_10_output.csv | 26 ++++++------ .../rom_test_data_case_mc_1_input.csv | 26 ++++++------ .../rom_test_data_case_mc_1_output.csv | 26 ++++++------ .../rom_test_data_case_mc_2_input.csv | 34 +++++++-------- .../rom_test_data_case_mc_2_output.csv | 34 +++++++-------- .../rom_test_data_case_mc_3_input.csv | 42 +++++++++---------- .../rom_test_data_case_mc_3_output.csv | 42 +++++++++---------- .../rom_test_data_case_mc_4_input.csv | 34 +++++++-------- .../rom_test_data_case_mc_4_output.csv | 34 +++++++-------- .../rom_test_data_case_mc_5_input.csv | 34 +++++++-------- .../rom_test_data_case_mc_5_output.csv | 34 +++++++-------- .../rom_test_data_case_mc_6_input.csv | 18 ++++---- .../rom_test_data_case_mc_6_output.csv | 18 ++++---- .../rom_test_data_case_mc_7_input.csv | 34 +++++++-------- .../rom_test_data_case_mc_7_output.csv | 34 +++++++-------- .../rom_test_data_case_mc_8_input.csv | 34 +++++++-------- .../rom_test_data_case_mc_8_output.csv | 34 +++++++-------- .../rom_test_data_case_mc_9_input.csv | 26 ++++++------ .../rom_test_data_case_mc_9_output.csv | 26 ++++++------ tests/test_ratio_of_means.py | 2 +- 21 files changed, 309 insertions(+), 309 deletions(-) diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_10_input.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_10_input.csv index 9947e9f6..9d0ac5b4 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_10_input.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_10_input.csv @@ -1,13 +1,13 @@ -identifier,date,group,question,other,question_man -30001,202001,100,8444,51, -30001,202002,100,7476,51, -30001,202003,100,2003,51, -30002,202001,100,9343,72, -30002,202002,100,7818,72, -30002,202003,100,4897,72, -30003,202001,100,7511,7, -30003,202002,100,1761,7, -30003,202003,100,6492,7, -30004,202001,100,64,81,4321 -30004,202002,100,2113,81, -30004,202003,100,,81, +identifier,date,group,question,other,question_man,question_no +30001,202001,100,8444,51,,42 +30001,202002,100,7476,51,,42 +30001,202003,100,2003,51,,42 +30002,202001,100,9343,72,,42 +30002,202002,100,7818,72,,42 +30002,202003,100,4897,72,,42 +30003,202001,100,7511,7,,42 +30003,202002,100,1761,7,,42 +30003,202003,100,6492,7,,42 +30004,202001,100,64,81,4321,42 +30004,202002,100,2113,81,,42 +30004,202003,100,,81,,42 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_10_output.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_10_output.csv index c1542969..202cf6cb 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_10_output.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_10_output.csv @@ -1,13 +1,13 @@ -backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier -1.3231427378965,120.1990521327014,4,4,0,False,False,True,1.0000000000000,100,R,8444.000000,202001,30001 -1.2735215053763,90.8436018957346,3,4,4,False,False,False,0.7557763583314,100,R,7476.000000,202002,30001 -1.0000000000000,103.0153846153846,0,3,3,True,False,False,0.7852242744063,100,R,2003.000000,202003,30001 -1.3231427378965,120.1990521327014,4,4,0,False,False,True,1.0000000000000,100,R,9343.000000,202001,30002 -1.2735215053763,90.8436018957346,3,4,4,False,False,False,0.7557763583314,100,R,7818.000000,202002,30002 -1.0000000000000,103.0153846153846,0,3,3,True,False,False,0.7852242744063,100,R,4897.000000,202003,30002 -1.3231427378965,120.1990521327014,4,4,0,False,False,True,1.0000000000000,100,R,7511.000000,202001,30003 -1.2735215053763,90.8436018957346,3,4,4,False,False,False,0.7557763583314,100,R,1761.000000,202002,30003 -1.0000000000000,103.0153846153846,0,3,3,True,False,False,0.7852242744063,100,R,6492.000000,202003,30003 -1.3231427378965,120.1990521327014,4,4,0,False,False,True,1.0000000000000,100,R,64.000000,202001,30004 -1.2735215053763,90.8436018957346,3,4,4,False,False,False,0.7557763583314,100,R,2113.000000,202002,30004 -1.0000000000000,103.0153846153846,0,3,3,True,False,False,0.7852242744063,100,FIR,1659.178892,202003,30004 +backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier,question_no +1.3231427378965,120.1990521327014,4,4,0,False,False,True,1.0000000000000,100,R,8444.000000,202001,30001,42 +1.2735215053763,90.8436018957346,3,4,4,False,False,False,0.7557763583314,100,R,7476.000000,202002,30001,42 +1.0000000000000,103.0153846153846,0,3,3,True,False,False,0.7852242744063,100,R,2003.000000,202003,30001,42 +1.3231427378965,120.1990521327014,4,4,0,False,False,True,1.0000000000000,100,R,9343.000000,202001,30002,42 +1.2735215053763,90.8436018957346,3,4,4,False,False,False,0.7557763583314,100,R,7818.000000,202002,30002,42 +1.0000000000000,103.0153846153846,0,3,3,True,False,False,0.7852242744063,100,R,4897.000000,202003,30002,42 +1.3231427378965,120.1990521327014,4,4,0,False,False,True,1.0000000000000,100,R,7511.000000,202001,30003,42 +1.2735215053763,90.8436018957346,3,4,4,False,False,False,0.7557763583314,100,R,1761.000000,202002,30003,42 +1.0000000000000,103.0153846153846,0,3,3,True,False,False,0.7852242744063,100,R,6492.000000,202003,30003,42 +1.3231427378965,120.1990521327014,4,4,0,False,False,True,1.0000000000000,100,R,64.000000,202001,30004,42 +1.2735215053763,90.8436018957346,3,4,4,False,False,False,0.7557763583314,100,R,2113.000000,202002,30004,42 +1.0000000000000,103.0153846153846,0,3,3,True,False,False,0.7852242744063,100,FIR,1659.178892,202003,30004,42 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_1_input.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_1_input.csv index f4a70d9b..64256452 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_1_input.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_1_input.csv @@ -1,13 +1,13 @@ -identifier,date,group,question,other,question_man -30001,202001,100,8444,51, -30001,202002,100,7476,51, -30001,202003,100,2003,51, -30002,202001,100,9343,72, -30002,202002,100,7818,72, -30002,202003,100,4897,72, -30003,202001,100,7511,7, -30003,202002,100,1761,7, -30003,202003,100,6492,7, -30004,202001,100,,81,4321 -30004,202002,100,2113,81, -30004,202003,100,,81,3189 +identifier,date,group,question,other,question_man,question_no +30001,202001,100,8444,51,,42 +30001,202002,100,7476,51,,42 +30001,202003,100,2003,51,,42 +30002,202001,100,9343,72,,42 +30002,202002,100,7818,72,,42 +30002,202003,100,4897,72,,42 +30003,202001,100,7511,7,,42 +30003,202002,100,1761,7,,42 +30003,202003,100,6492,7,,42 +30004,202001,100,,81,4321,42 +30004,202002,100,2113,81,,42 +30004,202003,100,,81,3189,42 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_1_output.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_1_output.csv index 3629e6d9..3756786d 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_1_output.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_1_output.csv @@ -1,13 +1,13 @@ -backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier -1.4833186748754,194.6000000000000,3,3,0,False,False,True,1.0000000000000,100,R,8444.000000,202001,30001 -1.2735215053763,90.8436018957346,3,4,3,False,False,False,0.6741639655309,100,R,7476.000000,202002,30001 -1.0000000000000,103.0153846153846,0,3,3,True,False,False,0.7852242744063,100,R,2003.000000,202003,30001 -1.4833186748754,194.6000000000000,3,3,0,False,False,True,1.0000000000000,100,R,9343.000000,202001,30002 -1.2735215053763,90.8436018957346,3,4,3,False,False,False,0.6741639655309,100,R,7818.000000,202002,30002 -1.0000000000000,103.0153846153846,0,3,3,True,False,False,0.7852242744063,100,R,4897.000000,202003,30002 -1.4833186748754,194.6000000000000,3,3,0,False,False,True,1.0000000000000,100,R,7511.000000,202001,30003 -1.2735215053763,90.8436018957346,3,4,3,False,False,False,0.6741639655309,100,R,1761.000000,202002,30003 -1.0000000000000,103.0153846153846,0,3,3,True,False,False,0.7852242744063,100,R,6492.000000,202003,30003 -1.4833186748754,194.6000000000000,3,3,0,False,False,True,1.0000000000000,100,MC,4321.000000,202001,30004 -1.2735215053763,90.8436018957346,3,4,3,False,False,False,0.6741639655309,100,R,2113.000000,202002,30004 -1.0000000000000,103.0153846153846,0,3,3,True,False,False,0.7852242744063,100,MC,3189.000000,202003,30004 +backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier,question_no +1.4833186748754,194.6000000000000,3,3,0,False,False,True,1.0000000000000,100,R,8444.000000,202001,30001,42 +1.2735215053763,90.8436018957346,3,4,3,False,False,False,0.6741639655309,100,R,7476.000000,202002,30001,42 +1.0000000000000,103.0153846153846,0,3,3,True,False,False,0.7852242744063,100,R,2003.000000,202003,30001,42 +1.4833186748754,194.6000000000000,3,3,0,False,False,True,1.0000000000000,100,R,9343.000000,202001,30002,42 +1.2735215053763,90.8436018957346,3,4,3,False,False,False,0.6741639655309,100,R,7818.000000,202002,30002,42 +1.0000000000000,103.0153846153846,0,3,3,True,False,False,0.7852242744063,100,R,4897.000000,202003,30002,42 +1.4833186748754,194.6000000000000,3,3,0,False,False,True,1.0000000000000,100,R,7511.000000,202001,30003,42 +1.2735215053763,90.8436018957346,3,4,3,False,False,False,0.6741639655309,100,R,1761.000000,202002,30003,42 +1.0000000000000,103.0153846153846,0,3,3,True,False,False,0.7852242744063,100,R,6492.000000,202003,30003,42 +1.4833186748754,194.6000000000000,3,3,0,False,False,True,1.0000000000000,100,MC,4321.000000,202001,30004,42 +1.2735215053763,90.8436018957346,3,4,3,False,False,False,0.6741639655309,100,R,2113.000000,202002,30004,42 +1.0000000000000,103.0153846153846,0,3,3,True,False,False,0.7852242744063,100,MC,3189.000000,202003,30004,42 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_2_input.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_2_input.csv index eb24ab23..433c974d 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_2_input.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_2_input.csv @@ -1,17 +1,17 @@ -identifier,date,group,question,other,question_man -40001,202001,100,9491,35, -40001,202002,100,4783,35, -40001,202003,100,7902,35, -40001,202004,100,4911,35, -40002,202001,100,2095,63, -40002,202002,100,442,63, -40002,202003,100,3136,63, -40002,202004,100,2115,63, -40003,202001,100,7863,16, -40003,202002,100,8121,16, -40003,202003,100,2151,16, -40003,202004,100,1377,16, -40004,202001,100,5131,78, -40004,202002,100,9836,78, -40004,202003,100,,78,7525 -40004,202004,100,,78, +identifier,date,group,question,other,question_man,question_no +40001,202001,100,9491,35,,42 +40001,202002,100,4783,35,,42 +40001,202003,100,7902,35,,42 +40001,202004,100,4911,35,,42 +40002,202001,100,2095,63,,42 +40002,202002,100,442,63,,42 +40002,202003,100,3136,63,,42 +40002,202004,100,2115,63,,42 +40003,202001,100,7863,16,,42 +40003,202002,100,8121,16,,42 +40003,202003,100,2151,16,,42 +40003,202004,100,1377,16,,42 +40004,202001,100,5131,78,,42 +40004,202002,100,9836,78,,42 +40004,202003,100,,78,7525,42 +40004,202004,100,,78,,42 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_2_output.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_2_output.csv index c42737de..2253f3ac 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_2_output.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_2_output.csv @@ -1,17 +1,17 @@ -backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier -1.0603054093693,128.0208333333333,4,4,0,False,False,True,1.0000000000000,100,R,9491.000000,202001,40001 -1.0119038592767,120.7395833333333,3,4,4,False,False,False,0.9431244914565,100,R,4783.000000,202002,40001 -1.5695584910151,115.6929824561404,3,3,3,False,False,False,0.9882361756331,100,R,7902.000000,202003,40001 -1.0000000000000,73.7105263157895,0,3,3,True,False,False,0.6371218439609,100,R,4911.000000,202004,40001 -1.0603054093693,128.0208333333333,4,4,0,False,False,True,1.0000000000000,100,R,2095.000000,202001,40002 -1.0119038592767,120.7395833333333,3,4,4,False,False,False,0.9431244914565,100,R,442.000000,202002,40002 -1.5695584910151,115.6929824561404,3,3,3,False,False,False,0.9882361756331,100,R,3136.000000,202003,40002 -1.0000000000000,73.7105263157895,0,3,3,True,False,False,0.6371218439609,100,R,2115.000000,202004,40002 -1.0603054093693,128.0208333333333,4,4,0,False,False,True,1.0000000000000,100,R,7863.000000,202001,40003 -1.0119038592767,120.7395833333333,3,4,4,False,False,False,0.9431244914565,100,R,8121.000000,202002,40003 -1.5695584910151,115.6929824561404,3,3,3,False,False,False,0.9882361756331,100,R,2151.000000,202003,40003 -1.0000000000000,73.7105263157895,0,3,3,True,False,False,0.6371218439609,100,R,1377.000000,202004,40003 -1.0603054093693,128.0208333333333,4,4,0,False,False,True,1.0000000000000,100,R,5131.000000,202001,40004 -1.0119038592767,120.7395833333333,3,4,4,False,False,False,0.9431244914565,100,R,9836.000000,202002,40004 -1.5695584910151,115.6929824561404,3,3,3,False,False,False,0.9882361756331,100,MC,7525.000000,202003,40004 -1.0000000000000,73.7105263157895,0,3,3,True,False,False,0.6371218439609,100,FIMC,4794.341876,202004,40004 +backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier,question_no +1.0603054093693,128.0208333333333,4,4,0,False,False,True,1.0000000000000,100,R,9491.000000,202001,40001,42 +1.0119038592767,120.7395833333333,3,4,4,False,False,False,0.9431244914565,100,R,4783.000000,202002,40001,42 +1.5695584910151,115.6929824561404,3,3,3,False,False,False,0.9882361756331,100,R,7902.000000,202003,40001,42 +1.0000000000000,73.7105263157895,0,3,3,True,False,False,0.6371218439609,100,R,4911.000000,202004,40001,42 +1.0603054093693,128.0208333333333,4,4,0,False,False,True,1.0000000000000,100,R,2095.000000,202001,40002,42 +1.0119038592767,120.7395833333333,3,4,4,False,False,False,0.9431244914565,100,R,442.000000,202002,40002,42 +1.5695584910151,115.6929824561404,3,3,3,False,False,False,0.9882361756331,100,R,3136.000000,202003,40002,42 +1.0000000000000,73.7105263157895,0,3,3,True,False,False,0.6371218439609,100,R,2115.000000,202004,40002,42 +1.0603054093693,128.0208333333333,4,4,0,False,False,True,1.0000000000000,100,R,7863.000000,202001,40003,42 +1.0119038592767,120.7395833333333,3,4,4,False,False,False,0.9431244914565,100,R,8121.000000,202002,40003,42 +1.5695584910151,115.6929824561404,3,3,3,False,False,False,0.9882361756331,100,R,2151.000000,202003,40003,42 +1.0000000000000,73.7105263157895,0,3,3,True,False,False,0.6371218439609,100,R,1377.000000,202004,40003,42 +1.0603054093693,128.0208333333333,4,4,0,False,False,True,1.0000000000000,100,R,5131.000000,202001,40004,42 +1.0119038592767,120.7395833333333,3,4,4,False,False,False,0.9431244914565,100,R,9836.000000,202002,40004,42 +1.5695584910151,115.6929824561404,3,3,3,False,False,False,0.9882361756331,100,MC,7525.000000,202003,40004,42 +1.0000000000000,73.7105263157895,0,3,3,True,False,False,0.6371218439609,100,FIMC,4794.341876,202004,40004,42 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_3_input.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_3_input.csv index 9e362884..a749e121 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_3_input.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_3_input.csv @@ -1,21 +1,21 @@ -identifier,date,group,question,other,question_man -50001,202001,100,6362,59, -50001,202002,100,8542,59, -50001,202003,100,5623,59, -50001,202004,100,7769,59, -50001,202005,100,4687,59, -50002,202001,100,4851,36, -50002,202002,100,8894,36, -50002,202003,100,3372,36, -50002,202004,100,3522,36, -50002,202005,100,2327,36, -50003,202001,100,2238,76, -50003,202002,100,769,76, -50003,202003,100,7722,76, -50003,202004,100,6445,76, -50003,202005,100,1521,76, -50004,202001,100,688,30, -50004,202002,100,3245,30, -50004,202003,100,,30,1487 -50004,202004,100,,30, -50004,202005,100,,30, +identifier,date,group,question,other,question_man,question_no +50001,202001,100,6362,59,,42 +50001,202002,100,8542,59,,42 +50001,202003,100,5623,59,,42 +50001,202004,100,7769,59,,42 +50001,202005,100,4687,59,,42 +50002,202001,100,4851,36,,42 +50002,202002,100,8894,36,,42 +50002,202003,100,3372,36,,42 +50002,202004,100,3522,36,,42 +50002,202005,100,2327,36,,42 +50003,202001,100,2238,76,,42 +50003,202002,100,769,76,,42 +50003,202003,100,7722,76,,42 +50003,202004,100,6445,76,,42 +50003,202005,100,1521,76,,42 +50004,202001,100,688,30,,42 +50004,202002,100,3245,30,,42 +50004,202003,100,,30,1487,42 +50004,202004,100,,30,,42 +50004,202005,100,,30,,42 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_3_output.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_3_output.csv index b22755b4..29318489 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_3_output.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_3_output.csv @@ -1,21 +1,21 @@ -backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier -0.6591608391608,70.3432835820896,4,4,0,False,False,True,1.0000000000000,100,R,6362.000000,202001,50001 -1.0890111862176,106.7164179104478,3,4,4,False,False,False,1.5170804158710,100,R,8542.000000,202002,50001 -0.9425462336491,97.7602339181287,3,3,3,False,False,False,0.9182642131283,100,R,5623.000000,202003,50001 -2.0780316344464,103.7192982456140,3,3,3,False,False,False,1.0609559131423,100,R,7769.000000,202004,50001 -1.0000000000000,49.9122807017544,0,3,3,True,False,False,0.4812246278755,100,R,4687.000000,202005,50001 -0.6591608391608,70.3432835820896,4,4,0,False,False,True,1.0000000000000,100,R,4851.000000,202001,50002 -1.0890111862176,106.7164179104478,3,4,4,False,False,False,1.5170804158710,100,R,8894.000000,202002,50002 -0.9425462336491,97.7602339181287,3,3,3,False,False,False,0.9182642131283,100,R,3372.000000,202003,50002 -2.0780316344464,103.7192982456140,3,3,3,False,False,False,1.0609559131423,100,R,3522.000000,202004,50002 -1.0000000000000,49.9122807017544,0,3,3,True,False,False,0.4812246278755,100,R,2327.000000,202005,50002 -0.6591608391608,70.3432835820896,4,4,0,False,False,True,1.0000000000000,100,R,2238.000000,202001,50003 -1.0890111862176,106.7164179104478,3,4,4,False,False,False,1.5170804158710,100,R,769.000000,202002,50003 -0.9425462336491,97.7602339181287,3,3,3,False,False,False,0.9182642131283,100,R,7722.000000,202003,50003 -2.0780316344464,103.7192982456140,3,3,3,False,False,False,1.0609559131423,100,R,6445.000000,202004,50003 -1.0000000000000,49.9122807017544,0,3,3,True,False,False,0.4812246278755,100,R,1521.000000,202005,50003 -0.6591608391608,70.3432835820896,4,4,0,False,False,True,1.0000000000000,100,R,688.000000,202001,50004 -1.0890111862176,106.7164179104478,3,4,4,False,False,False,1.5170804158710,100,R,3245.000000,202002,50004 -0.9425462336491,97.7602339181287,3,3,3,False,False,False,0.9182642131283,100,MC,1487.000000,202003,50004 -2.0780316344464,103.7192982456140,3,3,3,False,False,False,1.0609559131423,100,FIMC,1577.641443,202004,50004 -1.0000000000000,49.9122807017544,0,3,3,True,False,False,0.4812246278755,100,FIMC,759.199916,202005,50004 +backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier,question_no +0.6591608391608,70.3432835820896,4,4,0,False,False,True,1.0000000000000,100,R,6362.000000,202001,50001,42 +1.0890111862176,106.7164179104478,3,4,4,False,False,False,1.5170804158710,100,R,8542.000000,202002,50001,42 +0.9425462336491,97.7602339181287,3,3,3,False,False,False,0.9182642131283,100,R,5623.000000,202003,50001,42 +2.0780316344464,103.7192982456140,3,3,3,False,False,False,1.0609559131423,100,R,7769.000000,202004,50001,42 +1.0000000000000,49.9122807017544,0,3,3,True,False,False,0.4812246278755,100,R,4687.000000,202005,50001,42 +0.6591608391608,70.3432835820896,4,4,0,False,False,True,1.0000000000000,100,R,4851.000000,202001,50002,42 +1.0890111862176,106.7164179104478,3,4,4,False,False,False,1.5170804158710,100,R,8894.000000,202002,50002,42 +0.9425462336491,97.7602339181287,3,3,3,False,False,False,0.9182642131283,100,R,3372.000000,202003,50002,42 +2.0780316344464,103.7192982456140,3,3,3,False,False,False,1.0609559131423,100,R,3522.000000,202004,50002,42 +1.0000000000000,49.9122807017544,0,3,3,True,False,False,0.4812246278755,100,R,2327.000000,202005,50002,42 +0.6591608391608,70.3432835820896,4,4,0,False,False,True,1.0000000000000,100,R,2238.000000,202001,50003,42 +1.0890111862176,106.7164179104478,3,4,4,False,False,False,1.5170804158710,100,R,769.000000,202002,50003,42 +0.9425462336491,97.7602339181287,3,3,3,False,False,False,0.9182642131283,100,R,7722.000000,202003,50003,42 +2.0780316344464,103.7192982456140,3,3,3,False,False,False,1.0609559131423,100,R,6445.000000,202004,50003,42 +1.0000000000000,49.9122807017544,0,3,3,True,False,False,0.4812246278755,100,R,1521.000000,202005,50003,42 +0.6591608391608,70.3432835820896,4,4,0,False,False,True,1.0000000000000,100,R,688.000000,202001,50004,42 +1.0890111862176,106.7164179104478,3,4,4,False,False,False,1.5170804158710,100,R,3245.000000,202002,50004,42 +0.9425462336491,97.7602339181287,3,3,3,False,False,False,0.9182642131283,100,MC,1487.000000,202003,50004,42 +2.0780316344464,103.7192982456140,3,3,3,False,False,False,1.0609559131423,100,FIMC,1577.641443,202004,50004,42 +1.0000000000000,49.9122807017544,0,3,3,True,False,False,0.4812246278755,100,FIMC,759.199916,202005,50004,42 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_4_input.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_4_input.csv index 0f39eff2..28ba22e7 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_4_input.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_4_input.csv @@ -1,17 +1,17 @@ -identifier,date,group,question,other,question_man -100001,202001,100,3074,26, -100001,202002,100,9529,26, -100001,202003,100,615,26, -100001,202004,100,3540,26, -100002,202001,100,8084,19, -100002,202002,100,2422,19, -100002,202003,100,3058,19, -100002,202004,100,5608,19, -100003,202001,100,5161,46, -100003,202002,100,3648,46, -100003,202003,100,205,46, -100003,202004,100,2594,46, -100004,202001,100,,86, -100004,202002,100,,86, -100004,202003,100,,86,9352 -100004,202004,100,,86, +identifier,date,group,question,other,question_man,question_no +100001,202001,100,3074,26,,42 +100001,202002,100,9529,26,,42 +100001,202003,100,615,26,,42 +100001,202004,100,3540,26,,42 +100002,202001,100,8084,19,,42 +100002,202002,100,2422,19,,42 +100002,202003,100,3058,19,,42 +100002,202004,100,5608,19,,42 +100003,202001,100,5161,46,,42 +100003,202002,100,3648,46,,42 +100003,202003,100,205,46,,42 +100003,202004,100,2594,46,,42 +100004,202001,100,,86,,42 +100004,202002,100,,86,,42 +100004,202003,100,,86,9352,42 +100004,202004,100,,86,,42 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_4_output.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_4_output.csv index ab425397..1654d03a 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_4_output.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_4_output.csv @@ -1,17 +1,17 @@ -backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier -1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,R,3074.000000,202001,100001 -4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,R,9529.000000,202002,100001 -0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,R,615.000000,202003,100001 -1.0000000000000,129.0329670329670,0,3,3,True,False,False,3.0278494069108,100,R,3540.000000,202004,100001 -1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,R,8084.000000,202001,100002 -4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,R,2422.000000,202002,100002 -0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,R,3058.000000,202003,100002 -1.0000000000000,129.0329670329670,0,3,3,True,False,False,3.0278494069108,100,R,5608.000000,202004,100002 -1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,R,5161.000000,202001,100003 -4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,R,3648.000000,202002,100003 -0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,R,205.000000,202003,100003 -1.0000000000000,129.0329670329670,0,3,3,True,False,False,3.0278494069108,100,R,2594.000000,202004,100003 -1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,C,15422.351648,202001,100004 -4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,FIC,14741.912088,202002,100004 -0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,MC,9352.000000,202003,100004 -1.0000000000000,129.0329670329670,0,3,3,True,False,False,3.0278494069108,100,FIMC,28316.447653,202004,100004 +backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier,question_no +1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,R,3074.000000,202001,100001,42 +4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,R,9529.000000,202002,100001,42 +0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,R,615.000000,202003,100001,42 +1.0000000000000,129.0329670329670,0,3,3,True,False,False,3.0278494069108,100,R,3540.000000,202004,100001,42 +1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,R,8084.000000,202001,100002,42 +4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,R,2422.000000,202002,100002,42 +0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,R,3058.000000,202003,100002,42 +1.0000000000000,129.0329670329670,0,3,3,True,False,False,3.0278494069108,100,R,5608.000000,202004,100002,42 +1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,R,5161.000000,202001,100003,42 +4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,R,3648.000000,202002,100003,42 +0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,R,205.000000,202003,100003,42 +1.0000000000000,129.0329670329670,0,3,3,True,False,False,3.0278494069108,100,R,2594.000000,202004,100003,42 +1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,C,15422.351648,202001,100004,42 +4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,FIC,14741.912088,202002,100004,42 +0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,MC,9352.000000,202003,100004,42 +1.0000000000000,129.0329670329670,0,3,3,True,False,False,3.0278494069108,100,FIMC,28316.447653,202004,100004,42 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_5_input.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_5_input.csv index 1c8aedc0..e5cc895f 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_5_input.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_5_input.csv @@ -1,17 +1,17 @@ -identifier,date,group,question,other,question_man -100001,202001,100,3074,26, -100001,202002,100,9529,26, -100001,202003,100,615,26, -100001,202004,100,3540,26, -100002,202001,100,8084,19, -100002,202002,100,2422,19, -100002,202003,100,3058,19, -100002,202004,100,5608,19, -100003,202001,100,5161,46, -100003,202002,100,3648,46, -100003,202003,100,205,46, -100003,202004,100,2594,46, -100004,202001,100,,86, -100004,202002,100,,86, -100004,202003,100,,86,9352 -100004,202004,100,8762,86, +identifier,date,group,question,other,question_man,question_no +100001,202001,100,3074,26,,42 +100001,202002,100,9529,26,,42 +100001,202003,100,615,26,,42 +100001,202004,100,3540,26,,42 +100002,202001,100,8084,19,,42 +100002,202002,100,2422,19,,42 +100002,202003,100,3058,19,,42 +100002,202004,100,5608,19,,42 +100003,202001,100,5161,46,,42 +100003,202002,100,3648,46,,42 +100003,202003,100,205,46,,42 +100003,202004,100,2594,46,,42 +100004,202001,100,,86,,42 +100004,202002,100,,86,,42 +100004,202003,100,,86,9352,42 +100004,202004,100,8762,86,,42 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_5_output.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_5_output.csv index a5279908..e17e55ae 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_5_output.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_5_output.csv @@ -1,17 +1,17 @@ -backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier -1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,R,3074.000000,202001,100001 -4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,R,9529.000000,202002,100001 -0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,R,615.000000,202003,100001 -1.0000000000000,115.8418079096045,0,4,3,True,False,False,3.0278494069108,100,R,3540.000000,202004,100001 -1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,R,8084.000000,202001,100002 -4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,R,2422.000000,202002,100002 -0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,R,3058.000000,202003,100002 -1.0000000000000,115.8418079096045,0,4,3,True,False,False,3.0278494069108,100,R,5608.000000,202004,100002 -1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,R,5161.000000,202001,100003 -4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,R,3648.000000,202002,100003 -0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,R,205.000000,202003,100003 -1.0000000000000,115.8418079096045,0,4,3,True,False,False,3.0278494069108,100,R,2594.000000,202004,100003 -1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,C,15422.351648,202001,100004 -4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,FIC,14741.912088,202002,100004 -0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,MC,9352.000000,202003,100004 -1.0000000000000,115.8418079096045,0,4,3,True,False,False,3.0278494069108,100,R,8762.000000,202004,100004 +backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier,question_no +1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,R,3074.000000,202001,100001,42 +4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,R,9529.000000,202002,100001,42 +0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,R,615.000000,202003,100001,42 +1.0000000000000,115.8418079096045,0,4,3,True,False,False,3.0278494069108,100,R,3540.000000,202004,100001,42 +1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,R,8084.000000,202001,100002,42 +4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,R,2422.000000,202002,100002,42 +0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,R,3058.000000,202003,100002,42 +1.0000000000000,115.8418079096045,0,4,3,True,False,False,3.0278494069108,100,R,5608.000000,202004,100002,42 +1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,R,5161.000000,202001,100003,42 +4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,R,3648.000000,202002,100003,42 +0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,R,205.000000,202003,100003,42 +1.0000000000000,115.8418079096045,0,4,3,True,False,False,3.0278494069108,100,R,2594.000000,202004,100003,42 +1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,C,15422.351648,202001,100004,42 +4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,FIC,14741.912088,202002,100004,42 +0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,MC,9352.000000,202003,100004,42 +1.0000000000000,115.8418079096045,0,4,3,True,False,False,3.0278494069108,100,R,8762.000000,202004,100004,42 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_6_input.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_6_input.csv index e4d0c4bb..c687018e 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_6_input.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_6_input.csv @@ -1,9 +1,9 @@ -identifier,date,group,question,other,question_man -20001,202001,100,2536,35, -20001,202002,100,8283,35, -20002,202001,100,9113,72, -20002,202002,100,2970,72, -20003,202001,100,5644,77, -20003,202002,100,989,77, -20004,202001,100,,30, -20004,202002,100,,30,3021 +identifier,date,group,question,other,question_man,question_no +20001,202001,100,2536,35,,40 +20001,202002,100,8283,35,,40 +20002,202001,100,9113,72,,40 +20002,202002,100,2970,72,,40 +20003,202001,100,5644,77,,40 +20003,202002,100,989,77,,40 +20004,202001,100,,30,,40 +20004,202002,100,,30,3021,40 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_6_output.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_6_output.csv index bb57484d..2afaa475 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_6_output.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_6_output.csv @@ -1,9 +1,9 @@ -backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier -1.4125959810488,93.9836956521739,3,3,0,False,False,True,1.0000000000000,100,R,2536.000000,202001,20001 -1.0000000000000,66.5326086956522,0,3,3,True,False,False,0.7079164980050,100,R,8283.000000,202002,20001 -1.4125959810488,93.9836956521739,3,3,0,False,False,True,1.0000000000000,100,R,9113.000000,202001,20002 -1.0000000000000,66.5326086956522,0,3,3,True,False,False,0.7079164980050,100,R,2970.000000,202002,20002 -1.4125959810488,93.9836956521739,3,3,0,False,False,True,1.0000000000000,100,R,5644.000000,202001,20003 -1.0000000000000,66.5326086956522,0,3,3,True,False,False,0.7079164980050,100,R,989.000000,202002,20003 -1.4125959810488,93.9836956521739,3,3,0,False,False,True,1.0000000000000,100,C,2819.510870,202001,20004 -1.0000000000000,66.5326086956522,0,3,3,True,False,False,0.7079164980050,100,MC,3021.000000,202002,20004 +backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier,question_no +1.4125959810488,93.9836956521739,3,3,0,False,False,True,1.0000000000000,100,R,2536.000000,202001,20001,40 +1.0000000000000,66.5326086956522,0,3,3,True,False,False,0.7079164980050,100,R,8283.000000,202002,20001,40 +1.4125959810488,93.9836956521739,3,3,0,False,False,True,1.0000000000000,100,R,9113.000000,202001,20002,40 +1.0000000000000,66.5326086956522,0,3,3,True,False,False,0.7079164980050,100,R,2970.000000,202002,20002,40 +1.4125959810488,93.9836956521739,3,3,0,False,False,True,1.0000000000000,100,R,5644.000000,202001,20003,40 +1.0000000000000,66.5326086956522,0,3,3,True,False,False,0.7079164980050,100,R,989.000000,202002,20003,40 +1.4125959810488,93.9836956521739,3,3,0,False,False,True,1.0000000000000,100,C,2819.510870,202001,20004,40 +1.0000000000000,66.5326086956522,0,3,3,True,False,False,0.7079164980050,100,MC,3021.000000,202002,20004,40 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_7_input.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_7_input.csv index ad11a410..1e5ef3d3 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_7_input.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_7_input.csv @@ -1,17 +1,17 @@ -identifier,date,group,question,other,question_man -40001,202001,100,9491,35, -40001,202002,100,4783,35, -40001,202003,100,7902,35, -40001,202004,100,4911,35, -40002,202001,100,2095,63, -40002,202002,100,442,63, -40002,202003,100,3136,63, -40002,202004,100,2115,63, -40003,202001,100,7863,16, -40003,202002,100,8121,16, -40003,202003,100,2151,16, -40003,202004,100,1377,16, -40004,202001,100,5131,78, -40004,202002,100,,78, -40004,202003,100,,78,7525 -40004,202004,100,,78, +identifier,date,group,question,other,question_man,question_no +40001,202001,100,9491,35,,41 +40001,202002,100,4783,35,,41 +40001,202003,100,7902,35,,41 +40001,202004,100,4911,35,,41 +40002,202001,100,2095,63,,41 +40002,202002,100,442,63,,41 +40002,202003,100,3136,63,,41 +40002,202004,100,2115,63,,41 +40003,202001,100,7863,16,,41 +40003,202002,100,8121,16,,41 +40003,202003,100,2151,16,,41 +40003,202004,100,1377,16,,41 +40004,202001,100,5131,78,,41 +40004,202002,100,,78,,41 +40004,202003,100,,78,7525,41 +40004,202004,100,,78,,41 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_7_output.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_7_output.csv index baf3fb74..1fde1c4e 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_7_output.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_7_output.csv @@ -1,17 +1,17 @@ -backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier -1.4572905739547,128.0208333333333,3,4,0,False,False,True,1.0000000000000,100,R,9491.000000,202001,40001 -1.0119038592767,117.0701754385965,3,3,3,False,False,False,0.6862049462697,100,R,4783.000000,202002,40001 -1.5695584910151,115.6929824561404,3,3,3,False,False,False,0.9882361756331,100,R,7902.000000,202003,40001 -1.0000000000000,73.7105263157895,0,3,3,True,False,False,0.6371218439609,100,R,4911.000000,202004,40001 -1.4572905739547,128.0208333333333,3,4,0,False,False,True,1.0000000000000,100,R,2095.000000,202001,40002 -1.0119038592767,117.0701754385965,3,3,3,False,False,False,0.6862049462697,100,R,442.000000,202002,40002 -1.5695584910151,115.6929824561404,3,3,3,False,False,False,0.9882361756331,100,R,3136.000000,202003,40002 -1.0000000000000,73.7105263157895,0,3,3,True,False,False,0.6371218439609,100,R,2115.000000,202004,40002 -1.4572905739547,128.0208333333333,3,4,0,False,False,True,1.0000000000000,100,R,7863.000000,202001,40003 -1.0119038592767,117.0701754385965,3,3,3,False,False,False,0.6862049462697,100,R,8121.000000,202002,40003 -1.5695584910151,115.6929824561404,3,3,3,False,False,False,0.9882361756331,100,R,2151.000000,202003,40003 -1.0000000000000,73.7105263157895,0,3,3,True,False,False,0.6371218439609,100,R,1377.000000,202004,40003 -1.4572905739547,128.0208333333333,3,4,0,False,False,True,1.0000000000000,100,R,5131.000000,202001,40004 -1.0119038592767,117.0701754385965,3,3,3,False,False,False,0.6862049462697,100,FIR,3520.917579,202002,40004 -1.5695584910151,115.6929824561404,3,3,3,False,False,False,0.9882361756331,100,MC,7525.000000,202003,40004 -1.0000000000000,73.7105263157895,0,3,3,True,False,False,0.6371218439609,100,FIMC,4794.341876,202004,40004 +backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier,question_no +1.4572905739547,128.0208333333333,3,4,0,False,False,True,1.0000000000000,100,R,9491.000000,202001,40001,41 +1.0119038592767,117.0701754385965,3,3,3,False,False,False,0.6862049462697,100,R,4783.000000,202002,40001,41 +1.5695584910151,115.6929824561404,3,3,3,False,False,False,0.9882361756331,100,R,7902.000000,202003,40001,41 +1.0000000000000,73.7105263157895,0,3,3,True,False,False,0.6371218439609,100,R,4911.000000,202004,40001,41 +1.4572905739547,128.0208333333333,3,4,0,False,False,True,1.0000000000000,100,R,2095.000000,202001,40002,41 +1.0119038592767,117.0701754385965,3,3,3,False,False,False,0.6862049462697,100,R,442.000000,202002,40002,41 +1.5695584910151,115.6929824561404,3,3,3,False,False,False,0.9882361756331,100,R,3136.000000,202003,40002,41 +1.0000000000000,73.7105263157895,0,3,3,True,False,False,0.6371218439609,100,R,2115.000000,202004,40002,41 +1.4572905739547,128.0208333333333,3,4,0,False,False,True,1.0000000000000,100,R,7863.000000,202001,40003,41 +1.0119038592767,117.0701754385965,3,3,3,False,False,False,0.6862049462697,100,R,8121.000000,202002,40003,41 +1.5695584910151,115.6929824561404,3,3,3,False,False,False,0.9882361756331,100,R,2151.000000,202003,40003,41 +1.0000000000000,73.7105263157895,0,3,3,True,False,False,0.6371218439609,100,R,1377.000000,202004,40003,41 +1.4572905739547,128.0208333333333,3,4,0,False,False,True,1.0000000000000,100,R,5131.000000,202001,40004,41 +1.0119038592767,117.0701754385965,3,3,3,False,False,False,0.6862049462697,100,FIR,3520.917579,202002,40004,41 +1.5695584910151,115.6929824561404,3,3,3,False,False,False,0.9882361756331,100,MC,7525.000000,202003,40004,41 +1.0000000000000,73.7105263157895,0,3,3,True,False,False,0.6371218439609,100,FIMC,4794.341876,202004,40004,41 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_8_input.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_8_input.csv index 30ffadc7..b9a43f32 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_8_input.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_8_input.csv @@ -1,17 +1,17 @@ -identifier,date,group,question,other,question_man -100001,202001,100,3074,26, -100001,202002,100,9529,26, -100001,202003,100,615,26, -100001,202004,100,3540,26, -100002,202001,100,8084,19, -100002,202002,100,2422,19, -100002,202003,100,3058,19, -100002,202004,100,5608,19, -100003,202001,100,5161,46, -100003,202002,100,3648,46, -100003,202003,100,205,46, -100003,202004,100,2594,46, -100004,202001,100,,86, -100004,202002,100,,86, -100004,202003,100,,86,11085 -100004,202004,100,,86, +identifier,date,group,question,other,question_man,question_no +100001,202001,100,3074,26,,41 +100001,202002,100,9529,26,,41 +100001,202003,100,615,26,,41 +100001,202004,100,3540,26,,41 +100002,202001,100,8084,19,,41 +100002,202002,100,2422,19,,41 +100002,202003,100,3058,19,,41 +100002,202004,100,5608,19,,41 +100003,202001,100,5161,46,,41 +100003,202002,100,3648,46,,41 +100003,202003,100,205,46,,41 +100003,202004,100,2594,46,,41 +100004,202001,100,,86,,41 +100004,202002,100,,86,,41 +100004,202003,100,,86,11085,41 +100004,202004,100,,86,,41 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_8_output.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_8_output.csv index 76a3221f..ef276b49 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_8_output.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_8_output.csv @@ -1,17 +1,17 @@ -backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier -1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,R,3074.000000,202001,100001 -4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,R,9529.000000,202002,100001 -0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,R,615.000000,202003,100001 -1.0000000000000,129.0329670329670,0,3,3,True,False,False,3.0278494069108,100,R,3540.000000,202004,100001 -1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,R,8084.000000,202001,100002 -4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,R,2422.000000,202002,100002 -0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,R,3058.000000,202003,100002 -1.0000000000000,129.0329670329670,0,3,3,True,False,False,3.0278494069108,100,R,5608.000000,202004,100002 -1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,R,5161.000000,202001,100003 -4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,R,3648.000000,202002,100003 -0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,R,205.000000,202003,100003 -1.0000000000000,129.0329670329670,0,3,3,True,False,False,3.0278494069108,100,R,2594.000000,202004,100003 -1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,C,15422.351648,202001,100004 -4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,FIC,14741.912088,202002,100004 -0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,MC,11085.000000,202003,100004 -1.0000000000000,129.0329670329670,0,3,3,True,False,False,3.0278494069108,100,FIMC,33563.710676,202004,100004 +backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier,question_no +1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,R,3074.000000,202001,100001,41 +4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,R,9529.000000,202002,100001,41 +0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,R,615.000000,202003,100001,41 +1.0000000000000,129.0329670329670,0,3,3,True,False,False,3.0278494069108,100,R,3540.000000,202004,100001,41 +1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,R,8084.000000,202001,100002,41 +4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,R,2422.000000,202002,100002,41 +0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,R,3058.000000,202003,100002,41 +1.0000000000000,129.0329670329670,0,3,3,True,False,False,3.0278494069108,100,R,5608.000000,202004,100002,41 +1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,R,5161.000000,202001,100003,41 +4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,R,3648.000000,202002,100003,41 +0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,R,205.000000,202003,100003,41 +1.0000000000000,129.0329670329670,0,3,3,True,False,False,3.0278494069108,100,R,2594.000000,202004,100003,41 +1.0461568049234,179.3296703296703,3,3,0,False,False,True,1.0000000000000,100,C,15422.351648,202001,100004,41 +4.0224342444559,171.4175824175824,3,3,3,False,False,False,0.9558796494883,100,FIC,14741.912088,202002,100004,41 +0.3302674161131,42.6153846153846,3,3,3,False,False,False,0.2486056798513,100,MC,11085.000000,202003,100004,41 +1.0000000000000,129.0329670329670,0,3,3,True,False,False,3.0278494069108,100,FIMC,33563.710676,202004,100004,41 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_9_input.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_9_input.csv index 481ec390..d671fdcc 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_9_input.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_9_input.csv @@ -1,13 +1,13 @@ -identifier,date,group,question,other,question_man -60001,202001,100,5077,15, -60001,202002,100,7830,15, -60001,202003,100,1046,15, -60002,202001,100,1588,71, -60002,202002,100,1213,71, -60002,202003,100,3807,71, -60003,202001,100,6541,26, -60003,202002,100,336,26, -60003,202003,100,6351,26, -60004,202001,100,,3,839 -60004,202002,100,,3, -60004,202003,100,401,3, +identifier,date,group,question,other,question_man,question_no +60001,202001,100,5077,15,,42 +60001,202002,100,7830,15,,42 +60001,202003,100,1046,15,,42 +60002,202001,100,1588,71,,42 +60002,202002,100,1213,71,,42 +60002,202003,100,3807,71,,42 +60003,202001,100,6541,26,,42 +60003,202002,100,336,26,,42 +60003,202003,100,6351,26,,42 +60004,202001,100,,3,839,42 +60004,202002,100,,3,,42 +60004,202003,100,401,3,,42 diff --git a/tests/data/ratio_of_means/rom_test_data_case_mc_9_output.csv b/tests/data/ratio_of_means/rom_test_data_case_mc_9_output.csv index e91fefb1..b11b6fde 100644 --- a/tests/data/ratio_of_means/rom_test_data_case_mc_9_output.csv +++ b/tests/data/ratio_of_means/rom_test_data_case_mc_9_output.csv @@ -1,13 +1,13 @@ -backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier -1.4080392365924,117.9107142857143,3,3,0,False,False,True,1.0000000000000,100,R,5077.000000,202001,60001 -0.8371117458051,83.7410714285714,3,3,3,False,False,False,0.7102074814478,100,R,7830.000000,202002,60001 -1.0000000000000,100.9130434782609,0,4,3,True,False,False,1.1945836443118,100,R,1046.000000,202003,60001 -1.4080392365924,117.9107142857143,3,3,0,False,False,True,1.0000000000000,100,R,1588.000000,202001,60002 -0.8371117458051,83.7410714285714,3,3,3,False,False,False,0.7102074814478,100,R,1213.000000,202002,60002 -1.0000000000000,100.9130434782609,0,4,3,True,False,False,1.1945836443118,100,R,3807.000000,202003,60002 -1.4080392365924,117.9107142857143,3,3,0,False,False,True,1.0000000000000,100,R,6541.000000,202001,60003 -0.8371117458051,83.7410714285714,3,3,3,False,False,False,0.7102074814478,100,R,336.000000,202002,60003 -1.0000000000000,100.9130434782609,0,4,3,True,False,False,1.1945836443118,100,R,6351.000000,202003,60003 -1.4080392365924,117.9107142857143,3,3,0,False,False,True,1.0000000000000,100,MC,839.000000,202001,60004 -0.8371117458051,83.7410714285714,3,3,3,False,False,False,0.7102074814478,100,BI,335.681810,202002,60004 -1.0000000000000,100.9130434782609,0,4,3,True,False,False,1.1945836443118,100,R,401.000000,202003,60004 +backward,construction,count_backward,count_construction,count_forward,default_backward,default_construction,default_forward,forward,group,marker,output,date,identifier,question_no +1.4080392365924,117.9107142857143,3,3,0,False,False,True,1.0000000000000,100,R,5077.000000,202001,60001,42 +0.8371117458051,83.7410714285714,3,3,3,False,False,False,0.7102074814478,100,R,7830.000000,202002,60001,42 +1.0000000000000,100.9130434782609,0,4,3,True,False,False,1.1945836443118,100,R,1046.000000,202003,60001,42 +1.4080392365924,117.9107142857143,3,3,0,False,False,True,1.0000000000000,100,R,1588.000000,202001,60002,42 +0.8371117458051,83.7410714285714,3,3,3,False,False,False,0.7102074814478,100,R,1213.000000,202002,60002,42 +1.0000000000000,100.9130434782609,0,4,3,True,False,False,1.1945836443118,100,R,3807.000000,202003,60002,42 +1.4080392365924,117.9107142857143,3,3,0,False,False,True,1.0000000000000,100,R,6541.000000,202001,60003,42 +0.8371117458051,83.7410714285714,3,3,3,False,False,False,0.7102074814478,100,R,336.000000,202002,60003,42 +1.0000000000000,100.9130434782609,0,4,3,True,False,False,1.1945836443118,100,R,6351.000000,202003,60003,42 +1.4080392365924,117.9107142857143,3,3,0,False,False,True,1.0000000000000,100,MC,839.000000,202001,60004,42 +0.8371117458051,83.7410714285714,3,3,3,False,False,False,0.7102074814478,100,BI,335.681810,202002,60004,42 +1.0000000000000,100.9130434782609,0,4,3,True,False,False,1.1945836443118,100,R,401.000000,202003,60004,42 diff --git a/tests/test_ratio_of_means.py b/tests/test_ratio_of_means.py index ea24808a..3076d69b 100644 --- a/tests/test_ratio_of_means.py +++ b/tests/test_ratio_of_means.py @@ -196,7 +196,7 @@ def test_manual_construction_input(self, base_file_name): scenario_path_prefix + "ratio_of_means/" + base_file_name + "_output.csv" ) - manual_constructions = df.copy()[["identifier", "date", "question_man"]] + manual_constructions = df.copy()[["identifier", "date", "question_man","question_no"]] manual_constructions.rename(columns={"question_man": "question"}, inplace=True) df.drop(columns=["question_man"], inplace=True) From 2b600f93a578ec9949648cd03df60da7767ae4ff Mon Sep 17 00:00:00 2001 From: Jordan Day Date: Wed, 4 Sep 2024 13:37:12 +0100 Subject: [PATCH 7/8] commit hooks --- tests/test_ratio_of_means.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_ratio_of_means.py b/tests/test_ratio_of_means.py index 3076d69b..410b2faa 100644 --- a/tests/test_ratio_of_means.py +++ b/tests/test_ratio_of_means.py @@ -196,7 +196,9 @@ def test_manual_construction_input(self, base_file_name): scenario_path_prefix + "ratio_of_means/" + base_file_name + "_output.csv" ) - manual_constructions = df.copy()[["identifier", "date", "question_man","question_no"]] + manual_constructions = df.copy()[ + ["identifier", "date", "question_man", "question_no"] + ] manual_constructions.rename(columns={"question_man": "question"}, inplace=True) df.drop(columns=["question_man"], inplace=True) From 170f8d1d8950677fe0303fec6b567dc8c72696f1 Mon Sep 17 00:00:00 2001 From: dayj1 Date: Wed, 4 Sep 2024 13:41:10 +0100 Subject: [PATCH 8/8] commit hooks changes --- mbs_results/data_cleaning.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mbs_results/data_cleaning.py b/mbs_results/data_cleaning.py index 0214e9df..05c385ca 100755 --- a/mbs_results/data_cleaning.py +++ b/mbs_results/data_cleaning.py @@ -228,7 +228,7 @@ def join_manual_constructions( # as manual construction return df else: - manual_constructions_filter.drop(columns = [question_no],inplace=True) + manual_constructions_filter.drop(columns=[question_no], inplace=True) if period not in df.columns or reference not in df.columns: df = df.reset_index() @@ -238,9 +238,9 @@ def join_manual_constructions( ) if not is_same_dtype(df, manual_constructions_filter, reference): - manual_constructions_filter[reference] = manual_constructions_filter[reference].astype( - df[reference].dtype - ) + manual_constructions_filter[reference] = manual_constructions_filter[ + reference + ].astype(df[reference].dtype) manual_constructions_filter.set_index([reference, period], inplace=True) df.set_index([reference, period], inplace=True)