From ded8950d30febd922d9ec7806e94678e7c52bfb4 Mon Sep 17 00:00:00 2001 From: martinholmer Date: Wed, 18 Jan 2017 09:39:11 -0500 Subject: [PATCH 1/7] Generalize name of Calculator convert_reform_dict() method. --- taxcalc/calculate.py | 32 ++++++++++++++++---------------- taxcalc/tests/test_calculate.py | 10 +++++----- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/taxcalc/calculate.py b/taxcalc/calculate.py index cdd6bfadc..9cf7c9e3a 100644 --- a/taxcalc/calculate.py +++ b/taxcalc/calculate.py @@ -477,33 +477,33 @@ def repl(mat): for param, code in paramcode.items(): raw_dict['policy'][param] = {'0': code} # convert raw_dict component dictionaries - pol_dict = Calculator.convert_reform_dict(raw_dict['policy']) - beh_dict = Calculator.convert_reform_dict(raw_dict['behavior']) - gro_dict = Calculator.convert_reform_dict(raw_dict['growth']) - con_dict = Calculator.convert_reform_dict(raw_dict['consumption']) + pol_dict = Calculator.convert_parameter_dict(raw_dict['policy']) + beh_dict = Calculator.convert_parameter_dict(raw_dict['behavior']) + con_dict = Calculator.convert_parameter_dict(raw_dict['consumption']) + gro_dict = Calculator.convert_parameter_dict(raw_dict['growth']) return (pol_dict, beh_dict, gro_dict, con_dict) @staticmethod - def convert_reform_dict(param_key_dict): + def convert_parameter_dict(param_key_dict): """ Converts specified param_key_dict into a dictionary whose primary keys are calendary years, and hence, is suitable as the argument to the Policy implement_reform(reform_policy) method, or the Behavior update_behavior(reform_behavior) method, or - the Growth update_growth(reform_growth) method, or - the Consumption update_consumption(reform_consumption) method. + the Consumption update_consumption(reform_consumption) method, or + the Growth update_growth(reform_growth) method. Specified input dictionary has string parameter primary keys and string years as secondary keys. Returned dictionary has integer years as primary keys and string parameters as secondary keys. """ # convert year skey strings to integers and lists into np.arrays - reform_pkey_param = {} + year_param = dict() for pkey, sdict in param_key_dict.items(): if not isinstance(pkey, six.string_types): msg = 'pkey {} in reform is not a string' raise ValueError(msg.format(pkey)) - rdict = {} + rdict = dict() if not isinstance(sdict, dict): msg = 'pkey {} in reform is not paired with a dict' raise ValueError(msg.format(pkey)) @@ -515,14 +515,14 @@ def convert_reform_dict(param_key_dict): year = int(skey) rdict[year] = (np.array(val) if isinstance(val, list) else val) - reform_pkey_param[pkey] = rdict - # convert reform_pkey_param dictionary to reform_pkey_year dictionary + year_param[pkey] = rdict + # convert year_param dictionary to year_key_dict dictionary + year_key_dict = dict() years = set() - reform_pkey_year = dict() - for param, sdict in reform_pkey_param.items(): + for param, sdict in year_param.items(): for year, val in sdict.items(): if year not in years: years.add(year) - reform_pkey_year[year] = {} - reform_pkey_year[year][param] = val - return reform_pkey_year + year_key_dict[year] = dict() + year_key_dict[year][param] = val + return year_key_dict diff --git a/taxcalc/tests/test_calculate.py b/taxcalc/tests/test_calculate.py index 9d91935a5..aed067978 100644 --- a/taxcalc/tests/test_calculate.py +++ b/taxcalc/tests/test_calculate.py @@ -596,15 +596,15 @@ def test_read_bad_json_reform_file(bad1reformfile, bad2reformfile): Calculator.read_json_reform_file(bad2reformfile.name) -def test_convert_reform_dict(): +def test_convert_parameter_dict(): with pytest.raises(ValueError): - rdict = Calculator.convert_reform_dict({2013: {'2013': [40000]}}) + rdict = Calculator.convert_parameter_dict({2013: {'2013': [40000]}}) with pytest.raises(ValueError): - rdict = Calculator.convert_reform_dict({'_II_em': {2013: [40000]}}) + rdict = Calculator.convert_parameter_dict({'_II_em': {2013: [40000]}}) with pytest.raises(ValueError): - rdict = Calculator.convert_reform_dict({4567: {2013: [40000]}}) + rdict = Calculator.convert_parameter_dict({4567: {2013: [40000]}}) with pytest.raises(ValueError): - rdict = Calculator.convert_reform_dict({'_II_em': 40000}) + rdict = Calculator.convert_parameter_dict({'_II_em': 40000}) def test_param_code_calc_all(reform_file, rawinputfile): From e70da41c3c078605e35b0867f951e2fb488a750c Mon Sep 17 00:00:00 2001 From: martinholmer Date: Wed, 18 Jan 2017 12:23:32 -0500 Subject: [PATCH 2/7] Refactor read_json_* functions in calculate.py file. --- taxcalc/calculate.py | 149 +++++++++++++++++++++--------- taxcalc/incometaxio.py | 6 +- taxcalc/simpletaxio.py | 4 +- taxcalc/tests/test_calculate.py | 37 +++----- taxcalc/tests/test_dropq.py | 4 +- taxcalc/tests/test_incometaxio.py | 6 +- taxcalc/tests/test_policy.py | 19 ++-- taxcalc/tests/test_reforms.py | 48 +++++----- taxcalc/tests/test_simpletaxio.py | 12 +-- 9 files changed, 163 insertions(+), 122 deletions(-) diff --git a/taxcalc/calculate.py b/taxcalc/calculate.py index 9cf7c9e3a..8ecb7ca94 100644 --- a/taxcalc/calculate.py +++ b/taxcalc/calculate.py @@ -398,53 +398,62 @@ def current_law_version(self): return calc @staticmethod - def read_json_reform_file(reform_filename): + def read_json_param_files(reform_filename, assump_filename): """ - Read JSON reform file and call Calculator.read_json_reform_text method. + Read JSON files and call Calculator.read_json_*_text methods. """ - if os.path.isfile(reform_filename): + if reform_filename is None: + rpol_dict = dict() + elif os.path.isfile(reform_filename): txt = open(reform_filename, 'r').read() - return Calculator.read_json_reform_text(txt) + rpol_dict = Calculator.read_json_policy_reform_text(txt) else: - msg = 'reform file {} could not be found' + msg = 'policy reform file {} could not be found' raise ValueError(msg.format(reform_filename)) + if assump_filename is None: + behv_dict = dict() + cons_dict = dict() + grow_dict = dict() + elif os.path.isfile(assump_filename): + txt = open(assump_filename, 'r').read() + (behv_dict, + cons_dict, + grow_dict) = Calculator.read_json_econ_assump_text(txt) + else: + msg = 'economic assumption file {} could not be found' + raise ValueError(msg.format(assump_filename)) + return (rpol_dict, behv_dict, cons_dict, grow_dict) @staticmethod - def read_json_reform_text(text_string): + def read_json_policy_reform_text(text_string): """ - Strip //-comments from text_string and return 4 dict based on the JSON. - The reform text is JSON with four high-level string:object pairs: - a "policy": {...} pair, - a "behavior": {...} pair, - a "growth": {...} pair, and - a "consumption": {...} pair. - In all four cases the {...} object may be empty (that is, be {}), - or may contain one or more pairs with parameter string primary keys - and string years as secondary keys. See tests/test_calculate.py for - an extended example of a commented JSON reform text that can be read - by this method. Note that parameter code in the policy object is - enclosed inside a pair of double pipe characters (||) as shown - in the REFORM_CONTENTS string in the tests/test_calculate.py file. - Returned dictionaries (reform_policy, reform_behavior, - reform_growth reform_consumption) - have integer years as primary keys + Strip //-comments from text_string and return 1 dict based on the JSON. + Specified text is JSON with at least 1 high-level string:object pair: + a "policy": {...} pair. + Other high-level pairs will be ignored by this method. + The {...} object may be empty (that is, be {}), or + may contain one or more pairs with parameter string primary keys + and string years as secondary keys. See tests/test_calculate.py for + an extended example of a commented JSON policy reform text + that can be read by this method. + Note that parameter code in the policy object is enclosed inside a + pair of double pipe characters (||) as shown in the REFORM_CONTENTS + string in the tests/test_calculate.py file. + Returned dictionary rpol_dict + has integer years as primary keys and string parameters as secondary keys. - The returned dictionaries are suitable as the argument to - the Policy implement_reform(reform_policy) method, or - the Behavior update_behavior(reform_behavior) method, or - the Growth update_growth(reform_growth) method, or - the Consumption update_consumption(reform_consumption) method. + The returned dictionary is suitable as the argument to + the Policy implement_reform(rpol_dict) method. """ + # define function used by re.sub to process parameter code + def repl_func(mat): + code = mat.group(2).replace('\r', '\\r').replace('\n', '\\n') + return '"' + code + '"' # strip out //-comments without changing line numbers json_without_comments = re.sub('//.*', ' ', text_string) # convert multi-line string between pairs of || into a simple string - - def repl(mat): - code = mat.group(2).replace('\r', '\\r').replace('\n', '\\n') - return '"' + code + '"' - json_str = re.sub('(\|\|)(.*?)(\|\|)', # pylint: disable=W1401 - repl, json_without_comments, flags=re.DOTALL) + repl_func, json_without_comments, flags=re.DOTALL) # convert JSON text into a Python dictionary try: raw_dict = json.loads(json_str) @@ -463,11 +472,12 @@ def repl(mat): msg += bline + '\n' raise ValueError(msg) # check contents of dictionary - expect_keys = set(['policy', 'behavior', 'growth', 'consumption']) - actual_keys = set(raw_dict.keys()) - if actual_keys != expect_keys: - msg = 'reform keys {} not equal to {}' - raise ValueError(msg.format(actual_keys, expect_keys)) + required_keys = ['policy'] + actual_keys = raw_dict.keys() + for rkey in required_keys: + if rkey not in actual_keys: + msg = 'policy reform key "{}" not among high-level keys' + raise ValueError(msg.format(rkey)) # handle special param_code key in raw_dict policy component dictionary paramcode = raw_dict['policy'].pop('param_code', None) if paramcode: @@ -476,12 +486,63 @@ def repl(mat): raise ValueError(msg) for param, code in paramcode.items(): raw_dict['policy'][param] = {'0': code} - # convert raw_dict component dictionaries - pol_dict = Calculator.convert_parameter_dict(raw_dict['policy']) - beh_dict = Calculator.convert_parameter_dict(raw_dict['behavior']) - con_dict = Calculator.convert_parameter_dict(raw_dict['consumption']) - gro_dict = Calculator.convert_parameter_dict(raw_dict['growth']) - return (pol_dict, beh_dict, gro_dict, con_dict) + # convert the policy dictionary in raw_dict + rpol_dict = Calculator.convert_parameter_dict(raw_dict['policy']) + return rpol_dict + + @staticmethod + def read_json_econ_assump_text(text_string): + """ + Strip //-comments from text_string and return 3 dict based on the JSON. + Specified text is JSON with at least 3 high-level string:object pairs: + a "behavior": {...} pair, + a "consumption": {...} pair, and + a "growth": {...} pair. + Other high-level pairs will be ignored by this method. + The {...} object may be empty (that is, be {}), or + may contain one or more pairs with parameter string primary keys + and string years as secondary keys. See tests/test_calculate.py for + an extended example of a commented JSON economic assumption text + that can be read by this method. + Returned dictionaries (behv_dict, cons_dict, grow_dict) + have integer years as primary keys + and string parameters as secondary keys. + The returned dictionaries are suitable as the arguments to + the Behavior update_behavior(behv_dict) method, or + the Consumption update_consumption(cons_dict) method, or + the Growth update_growth(grow_dict) method. + """ + # strip out //-comments without changing line numbers + json_str = re.sub('//.*', ' ', text_string) + # convert JSON text into a Python dictionary + try: + raw_dict = json.loads(json_str) + except ValueError as valerr: + msg = 'Economic assumption text below contains invalid JSON:\n' + msg += str(valerr) + '\n' + msg += 'Above location of the first error may be approximate.\n' + msg += 'The invalid JSON asssump text is between the lines:\n' + bline = 'XX----.----1----.----2----.----3----.----4' + bline += '----.----5----.----6----.----7' + msg += bline + '\n' + linenum = 0 + for line in json_str.split('\n'): + linenum += 1 + msg += '{:02d}{}'.format(linenum, line) + '\n' + msg += bline + '\n' + raise ValueError(msg) + # check contents of dictionary + required_keys = ['behavior', 'consumption', 'growth'] + actual_keys = raw_dict.keys() + for rkey in required_keys: + if rkey not in actual_keys: + msg = 'economic assumption key "{}" not among high-level keys' + raise ValueError(msg.format(rkey)) + # convert the assumption dictionaries in raw_dict + behv_dict = Calculator.convert_parameter_dict(raw_dict['behavior']) + cons_dict = Calculator.convert_parameter_dict(raw_dict['consumption']) + grow_dict = Calculator.convert_parameter_dict(raw_dict['growth']) + return (behv_dict, cons_dict, grow_dict) @staticmethod def convert_parameter_dict(param_key_dict): diff --git a/taxcalc/incometaxio.py b/taxcalc/incometaxio.py index e9209c5e3..360599413 100644 --- a/taxcalc/incometaxio.py +++ b/taxcalc/incometaxio.py @@ -149,8 +149,10 @@ def __init__(self, input_data, tax_year, reform, # implement reform if one is specified if self._reform: if self._using_reform_file: - (r_pol, r_beh, - r_gro, r_con) = Calculator.read_json_reform_file(reform) + (r_pol, + r_beh, + r_con, + r_gro) = Calculator.read_json_param_files(reform, None) else: r_pol = reform pol.implement_reform(r_pol) diff --git a/taxcalc/simpletaxio.py b/taxcalc/simpletaxio.py index 895435f18..b0545e0e1 100644 --- a/taxcalc/simpletaxio.py +++ b/taxcalc/simpletaxio.py @@ -102,10 +102,10 @@ def __init__(self, # read input file contents into self._input dictionary self._read_input(input_filename) self._policy = Policy() - # implement reform if reform is specified (no behavior, growth or cons) + # implement reform if reform is specified if reform: if self._using_reform_file: - r_pol, _, _, _ = Calculator.read_json_reform_file(reform) + r_pol, _, _, _ = Calculator.read_json_param_files(reform, None) else: r_pol = reform self._policy.implement_reform(r_pol) diff --git a/taxcalc/tests/test_calculate.py b/taxcalc/tests/test_calculate.py index aed067978..8d314c5c8 100644 --- a/taxcalc/tests/test_calculate.py +++ b/taxcalc/tests/test_calculate.py @@ -427,7 +427,7 @@ def test_Calculator_using_nonstd_input(rawinputfile): REFORM_CONTENTS = """ -// Example of a reform file suitable for the read_json_reform_file function. +// Example of a reform file suitable for read_json_parameter_files function. // This JSON file can contain any number of trailing //-style comments, which // will be removed before the contents are converted from JSON to a dictionary. // Within each "policy", "behavior", "growth", and "consumption" object, the @@ -437,6 +437,9 @@ def test_Calculator_using_nonstd_input(rawinputfile): // Parameter code in the policy object is enclosed inside a pair of double // pipe characters (||). { + "title": "", + "author": "", + "date": "", "policy": { "param_code": { // all the parameter code must go in one place "ALD_InvInc_ec_base_code": @@ -473,12 +476,6 @@ def test_Calculator_using_nonstd_input(rawinputfile): {"2017": false, // values in future years are same as this year value "2020": true // values in future years indexed with this year as base } - }, - "behavior": { - }, - "growth": { - }, - "consumption": { } } """ @@ -487,7 +484,7 @@ def test_Calculator_using_nonstd_input(rawinputfile): @pytest.yield_fixture def reform_file(): """ - Temporary reform file for read_json_reform_file function. + Temporary reform file for read_json_policy_reform_file function. """ rfile = tempfile.NamedTemporaryFile(mode='a', delete=False) rfile.write(REFORM_CONTENTS) @@ -508,7 +505,7 @@ def test_read_json_reform_file_and_implement_reform(reform_file, set_year): that is then used to call implement_reform method and Calculate.calc_all() NOTE: implement_reform called when policy.current_year == policy.start_year """ - reform, _, _, _ = Calculator.read_json_reform_file(reform_file.name) + reform, _, _, _ = Calculator.read_json_param_files(reform_file.name, None) policy = Policy() if set_year: policy.set_year(2015) @@ -549,12 +546,6 @@ def bad1reformfile(): "policy": { // example of incorrect JSON because 'x' must be "x" 'x': {"2014": [4000]} }, - "behavior": { - }, - "growth": { - }, - "consumption": { - } } """ f = tempfile.NamedTemporaryFile(mode='a', delete=False) @@ -570,15 +561,9 @@ def bad2reformfile(): # specify JSON text for reform txt = """ { - "policy": { + "policyx": { // example of reform file not containing "policy" key "_SS_Earnings_c": {"2018": [9e99]} }, - "behavior": { - }, - "growthx": { - }, - "consumption": { - } } """ f = tempfile.NamedTemporaryFile(mode='a', delete=False) @@ -591,9 +576,9 @@ def bad2reformfile(): def test_read_bad_json_reform_file(bad1reformfile, bad2reformfile): with pytest.raises(ValueError): - Calculator.read_json_reform_file(bad1reformfile.name) + Calculator.read_json_param_files(bad1reformfile.name, None) with pytest.raises(ValueError): - Calculator.read_json_reform_file(bad2reformfile.name) + Calculator.read_json_param_files(bad2reformfile.name, None) def test_convert_parameter_dict(): @@ -609,9 +594,9 @@ def test_convert_parameter_dict(): def test_param_code_calc_all(reform_file, rawinputfile): cyr = 2016 - reform, _, _, _ = Calculator.read_json_reform_file(reform_file.name) + (ref, _, _, _) = Calculator.read_json_param_files(reform_file.name, None) policy = Policy() - policy.implement_reform(reform) + policy.implement_reform(ref) policy.set_year(cyr) nonpuf = Records(data=rawinputfile.name, blowup_factors=None, weights=None, start_year=cyr) diff --git a/taxcalc/tests/test_dropq.py b/taxcalc/tests/test_dropq.py index 86a107c76..3a2a36b81 100644 --- a/taxcalc/tests/test_dropq.py +++ b/taxcalc/tests/test_dropq.py @@ -109,7 +109,7 @@ def test_run_dropq_nth_year(is_strict, rjson, growth_params, @pytest.mark.parametrize("is_strict", [True, False]) def test_run_dropq_nth_year_from_file(is_strict, puf_1991_path, reform_file): - user_reform = Calculator.read_json_reform_file(reform_file.name) + user_reform = Calculator.read_json_param_files(reform_file.name, None) user_mods = user_reform # Create a Public Use File object @@ -130,7 +130,7 @@ def test_run_dropq_nth_year_from_file(is_strict, puf_1991_path, reform_file): def test_run_dropq_nth_year_mtr_from_file(puf_1991_path, reform_file): - user_reform = Calculator.read_json_reform_file(reform_file.name) + user_reform = Calculator.read_json_param_files(reform_file.name, None) first_year = 2016 elast_params = {'elastic_gdp': [.54, .56, .58]} user_reform[0][first_year].update(elast_params) diff --git a/taxcalc/tests/test_incometaxio.py b/taxcalc/tests/test_incometaxio.py index 32dcadc81..83155735b 100644 --- a/taxcalc/tests/test_incometaxio.py +++ b/taxcalc/tests/test_incometaxio.py @@ -144,11 +144,11 @@ def test_2(rawinputfile): # pylint: disable=redefined-outer-name REFORM_CONTENTS = """ -// Example of a reform file suitable for the read_json_reform_file function. +// Example of a reform file suitable for the read_json_param_files function. // This JSON file can contain any number of trailing //-style comments, which // will be removed before the contents are converted from JSON to a dictionary. -// Within each "policy", "behavior", "growth", and "consumption" object, the -// primary keys are parameters and secondary keys are years. +// Within the "policy" object, the primary keys are parameters and +// secondary keys are years. // Both the primary and secondary key values must be enclosed in quotes ("). // Boolean variables are specified as true or false (no quotes; all lowercase). { diff --git a/taxcalc/tests/test_policy.py b/taxcalc/tests/test_policy.py index ee9e33754..5f1157f42 100644 --- a/taxcalc/tests/test_policy.py +++ b/taxcalc/tests/test_policy.py @@ -501,7 +501,7 @@ def test_parameters_get_default_start_year(): REFORM_CONTENTS = """ -// Example of a reform file suitable for Calculator read_json_reform_file(). +// Example of a reform file suitable for Calculator read_json_param_files(). // This JSON file can contain any number of trailing //-style comments, which // will be removed before the contents are converted from JSON to a dictionary. // The primary keys are policy parameters and secondary keys are years. @@ -546,12 +546,6 @@ def test_parameters_get_default_start_year(): {"2017": false, // values in future years are same as this year value "2020": true // values in future years indexed with this year as base } -}, -"behavior": { -}, -"growth": { -}, -"consumption": { } } """ @@ -560,7 +554,7 @@ def test_parameters_get_default_start_year(): @pytest.yield_fixture def reform_file(): """ - Temporary reform file for Calculator read_json_reform_file function. + Temporary reform file for Calculator read_json_param_files function. """ rfile = tempfile.NamedTemporaryFile(mode='a', delete=False) rfile.write(REFORM_CONTENTS) @@ -577,22 +571,22 @@ def reform_file(): def test_prohibit_param_code(reform_file): Policy.PROHIBIT_PARAM_CODE = True with pytest.raises(ValueError): - Calculator.read_json_reform_file(reform_file.name) + Calculator.read_json_param_files(reform_file.name, None) Policy.PROHIBIT_PARAM_CODE = False @pytest.mark.parametrize("set_year", [False, True]) -def test_read_json_reform_file_and_implement_reform(reform_file, set_year): +def test_read_json_param_files_and_implement_reform(reform_file, set_year): """ Test reading and translation of reform file into a reform dictionary that is then used to call implement_reform method. NOTE: implement_reform called when policy.current_year == policy.start_year """ - reform, rb, rg, rc = Calculator.read_json_reform_file(reform_file.name) + ref, _, _, _ = Calculator.read_json_param_files(reform_file.name, None) policy = Policy() if set_year: policy.set_year(2015) - policy.implement_reform(reform) + policy.implement_reform(ref) syr = policy.start_year amt_brk1 = policy._AMT_brk1 assert amt_brk1[2015 - syr] == 200000 @@ -729,7 +723,6 @@ def test_scan_param_code(): Policy.scan_param_code('9999**99999999') -@pytest.mark.one def test_cpi_for_param_code(): """ Test cpi_for_param_code function. diff --git a/taxcalc/tests/test_reforms.py b/taxcalc/tests/test_reforms.py index f514b3bfb..5a05af1f1 100644 --- a/taxcalc/tests/test_reforms.py +++ b/taxcalc/tests/test_reforms.py @@ -1,6 +1,6 @@ """ -Tests all the example JSON reform files located in taxcalc/reforms directory - and all the JSON reform files located in taxcalc/taxbrain directory +Tests all the example JSON parameter files located in taxcalc/reforms directory + and all the JSON parameter files located in taxcalc/taxbrain directory """ # CODING-STYLE CHECKS: # pep8 --ignore=E402 test_reforms.py @@ -24,9 +24,8 @@ def reforms_path(tests_path): def test_reforms(reforms_path): # pylint: disable=redefined-outer-name """ - Check that each JSON reform file can be converted into a reform dictionary, - the policy component part of which can then be passed to the Policy class - implement_reform() method. + Check that each JSON reform file can be converted into a reform dictionary + that can then be passed to the Policy class implement_reform() method. While doing this, construct a set of Policy parameters (other than those ending in '_cpi') included in the JSON reform files. """ @@ -36,7 +35,7 @@ def test_reforms(reforms_path): # pylint: disable=redefined-outer-name with open(jrf) as jrfile: jrf_text = jrfile.read() # check that jrf_text has "policy" that can be implemented as a reform - policy_dict, _, _, _ = Calculator.read_json_reform_text(jrf_text) + policy_dict = Calculator.read_json_policy_reform_text(jrf_text) policy = Policy() policy.implement_reform(policy_dict) # identify "policy" parameters included in jrf @@ -62,20 +61,27 @@ def taxbrain_path(tests_path): def test_taxbrain_json(taxbrain_path): # pylint: disable=redefined-outer-name """ - Check that each JSON reform file can be converted into a four dictionaries + Check that each JSON parameter file can be converted into dictionaries that can be used to construct objects needed for a Calculator object. """ - for jrf in glob.glob(taxbrain_path): - # read contents of jrf (JSON reform filename) - with open(jrf) as jrfile: - jrf_text = jrfile.read() - # check that jrf_text can be used to instantiate Calculator object - pol, beh, gro, con = Calculator.read_json_reform_text(jrf_text) - policy = Policy() - policy.implement_reform(pol) - behv = Behavior() - behv.update_behavior(beh) - grow = Growth() - grow.update_growth(gro) - cons = Consumption() - cons.update_consumption(con) + for jpf in glob.glob(taxbrain_path): + # read contents of jpf (JSON parameter filename) + with open(jpf) as jpfile: + jpf_text = jpfile.read() + # check that jpf_text can be used to instantiate Calculator object + if '"policy"' in jpf_text: + pol = Calculator.read_json_policy_reform_text(jpf_text) + policy = Policy() + policy.implement_reform(pol) + elif ('"behavior"' in jpf_text and + '"consumption"' in jpf_text and + '"growth"' in jpf_text): + beh, con, gro = Calculator.read_json_econ_assump_text(jpf_text) + behv = Behavior() + behv.update_behavior(beh) + cons = Consumption() + cons.update_consumption(con) + grow = Growth() + grow.update_growth(gro) + else: # jpf_text is not a valid JSON parameter file + assert False diff --git a/taxcalc/tests/test_simpletaxio.py b/taxcalc/tests/test_simpletaxio.py index 91546f563..d2e809cb5 100644 --- a/taxcalc/tests/test_simpletaxio.py +++ b/taxcalc/tests/test_simpletaxio.py @@ -21,11 +21,11 @@ '4 2015 0 2 0 4039 15000 0 0 0 50000 70000 0 0 0 0 0 0 0 0 0 -3000\n' ) REFORM_CONTENTS = """ -// Example of a reform file suitable for the read_json_reform_file function. +// Example of a reform file suitable for the read_json_param_files function. // This JSON file can contain any number of trailing //-style comments, which // will be removed before the contents are converted from JSON to a dictionary. -// Within each "policy", "behavior", "growth", and "consumption" object, the -// primary keys are parameters and secondary keys are years. +// Within the "policy" object, the primary keys are parameters and +// secondary keys are years. // Both the primary and secondary key values must be enclosed in quotes ("). // Boolean variables are specified as true or false (no quotes; all lowercase). { @@ -56,12 +56,6 @@ {"2017": false, // values in future years are same as this year value "2020": true // values in future years indexed with this year as base } - }, - "behavior": { - }, - "growth": { - }, - "consumption": { } } """ From 727709c40c7650b18070037980e350e5d8453c5d Mon Sep 17 00:00:00 2001 From: martinholmer Date: Wed, 18 Jan 2017 16:29:17 -0500 Subject: [PATCH 3/7] Add more tests in order to maintain code coverage. --- taxcalc/calculate.py | 2 + taxcalc/tests/test_calculate.py | 104 +++++++++++++++++++++++++++++--- taxcalc/tests/test_dropq.py | 37 +++++++++++- 3 files changed, 133 insertions(+), 10 deletions(-) diff --git a/taxcalc/calculate.py b/taxcalc/calculate.py index 8ecb7ca94..736c3a44b 100644 --- a/taxcalc/calculate.py +++ b/taxcalc/calculate.py @@ -504,6 +504,8 @@ def read_json_econ_assump_text(text_string): and string years as secondary keys. See tests/test_calculate.py for an extended example of a commented JSON economic assumption text that can be read by this method. + Note that an example is shown in the ASSUMP_CONTENTS string in + tests/test_calculate.py file. Returned dictionaries (behv_dict, cons_dict, grow_dict) have integer years as primary keys and string parameters as secondary keys. diff --git a/taxcalc/tests/test_calculate.py b/taxcalc/tests/test_calculate.py index 8d314c5c8..e207dde9c 100644 --- a/taxcalc/tests/test_calculate.py +++ b/taxcalc/tests/test_calculate.py @@ -427,11 +427,11 @@ def test_Calculator_using_nonstd_input(rawinputfile): REFORM_CONTENTS = """ -// Example of a reform file suitable for read_json_parameter_files function. +// Example of a reform file suitable for read_json_param_files() function. // This JSON file can contain any number of trailing //-style comments, which // will be removed before the contents are converted from JSON to a dictionary. -// Within each "policy", "behavior", "growth", and "consumption" object, the -// primary keys are parameters and secondary keys are years. +// Within each "policy" object, the primary keys are parameters and +// the secondary keys are years. // Both the primary and secondary key values must be enclosed in quotes ("). // Boolean variables are specified as true or false (no quotes; all lowercase). // Parameter code in the policy object is enclosed inside a pair of double @@ -484,7 +484,7 @@ def test_Calculator_using_nonstd_input(rawinputfile): @pytest.yield_fixture def reform_file(): """ - Temporary reform file for read_json_policy_reform_file function. + Temporary reform file for read_json_param_files() function. """ rfile = tempfile.NamedTemporaryFile(mode='a', delete=False) rfile.write(REFORM_CONTENTS) @@ -498,14 +498,53 @@ def reform_file(): pass # sometimes we can't remove a generated temporary file +ASSUMP_CONTENTS = """ +// Example of an assump file suitable for the read_json_param_files() function. +// This JSON file can contain any number of trailing //-style comments, which +// will be removed before the contents are converted from JSON to a dictionary. +// Within each "behavior", "consumption" and "growth" object, the +// primary keys are parameters and the secondary keys are years. +// Both the primary and secondary key values must be enclosed in quotes ("). +// Boolean variables are specified as true or false (no quotes; all lowercase). +{ + "title": "", + "author": "", + "date": "", + "behavior": {}, + "consumption": { "_MPC_e18400": {"2018": [0.05]} }, + "growth": {} +} +""" + + +@pytest.yield_fixture +def assump_file(): + """ + Temporary assumption file for read_json_params_files() function. + """ + afile = tempfile.NamedTemporaryFile(mode='a', delete=False) + afile.write(ASSUMP_CONTENTS) + afile.close() + # must close and then yield for Windows platform + yield afile + if os.path.isfile(afile.name): + try: + os.remove(afile.name) + except OSError: + pass # sometimes we can't remove a generated temporary file + + @pytest.mark.parametrize("set_year", [False, True]) -def test_read_json_reform_file_and_implement_reform(reform_file, set_year): +def test_read_json_reform_file_and_implement_reform(reform_file, + assump_file, + set_year): """ Test reading and translation of reform file into a reform dictionary that is then used to call implement_reform method and Calculate.calc_all() NOTE: implement_reform called when policy.current_year == policy.start_year """ - reform, _, _, _ = Calculator.read_json_param_files(reform_file.name, None) + reform, _, _, _ = Calculator.read_json_param_files(reform_file.name, + assump_file.name) policy = Policy() if set_year: policy.set_year(2015) @@ -545,7 +584,7 @@ def bad1reformfile(): { "policy": { // example of incorrect JSON because 'x' must be "x" 'x': {"2014": [4000]} - }, + } } """ f = tempfile.NamedTemporaryFile(mode='a', delete=False) @@ -561,9 +600,10 @@ def bad2reformfile(): # specify JSON text for reform txt = """ { + "title": "", "policyx": { // example of reform file not containing "policy" key "_SS_Earnings_c": {"2018": [9e99]} - }, + } } """ f = tempfile.NamedTemporaryFile(mode='a', delete=False) @@ -581,6 +621,54 @@ def test_read_bad_json_reform_file(bad1reformfile, bad2reformfile): Calculator.read_json_param_files(bad2reformfile.name, None) +@pytest.yield_fixture +def bad1assumpfile(): + # specify JSON text for assumptions + txt = """ + { + "behavior": { // example of incorrect JSON because 'x' must be "x" + 'x': {"2014": [0.25]} + } + } + """ + f = tempfile.NamedTemporaryFile(mode='a', delete=False) + f.write(txt + '\n') + f.close() + # Must close and then yield for Windows platform + yield f + os.remove(f.name) + + +@pytest.yield_fixture +def bad2assumpfile(): + # specify JSON text for assumptions + txt = """ + { + "title": "", + "author": "", + "date": "", + "behaviorx": {}, // example of assump file not containing "behavior" key + "consumption": {}, + "growth": {} + } + """ + f = tempfile.NamedTemporaryFile(mode='a', delete=False) + f.write(txt + '\n') + f.close() + # Must close and then yield for Windows platform + yield f + os.remove(f.name) + + +def test_read_bad_json_assump_file(bad1assumpfile, bad2assumpfile): + with pytest.raises(ValueError): + Calculator.read_json_param_files(None, bad1assumpfile.name) + with pytest.raises(ValueError): + Calculator.read_json_param_files(None, bad2assumpfile.name) + with pytest.raises(ValueError): + Calculator.read_json_param_files(None, 'unknown_file_name') + + def test_convert_parameter_dict(): with pytest.raises(ValueError): rdict = Calculator.convert_parameter_dict({2013: {'2013': [40000]}}) diff --git a/taxcalc/tests/test_dropq.py b/taxcalc/tests/test_dropq.py index 3a2a36b81..ae839e9c7 100644 --- a/taxcalc/tests/test_dropq.py +++ b/taxcalc/tests/test_dropq.py @@ -33,10 +33,24 @@ "behavior": { "_BE_sub": {"2016": [0.25]} }, + "consumption": { + "_MPC_e20400": {"2016": [0.01]} + }, "growth": { + } +} +""" + + +ASSUMP_CONTENTS = """ +{ + "behavior": { + "_BE_sub": {"2016": [0.25]} }, "consumption": { "_MPC_e20400": {"2016": [0.01]} + }, + "growth": { } } """ @@ -59,6 +73,23 @@ def reform_file(): pass # sometimes we can't remove a generated temporary file +@pytest.yield_fixture +def assump_file(): + """ + Temporary assump file for testing dropq functions with assump file. + """ + afile = tempfile.NamedTemporaryFile(mode='a', delete=False) + afile.write(ASSUMP_CONTENTS) + afile.close() + # must close and then yield for Windows platform + yield afile + if os.path.isfile(afile.name): + try: + os.remove(afile.name) + except OSError: + pass # sometimes we can't remove a generated temporary file + + @pytest.fixture(scope='session') def puf_path(tests_path): """ @@ -107,9 +138,11 @@ def test_run_dropq_nth_year(is_strict, rjson, growth_params, @pytest.mark.parametrize("is_strict", [True, False]) -def test_run_dropq_nth_year_from_file(is_strict, puf_1991_path, reform_file): +def test_run_dropq_nth_year_from_file(is_strict, puf_1991_path, + reform_file, assump_file): - user_reform = Calculator.read_json_param_files(reform_file.name, None) + user_reform = Calculator.read_json_param_files(reform_file.name, + assump_file.name) user_mods = user_reform # Create a Public Use File object From d018087bffa4307d052f5f4e6cb8603bf7b593e9 Mon Sep 17 00:00:00 2001 From: martinholmer Date: Wed, 18 Jan 2017 18:36:35 -0500 Subject: [PATCH 4/7] Add --assump option to inctax.py script and related tests. --- inctax.py | 23 +++++-- taxcalc/incometaxio.py | 51 +++++++++++---- taxcalc/tests/test_functions.py | 2 + taxcalc/tests/test_incometaxio.py | 103 +++++++++++++++++++++++------- 4 files changed, 138 insertions(+), 41 deletions(-) diff --git a/inctax.py b/inctax.py index ebb5c96f9..18ff63cef 100644 --- a/inctax.py +++ b/inctax.py @@ -46,11 +46,18 @@ def main(): action="store_true") parser.add_argument('--reform', help=('REFORM is name of optional file that contains ' - 'tax reform "policy" parameters and "behavior" ' - 'parameters and "growth" parameters; the ' - 'REFORM file is specified using JSON that may ' - 'include //-comments. No --reform implies use ' - 'of current-law policy.'), + 'reform "policy" parameters; the REFORM file ' + 'is specified using JSON that may include ' + '//-comments. No --reform implies use of ' + 'current-law policy.'), + default=None) + parser.add_argument('--assump', + help=('ASSUMP is name of optional file that contains ' + 'economic assumption parameters ("behavior", ' + '"consumption" and "growth" parameters); the ' + 'ASSUMP file is specified using JSON that may ' + 'include //-comments. No --assump implies use ' + 'of static analysis assumptions.'), default=None) parser.add_argument('--exact', help=('optional flag to suppress smoothing in income ' @@ -150,10 +157,16 @@ def main(): sys.stderr.write('ERROR: must specify TAXYEAR >= 2013;\n') sys.stderr.write('USAGE: python inctax.py --help\n') return 1 + # check consistency of REFORM and ASSUMP options + if args.assump and not args.reform: + sys.stderr.write('ERROR: cannot specify ASSUMP without a REFORM\n') + sys.stderr.write('USAGE: python inctax.py --help\n') + return 1 # instantiate IncometaxIO object and do federal income tax calculations inctax = IncomeTaxIO(input_data=args.INPUT, tax_year=args.TAXYEAR, reform=args.reform, + assump=args.assump, exact_calculations=args.exact, blowup_input_data=args.blowup, output_weights=args.weights, diff --git a/taxcalc/incometaxio.py b/taxcalc/incometaxio.py index 360599413..7a4c4c81c 100644 --- a/taxcalc/incometaxio.py +++ b/taxcalc/incometaxio.py @@ -47,6 +47,11 @@ class IncomeTaxIO(object): string is name of optional REFORM file, or dictionary suitable for passing to Policy.implement_reform() method. + assump: None or string + None implies economic assumptions are baseline and statuc analysis + of reform is conducted, or + string is name of optional ASSUMP file. + exact_calculations: boolean specifies whether or not exact tax calculations are done without any smoothing of "stair-step" provisions in income tax law. @@ -72,6 +77,7 @@ class IncomeTaxIO(object): ValueError: if file specified by input_data string does not exist. if reform is neither None, string, nor dictionary. + if assump is neither None nor string. if tax_year before Policy start_year. if tax_year after Policy end_year. @@ -80,7 +86,7 @@ class IncomeTaxIO(object): class instance: IncomeTaxIO """ - def __init__(self, input_data, tax_year, reform, + def __init__(self, input_data, tax_year, reform, assump, exact_calculations, blowup_input_data, output_weights, output_records, csv_dump): @@ -106,6 +112,18 @@ def __init__(self, input_data, tax_year, reform, msg = 'INPUT is neither string nor Pandas DataFrame' raise ValueError(msg) # construct output_filename and delete old output file if it exists + if assump is None: + asm = '' + self._assump = False + elif isinstance(assump, six.string_types): + if assump.endswith('.json'): + asm = '-{}'.format(assump[:-5]) + else: + asm = '-{}'.format(assump) + self._assump = True + else: + msg = 'IncomeTaxIO.ctor assump is neither None nor str' + raise ValueError(msg) if reform is None: ref = '' self._reform = False @@ -125,11 +143,11 @@ def __init__(self, input_data, tax_year, reform, msg = 'IncomeTaxIO.ctor reform is neither None, str, nor dict' raise ValueError(msg) if output_records: - self._output_filename = '{}.records{}'.format(inp, ref) + self._output_filename = '{}.records{}{}'.format(inp, ref, asm) elif csv_dump: - self._output_filename = '{}.csvdump{}'.format(inp, ref) + self._output_filename = '{}.csvdump{}{}'.format(inp, ref, asm) else: - self._output_filename = '{}.out-inctax{}'.format(inp, ref) + self._output_filename = '{}.out-inctax{}{}'.format(inp, ref, asm) if os.path.isfile(self._output_filename): os.remove(self._output_filename) # check for existence of INPUT file @@ -146,16 +164,21 @@ def __init__(self, input_data, tax_year, reform, if tax_year > pol.end_year: msg = 'tax_year {} greater than policy.end_year {}' raise ValueError(msg.format(tax_year, pol.end_year)) - # implement reform if one is specified + # implement reform and assump if specified + ref_d = dict() + beh_d = dict() + con_d = dict() + gro_d = dict() if self._reform: if self._using_reform_file: - (r_pol, - r_beh, - r_con, - r_gro) = Calculator.read_json_param_files(reform, None) + (ref_d, beh_d, con_d, + gro_d) = Calculator.read_json_param_files(reform, assump) else: - r_pol = reform - pol.implement_reform(r_pol) + ref_d = reform + beh_d = dict() + con_d = dict() + gro_d = dict() + pol.implement_reform(ref_d) # set tax policy parameters to specified tax_year pol.set_year(tax_year) # read input file contents into Records object @@ -185,16 +208,16 @@ def __init__(self, input_data, tax_year, reform, clp.set_year(tax_year) recs_clp = copy.deepcopy(recs) con = Consumption() - con.update_consumption(r_con) + con.update_consumption(con_d) gro = Growth() - gro.update_growth(r_gro) + gro.update_growth(gro_d) self._calc_clp = Calculator(policy=clp, records=recs_clp, verbose=False, consumption=con, growth=gro, sync_years=blowup_input_data) beh = Behavior() - beh.update_behavior(r_beh) + beh.update_behavior(beh_d) self._calc = Calculator(policy=pol, records=recs, verbose=True, behavior=beh, diff --git a/taxcalc/tests/test_functions.py b/taxcalc/tests/test_functions.py index 356e78e3b..ade46f778 100644 --- a/taxcalc/tests/test_functions.py +++ b/taxcalc/tests/test_functions.py @@ -207,6 +207,7 @@ def test_1(): inctax = IncomeTaxIO(input_data=input_dataframe, tax_year=2015, reform=policy_reform, + assump=None, exact_calculations=False, blowup_input_data=False, output_weights=False, @@ -257,6 +258,7 @@ def test_2(): inctax = IncomeTaxIO(input_data=input_dataframe, tax_year=2015, reform=policy_reform, + assump=None, exact_calculations=False, blowup_input_data=False, output_weights=False, diff --git a/taxcalc/tests/test_incometaxio.py b/taxcalc/tests/test_incometaxio.py index 83155735b..b26d90ca8 100644 --- a/taxcalc/tests/test_incometaxio.py +++ b/taxcalc/tests/test_incometaxio.py @@ -67,6 +67,7 @@ def test_incorrect_creation_1(input_data, exact): input_data=input_data, tax_year=2013, reform=None, + assump=None, exact_calculations=exact, blowup_input_data=True, output_weights=False, @@ -75,21 +76,25 @@ def test_incorrect_creation_1(input_data, exact): ) -@pytest.mark.parametrize("year, reform", [ - (2013, list()), - (2001, None), - (2099, None), +# for fixture args, pylint: disable=redefined-outer-name + + +@pytest.mark.parametrize("year, reform, assump", [ + (2013, list(), None), + (2013, None, list()), + (2001, None, None), + (2099, None, None), ]) -def test_incorrect_creation_2(rawinputfile, year, reform): +def test_incorrect_creation_2(rawinputfile, year, reform, assump): """ - Ensure a ValueError is raised when created with invalid reform params + Ensure a ValueError is raised when created with invalid parameters """ - # for fixture args, pylint: disable=redefined-outer-name with pytest.raises(ValueError): IncomeTaxIO( input_data=rawinputfile.name, tax_year=year, reform=reform, + assump=assump, exact_calculations=False, blowup_input_data=True, output_weights=False, @@ -107,12 +112,12 @@ def test_creation_with_blowup(rawinputfile, blowup, weights_out): """ Test IncomeTaxIO instantiation with no policy reform and with blowup. """ - # for fixture args, pylint: disable=redefined-outer-name IncomeTaxIO.show_iovar_definitions() taxyear = 2021 inctax = IncomeTaxIO(input_data=rawinputfile.name, tax_year=taxyear, reform=None, + assump=None, exact_calculations=False, blowup_input_data=blowup, output_weights=weights_out, @@ -121,7 +126,7 @@ def test_creation_with_blowup(rawinputfile, blowup, weights_out): assert inctax.tax_year() == taxyear -def test_2(rawinputfile): # pylint: disable=redefined-outer-name +def test_2(rawinputfile): """ Test IncomeTaxIO calculate method with no output writing and no blowup. """ @@ -134,6 +139,7 @@ def test_2(rawinputfile): # pylint: disable=redefined-outer-name inctax = IncomeTaxIO(input_data=rawinputfile.name, tax_year=taxyear, reform=reform_dict, + assump=None, exact_calculations=False, blowup_input_data=False, output_weights=False, @@ -144,7 +150,7 @@ def test_2(rawinputfile): # pylint: disable=redefined-outer-name REFORM_CONTENTS = """ -// Example of a reform file suitable for the read_json_param_files function. +// Example of a reform file suitable for the read_json_param_files() function. // This JSON file can contain any number of trailing //-style comments, which // will be removed before the contents are converted from JSON to a dictionary. // Within the "policy" object, the primary keys are parameters and @@ -182,12 +188,6 @@ def test_2(rawinputfile): # pylint: disable=redefined-outer-name "_LST": // Lump-Sum Tax {"2013": [500] } - }, - "behavior": { - }, - "growth": { - }, - "consumption": { } } """ @@ -227,7 +227,60 @@ def reformfile2(): pass # sometimes we can't remove a generated temporary file -def test_3(rawinputfile, reformfile1): # pylint: disable=redefined-outer-name +ASSUMP_CONTENTS = """ +// Example of an assump file suitable for the read_json_param_files() function. +// This JSON file can contain any number of trailing //-style comments, which +// will be removed before the contents are converted from JSON to a dictionary. +// Within each "behavior", "consumption" and "growth" object, the +// primary keys are parameters and the secondary keys are years. +// Both the primary and secondary key values must be enclosed in quotes ("). +// Boolean variables are specified as true or false (no quotes; all lowercase). +{ + "title": "", + "author": "", + "date": "", + "behavior": {}, + "consumption": { "_MPC_e18400": {"2018": [0.05]} }, + "growth": {} +} +""" + + +@pytest.yield_fixture +def assumpfile1(): + """ + Temporary assumption file with .json extension. + """ + afile = tempfile.NamedTemporaryFile(suffix='.json', mode='a', delete=False) + afile.write(ASSUMP_CONTENTS) + afile.close() + # must close and then yield for Windows platform + yield afile + if os.path.isfile(afile.name): + try: + os.remove(afile.name) + except OSError: + pass # sometimes we can't remove a generated temporary file + + +@pytest.yield_fixture +def assumpfile2(): + """ + Temporary assumption file without .json extension. + """ + afile = tempfile.NamedTemporaryFile(mode='a', delete=False) + afile.write(ASSUMP_CONTENTS) + afile.close() + # must close and then yield for Windows platform + yield afile + if os.path.isfile(afile.name): + try: + os.remove(afile.name) + except OSError: + pass # sometimes we can't remove a generated temporary file + + +def test_3(rawinputfile, reformfile1, assumpfile1): """ Test IncomeTaxIO calculate method with no output writing and no blowup, using file name for IncomeTaxIO constructor input_data. @@ -236,6 +289,7 @@ def test_3(rawinputfile, reformfile1): # pylint: disable=redefined-outer-name inctax = IncomeTaxIO(input_data=rawinputfile.name, tax_year=taxyear, reform=reformfile1.name, + assump=assumpfile1.name, exact_calculations=False, blowup_input_data=False, output_weights=False, @@ -245,7 +299,7 @@ def test_3(rawinputfile, reformfile1): # pylint: disable=redefined-outer-name assert output == EXPECTED_OUTPUT -def test_4(reformfile2): # pylint: disable=redefined-outer-name +def test_4(reformfile2, assumpfile2): """ Test IncomeTaxIO calculate method with no output writing and no blowup, using DataFrame for IncomeTaxIO constructor input_data. @@ -256,6 +310,7 @@ def test_4(reformfile2): # pylint: disable=redefined-outer-name inctax = IncomeTaxIO(input_data=input_dataframe, tax_year=taxyear, reform=reformfile2.name, + assump=assumpfile2.name, exact_calculations=False, blowup_input_data=False, output_weights=False, @@ -265,7 +320,7 @@ def test_4(reformfile2): # pylint: disable=redefined-outer-name assert output == EXPECTED_OUTPUT -def test_5(rawinputfile): # pylint: disable=redefined-outer-name +def test_5(rawinputfile): """ Test IncomeTaxIO calculate method with no output writing and no blowup and no reform, using the output_records option. @@ -274,6 +329,7 @@ def test_5(rawinputfile): # pylint: disable=redefined-outer-name inctax = IncomeTaxIO(input_data=rawinputfile.name, tax_year=taxyear, reform=None, + assump=None, exact_calculations=False, blowup_input_data=False, output_weights=False, @@ -283,7 +339,7 @@ def test_5(rawinputfile): # pylint: disable=redefined-outer-name assert inctax.tax_year() == taxyear -def test_6(rawinputfile): # pylint: disable=redefined-outer-name +def test_6(rawinputfile): """ Test IncomeTaxIO calculate method with no output writing and no blowup and no reform, using the csv_dump option. @@ -292,6 +348,7 @@ def test_6(rawinputfile): # pylint: disable=redefined-outer-name inctax = IncomeTaxIO(input_data=rawinputfile.name, tax_year=taxyear, reform=None, + assump=None, exact_calculations=False, blowup_input_data=False, output_weights=False, @@ -301,7 +358,7 @@ def test_6(rawinputfile): # pylint: disable=redefined-outer-name assert inctax.tax_year() == taxyear -def test_7(rawinputfile, reformfile1): # pylint: disable=redefined-outer-name +def test_7(rawinputfile, reformfile1): """ Test IncomeTaxIO calculate method with no output writing using ceeu option. """ @@ -309,6 +366,7 @@ def test_7(rawinputfile, reformfile1): # pylint: disable=redefined-outer-name inctax = IncomeTaxIO(input_data=rawinputfile.name, tax_year=taxyear, reform=reformfile1.name, + assump=None, exact_calculations=False, blowup_input_data=False, output_weights=False, @@ -318,7 +376,7 @@ def test_7(rawinputfile, reformfile1): # pylint: disable=redefined-outer-name assert inctax.tax_year() == taxyear -def test_8(reformfile1): # pylint: disable=redefined-outer-name +def test_8(reformfile1): """ Test IncomeTaxIO calculate method with no output writing using ceeu option. """ @@ -329,6 +387,7 @@ def test_8(reformfile1): # pylint: disable=redefined-outer-name inctax = IncomeTaxIO(input_data=recdf, tax_year=taxyear, reform=reformfile1.name, + assump=None, exact_calculations=False, blowup_input_data=False, output_weights=True, From 87bbc835928dfa318566559b3a9014a3b5d2441c Mon Sep 17 00:00:00 2001 From: martinholmer Date: Thu, 19 Jan 2017 09:01:42 -0500 Subject: [PATCH 5/7] Refactor taxcalc/taxbrain/*json files. --- taxcalc/taxbrain/README.md | 2 +- taxcalc/taxbrain/a1.json | 20 +++++++++++ taxcalc/taxbrain/file1.json | 44 ----------------------- taxcalc/taxbrain/file2.json | 44 ----------------------- taxcalc/taxbrain/file3.json | 45 ------------------------ taxcalc/taxbrain/{file0.json => r0.json} | 23 +++++------- 6 files changed, 30 insertions(+), 148 deletions(-) create mode 100644 taxcalc/taxbrain/a1.json delete mode 100644 taxcalc/taxbrain/file1.json delete mode 100644 taxcalc/taxbrain/file2.json delete mode 100644 taxcalc/taxbrain/file3.json rename taxcalc/taxbrain/{file0.json => r0.json} (70%) diff --git a/taxcalc/taxbrain/README.md b/taxcalc/taxbrain/README.md index 9e5509f68..d80d3fb1a 100644 --- a/taxcalc/taxbrain/README.md +++ b/taxcalc/taxbrain/README.md @@ -12,5 +12,5 @@ The contents of this directory include JSON reform files used in the cross-checking, which should be done immediately after each TaxBrain upgrade to a new Tax-Calculator release. -See the comments in `file0.json` for an example of how the +See the comments in `r0.json` and `a1.json` for examples of how the cross-checking is done. diff --git a/taxcalc/taxbrain/a1.json b/taxcalc/taxbrain/a1.json new file mode 100644 index 000000000..5e249880b --- /dev/null +++ b/taxcalc/taxbrain/a1.json @@ -0,0 +1,20 @@ +// 2022 RESULTS from taxcalc-inctax and taxbrain-upload version 0.7.4 +// INCOME TAX ($B) 1798.31 [1,651.7] <-- ???????????? +// PAYROLL TAX ($B) 1465.08 [1,475.6] <-- ???????????? +// taxcalc-inctax results from: +// python ../../inctax.py puf.csv 2022 --blowup --weights --reform r0.json --assump a1.json +// awk '{r+=$4*$29}END{print r*1e-9}' puf-22.out-inctax-r0-a1 +// awk '{r+=$6*$29}END{print r*1e-9}' puf-22.out-inctax-r0-a1 +// taxbrain-upload results from: +// http://www.ospc.org/taxbrain/[????]/ ????????????????????????????? +{ + "title": "", + "behavior": { + "_BE_sub": {"2018": [0.25]} + }, + "consumption": { + "_MPC_e18400": {"2018": [0.05]} + }, + "growth": { + } +} diff --git a/taxcalc/taxbrain/file1.json b/taxcalc/taxbrain/file1.json deleted file mode 100644 index 011140dd4..000000000 --- a/taxcalc/taxbrain/file1.json +++ /dev/null @@ -1,44 +0,0 @@ -// 2022 RESULTS from taxcalc-inctax and taxbrain-upload version 0.7.3 -// INCOME TAX ($B) 1651.70 1,651.7 <-- same -// PAYROLL TAX ($B) 1475.65 1,475.6 <-- same -// taxcalc-inctax results from: -// python ../../inctax.py puf.csv 2022 --blowup --weights --reform file1.json -// awk '{r+=$4*$29}END{print r*1e-9}' puf-22.out-inctax-file1 -// awk '{r+=$6*$29}END{print r*1e-9}' puf-22.out-inctax-file1 -// taxbrain-upload results from: -// http://www.ospc.org/taxbrain/9462/ -{ - "policy": { - "_SS_Earnings_c": // social security (OASDI) maximum taxable earnings - {"2018": [400000], - "2019": [500000], - "2020": [600000] - }, - "_II_em": // personal exemption amount - {"2018": [8000] - }, - "_II_em_cpi": // personal exemption amount indexing status - {"2018": false, // values in future years are same as this year value - "2020": true // values in future years indexed with this year as base - }, - "_ALD_InvInc_ec_rt": // investment income AGI exclusion rate - {"2019": [0.20] - }, - // investment income AGI exclusion base is not all investment income - // but rather the sum of taxable interest, qualified dividends, and - // long-term capital gains - "_ALD_InvInc_ec_base_code_active": - {"2019": [true] - }, - "param_code": - {"ALD_InvInc_ec_base_code": "e00300 + e00650 + p23250" - } - }, - "behavior": { - "_BE_sub": {"2018": [0.25]} - }, - "consumption": { - }, - "growth": { - } -} diff --git a/taxcalc/taxbrain/file2.json b/taxcalc/taxbrain/file2.json deleted file mode 100644 index 3cc3c455b..000000000 --- a/taxcalc/taxbrain/file2.json +++ /dev/null @@ -1,44 +0,0 @@ -// 2022 RESULTS from taxcalc-inctax and taxbrain-upload version 0.7.3 -// INCOME TAX ($B) 1672.23 1,672.2 <-- same -// PAYROLL TAX ($B) 1485.37 1,485.4 <-- same -// taxcalc-inctax results from: -// python ../../inctax.py puf.csv 2022 --blowup --weights --reform file2.json -// awk '{r+=$4*$29}END{print r*1e-9}' puf-22.out-inctax-file2 -// awk '{r+=$6*$29}END{print r*1e-9}' puf-22.out-inctax-file2 -// taxbrain-upload results from: -// http://www.ospc.org/taxbrain/9465/ -{ - "policy": { - "_SS_Earnings_c": // social security (OASDI) maximum taxable earnings - {"2018": [400000], - "2019": [500000], - "2020": [600000] - }, - "_II_em": // personal exemption amount - {"2018": [8000] - }, - "_II_em_cpi": // personal exemption amount indexing status - {"2018": false, // values in future years are same as this year value - "2020": true // values in future years indexed with this year as base - }, - "_ALD_InvInc_ec_rt": // investment income AGI exclusion rate - {"2019": [0.20] - }, - // investment income AGI exclusion base is not all investment income - // but rather the sum of taxable interest, qualified dividends, and - // long-term capital gains - "_ALD_InvInc_ec_base_code_active": - {"2019": [true] - }, - "param_code": - {"ALD_InvInc_ec_base_code": "e00300 + e00650 + p23250" - } - }, - "behavior": { - }, - "consumption": { - "_MPC_e18400": {"2018": [0.05]} - }, - "growth": { - } -} diff --git a/taxcalc/taxbrain/file3.json b/taxcalc/taxbrain/file3.json deleted file mode 100644 index f04f38465..000000000 --- a/taxcalc/taxbrain/file3.json +++ /dev/null @@ -1,45 +0,0 @@ -// 2022 RESULTS from taxcalc-inctax and taxbrain-upload version 0.7.3 -// INCOME TAX ($B) 1650.91 1,650.9 <-- same -// PAYROLL TAX ($B) 1475.24 1,475.2 <-- same -// taxcalc-inctax results from: -// python ../../inctax.py puf.csv 2022 --blowup --weights --reform file3.json -// awk '{r+=$4*$29}END{print r*1e-9}' puf-22.out-inctax-file3 -// awk '{r+=$6*$29}END{print r*1e-9}' puf-22.out-inctax-file3 -// taxbrain-upload results from: -// http://www.ospc.org/taxbrain/9467/ -{ - "policy": { - "_SS_Earnings_c": // social security (OASDI) maximum taxable earnings - {"2018": [400000], - "2019": [500000], - "2020": [600000] - }, - "_II_em": // personal exemption amount - {"2018": [8000] - }, - "_II_em_cpi": // personal exemption amount indexing status - {"2018": false, // values in future years are same as this year value - "2020": true // values in future years indexed with this year as base - }, - "_ALD_InvInc_ec_rt": // investment income AGI exclusion rate - {"2019": [0.20] - }, - // investment income AGI exclusion base is not all investment income - // but rather the sum of taxable interest, qualified dividends, and - // long-term capital gains - "_ALD_InvInc_ec_base_code_active": - {"2019": [true] - }, - "param_code": - {"ALD_InvInc_ec_base_code": "e00300 + e00650 + p23250" - } - }, - "behavior": { - "_BE_sub": {"2018": [0.25]} - }, - "consumption": { - "_MPC_e18400": {"2018": [0.05]} - }, - "growth": { - } -} diff --git a/taxcalc/taxbrain/file0.json b/taxcalc/taxbrain/r0.json similarity index 70% rename from taxcalc/taxbrain/file0.json rename to taxcalc/taxbrain/r0.json index 48d69d39f..191f1d188 100644 --- a/taxcalc/taxbrain/file0.json +++ b/taxcalc/taxbrain/r0.json @@ -1,12 +1,12 @@ -// 2022 RESULTS from taxcalc-inctax and taxbrain-upload version 0.7.3 -// INCOME TAX ($B) 1672.23 1,672.2 <-- same -// PAYROLL TAX ($B) 1485.37 1,485.4 <-- same +// 2022 RESULTS from taxcalc-inctax and taxbrain-upload version 0.7.4 +// INCOME TAX ($B) 1818.54 [1,672.2] <-- ?????????? +// PAYROLL TAX ($B) 1475.43 [1,485.4] <-- ?????????? // taxcalc-inctax results from: -// python ../../inctax.py puf.csv 2022 --blowup --weights --reform file0.json -// awk '{r+=$4*$29}END{print r*1e-9}' puf-22.out-inctax-file0 -// awk '{r+=$6*$29}END{print r*1e-9}' puf-22.out-inctax-file0 +// python ../../inctax.py puf.csv 2022 --blowup --weights --reform r0.json +// awk '{r+=$4*$29}END{print r*1e-9}' puf-22.out-inctax-r0 +// awk '{r+=$6*$29}END{print r*1e-9}' puf-22.out-inctax-r0 // taxbrain-upload results from: -// http://www.ospc.org/taxbrain/9461/ +// http://www.ospc.org/taxbrain/????/ ???????????????????????????? { "policy": { "_SS_Earnings_c": // social security (OASDI) maximum taxable earnings @@ -31,13 +31,8 @@ {"2019": [true] }, "param_code": - {"ALD_InvInc_ec_base_code": "e00300 + e00650 + p23250" + {"ALD_InvInc_ec_base_code": + "returned_value = e00300 + e00650 + p23250" } - }, - "behavior": { - }, - "consumption": { - }, - "growth": { } } From 4e5af19d0669faa704a457c6f7bc2bf4dac3ec8c Mon Sep 17 00:00:00 2001 From: martinholmer Date: Fri, 20 Jan 2017 10:03:46 -0500 Subject: [PATCH 6/7] Fix test_dropq.py REFORM_CONTENTS and ASSUMP_CONTENTS. --- taxcalc/tests/test_dropq.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/taxcalc/tests/test_dropq.py b/taxcalc/tests/test_dropq.py index ae839e9c7..7580ae5f5 100644 --- a/taxcalc/tests/test_dropq.py +++ b/taxcalc/tests/test_dropq.py @@ -15,6 +15,9 @@ REFORM_CONTENTS = """ // Specify AGI exclusion of some fraction of investment income { + "title": "", + "author": "", + "date": "", "policy": { // specify fraction of investment income that can be excluded from AGI "_ALD_InvInc_ec_rt": {"2016": [0.50]}, @@ -29,14 +32,6 @@ "_ALD_InvInc_ec_base_code_active": {"2016": [true]} // the dollar exclusion is the product of the base defined by code // and the fraction defined above - }, - "behavior": { - "_BE_sub": {"2016": [0.25]} - }, - "consumption": { - "_MPC_e20400": {"2016": [0.01]} - }, - "growth": { } } """ @@ -44,6 +39,9 @@ ASSUMP_CONTENTS = """ { + "title": "", + "author": "", + "date": "", "behavior": { "_BE_sub": {"2016": [0.25]} }, From 3305f62b296540f4edf78da615d9434e96d741a1 Mon Sep 17 00:00:00 2001 From: martinholmer Date: Sat, 21 Jan 2017 08:02:18 -0500 Subject: [PATCH 7/7] Add error checking for misplaced keys in reform/assump files. --- taxcalc/calculate.py | 31 ++++++++++++------ taxcalc/reforms/adjust0.txt | 6 ---- taxcalc/reforms/ptaxes0.txt | 6 ---- taxcalc/reforms/ptaxes1.txt | 6 ---- taxcalc/reforms/ptaxes2.txt | 6 ---- taxcalc/reforms/ptaxes3.txt | 6 ---- taxcalc/tests/test_calculate.py | 57 +++++++++++++++++++++++++++++++-- 7 files changed, 75 insertions(+), 43 deletions(-) diff --git a/taxcalc/calculate.py b/taxcalc/calculate.py index 736c3a44b..967d160d1 100644 --- a/taxcalc/calculate.py +++ b/taxcalc/calculate.py @@ -424,13 +424,17 @@ def read_json_param_files(reform_filename, assump_filename): raise ValueError(msg.format(assump_filename)) return (rpol_dict, behv_dict, cons_dict, grow_dict) + REQUIRED_REFORM_KEYS = set(['policy']) + REQUIRED_ASSUMP_KEYS = set(['behavior', 'consumption', 'growth']) + @staticmethod def read_json_policy_reform_text(text_string): """ Strip //-comments from text_string and return 1 dict based on the JSON. Specified text is JSON with at least 1 high-level string:object pair: a "policy": {...} pair. - Other high-level pairs will be ignored by this method. + Other high-level pairs will be ignored by this method, except that + a "behavior", "consumption" or "growth" key will raise a ValueError. The {...} object may be empty (that is, be {}), or may contain one or more pairs with parameter string primary keys and string years as secondary keys. See tests/test_calculate.py for @@ -471,12 +475,15 @@ def repl_func(mat): msg += '{:02d}{}'.format(linenum, line) + '\n' msg += bline + '\n' raise ValueError(msg) - # check contents of dictionary - required_keys = ['policy'] + # check key contents of dictionary actual_keys = raw_dict.keys() - for rkey in required_keys: + for rkey in Calculator.REQUIRED_REFORM_KEYS: if rkey not in actual_keys: - msg = 'policy reform key "{}" not among high-level keys' + msg = 'key "{}" is not in policy reform file' + raise ValueError(msg.format(rkey)) + for rkey in actual_keys: + if rkey in Calculator.REQUIRED_ASSUMP_KEYS: + msg = 'key "{}" should be in economic assumption file' raise ValueError(msg.format(rkey)) # handle special param_code key in raw_dict policy component dictionary paramcode = raw_dict['policy'].pop('param_code', None) @@ -498,7 +505,8 @@ def read_json_econ_assump_text(text_string): a "behavior": {...} pair, a "consumption": {...} pair, and a "growth": {...} pair. - Other high-level pairs will be ignored by this method. + Other high-level pairs will be ignored by this method, except that + a "policy" key will raise a ValueError. The {...} object may be empty (that is, be {}), or may contain one or more pairs with parameter string primary keys and string years as secondary keys. See tests/test_calculate.py for @@ -533,12 +541,15 @@ def read_json_econ_assump_text(text_string): msg += '{:02d}{}'.format(linenum, line) + '\n' msg += bline + '\n' raise ValueError(msg) - # check contents of dictionary - required_keys = ['behavior', 'consumption', 'growth'] + # check key contents of dictionary actual_keys = raw_dict.keys() - for rkey in required_keys: + for rkey in Calculator.REQUIRED_ASSUMP_KEYS: if rkey not in actual_keys: - msg = 'economic assumption key "{}" not among high-level keys' + msg = 'key "{}" is not in economic assumption file' + raise ValueError(msg.format(rkey)) + for rkey in actual_keys: + if rkey in Calculator.REQUIRED_REFORM_KEYS: + msg = 'key "{}" should be in policy reform file' raise ValueError(msg.format(rkey)) # convert the assumption dictionaries in raw_dict behv_dict = Calculator.convert_parameter_dict(raw_dict['behavior']) diff --git a/taxcalc/reforms/adjust0.txt b/taxcalc/reforms/adjust0.txt index fb45cfa82..a9f9ace4e 100644 --- a/taxcalc/reforms/adjust0.txt +++ b/taxcalc/reforms/adjust0.txt @@ -17,11 +17,5 @@ returned_value = e00300 + e00650 + p23250 "_ALD_InvInc_ec_base_code_active": {"2016": [true]} // the dollar exclusion is the product of the base defined by code // and the fraction defined above - }, - "behavior": { - }, - "growth": { - }, - "consumption": { } } diff --git a/taxcalc/reforms/ptaxes0.txt b/taxcalc/reforms/ptaxes0.txt index c2b71ba11..c51b65402 100644 --- a/taxcalc/reforms/ptaxes0.txt +++ b/taxcalc/reforms/ptaxes0.txt @@ -9,11 +9,5 @@ "2019": [0.030], // raise by 0.1 percentage point "2021": [0.032] // raise by another 0.2 percentage point } - }, - "behavior": { - }, - "growth": { - }, - "consumption": { } } diff --git a/taxcalc/reforms/ptaxes1.txt b/taxcalc/reforms/ptaxes1.txt index a89c59ebf..2e0da43e7 100644 --- a/taxcalc/reforms/ptaxes1.txt +++ b/taxcalc/reforms/ptaxes1.txt @@ -5,12 +5,6 @@ "2018": [200000], // raise to $200,000 and wage index after that "2020": [250000] // raise to $250,000 and wage index after that } - }, - "behavior": { - }, - "growth": { - }, - "consumption": { } } diff --git a/taxcalc/reforms/ptaxes2.txt b/taxcalc/reforms/ptaxes2.txt index 3951c69b3..88d1b5f62 100644 --- a/taxcalc/reforms/ptaxes2.txt +++ b/taxcalc/reforms/ptaxes2.txt @@ -4,11 +4,5 @@ "_SS_Earnings_c": { // at $118,500 in 2016; wage indexed after that "2022": [9e99] // raise to essentially infinity } - }, - "behavior": { - }, - "growth": { - }, - "consumption": { } } diff --git a/taxcalc/reforms/ptaxes3.txt b/taxcalc/reforms/ptaxes3.txt index ba2eb663b..4f9b2cc10 100644 --- a/taxcalc/reforms/ptaxes3.txt +++ b/taxcalc/reforms/ptaxes3.txt @@ -12,11 +12,5 @@ "_AMEDT_ec_cpi": { // start price-indexing the exclusion "2019": true // values in future years price indexed } - }, - "behavior": { - }, - "growth": { - }, - "consumption": { } } diff --git a/taxcalc/tests/test_calculate.py b/taxcalc/tests/test_calculate.py index e207dde9c..659b7a432 100644 --- a/taxcalc/tests/test_calculate.py +++ b/taxcalc/tests/test_calculate.py @@ -614,11 +614,35 @@ def bad2reformfile(): os.remove(f.name) -def test_read_bad_json_reform_file(bad1reformfile, bad2reformfile): +@pytest.yield_fixture +def bad3reformfile(): + # specify JSON text for reform + txt = """ + { + "title": "", + "policy": { + "_SS_Earnings_c": {"2018": [9e99]} + }, + "behavior": { // example of misplaced "behavior" key + } + } + """ + f = tempfile.NamedTemporaryFile(mode='a', delete=False) + f.write(txt + '\n') + f.close() + # Must close and then yield for Windows platform + yield f + os.remove(f.name) + + +def test_read_bad_json_reform_file(bad1reformfile, bad2reformfile, + bad3reformfile): with pytest.raises(ValueError): Calculator.read_json_param_files(bad1reformfile.name, None) with pytest.raises(ValueError): Calculator.read_json_param_files(bad2reformfile.name, None) + with pytest.raises(ValueError): + Calculator.read_json_param_files(bad3reformfile.name, None) @pytest.yield_fixture @@ -628,7 +652,9 @@ def bad1assumpfile(): { "behavior": { // example of incorrect JSON because 'x' must be "x" 'x': {"2014": [0.25]} - } + }, + "consumption": {}, + "growth": {} } """ f = tempfile.NamedTemporaryFile(mode='a', delete=False) @@ -660,11 +686,36 @@ def bad2assumpfile(): os.remove(f.name) -def test_read_bad_json_assump_file(bad1assumpfile, bad2assumpfile): +@pytest.yield_fixture +def bad3assumpfile(): + # specify JSON text for assump + txt = """ + { + "title": "", + "behavior": {}, + "consumption": {}, + "growth": {}, + "policy": { // example of misplaced policy key + "_SS_Earnings_c": {"2018": [9e99]} + } + } + """ + f = tempfile.NamedTemporaryFile(mode='a', delete=False) + f.write(txt + '\n') + f.close() + # Must close and then yield for Windows platform + yield f + os.remove(f.name) + + +def test_read_bad_json_assump_file(bad1assumpfile, bad2assumpfile, + bad3assumpfile): with pytest.raises(ValueError): Calculator.read_json_param_files(None, bad1assumpfile.name) with pytest.raises(ValueError): Calculator.read_json_param_files(None, bad2assumpfile.name) + with pytest.raises(ValueError): + Calculator.read_json_param_files(None, bad3assumpfile.name) with pytest.raises(ValueError): Calculator.read_json_param_files(None, 'unknown_file_name')