Merge pull request PolicyEngine#56 from noman404/noman404/fix22

Fix No state variable or 0 state input issue

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.10", 3.11, 3.12]
+        python-version: ["3.10", 3.11]
 
     steps:
       - uses: actions/checkout@v2

.gitignore

@@ -168,4 +168,3 @@ cython_debug/
 .idea/
 input/*
 output.csv
-taxsim_input.csv

README.md

@@ -6,7 +6,7 @@ The emulator takes a .csv file in the form of a csv. This is the same form of in
 
 **Open your terminal in the parent directory**
 
-1. Install necessary dependencies
+1. To install this package
 
 `pip install -e .`
 
@@ -16,6 +16,13 @@ The emulator takes a .csv file in the form of a csv. This is the same form of in
 
 Output will be generated as `output.csv` in the same directory
 
+
+**For directly installing from pip**
+
+You can install through to execute directly
+
+`pip install git+https://github.com/PolicyEngine/policyengine-taxsim.git`
+
 ### Example ##
 input file:
 

policyengine_taxsim/cli.py

@@ -1,9 +1,12 @@
 import click
 import pandas as pd
 from pathlib import Path
-from policyengine_taxsim.core.input_mapper import import_single_household
-from policyengine_taxsim.core.output_mapper import export_single_household
-
+try:
+    from .core.input_mapper import import_single_household
+    from .core.output_mapper import export_single_household
+except ImportError:
+    from policyengine_taxsim.core.input_mapper import import_single_household
+    from policyengine_taxsim.core.output_mapper import export_single_household
 
 @click.command()
 @click.argument("input_file", type=click.Path(exists=True))

policyengine_taxsim/core/input_mapper.py

@@ -1,4 +1,4 @@
-from policyengine_taxsim.core.utils import (
+from .utils import (
     load_variable_mappings,
     get_state_code,
 )
@@ -17,10 +17,8 @@ def import_single_household(taxsim_vars):
     mappings = load_variable_mappings()["taxsim_to_policyengine"]
 
     year = str(int(taxsim_vars["year"]))  # Ensure year is an integer string
-
-    if "state" not in taxsim_vars: # If state is not provided set it to AL as default state
-        taxsim_vars["state"] = 2
 
+    taxsim_vars["state"] = taxsim_vars.get("state", 44) or 44 #set TX texas as default is no state has passed or passed as 0
     state = get_state_code(taxsim_vars["state"])
 
     situation = {

policyengine_taxsim/core/output_mapper.py

@@ -1,4 +1,4 @@
-from policyengine_taxsim.core.utils import (
+from .utils import (
     load_variable_mappings,
     get_state_number, to_roundedup_number,
 )

pyproject.toml

@@ -47,6 +47,7 @@ dependencies = [
     "pytest",
     "pytest-cov",
 ]
+
 [tool.hatch.envs.default.scripts]
 test = "pytest tests/ {args}"
 test-cov = "pytest tests/ --cov-report=term-missing --cov-config=pyproject.toml --cov=policyengine_taxsim {args}"
@@ -71,4 +72,22 @@ minversion = "6.0"
 addopts = "-ra -q"
 testpaths = [
     "tests",
+]
+
+[tool.nuitka]
+follow-imports = true
+standalone = true
+onefile = true
+# Remove or set to false since we need console
+# windows-disable-console = true
+include-data-dir = ["resources/taxsim35=resources/taxsim35"]
+include-package = [
+    "policyengine_taxsim",
+    "click",
+    "pandas",
+    "numpy"
+]
+plugin-enable = [
+    "numpy",
+    "pandas"
 ]

requirements.txt

@@ -1,97 +1,6 @@
-annotated-types==0.7.0
-backcall==0.2.0
-black==24.4.2
-blosc2==2.0.0
-cachetools==5.3.3
-certifi==2024.6.2
-charset-normalizer==3.3.2
+policyengine-us==1.45.1
 click==8.1.3
-colorama==0.4.6
-contourpy==1.2.1
-coverage==7.5.3
-cycler==0.12.1
-Cython==3.0.10
-DateTime==5.5
-decorator==5.1.1
-dpath==2.1.6
-fonttools==4.53.0
-google-ai-generativelanguage==0.6.4
-google-api-core==2.19.0
-google-api-python-client==2.132.0
-google-auth==2.29.0
-google-auth-httplib2==0.2.0
-google-generativeai==0.6.0
-googleapis-common-protos==1.63.1
-grpcio==1.64.1
-grpcio-status==1.62.2
-h5py==3.11.0
-httplib2==0.22.0
-idna==3.7
-iniconfig==2.0.0
-ipython==7.34.0
-jedi==0.19.1
-joblib==1.4.2
-kiwisolver==1.4.5
-linecheck==0.1.0
 matplotlib==3.9.0
-matplotlib-inline==0.1.7
-matplotlib-label-lines==0.7.0
-microdf-python==0.3.0
-more-itertools==10.2.0
-msgpack==1.0.8
-mypy-extensions==1.0.0
-nptyping==1.4.4
-numexpr==2.10.0
-numpy==1.24.4
-packaging==24.0
 pandas==2.2.2
-parso==0.8.4
-pathlib==1.0.1
-pathspec==0.12.1
-patsy==0.5.6
-pickleshare==0.7.5
-pillow==10.3.0
-platformdirs==4.2.2
-plotly==5.22.0
-pluggy==1.5.0
-policyengine-core==2.21.8
-policyengine-us==1.45.1
-prompt_toolkit==3.0.46
-proto-plus==1.23.0
-protobuf==4.25.3
-psutil==5.9.8
-py-cpuinfo==9.0.0
-pyasn1==0.6.0
-pyasn1_modules==0.4.0
-pydantic==2.7.3
-pydantic_core==2.18.4
-Pygments==2.18.0
-pyparsing==3.1.2
-pytest==8.2.2
-pytest-dependency==0.6.0
-python-dateutil==2.9.0.post0
-pytz==2024.1
-PyYAML==6.0.1
-requests==2.32.3
-rsa==4.9
-scikit-learn==1.5.0
-scipy==1.10.1
-seaborn==0.13.2
-six==1.16.0
-sortedcontainers==2.4.0
-statsmodels==0.14.2
-synthimpute==0.1
-tables==3.8.0
-tabulate==0.9.0
-tenacity==8.3.0
-threadpoolctl==3.5.0
-tqdm==4.66.4
-traitlets==5.14.3
-typing_extensions==4.12.1
-typish==1.9.3
-tzdata==2024.1
-uritemplate==4.1.1
-urllib3==2.2.1
-wcwidth==0.2.13
-yaml-changelog==0.3.0
-zope.interface==6.4.post2
+numpy==1.26.4
+pathlib==1.0.1

tests/test_e2e.py

@@ -11,11 +11,39 @@
 class E2ETest(unittest.TestCase):
 
     def setUp(self) -> None:
-        self.project_root = Path(__file__).parent.parent
-        self.taxsim_dir = self.project_root / "resources" / "taxsim35"
-        self.output_dir = self.project_root / "output"
+        import importlib.resources as pkg_resources
+        import policyengine_taxsim
+        from importlib.metadata import distribution
+
+        # Get the correct path to shared data
+        dist = distribution('policyengine-taxsim')
+        # Print for debugging
+        print(f"Distribution location: {dist.locate_file('share')}")
+
+        # Try different methods to locate the file
+        possible_paths = [
+            Path(dist.locate_file('share')) / 'policyengine_taxsim' / 'taxsim35',
+            Path(sys.prefix) / 'share' / 'policyengine_taxsim' / 'taxsim35',
+            Path(policyengine_taxsim.__file__).parent.parent / 'share' / 'policyengine_taxsim' / 'taxsim35'
+        ]
+
+        # Find the first path that exists and contains our files
+        for path in possible_paths:
+            if (path / 'taxsim_input.csv').exists():
+                self.taxsim_dir = path
+                break
+        else:
+            print("Searched in the following locations:")
+            for path in possible_paths:
+                print(f"  {path}")
+            raise FileNotFoundError("Could not find taxsim directory")
+
+        self.output_dir = Path.cwd() / "output"
         self.output_dir.mkdir(exist_ok=True)
 
+        # Get CLI path
+        self.cli_path = Path(policyengine_taxsim.__file__).parent / "cli.py"
+
         # Determine the correct TAXSIM executable based on the OS
         system = platform.system().lower()
         if system == "darwin":
@@ -29,12 +57,44 @@ def setUp(self) -> None:
 
         self.input_file = self.taxsim_dir / "taxsim_input.csv"
 
-    def test_generate_policyengine_taxsim(self):
+        # Verify and print paths for debugging
+        print(f"\nDebug Information:")
+        print(f"Taxsim Directory: {self.taxsim_dir}")
+        print(f"Input File Path: {self.input_file}")
+        print(f"Input File Exists: {self.input_file.exists()}")
+        if self.input_file.exists():
+            print(f"Input File is Readable: {os.access(self.input_file, os.R_OK)}")
+
+    def test_generate_policyengine_taxsim_output(self):
         output_file = self.output_dir / "policyengine_taxsim_output.csv"
 
-        cmd = f"{sys.executable} {self.project_root}/policyengine_taxsim/cli.py {self.input_file} -o {output_file}"
+        # Use list form and absolute paths
+        cmd = [
+            sys.executable,
+            str(self.cli_path.absolute()),
+            str(self.input_file.absolute()),
+            "-o",
+            str(output_file.absolute())
+        ]
+
+        # Print command for debugging
+        print(f"Running command: {' '.join(cmd)}")
+
+        creation_flags = 0
+        if platform.system().lower() == "windows":
+            if hasattr(subprocess, 'CREATE_NO_WINDOW'):
+                creation_flags = subprocess.CREATE_NO_WINDOW
+            else:
+                # For Python < 3.11 on Windows
+                # DETACHED_PROCESS = 0x00000008
+                creation_flags = 0x00000008
+
         process = subprocess.run(
-            cmd, shell=True, capture_output=True, text=True
+            cmd,
+            shell=False,
+            capture_output=True,
+            text=True,
+            creationflags=creation_flags
         )
 
         print(f"PolicyEngine TAXSIM CLI output:\n{process.stdout}")
@@ -51,29 +111,55 @@ def test_generate_policyengine_taxsim(self):
         with open(output_file, "r") as f:
             print(f.read())
 
-    def test_generate_taxsim_output(self):
-        output_file = self.output_dir / "taxsim35_output.csv"
+    def test_generate_taxsim35_output(self):
+        import tempfile
+        import shutil
 
+        output_file = self.output_dir / "taxsim35_output.csv"
         taxsim_path = self.taxsim_dir / self.taxsim_exe
 
-        if platform.system().lower() != "windows":
-            # Make the file executable on Unix-like systems
-            os.chmod(taxsim_path, 0o755)
-
-        cmd = f"{taxsim_path} < {self.input_file} > {output_file}"
-        process = subprocess.run(
-            cmd, shell=True, capture_output=True, text=True
-        )
+        # Create a temporary directory for execution
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Copy executable and input to temp directory
+            temp_exe = Path(temp_dir) / self.taxsim_exe
+            temp_input = Path(temp_dir) / "input.csv"
+
+            shutil.copy2(taxsim_path, temp_exe)
+            shutil.copy2(self.input_file, temp_input)
+
+            if platform.system().lower() != "windows":
+                os.chmod(temp_exe, 0o755)
+                cmd = f'cat "{str(temp_input)}" | "{str(temp_exe)}" > "{str(output_file)}"'
+            else:
+                # Windows specific handling
+                cmd = f'cmd.exe /c "type "{str(temp_input)}" | "{str(temp_exe)}" > "{str(output_file)}""'
+
+            creation_flags = 0
+            if platform.system().lower() == "windows":
+                if hasattr(subprocess, 'CREATE_NO_WINDOW'):
+                    creation_flags = subprocess.CREATE_NO_WINDOW
+                else:
+                    # For Python < 3.11 on Windows
+                    # DETACHED_PROCESS = 0x00000008
+                    creation_flags = 0x00000008
+
+            process = subprocess.run(
+                cmd,
+                shell=True,
+                capture_output=True,
+                text=True,
+                creationflags=creation_flags if platform.system().lower() == "windows" else 0
+            )
 
-        print(f"TAXSIM35 output:\n{process.stdout}")
-        if process.returncode != 0:
-            print(f"TAXSIM35 failed with error:\n{process.stderr}")
-            raise Exception(f"TAXSIM35 failed: {process.returncode}")
+            print(f"TAXSIM35 output:\n{process.stdout}")
+            if process.returncode != 0:
+                print(f"TAXSIM35 failed with error:\n{process.stderr}")
+                raise Exception(f"TAXSIM35 failed: {process.returncode}")
 
-        self.assertTrue(output_file.is_file())
-        print(f"Content of {output_file}:")
-        with open(output_file, "r") as f:
-            print(f.read())
+            self.assertTrue(output_file.is_file())
+            print(f"Content of {output_file}:")
+            with open(output_file, "r") as f:
+                print(f.read())
 
     def test_match_both_output(self):
         taxsim35_csv = pd.read_csv(self.output_dir / "taxsim35_output.csv")
@@ -154,4 +240,4 @@ def test_match_both_output(self):
         self.assertTrue(all_matched, f"Columns with missmatches: {[col for col, matched in comparison_results.items() if not matched]}")
 
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main()