Streamline API call flow (#142)

.ci/py36.yml

@@ -5,11 +5,9 @@ dependencies:
   - python=3.6
   - geopandas
   - fuzzywuzzy
-  - libpysal
   - numpy
   - pandas
   - pytest
   - requests
-  - rtree
-  - scipy
-  - six
+  - pip:
+    - git+https://github.com/ronnie-llamado/cenpy.git@development

.ci/py37.yml

@@ -5,11 +5,9 @@ dependencies:
   - python=3.7
   - geopandas
   - fuzzywuzzy
-  - libpysal
   - numpy
   - pandas
   - pytest
   - requests
-  - rtree
-  - scipy
-  - six
+  - pip:
+    - git+https://github.com/ronnie-llamado/cenpy.git@development

.ci/py38.yml

@@ -5,11 +5,9 @@ dependencies:
   - python=3.8
   - geopandas
   - fuzzywuzzy
-  - libpysal
   - numpy
   - pandas
   - pytest
   - requests
-  - rtree
-  - scipy
-  - six
+  - pip:
+    - git+https://github.com/ronnie-llamado/cenpy.git@development

.ci/py39.yml

@@ -5,11 +5,9 @@ dependencies:
   - python=3.9
   - geopandas
   - fuzzywuzzy
-  - libpysal
   - numpy
   - pandas
   - pytest
   - requests
-  - rtree
-  - scipy
-  - six
+  - pip:
+    - git+https://github.com/ronnie-llamado/cenpy.git@development

.github/workflows/build.yml

@@ -2,11 +2,13 @@ name: build
 
 on:
   push:
-    branches: 
+    branches:
       - master
+      - development
   pull_request:
-    branches: 
+    branches:
       - master
+      - development
   schedule:
     - cron: '0 0 * * 0'
 
@@ -19,23 +21,23 @@ jobs:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
         environment-file: [.ci/py36.yml, .ci/py37.yml, .ci/py38.yml, .ci/py39.yml]
-    
+
     steps:
       - name: Checkout repo
         uses: actions/checkout@v2
-      
+
       - name: Setup micromamba
         uses: mamba-org/provision-with-micromamba@main
         with:
           environment-file: ${{ matrix.environment-file }}
           micromamba-version: 'latest'
-      
+
       - name: Test with pytest - bash
         shell: bash -l {0}
-        run: pytest -v cenpy
+        run: pytest -v
         if: matrix.os != 'windows-latest'
-      
+
       - name: Test with pytest - powershell
         shell: powershell
-        run: pytest -v cenpy
+        run: pytest -v
         if: matrix.os == 'windows-latest'

.gitignore

@@ -10,3 +10,4 @@ docsrc/generated
 docsrc/_build
 cenpy-feedstock
 build/
+.venv

cenpy/__init__.py

@@ -1,11 +1,4 @@
-__version__ = "1.0.0post3"
+__version__ = "1.0.post3"
 __author__ = "Levi John Wolf [email protected]"
 
-from . import explorer
-from .remote import APIConnection as _APIConnection
-from .tools import _load_sitekey, set_sitekey
-from .products import *
-
-SITEKEY = _load_sitekey()
-
-# __all__ = ['explorer', 'base']
+from .product import ACS, Decennial

cenpy/census.py

@@ -0,0 +1,101 @@
+import numpy as np
+import pandas as pd
+
+from .utils import RestApiBase, lazy_property, chunks
+
+
+class CensusDataset(RestApiBase):
+    def __init__(self, url, key=None, session=None):
+        self.url = url
+        self.key = key
+        super(CensusDataset, self).__init__(session=session)
+
+    @lazy_property
+    def variables(self):
+        response = self._get(f"{self.url}/variables.json")
+        response.raise_for_status()
+        df = pd.DataFrame.from_dict(response.json()["variables"]).T
+        df.predicateType = df.predicateType.replace(["string", np.nan], "str")
+        return df
+
+    @lazy_property
+    def geographies(self):
+        response = self._get(f"{self.url}/geography.json")
+        return pd.DataFrame(response.json()["fips"])
+
+    def query(
+        self,
+        get: list,
+        for_dict: dict,
+        in_dict: dict = {},
+        key: str = "",
+    ):
+
+        result = None
+
+        # census api caps each query to 50 get variables
+        for c in chunks(sorted(get), 50):
+
+            params = {
+                "get": ",".join(c),
+                "for": "+".join([f"{k}:{v}" for k, v in for_dict.items()]),
+            }
+
+            if in_dict:
+                # convert lists to csv
+                params_in_dict = {}
+                params_in_dict.update(in_dict)
+                params_in_dict.update(
+                    {k: ",".join(v) for k, v in in_dict.items() if isinstance(v, list)}
+                )
+
+                params["in"] = "+".join([f"{k}:{v}" for k, v in params_in_dict.items()])
+
+            if key or self.key:
+                params["key"] = key if key else self.key
+
+            response = self._get(self.url, params=params)
+            response.raise_for_status()
+
+            # convert to DataFrame
+            chunk_result = pd.DataFrame.from_records(
+                response.json()[1:],
+                columns=response.json()[0],
+            )
+
+            # replace placeholder values
+            chunk_result.replace(
+                to_replace=r"\-\d{9}",
+                value=np.nan,
+                regex=True,
+                inplace=True,
+            )
+
+            # convert each variable to type provided in /variables.json
+            type_dict = {
+                k: eval(self.variables.loc[k, "predicateType"])
+                for k in chunk_result.columns
+                if k not in list(self.geographies.name)
+            }
+            chunk_result = chunk_result.astype(type_dict, errors="ignore")
+
+            # pull out geographies in columns - for merge and reorganize
+            geographies = [
+                i for i in chunk_result.columns if i in list(self.geographies.name)
+            ]
+
+            # combine results if multiple queries were necessary
+            if isinstance(result, pd.DataFrame):
+
+                # merge on geographies
+                result = pd.merge(result, chunk_result, on=geographies)
+
+            else:
+                result = chunk_result
+
+            # (optional) reorganize columns - move geographies first
+            result = result[
+                geographies + [i for i in result.columns if i not in geographies]
+            ]
+
+        return result

cenpy/conf/geographies.ini

@@ -0,0 +1,82 @@
+[variables]
+# https://tigerweb.geo.census.gov/tigerwebmain/TIGERweb_attribute_glossary.html
+# CENSUS_GEOGRAPHY: TIGER_VARIABLE
+alaska native regional corporation: ANRC
+american indian area (off-reservation trust land only)/hawaiian home land: AIANNHCOMP
+american indian area/alaska native area (reservation or statistical entity only): AIANNHCOMP
+american indian area/alaska native area/hawaiian home land: AIANNH
+american indian tribal subdivision: AITSCE
+block: BLOCK
+block group: BLKGRP
+combined new england city and town area: CNECTA
+combined statistical area: CSA
+congressional district: BASENAME
+consolidated city: CONCITY
+county: COUNTY
+county subdivision: COUSUB
+division: DIVISION
+metropolitan division: METDIV
+metropolitan statistical area/micropolitan statistical area: CBSA
+necta division: NECTADIV
+new england city and town area: NECTA
+place: PLACE
+place/remainder: PLACE
+principal city: PCI
+public use microdata area: PUMA
+region: REGION
+school district (elementary): SDELM
+school district (secondary): SDSEC
+school district (unified): SDUNI
+state: STATE
+state legislative district (lower chamber): SLDL
+state legislative district (upper chamber): SLDU
+subminor civil division: SUBMCD
+tract: TRACT
+tribal block group: TBLKGRP
+tribal census tract: TTRACT
+tribal subdivision/remainder: AITSCE
+urban area: UA
+urban rural: UR
+zip code tabulation area: ZCTA5
+
+[layers]
+alaska native regional corporation: Alaska Native Regional Corporations
+american indian area (off-reservation trust land only)/hawaiian home land: Hawaiian Home Lands,Off-Reservation Trust Lands
+american indian area/alaska native area (reservation or statistical entity only):Federal American Indian Reservations,American Indian Joint-Use Areas,Oklahoma Tribal Statistical Areas,Alaska Native Village Statistical Areas,Tribal Designated Statistical Areas,State Designated Tribal Statistical Areas,State American Indian Reservations
+american indian area/alaska native area/hawaiian home land: Federal American Indian Reservations,American Indian Joint-Use Areas,Hawaiian Home Lands,Oklahoma Tribal Statistical Areas,Alaska Native Village Statistical Areas,Tribal Designated Statistical Areas,State Designated Tribal Statistical Areas,State American Indian Reservations,Off-Reservation Trust Lands
+american indian tribal subdivision: Tribal Subdivisions
+block: Census Blocks
+block group: Census Block Groups
+combined new england city and town area: Combined New England City and Town Areas
+combined statistical area: Combined Statistical Areas
+congressional district: {congressional_district} Congressional Districts
+consolidated city: Consolidated Cities
+county: Counties
+county subdivision: County Subdivisions
+division: Census Divisions
+metropolitan division: Metropolitan Divisions
+metropolitan statistical area/micropolitan statistical area: Metropolitan Statistical Areas,Micropolitan Statistical Areas
+necta division: New England City and Town Area Divisions
+new england city and town area: Metropolitan New England City and Town Areas,Micropolitan New England City and Town Areas
+place: Census Designated Places,Incorporated Places
+place/remainder: Census Designated Places,Incorporated Places
+principal city: Census Designated Places,Incorporated Places
+public use microdata area: {census_year} Public Use Microdata Areas
+region: Census Regions
+school district (elementary): Elementary School Districts
+school district (secondary): Secondary School Districts
+school district (unified): Unified School Districts
+state: States
+state legislative district (lower chamber): {legislative_year} State Legislative Districts - Lower
+state legislative district (upper chamber): {legislative_year} State Legislative Districts - Upper
+subminor civil division: Subbarrios
+tract: Census Tracts
+tribal block group: Tribal Block Groups
+tribal census tract: Tribal Census Tracts
+tribal subdivision/remainder: Tribal Subdivisions
+urban area: {census_year} Urbanized Areas,{census_year} Urban Clusters
+zip code tabulation area: {census_year} ZIP Code Tabulation Areas
+
+### remaining layers ###
+#{census_year} Blocks
+#Estates