Merge pull request Revenue-Academy#12 from jdebacker/demog_patch

Merging

.github/workflows/build_and_test.yml

@@ -1,4 +1,4 @@
-name: Build and test [Python 3.7, 3.8, 3.9]
+name: Build and test [Python 3.8, 3.9, 3.10]
 
 on: [push, pull_request]
 
@@ -7,17 +7,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.8, 3.9, "3.10"]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           persist-credentials: false
 
       - name: Setup Miniconda using Python ${{ matrix.python-version }}
         uses: conda-incubator/setup-miniconda@v2
         with:
+          miniforge-variant: Mambaforge
+          auto-update-conda: true
           activate-environment: ogmys-dev
           environment-file: environment.yml
           python-version: ${{ matrix.python-version }}
@@ -35,7 +37,7 @@ jobs:
         run: |
           pytest -m 'not local and not regression' --cov=./ --cov-report=xml
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v3
         with:
           files: ./coverage.xml
           flags: unittests

environment.yml

@@ -2,12 +2,11 @@ name: ogmys-dev
 channels:
 - conda-forge
 dependencies:
-- python>=3.7.7, <3.11  # This restriction can be removed as soon as these packages support Python 3.11
+- python>=3.7.7
 - ipython
 - setuptools
 - psutil
-- openssl=1.1.1  # This is a requirement of the UN data portal which we are using in demographics.py
-- numpy<=1.21.2  # This restriction can be removed as soon as Numba supports NumPy 1.22
+- numpy
 - scipy>=1.7.1
 - pandas>=1.2.5
 - matplotlib

examples/run_og_mys.py

@@ -56,7 +56,6 @@ def main():
         "omega": d["omega"],
         "g_n_ss": d["g_n_ss"],
         "omega_SS": d["omega_SS"],
-        "surv_rate": d["surv_rate"],
         "rho": d["rho"],
         "g_n": d["g_n"],
         "imm_rates": d["imm_rates"],

ogmys/calibrate.py

@@ -22,7 +22,6 @@ def __init__(
         client=None,
         num_workers=1,
     ):
-
         self.estimate_tax_functions = estimate_tax_functions
         self.estimate_beta = estimate_beta
         self.estimate_chi_n = estimate_chi_n
@@ -62,17 +61,17 @@ def __init__(
             p.start_year,
             GraphDiag=True,
         )
-        # demographics for 80 period lives (needed for getting e below)
-        demog80 = demographics.get_pop_objs(
-            20, 80, p.T, 1, 100, p.start_year - 1, p.start_year
-        )
 
         # earnings profiles
+        # Find the e matrix for MYS by changing the earnings profiles for
+        # the USA to match the gini in MYS
+        # (=41.1 in WB data: https://data.worldbank.org/indicator/SI.POV.GINI))
         self.e = income.get_e_interp(
+            p.E,
             p.S,
-            self.demographic_params["omega_SS"],
-            demog80["omega_SS"],
+            p.J,
             p.lambdas,
+            self.demographic_params["omega_SS"],
             plot=False,
         )
 

ogmys/demographics.py

@@ -12,6 +12,9 @@
 import pandas as pd
 import matplotlib.pyplot as plt
 from ogcore import parameter_plots as pp
+from ogmys.utils import get_legacy_session
+from io import StringIO
+
 
 UN_COUNTRY_CODE = "458"  # UN code for MYS
 # create output director for figures
@@ -57,9 +60,15 @@ def get_un_data(
     )
 
     # get data from url
-    response = requests.get(target)
-    # Converts call into JSON
-    j = response.json()
+    response = get_legacy_session().get(target)
+    # Check if the request was successful before processing
+    if response.status_code == 200:
+        # Converts call into JSON
+        j = response.json()
+    else:
+        print(
+            f"Failed to retrieve population data. HTTP status code: {response.status_code}"
+        )
     # Convert JSON into a pandas DataFrame.
     # pd.json_normalize flattens the JSON to accommodate nested lists
     # within the JSON structure
@@ -69,9 +78,15 @@ def get_un_data(
         # Reset the target to the next page
         target = j["nextPage"]
         # call the API for the next page
-        response = requests.get(target)
+        response = get_legacy_session().get(target)
         # Convert response to JSON format
-        j = response.json()
+        if response.status_code == 200:
+            # Converts call into JSON
+            j = response.json()
+        else:
+            print(
+                f"Failed to retrieve population data. HTTP status code: {response.status_code}"
+            )
         # Store the next page in a data frame
         df_temp = pd.json_normalize(j["data"])
         # Append next page to the data frame
@@ -415,8 +430,6 @@ def get_pop_objs(
             g_n_SS (scalar): steady-state population growth rate
             omega_SS (Numpy array): normalized steady-state population
                 distribution, length S
-            surv_rates (Numpy array): survival rates that correspond to
-                each model period of life, length S
             mort_rates (Numpy array): mortality rates that correspond to
                 each model period of life, length S
             g_n_path (Numpy array): population growth rates over the time
@@ -675,14 +688,13 @@ def get_pop_objs(
             output_dir=OUTPUT_DIR,
         )
 
-    # return omega_path_S, g_n_SS, omega_SSfx, survival rates,
+    # return omega_path_S, g_n_SS, omega_SSfx,
     # mort_rates_S, and g_n_path
     pop_dict = {
         "omega": omega_path_S.T,
         "g_n_ss": g_n_SS,
         "omega_SS": omega_SSfx[-S:] / omega_SSfx[-S:].sum(),
-        "surv_rate": 1 - mort_rates_S,
-        "rho": mort_rates_S,
+        "rho": [mort_rates_S],
         "g_n": g_n_path,
         "imm_rates": imm_rates_mat.T,
         "omega_S_preTP": omega_S_preTP,