Merge pull request #23 from Urban-Analytics-Technology-Platform/21-co…

…nvert-to-scripts

Convert notebooks to scripts

.github/workflows/ci.yml

@@ -29,7 +29,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.10, 3.12]  # test oldest and latest supported versions
+        python-version: ['3.10', '3.11', '3.12']  # test oldest and latest supported versions
         runs-on: [ubuntu-latest]  # can be extended to other OSes, e.g. [ubuntu-latest, macos-latest]
 
     steps:

notebooks/1_prep_synthpop.ipynb

@@ -20,9 +20,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "#import json\n",
-    "import pandas as pd\n",
-    "\n",
     "#https://github.com/alan-turing-institute/uatk-spc/blob/55-output-formats-python/python/examples/spc_builder_example.ipynb\n",
     "from uatk_spc.builder import Builder"
    ]
@@ -76,34 +73,13 @@
     "spc_people_hh = (\n",
     "    Builder(path, region, backend=\"pandas\", input_type=\"parquet\")\n",
     "    .add_households()\n",
-    "    .unnest([\"health\", \"employment\", \"details\"])\n",
-    "    # remove nssec column\n",
+    "    .unnest([\"health\", \"employment\", \"details\", \"demographics\"], rsuffix=\"_household\")\n",
     "    .build()\n",
     ")\n",
     "\n",
     "spc_people_hh.head(5)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# we need to unnest the demographic data. If we do this above\n",
-    "# we get an error as there will be two \"nssec8\" columns.\n",
-    "\n",
-    "# Unnest the JSON column\n",
-    "demographics = pd.json_normalize(spc_people_hh['demographics'])\n",
-    "\n",
-    "# Remove the columns we don't want\n",
-    "spc_people_hh = spc_people_hh.drop(['demographics', 'nssec8'], axis = 1)\n",
-    "# Add the unnested demographics column\n",
-    "spc_people_hh = pd.concat([spc_people_hh, demographics], axis = 1)\n",
-    "\n",
-    "spc_people_hh.head()"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -194,6 +170,13 @@
     "# save the output\n",
     "spc_people_tu.write_parquet('../data/external/spc_output/' + region + '_people_tu.parquet')"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -212,7 +195,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "version": "3.11.8"
   }
  },
  "nbformat": 4,