Convert notebooks to scripts

notebooks/1_prep_synthpop.ipynb

@@ -76,34 +76,13 @@
     "spc_people_hh = (\n",
     "    Builder(path, region, backend=\"pandas\", input_type=\"parquet\")\n",
     "    .add_households()\n",
-    "    .unnest([\"health\", \"employment\", \"details\"])\n",
-    "    # remove nssec column\n",
+    "    .unnest([\"health\", \"employment\", \"details\", \"demographics\"], rsuffix=\"_household\")\n",
     "    .build()\n",
     ")\n",
     "\n",
     "spc_people_hh.head(5)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# we need to unnest the demographic data. If we do this above\n",
-    "# we get an error as there will be two \"nssec8\" columns.\n",
-    "\n",
-    "# Unnest the JSON column\n",
-    "demographics = pd.json_normalize(spc_people_hh['demographics'])\n",
-    "\n",
-    "# Remove the columns we don't want\n",
-    "spc_people_hh = spc_people_hh.drop(['demographics', 'nssec8'], axis = 1)\n",
-    "# Add the unnested demographics column\n",
-    "spc_people_hh = pd.concat([spc_people_hh, demographics], axis = 1)\n",
-    "\n",
-    "spc_people_hh.head()"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -194,6 +173,13 @@
     "# save the output\n",
     "spc_people_tu.write_parquet('../data/external/spc_output/' + region + '_people_tu.parquet')"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -212,7 +198,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "version": "3.11.8"
   }
  },
  "nbformat": 4,