merge cob lung cancer

Rafsan7238 · May 21, 2024 · 120f343 · 120f343
1 parent fd7a531
commit 120f343
Show file tree

Hide file tree

Showing 6 changed files with 144 additions and 34 deletions.
diff --git a/backend/backend.py b/backend/backend.py
@@ -156,6 +156,7 @@ def select_all_from_index():
     if index == AIR_QUALITY_HOURLY_AVG:
         return jsonify({'result': "Select all is not allowed for this index"}), 400
 
+
     query = f"""
         SELECT * FROM {index}
     """
@@ -189,6 +190,8 @@ def air_quality_endpoint():
             result = jsonify({'result': get_air_quality_hourly_for_statistical(es)}), 200
         elif resource == FOR_SPATIAL_ANALYSIS:
             result = jsonify({'result': get_air_quality_hourly_for_spatial(es)}), 200
+        elif resource == COB_CANCER_MERGE:
+            result = jsonify({'result': get_cob_merge_lung_cancer(es)}), 200
         else:
             result = jsonify({'Resource in headers is not valid': resource}), 400
     except Exception as e:

diff --git a/backend/fission_startup.sh b/backend/fission_startup.sh
@@ -20,14 +20,14 @@ fission timer create --name bom-harvester-repeater --function addobservations --
 ### MASTODON HARVESTER
 (   cd backend/harvesters/Mastodon/;   zip -r mharvester.zip .;   mv mharvester.zip ../; )
 fission package create --sourcearchive backend/harvesters/mharvester.zip  --env python  --name mharvester  --buildcmd './build.sh' --verbosity=0;
-fission fn create --name mharvester  --pkg mharvester  --env python  --entrypoint "mharvester.main" --fntimeout 3600 --verbosity=0;
+fission fn create --name mharvester  --pkg mharvester  --env python  --entrypoint "mharvester.main" --fntimeout 360 --verbosity=0;
 fission timer create --name mastodon-harvester-repeater --function mharvester --cron "@every 5m" --verbosity=0;
 
 ### Index Management
 fission fn create --name create-indexes --pkg backend --env python --entrypoint "backend.create_indexes_endpoint" --fntimeout 360 --verbosity=0;
 fission route create --method POST --url "/indexes/create/all" --function create-indexes --name create-indexes --createingress --verbosity=0;
 
-fission fn create --name insert-documents --pkg backend --env python --entrypoint "backend.insert_documents" --fntimeout 36000 --verbosity=0;
+fission fn create --name insert-documents --pkg backend --env python --entrypoint "backend.insert_documents" --fntimeout 360 --verbosity=0;
 (
   fission route create --name insert-documents --function insert-documents \
     --method POST \

diff --git a/backend/querying/air_quality_analysis_queries.py b/backend/querying/air_quality_analysis_queries.py
@@ -1,4 +1,26 @@
+from constants import *
 from querying.make_query import make_query
+import pandas as pd
+
+def get_cob_merge_lung_cancer(es):
+    query = f"""
+        SELECT * FROM {MORTALITY_PERSONS}
+    """
+    print('querying persons')
+    persons_json = make_query(es, query)
+    persons_df = pd.DataFrame(persons_json['rows'], columns=persons_json['columns'])
+
+    query = f"""
+        SELECT * FROM {ASTHMA_BY_REGION_INDEX_NAME}
+    """
+    print('querying asthma')
+
+    asthma_by_region = make_query(es, query)
+    asthma_by_region_df = pd.DataFrame(asthma_by_region['rows'], columns=asthma_by_region['columns'])
+    merged_data = persons_df.merge(asthma_by_region_df, how='inner', on=['gccsa_code', 'gccsa_name'])
+    merged_data_json = merged_data.to_dict(orient='split')
+    return {'columns': list(merged_data_json['columns']), 'rows': list(merged_data_json['data'])}
+
 
 def get_air_quality_hourly_for_spatial(es):
     query = """
@@ -73,4 +95,3 @@ def get_air_quality_data_dist(es):
         GROUP BY location_name
     """
     return make_query(es, query)
-    return result
diff --git a/backend/querying/query_path_constants.py b/backend/querying/query_path_constants.py
@@ -5,4 +5,5 @@
 FOR_SPATIAL_ANALYSIS = 'spatial_analysis'
 AVG_MONTHLY_ANALYSIS = 'avg_monthly_analysis'
 UPDATING_ANALYSIS = 'updateing_analysis'
-MESSAGE_COUNTS_HOURLY = 'message_counts_hourly'
+MESSAGE_COUNTS_HOURLY = 'message_counts_hourly'
+COB_CANCER_MERGE = "persons_merged_with_COB"
diff --git a/backend/refresh_environments.sh b/backend/refresh_environments.sh
@@ -2,5 +2,5 @@
 fission env delete -f --name python --verbosity=0;
 fission env delete -f --name nodejs --verbosity=0;
 
-fission env create --name python --builder fission/python-builder-3.9 --image fission/python-env-3.9 --verbosity=0;
+fission env create --name python --builder fission/python-builder-3.9 --image lmorandini/python39x:1.0.0 --verbosity=0;
 fission env create --name nodejs --image fission/node-env --builder fission/node-builder --verbosity=0;
diff --git a/frontend/Air Quality vs Lung Disease Analysis.ipynb b/frontend/Air Quality vs Lung Disease Analysis.ipynb
@@ -55,7 +55,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
@@ -514,13 +514,6 @@
     "plt.show()"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
   {
    "cell_type": "markdown",
    "metadata": {
@@ -683,25 +676,6 @@
     "plt.show()\n"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 15,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "24\n"
-     ]
-    }
-   ],
-   "source": [
-    "aif_time = \"20240518090000\"\n",
-    "year = aif_time[2:4]\n",
-    "print(year)\n"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {
@@ -1579,7 +1553,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 4,
    "metadata": {
     "collapsed": false,
     "jupyter": {
@@ -1614,6 +1588,98 @@
     "census_by_inc_emp_data = dataframe_from_url_ext(f'/data/{CENSUS_G21B}/all')"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'index': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14],\n",
+       " 'columns': ['australian asthma',\n",
+       "  'australian copd',\n",
+       "  'employed asthma',\n",
+       "  'foreigner asthma',\n",
+       "  'foreigner copd',\n",
+       "  'gccsa_code',\n",
+       "  'gccsa_name',\n",
+       "  'total asthma',\n",
+       "  'total copd'],\n",
+       " 'data': [[263188,\n",
+       "   40132,\n",
+       "   176461,\n",
+       "   88368,\n",
+       "   21806,\n",
+       "   '1GSYD',\n",
+       "   'Greater Sydney',\n",
+       "   353656,\n",
+       "   63101],\n",
+       "  [246105, 60438, 122833, 21667, 9994, '1RNSW', 'Rest of NSW', 271403, 72543],\n",
+       "  [292521,\n",
+       "   37523,\n",
+       "   208830,\n",
+       "   91496,\n",
+       "   20287,\n",
+       "   '2GMEL',\n",
+       "   'Greater Melbourne',\n",
+       "   386347,\n",
+       "   58925],\n",
+       "  [140299, 31287, 73533, 13405, 5734, '2RVIC', 'Rest of Vic', 156014, 38195],\n",
+       "  [182163,\n",
+       "   35004,\n",
+       "   116657,\n",
+       "   40601,\n",
+       "   11007,\n",
+       "   '3GBRI',\n",
+       "   'Greater Brisbane',\n",
+       "   224233,\n",
+       "   46859],\n",
+       "  [181605, 46620, 102552, 29017, 11730, '3RQLD', 'Rest of Qld', 212875, 59756],\n",
+       "  [99461,\n",
+       "   17031,\n",
+       "   60136,\n",
+       "   20816,\n",
+       "   7994,\n",
+       "   '4GADE',\n",
+       "   'Greater Adelaide',\n",
+       "   121331,\n",
+       "   25546],\n",
+       "  [33572, 8168, 16482, 3384, 1916, '4RSAU', 'Rest of SA', 37619, 10432],\n",
+       "  [113226,\n",
+       "   18949,\n",
+       "   82550,\n",
+       "   40854,\n",
+       "   12958,\n",
+       "   '5GPER',\n",
+       "   'Greater Perth',\n",
+       "   155026,\n",
+       "   32447],\n",
+       "  [33597, 6959, 20169, 5787, 2310, '5RWAU', 'Rest of WA', 39893, 9532],\n",
+       "  [19272, 4002, 10485, 2128, 669, '6GHOB', 'Greater Hobart', 21649, 4835],\n",
+       "  [27371, 7537, 12927, 2473, 1277, '6RTAS', 'Rest of Tas', 30398, 9157],\n",
+       "  [6878, 1276, 5102, 1513, 410, '7GDAR', 'Greater Darwin', 8465, 1719],\n",
+       "  [3906, 819, 2318, 492, 114, '7RNTE', 'Rest of NT', 4429, 942],\n",
+       "  [33743,\n",
+       "   4317,\n",
+       "   24215,\n",
+       "   6897,\n",
+       "   1583,\n",
+       "   '8ACTE',\n",
+       "   'Australian Capital Territory',\n",
+       "   40783,\n",
+       "   5990]]}"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "census_by_cob_data.to_dict(orient='split')"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {
@@ -1793,6 +1859,25 @@
     "merged_data_1.head()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Fetching from http://localhost:9090/data/air_quality/persons_merged_with_COB\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Merge mortality and census datasets based on location\n",
+    "merged_data_1 = dataframe_from_url_ext(f'/data/air_quality/{query_constants.COB_CANCER_MERGE}')\n",
+    "merged_data_1.head()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 8,
@@ -2948,7 +3033,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.6"
+   "version": "3.11.9"
   }
  },
  "nbformat": 4,