diff --git a/backend/backend.py b/backend/backend.py index 0fe505c..82f52a8 100644 --- a/backend/backend.py +++ b/backend/backend.py @@ -156,6 +156,7 @@ def select_all_from_index(): if index == AIR_QUALITY_HOURLY_AVG: return jsonify({'result': "Select all is not allowed for this index"}), 400 + query = f""" SELECT * FROM {index} """ @@ -189,6 +190,8 @@ def air_quality_endpoint(): result = jsonify({'result': get_air_quality_hourly_for_statistical(es)}), 200 elif resource == FOR_SPATIAL_ANALYSIS: result = jsonify({'result': get_air_quality_hourly_for_spatial(es)}), 200 + elif resource == COB_CANCER_MERGE: + result = jsonify({'result': get_cob_merge_lung_cancer(es)}), 200 else: result = jsonify({'Resource in headers is not valid': resource}), 400 except Exception as e: diff --git a/backend/fission_startup.sh b/backend/fission_startup.sh index 768e83c..53a8c8f 100755 --- a/backend/fission_startup.sh +++ b/backend/fission_startup.sh @@ -20,14 +20,14 @@ fission timer create --name bom-harvester-repeater --function addobservations -- ### MASTODON HARVESTER ( cd backend/harvesters/Mastodon/; zip -r mharvester.zip .; mv mharvester.zip ../; ) fission package create --sourcearchive backend/harvesters/mharvester.zip --env python --name mharvester --buildcmd './build.sh' --verbosity=0; -fission fn create --name mharvester --pkg mharvester --env python --entrypoint "mharvester.main" --fntimeout 3600 --verbosity=0; +fission fn create --name mharvester --pkg mharvester --env python --entrypoint "mharvester.main" --fntimeout 360 --verbosity=0; fission timer create --name mastodon-harvester-repeater --function mharvester --cron "@every 5m" --verbosity=0; ### Index Management fission fn create --name create-indexes --pkg backend --env python --entrypoint "backend.create_indexes_endpoint" --fntimeout 360 --verbosity=0; fission route create --method POST --url "/indexes/create/all" --function create-indexes --name create-indexes --createingress --verbosity=0; -fission fn create --name insert-documents --pkg backend --env python --entrypoint "backend.insert_documents" --fntimeout 36000 --verbosity=0; +fission fn create --name insert-documents --pkg backend --env python --entrypoint "backend.insert_documents" --fntimeout 360 --verbosity=0; ( fission route create --name insert-documents --function insert-documents \ --method POST \ diff --git a/backend/querying/air_quality_analysis_queries.py b/backend/querying/air_quality_analysis_queries.py index f399d83..9c60dbd 100644 --- a/backend/querying/air_quality_analysis_queries.py +++ b/backend/querying/air_quality_analysis_queries.py @@ -1,4 +1,26 @@ +from constants import * from querying.make_query import make_query +import pandas as pd + +def get_cob_merge_lung_cancer(es): + query = f""" + SELECT * FROM {MORTALITY_PERSONS} + """ + print('querying persons') + persons_json = make_query(es, query) + persons_df = pd.DataFrame(persons_json['rows'], columns=persons_json['columns']) + + query = f""" + SELECT * FROM {ASTHMA_BY_REGION_INDEX_NAME} + """ + print('querying asthma') + + asthma_by_region = make_query(es, query) + asthma_by_region_df = pd.DataFrame(asthma_by_region['rows'], columns=asthma_by_region['columns']) + merged_data = persons_df.merge(asthma_by_region_df, how='inner', on=['gccsa_code', 'gccsa_name']) + merged_data_json = merged_data.to_dict(orient='split') + return {'columns': list(merged_data_json['columns']), 'rows': list(merged_data_json['data'])} + def get_air_quality_hourly_for_spatial(es): query = """ @@ -73,4 +95,3 @@ def get_air_quality_data_dist(es): GROUP BY location_name """ return make_query(es, query) - return result diff --git a/backend/querying/query_path_constants.py b/backend/querying/query_path_constants.py index aaab5b2..97565da 100644 --- a/backend/querying/query_path_constants.py +++ b/backend/querying/query_path_constants.py @@ -5,4 +5,5 @@ FOR_SPATIAL_ANALYSIS = 'spatial_analysis' AVG_MONTHLY_ANALYSIS = 'avg_monthly_analysis' UPDATING_ANALYSIS = 'updateing_analysis' -MESSAGE_COUNTS_HOURLY = 'message_counts_hourly' \ No newline at end of file +MESSAGE_COUNTS_HOURLY = 'message_counts_hourly' +COB_CANCER_MERGE = "persons_merged_with_COB" \ No newline at end of file diff --git a/backend/refresh_environments.sh b/backend/refresh_environments.sh index 304a6e0..5279c43 100644 --- a/backend/refresh_environments.sh +++ b/backend/refresh_environments.sh @@ -2,5 +2,5 @@ fission env delete -f --name python --verbosity=0; fission env delete -f --name nodejs --verbosity=0; -fission env create --name python --builder fission/python-builder-3.9 --image fission/python-env-3.9 --verbosity=0; +fission env create --name python --builder fission/python-builder-3.9 --image lmorandini/python39x:1.0.0 --verbosity=0; fission env create --name nodejs --image fission/node-env --builder fission/node-builder --verbosity=0; \ No newline at end of file diff --git a/frontend/Air Quality vs Lung Disease Analysis.ipynb b/frontend/Air Quality vs Lung Disease Analysis.ipynb index fa3003a..f96a406 100644 --- a/frontend/Air Quality vs Lung Disease Analysis.ipynb +++ b/frontend/Air Quality vs Lung Disease Analysis.ipynb @@ -55,7 +55,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": { "pycharm": { "name": "#%%\n" @@ -514,13 +514,6 @@ "plt.show()" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "markdown", "metadata": { @@ -683,25 +676,6 @@ "plt.show()\n" ] }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "24\n" - ] - } - ], - "source": [ - "aif_time = \"20240518090000\"\n", - "year = aif_time[2:4]\n", - "print(year)\n" - ] - }, { "cell_type": "markdown", "metadata": { @@ -1579,7 +1553,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 4, "metadata": { "collapsed": false, "jupyter": { @@ -1614,6 +1588,98 @@ "census_by_inc_emp_data = dataframe_from_url_ext(f'/data/{CENSUS_G21B}/all')" ] }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'index': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14],\n", + " 'columns': ['australian asthma',\n", + " 'australian copd',\n", + " 'employed asthma',\n", + " 'foreigner asthma',\n", + " 'foreigner copd',\n", + " 'gccsa_code',\n", + " 'gccsa_name',\n", + " 'total asthma',\n", + " 'total copd'],\n", + " 'data': [[263188,\n", + " 40132,\n", + " 176461,\n", + " 88368,\n", + " 21806,\n", + " '1GSYD',\n", + " 'Greater Sydney',\n", + " 353656,\n", + " 63101],\n", + " [246105, 60438, 122833, 21667, 9994, '1RNSW', 'Rest of NSW', 271403, 72543],\n", + " [292521,\n", + " 37523,\n", + " 208830,\n", + " 91496,\n", + " 20287,\n", + " '2GMEL',\n", + " 'Greater Melbourne',\n", + " 386347,\n", + " 58925],\n", + " [140299, 31287, 73533, 13405, 5734, '2RVIC', 'Rest of Vic', 156014, 38195],\n", + " [182163,\n", + " 35004,\n", + " 116657,\n", + " 40601,\n", + " 11007,\n", + " '3GBRI',\n", + " 'Greater Brisbane',\n", + " 224233,\n", + " 46859],\n", + " [181605, 46620, 102552, 29017, 11730, '3RQLD', 'Rest of Qld', 212875, 59756],\n", + " [99461,\n", + " 17031,\n", + " 60136,\n", + " 20816,\n", + " 7994,\n", + " '4GADE',\n", + " 'Greater Adelaide',\n", + " 121331,\n", + " 25546],\n", + " [33572, 8168, 16482, 3384, 1916, '4RSAU', 'Rest of SA', 37619, 10432],\n", + " [113226,\n", + " 18949,\n", + " 82550,\n", + " 40854,\n", + " 12958,\n", + " '5GPER',\n", + " 'Greater Perth',\n", + " 155026,\n", + " 32447],\n", + " [33597, 6959, 20169, 5787, 2310, '5RWAU', 'Rest of WA', 39893, 9532],\n", + " [19272, 4002, 10485, 2128, 669, '6GHOB', 'Greater Hobart', 21649, 4835],\n", + " [27371, 7537, 12927, 2473, 1277, '6RTAS', 'Rest of Tas', 30398, 9157],\n", + " [6878, 1276, 5102, 1513, 410, '7GDAR', 'Greater Darwin', 8465, 1719],\n", + " [3906, 819, 2318, 492, 114, '7RNTE', 'Rest of NT', 4429, 942],\n", + " [33743,\n", + " 4317,\n", + " 24215,\n", + " 6897,\n", + " 1583,\n", + " '8ACTE',\n", + " 'Australian Capital Territory',\n", + " 40783,\n", + " 5990]]}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "census_by_cob_data.to_dict(orient='split')" + ] + }, { "cell_type": "markdown", "metadata": { @@ -1793,6 +1859,25 @@ "merged_data_1.head()" ] }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fetching from http://localhost:9090/data/air_quality/persons_merged_with_COB\n" + ] + } + ], + "source": [ + "# Merge mortality and census datasets based on location\n", + "merged_data_1 = dataframe_from_url_ext(f'/data/air_quality/{query_constants.COB_CANCER_MERGE}')\n", + "merged_data_1.head()" + ] + }, { "cell_type": "code", "execution_count": 8, @@ -2948,7 +3033,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.6" + "version": "3.11.9" } }, "nbformat": 4,