diff --git a/SDG_15_4_2_Sub_A_Default_values _SEPAL.ipynb b/SDG_15_4_2_Sub_A_Default_values _SEPAL.ipynb new file mode 100644 index 0000000..8e79479 --- /dev/null +++ b/SDG_15_4_2_Sub_A_Default_values _SEPAL.ipynb @@ -0,0 +1,759 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "source": [ + "# **SDG 15.4.2 Sub-indicator A: Calculate Global Default Values**\n", + "\n", + "* This script allows batch processing for this indicator for all countries.\n", + "\n", + "* Output is a combined excel file on your Google Drive.\n", + "\n", + "* Runs on the cloud using [Google Colab](https://research.google.com/colaboratory/faq.html)\n", + "\n", + "* Requires: [Google Earth Engine](https://earthengine.google.com/) (GEE) account and project and access to Google Drive\n" + ], + "metadata": { + "id": "82FWR5yMh0HV" + } + }, + { + "cell_type": "markdown", + "source": [ + "### 1) Install required packages" + ], + "metadata": { + "id": "YdoCOI_1yWY0" + } + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "Kg3K1EPJu_f5", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 208 + }, + "outputId": "460f7ba2-8668-4c4d-824b-6d5068884129" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.1/6.1 MB\u001b[0m \u001b[31m34.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.7/2.7 MB\u001b[0m \u001b[31m54.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m38.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hipyvuetify has been installed.\n", + "ee is already installed.\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m235.5/235.5 kB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hunidecode has been installed.\n", + "google-api-python-client has been installed.\n", + "google-auth-httplib2 has been installed.\n", + "google-auth-oauthlib has been installed.\n", + "geemap is already installed.\n" + ] + } + ], + "source": [ + "# to automatically reload modules.\n", + "%load_ext autoreload\n", + "\n", + "# Set to reload all modules before executing code.\n", + "%autoreload 2\n", + "\n", + "# Function to install a package if it's not already installed\n", + "def install_if_not_exists(package_name):\n", + " try:\n", + " __import__(package_name)\n", + " print(f\"{package_name} is already installed.\")\n", + " except ImportError:\n", + " !pip install -q {package_name}\n", + " print(f\"{package_name} has been installed.\")\n", + "\n", + "# List of packages to install if not already installed\n", + "packages_to_install = ['ipyvuetify','ee', 'unidecode', 'google-api-python-client',\n", + " 'google-auth-httplib2', 'google-auth-oauthlib','geemap'] #admin_boundaries\n", + "\n", + "# Install necessary packages\n", + "for package in packages_to_install:\n", + " install_if_not_exists(package)" + ] + }, + { + "cell_type": "markdown", + "source": [ + "### 2) Access GitHub repository\n", + "Clones repository for SDG 15.4.2 into colab.\n", + "Provides functions and lookup tables etc." + ], + "metadata": { + "id": "qGfLFLEkwS1n" + } + }, + { + "cell_type": "code", + "source": [ + "# Change the current working directory to \"/content\" for cloning the repo into.\n", + "%cd \"/content\"\n", + "\n", + "# Clone the GitHub repository \"sepalz_mgci\" into the current directory.\n", + "# NB 'fatal' error on reruns are typically just saying it already exists\n", + "!git clone https://github.com/sepal-contrib/sepal_mgci\n", + "\n", + "# Change working directory to the cloned sepal_mgci github repository\n", + "%cd \"/content/sepal_mgci\"" + ], + "metadata": { + "id": "TNohZrOTvNqT", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 173 + }, + "outputId": "1ad0a56a-2b6e-4966-ab95-a55a1a612aae" + }, + "execution_count": 2, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "\n", + " \n", + " " + ] + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "/content\n", + "Cloning into 'sepal_mgci'...\n", + "remote: Enumerating objects: 3071, done.\u001b[K\n", + "remote: Counting objects: 100% (1009/1009), done.\u001b[K\n", + "remote: Compressing objects: 100% (426/426), done.\u001b[K\n", + "remote: Total 3071 (delta 663), reused 807 (delta 582), pack-reused 2062\u001b[K\n", + "Receiving objects: 100% (3071/3071), 5.02 MiB | 17.37 MiB/s, done.\n", + "Resolving deltas: 100% (1939/1939), done.\n", + "/content/sepal_mgci\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "### 3) Setup Google Earth Engine\n", + "Launches access request pop up window" + ], + "metadata": { + "id": "Lbg9P7frZRy1" + } + }, + { + "cell_type": "code", + "source": [ + "# Google Earth Engine project\n", + "gee_project_name = \"ee-andyarnellgee\" # \"insert cloud project here\" # a registered cloud project (if unsure of name see pic here: https://developers.google.com/earth-engine/cloud/assets)\n", + "\n", + "import ee # google earth engine\n", + "\n", + "ee.Authenticate()\n", + "\n", + "ee.Initialize(project=gee_project_name) # NB gee project name is defined in parameters section" + ], + "metadata": { + "id": "SLLcPONnZRy2" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### 4) Setup Google Drive\n", + "Launches access request pop up window" + ], + "metadata": { + "id": "cxXsgaXEZRy3" + } + }, + { + "cell_type": "code", + "source": [ + "# for accessing google drive\n", + "from google.colab import auth, drive\n", + "from googleapiclient.discovery import build\n", + "\n", + "drive.mount('/content/drive')" + ], + "metadata": { + "id": "e2NxIhcgZRy3" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### 5) Import remaining packages\n", + "- imports required packages and functions listed in 'colab_imports.py' script (found here: sepal_mgci/component/scripts)\n", + "\n" + ], + "metadata": { + "id": "MsQu9VfqZRy3" + } + }, + { + "cell_type": "code", + "source": [ + "from component.scripts.colab_imports import *" + ], + "metadata": { + "id": "NQ9flKZGZRy4" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### 6) Set parameters\n" + ], + "metadata": { + "id": "Bi0XpRRDo6V1" + } + }, + { + "cell_type": "markdown", + "source": [ + "Input parameters" + ], + "metadata": { + "id": "5aIr_OxkG-Ky" + } + }, + { + "cell_type": "code", + "source": [ + "# Google Earth Engine project\n", + "gee_project_name = \"ee-andyarnellgee\" # \"insert cloud project here\" # a registered cloud project (if unsure of name see pic here: https://developers.google.com/earth-engine/cloud/assets)\n", + "\n", + "\n", + "# Admin boundaries asset\n", + "admin_asset_id = \"FAO/GAUL/2015/level0\" # administrative units feature collection\n", + "\n", + "admin_asset_property_name = \"ADM0_NAME\" # property/column name for selecting admin boundaries (e.g. ISO3 code or country name)\n", + "\n", + "\n", + "# Land cover assets\n", + "\n", + "# For Sub-indicator A (sub_a), we need to set the following structure.\n", + "a_years = {\n", + " 1: {\"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2000\", \"year\": 2000}, # baseline\n", + " 2: {\"year\": 2003, \"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2003\"}, # subsequent reporting years...\n", + " 3: {\"year\": 2007, \"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2007\"},\n", + " 4: {\"year\": 2010, \"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2010\"},\n", + "}\n", + "\n" + ], + "metadata": { + "id": "t9C1qT5bGoqt" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "Output parameters\n", + "\n", + "---\n", + "\n" + ], + "metadata": { + "id": "naz3Qe4JHFER" + } + }, + { + "cell_type": "code", + "source": [ + "final_report_folder = \"sdg_15_4_2_A_combined_report\" # folder name in Google Drive for final output (if doesnt exist creates one)\n", + "\n", + "final_report_name = \"sdg_15_4_2_A_default_global.xlsx\" # file name for final excel output\n", + "\n", + "# export GEE tasks or not\n", + "export = False # default: True. Set to False if debugging or limiting accidental re-exporting of tasks\n", + "\n", + "# prints more messages\n", + "debug = False # default: False. Set to True if debugging code" + ], + "metadata": { + "id": "mSvTM29TnndR" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "Temporary output parameters\n" + ], + "metadata": { + "id": "iV8-wbddOJuT" + } + }, + { + "cell_type": "code", + "source": [ + "stats_csv_folder = \"sdg_15_4_2_A_csvs\" # for storing stats tables exported from GEE for each admin boundary/AOI\n", + "\n", + "excel_reports_folder = \"sdg_15_4_2_A_reports\" # for storing formatted excel tables for each admin boundary/AOI\n", + "\n", + "drive_home =\"/content/drive/MyDrive/\" # Google Drive location. Don't change unless you know this is incorrect\n", + "\n", + "error_log_file_path = drive_home + excel_reports_folder + \"/\"+\"1_error_log\" +\".csv\" # for storing errors\n" + ], + "metadata": { + "id": "TCBNkALuOL1N" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "c_8UzlUnu_f6" + }, + "source": [ + "### 7) Setup inputs for processing" + ] + }, + { + "cell_type": "markdown", + "source": [ + "Create list of boundaries to process" + ], + "metadata": { + "id": "qi5zNGTWN3df" + } + }, + { + "cell_type": "code", + "source": [ + "# admin boundary feature collection\n", + "admin_boundaries = ee.FeatureCollection(admin_asset_id)\n", + "\n", + "# list to process\n", + "list_of_countries = admin_boundaries.aggregate_array(admin_asset_property_name).getInfo()\n", + "\n", + "print (\"Length of admin boundaries to process\", len(list_of_countries))\n", + "\n", + "list_of_countries = list(set(list_of_countries)) # remove dupicates\n", + "\n", + "print (\"Length of distinct admin boundaries to process\", (len(set(list_of_countries))))\n" + ], + "metadata": { + "id": "BiuBEJwPue2v" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "Read the default land cover remapping table and convert it to a dictionary" + ], + "metadata": { + "id": "2G1q9TSiUsc1" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PRSEqq5bu_f7" + }, + "outputs": [], + "source": [ + "default_map_matrix = map_matrix_to_dict(LC_MAP_MATRIX)" + ] + }, + { + "cell_type": "markdown", + "source": [ + "Select years of land cover to process" + ], + "metadata": { + "id": "4-Ut_-S35Yg8" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PwqJFWR4u_f7", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 17 + }, + "outputId": "09d3ab98-6b20-4212-b5ed-f3386c3a32bf" + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "\n", + " \n", + " " + ] + }, + "metadata": {} + } + ], + "source": [ + "# extracts the years from the a_years dictionary (as defined in parameters)\n", + "single_years = [y[\"year\"] for y in a_years.values()]" + ] + }, + { + "cell_type": "markdown", + "source": [ + "### 8) Calculate area statistics by country\n", + "* Runs for each country and each mountain biobelt\n", + "* Gets area of land cover reclassified into the 10 SEAM classes\n", + "* Repeat for each year specified\n" + ], + "metadata": { + "id": "iNxHtR984cNk" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "VftKLuY4u_f7" + }, + "outputs": [], + "source": [ + "# you can monitor your GEE tasks here : https://code.earthengine.google.com/tasks\n", + "create_folder_if_not_exists(stats_csv_folder) # to store outputs in google drive\n", + "\n", + "counter=0 # starting place of counter used to keep track of number of tasks that are being run\n", + "\n", + "for aoi_name in list_of_countries:\n", + "\n", + " aoi = admin_boundaries.filter(ee.Filter.eq(admin_asset_property_name,aoi_name))#.first()\n", + "\n", + " # gets areas of landcover in each mountain belt in each country\n", + " # uses reduce_regions function imported from the cloned sepal_mgci git hub repository (see Imports section)\n", + " # pixels counted at native resolution (scale) of input land cover (or DEM if RSA implementation)\n", + " process = ee.FeatureCollection([\n", + " ee.Feature(\n", + " None,\n", + " reduce_regions(\n", + " aoi,\n", + " remap_matrix=default_map_matrix,\n", + " rsa=False,\n", + " # dem=param.DEM_DEFAULT,\n", + " dem=DEM_DEFAULT, #default digital elevation model (DEM). Relevant for the real surface area (RSA) implementation.\n", + " lc_years= year,\n", + " transition_matrix=False\n", + " )\n", + " ).set(\"process_id\", year[0][\"year\"])\n", + " for year in get_a_years(a_years) # creates GEE images and runs stats on each. Images to run are in the 'a_years\" dictionary (above)\n", + " ])\n", + "\n", + " #make name acceptable for running tasks (i.e., removes special characters)\n", + " task_name = str(sanitize_description(unidecode(aoi_name)))\n", + "\n", + "\n", + " task = ee.batch.Export.table.toDrive(\n", + " **{ #asterisks unpack dictionary into keyword arguments format\n", + " \"collection\": process,\n", + " \"description\": task_name,\n", + " \"fileFormat\": \"CSV\",\n", + " \"folder\":stats_csv_folder,\n", + " \"selectors\": [\n", + " \"process_id\",\n", + " \"sub_a\",\n", + " ],\n", + " }\n", + " )\n", + "\n", + " counter+=1\n", + "\n", + " print (f\"\\r process {counter}/{len(list_of_countries)} {aoi_name} \", end=\"\") #print in place (remove \\r and end=\"\" for verbose version)\n", + "\n", + " if export:\n", + " task.start()\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vM0SIvJtu_f8" + }, + "source": [ + "### 9) Read and translate results into report tables" + ] + }, + { + "cell_type": "markdown", + "source": [ + "#####NOTE: you will need to wait until results files for each country have been created in your google drive (from previous step).\n", + "- see here to monitor the tasks https://code.earthengine.google.com/tasks\n", + "- once tasks are complete, you can run the cell below\n", + "\n", + "This cell formats individual excel reports for each country.\n", + "See Error_log.csv for missing files/errors" + ], + "metadata": { + "id": "3x7jwkZJWwE-" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qkpDfHqQu_f9" + }, + "outputs": [], + "source": [ + "# Initialize the counter\n", + "counter = 0\n", + "\n", + "# to store outputs in google drive\n", + "create_folder_if_not_exists(excel_reports_folder)\n", + "\n", + "# Loop over each AOI name in the list of countries\n", + "for aoi_name in list_of_countries:\n", + " counter += 1\n", + "\n", + " # Clean the AOI name\n", + " aoi_name_clean = str(sanitize_description(unidecode(aoi_name)))\n", + "\n", + " # Construct the file path for the stats CSV file\n", + " stats_csv_file = aoi_name_clean + \".csv\"\n", + " stats_csv_file_path = os.path.join(drive_home, stats_csv_folder, stats_csv_file)\n", + "\n", + " message = f\"Process {counter}, {stats_csv_file}\"\n", + "\n", + " try:\n", + " # Read the results from the CSV file and parse it to a dictionary\n", + " dict_results = read_from_csv(stats_csv_file_path)\n", + "\n", + " details = {\n", + " \"geo_area_name\": aoi_name,\n", + " \"ref_area\": \" \",\n", + " \"source_detail\": \" \",\n", + " }\n", + "\n", + " # Generate reports for the sub_a and mtn indicators\n", + " sub_a_reports = [sub_a.get_reports(parse_result(dict_results[year][\"sub_a\"], single=True), year, **details) for year in single_years]\n", + " mtn_reports = [mntn.get_report(parse_result(dict_results[year][\"sub_a\"], single=True), year, **details) for year in single_years]\n", + "\n", + " # Concatenate the mtn reports\n", + " mtn_reports_df = pd.concat(mtn_reports)\n", + "\n", + " # Concatenate the sub a reports\n", + " er_mtn_grnvi_df = pd.concat([report[0] for report in sub_a_reports])\n", + " er_mtn_grncov_df = pd.concat([report[1] for report in sub_a_reports])\n", + "\n", + " # Define the output report file path\n", + " report_file_path = os.path.join(drive_home, excel_reports_folder, aoi_name_clean + \".xlsx\")\n", + "\n", + " # Create the Excel file with the reports\n", + " with pd.ExcelWriter(report_file_path) as writer:\n", + " mtn_reports_df.to_excel(writer, sheet_name=\"Table1_ER_MTN_TOTL\", index=False)\n", + " er_mtn_grncov_df.to_excel(writer, sheet_name=\"Table2_ER_MTN_GRNCOV\", index=False)\n", + " er_mtn_grnvi_df.to_excel(writer, sheet_name=\"Table3_ER_MTN_GRNCVI\", index=False)\n", + "\n", + " # Adjust column widths and alignment for each sheet\n", + " for sheetname in writer.sheets:\n", + " worksheet = writer.sheets[sheetname]\n", + " for col in worksheet.columns:\n", + " max_length = max(len(str(cell.value)) for cell in col)\n", + " column = col[0]\n", + " adjusted_width = max(max_length, len(str(column.value))) + 4\n", + " worksheet.column_dimensions[get_column_letter(column.column)].width = adjusted_width\n", + "\n", + " # Align \"obs_value\" column to the right\n", + " if \"OBS\" in column.value:\n", + " for cell in col:\n", + " cell.alignment = Alignment(horizontal=\"right\")\n", + "\n", + " except Exception as e:\n", + " # If an error occurs, catch the exception and handle it\n", + " message = f\"process {counter}, {stats_csv_file}, Error: {e}\"\n", + "\n", + " # Get the current time\n", + " current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')\n", + "\n", + " # Write the error message and file name to the error log file\n", + " error_info = pd.DataFrame([[stats_csv_file, str(e), current_time]], columns=['File Name', 'Error Message', 'Time'])\n", + "\n", + " mode = 'w' if not os.path.exists(error_log_file_path) else 'a'\n", + " header = False if os.path.exists(error_log_file_path) else True\n", + "\n", + " # Append or write to the error log file\n", + " error_info.to_csv(error_log_file_path, mode=mode, header=header, index=False)\n", + "\n", + " print(message)\n" + ] + }, + { + "cell_type": "markdown", + "source": [ + "### 10) Combine excel report files into one" + ], + "metadata": { + "id": "qTqI3Ag08k5b" + } + }, + { + "cell_type": "markdown", + "source": [ + "Make a list of files to combine" + ], + "metadata": { + "id": "sVpLtG4Z7Jbe" + } + }, + { + "cell_type": "code", + "source": [ + "# Directory path where Excel reports are stored\n", + "directory_path = os.path.join(drive_home, excel_reports_folder)\n", + "\n", + "# List files in the directory with '.xlsx' extension\n", + "files = [file for file in os.listdir(directory_path) if file.endswith('.xlsx')]\n", + "\n", + "# Create a list of full file paths\n", + "full_file_paths = [os.path.join(directory_path, file) for file in files]\n", + "\n", + "# Print the number of Excel files found in the folder\n", + "print(f\"Number of Excel files in folder: {len(full_file_paths)}\")\n", + "\n", + "# folder to store outputs in google drive\n", + "create_folder_if_not_exists(final_report_folder)\n", + "\n", + "# File path for the combined final report\n", + "reports_combined_file_path = os.path.join(drive_home, final_report_folder, final_report_name)\n" + ], + "metadata": { + "id": "sTyhwne7rr9D" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "##### Run function to combine into a single report" + ], + "metadata": { + "id": "C77flAQu7xWX" + } + }, + { + "cell_type": "code", + "source": [ + "append_excel_files(file_paths=full_file_paths,num_sheets=3,output_file_path=reports_combined_file_path)\n", + "\n", + "print (f\"\\n Complete! Output file for SDG 15.4.2 Sub-indicator A here: {reports_combined_file_path}\")" + ], + "metadata": { + "id": "FkS1ErA9AYj6" + }, + "execution_count": null, + "outputs": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "(test) test-sepal_mgci", + "language": "python", + "name": "test-sepal_mgci" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + }, + "colab": { + "provenance": [], + "include_colab_link": true + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/SDG_15_4_2_Sub_A_Default_values_sepal.ipynb b/SDG_15_4_2_Sub_A_Default_values_sepal.ipynb new file mode 100644 index 0000000..e1f1591 --- /dev/null +++ b/SDG_15_4_2_Sub_A_Default_values_sepal.ipynb @@ -0,0 +1,750 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "view-in-github" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "82FWR5yMh0HV" + }, + "source": [ + "# **SDG 15.4.2 Sub-indicator A: Calculate Global Default Values**\n", + "\n", + "* This script allows batch processing for this indicator for all countries.\n", + "\n", + "* Output is a combined excel file on your Google Drive.\n", + "\n", + "* Runs on the cloud using [Google Colab](https://research.google.com/colaboratory/faq.html)\n", + "\n", + "* Requires: [Google Earth Engine](https://earthengine.google.com/) (GEE) account and project and access to Google Drive\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import ee\n", + "ee.Initialize()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Create a base directory\n", + "\n", + "base_dir = Path(\"content/sepal_mgci\")\n", + "base_dir.mkdir(parents=True, exist_ok=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MsQu9VfqZRy3" + }, + "source": [ + "### 5) Import remaining packages\n", + "- imports required packages and functions listed in 'colab_imports.py' script (found here: sepal_mgci/component/scripts)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "NQ9flKZGZRy4" + }, + "outputs": [ + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": "/*******************************************************************************\n * remove any links from fontawesome 5 created by jupyter in favor of\n * fontawesome 6. to be removed when Jupyter updates it\n */\n\nfunction remove_fa5() {\n let links = document.querySelectorAll(\n \"link[href^='https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@^5']\"\n );\n\n links.forEach((link) => link.remove());\n}\n\nif (document.readyState != \"loading\") remove_fa5();\nelse document.addEventListener(\"DOMContentLoaded\", remove_fa5);\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import os\n", + "\n", + "from datetime import datetime # for time stamping error log\n", + "import pandas as pd # pandas library for tabular data manipulation\n", + "from unidecode import (\n", + " unidecode,\n", + ") # converting symbols in country names to ascii compliant (required for naming GEE tasks)\n", + "\n", + "# formatting excel report file\n", + "from openpyxl.utils import get_column_letter\n", + "from openpyxl.styles import Alignment\n", + "\n", + "from pathlib import Path\n", + "\n", + "# # # # Import scripts and modules from cloned GitHub repository (i.e., functions for indicator calculation and formatting)\n", + "from component.scripts.gee import (\n", + " reduce_regions,\n", + ") # for running summary statistics in GEE\n", + "\n", + "from component.scripts.scripts import (\n", + " get_a_years,\n", + " map_matrix_to_dict,\n", + " parse_result,\n", + " read_from_csv,\n", + " map_matrix_to_dict,\n", + ") # parameter prep and reformatting\n", + "from component.scripts import (\n", + " sub_a,\n", + " mountain_area as mntn,\n", + ") ###TO DO: ADD DESCRIPTIONS\n", + "\n", + "from component.scripts.colab_combining_files import (\n", + " sanitize_description,\n", + " append_excel_files,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Bi0XpRRDo6V1" + }, + "source": [ + "### 6) Set parameters\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5aIr_OxkG-Ky" + }, + "source": [ + "Input parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "DEM_DEFAULT = \"CGIAR/SRTM90_V4\"\n", + "\n", + "# Define the translation matrix between ESA and MGCI LC classes\n", + "\n", + "LC_MAP_MATRIX = Path(\"content/corine_lc_map_matrix2.csv\")\n", + "TRANSITION_MATRIX_FILE = Path(\"component/parameter/transition_matrix.csv\")\n", + "\n", + "# Check they both exist\n", + "assert LC_MAP_MATRIX.exists()\n", + "assert TRANSITION_MATRIX_FILE.exists()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "t9C1qT5bGoqt" + }, + "outputs": [], + "source": [ + "# Admin boundaries asset\n", + "admin_asset_id = \"FAO/GAUL/2015/level0\" # administrative units feature collection\n", + "\n", + "admin_asset_property_name = \"ADM0_NAME\" # property/column name for selecting admin boundaries (e.g. ISO3 code or country name)\n", + "\n", + "\n", + "# Land cover assets\n", + "\n", + "# For Sub-indicator A (sub_a), we need to set the following structure.\n", + "a_years = {\n", + " 1: {\"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2000\", \"year\": 2000}, # baseline\n", + " 2: {\"year\": 2003, \"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2003\"}, # subsequent reporting years...\n", + " 3: {\"year\": 2007, \"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2007\"},\n", + " 4: {\"year\": 2010, \"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2010\"},\n", + "}\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "naz3Qe4JHFER" + }, + "source": [ + "Output parameters\n", + "\n", + "---\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mSvTM29TnndR" + }, + "outputs": [], + "source": [ + "final_report_folder = \"sdg_15_4_2_A_combined_report\" # folder name in Google Drive for final output (if doesnt exist creates one)\n", + "\n", + "final_report_name = \"sdg_15_4_2_A_default_global.xlsx\" # file name for final excel output\n", + "\n", + "# export GEE tasks or not\n", + "export = False # default: True. Set to False if debugging or limiting accidental re-exporting of tasks\n", + "\n", + "# prints more messages\n", + "debug = False # default: False. Set to True if debugging code" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iV8-wbddOJuT" + }, + "source": [ + "Temporary output parameters\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "TCBNkALuOL1N" + }, + "outputs": [], + "source": [ + "stats_csv_folder = \"sdg_15_4_2_A_csvs\" # for storing stats tables exported from GEE for each admin boundary/AOI\n", + "\n", + "excel_reports_folder = \"sdg_15_4_2_A_reports\" # for storing formatted excel tables for each admin boundary/AOI\n", + "\n", + "drive_home =\"/content/drive/MyDrive/\" # Google Drive location. Don't change unless you know this is incorrect\n", + "\n", + "error_log_file_path = drive_home + excel_reports_folder + \"/\"+\"1_error_log\" +\".csv\" # for storing errors\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "c_8UzlUnu_f6" + }, + "source": [ + "### 7) Setup inputs for processing" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qi5zNGTWN3df" + }, + "source": [ + "Create list of boundaries to process" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BiuBEJwPue2v" + }, + "outputs": [], + "source": [ + "# admin boundary feature collection\n", + "admin_boundaries = ee.FeatureCollection(admin_asset_id)\n", + "\n", + "# list to process\n", + "list_of_countries = admin_boundaries.aggregate_array(admin_asset_property_name).getInfo()\n", + "\n", + "print (\"Length of admin boundaries to process\", len(list_of_countries))\n", + "\n", + "list_of_countries = list(set(list_of_countries)) # remove dupicates\n", + "\n", + "print (\"Length of distinct admin boundaries to process\", (len(set(list_of_countries))))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2G1q9TSiUsc1" + }, + "source": [ + "Read the default land cover remapping table and convert it to a dictionary" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PRSEqq5bu_f7" + }, + "outputs": [], + "source": [ + "default_map_matrix = map_matrix_to_dict(LC_MAP_MATRIX)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4-Ut_-S35Yg8" + }, + "source": [ + "Select years of land cover to process" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 17 + }, + "id": "PwqJFWR4u_f7", + "outputId": "09d3ab98-6b20-4212-b5ed-f3386c3a32bf" + }, + "outputs": [], + "source": [ + "# extracts the years from the a_years dictionary (as defined in parameters)\n", + "single_years = [y[\"year\"] for y in a_years.values()]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iNxHtR984cNk" + }, + "source": [ + "### 8) Calculate area statistics by country\n", + "* Runs for each country and each mountain biobelt\n", + "* Gets area of land cover reclassified into the 10 SEAM classes\n", + "* Repeat for each year specified\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "VftKLuY4u_f7" + }, + "outputs": [], + "source": [ + "# you can monitor your GEE tasks here : https://code.earthengine.google.com/tasks\n", + "create_folder_if_not_exists(stats_csv_folder) # to store outputs in google drive\n", + "\n", + "counter=0 # starting place of counter used to keep track of number of tasks that are being run\n", + "\n", + "for aoi_name in list_of_countries:\n", + "\n", + " aoi = admin_boundaries.filter(ee.Filter.eq(admin_asset_property_name,aoi_name))#.first()\n", + "\n", + " # gets areas of landcover in each mountain belt in each country\n", + " # uses reduce_regions function imported from the cloned sepal_mgci git hub repository (see Imports section)\n", + " # pixels counted at native resolution (scale) of input land cover (or DEM if RSA implementation)\n", + " process = ee.FeatureCollection([\n", + " ee.Feature(\n", + " None,\n", + " reduce_regions(\n", + " aoi,\n", + " remap_matrix=default_map_matrix,14 test sepal_mgci /home/dguerrero/module-venv/sepal_mgci\n", + " dem=DEM_DEFAULT, #default digital elevation model (DEM). Relevant for the real surface area (RSA) implementation.\n", + " lc_years= year,\n", + " transition_matrix=False\n", + " )\n", + " ).set(\"process_id\", year[0][\"year\"])\n", + " for year in get_a_years(a_years) # creates GEE images and runs stats on each. Images to run are in the 'a_years\" dictionary (above)\n", + " ])\n", + "\n", + " #make name acceptable for running tasks (i.e., removes special characters)\n", + " task_name = str(sanitize_description(unidecode(aoi_name)))\n", + "\n", + "\n", + " task = ee.batch.Export.table.toDrive(\n", + " **{ #asterisks unpack dictionary into keyword arguments format\n", + " \"collection\": process,\n", + " \"description\": task_name,\n", + " \"fileFormat\": \"CSV\",\n", + " \"folder\":stats_csv_folder,\n", + " \"selectors\": [\n", + " \"process_id\",\n", + " \"sub_a\",\n", + " ],\n", + " }\n", + " )\n", + "\n", + " counter+=1\n", + "\n", + " print (f\"\\r process {counter}/{len(list_of_countries)} {aoi_name} \", end=\"\") #print in place (remove \\r and end=\"\" for verbose version)\n", + "\n", + " if export:\n", + " task.start()\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vM0SIvJtu_f8" + }, + "source": [ + "### 9) Read and translate results into report tables" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3x7jwkZJWwE-" + }, + "source": [ + "#####NOTE: you will need to wait until results files for each country have been created in your google drive (from previous step).\n", + "- see here to monitor the tasks https://code.earthengine.google.com/tasks\n", + "- once tasks are complete, you can run the cell below\n", + "\n", + "This cell formats individual excel reports for each country.\n", + "See Error_log.csv for missing files/errors" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qkpDfHqQu_f9" + }, + "outputs": [], + "source": [ + "# Initialize the counter\n", + "counter = 0\n", + "\n", + "# to store outputs in google drive\n", + "create_folder_if_not_exists(excel_reports_folder)\n", + "\n", + "# Loop over each AOI name in the list of countries\n", + "for aoi_name in list_of_countries:\n", + " counter += 1\n", + "\n", + " # Clean the AOI name\n", + " aoi_name_clean = str(sanitize_description(unidecode(aoi_name)))\n", + "\n", + " # Construct the file path for the stats CSV file\n", + " stats_csv_file = aoi_name_clean + \".csv\"\n", + " stats_csv_file_path = os.path.join(drive_home, stats_csv_folder, stats_csv_file)\n", + "\n", + " message = f\"Process {counter}, {stats_csv_file}\"\n", + "\n", + " try:\n", + " # Read the results from the CSV file and parse it to a dictionary\n", + " dict_results = read_from_csv(stats_csv_file_path)\n", + "\n", + " details = {\n", + " \"geo_area_name\": aoi_name,\n", + " \"ref_area\": \" \",\n", + " \"source_detail\": \" \",\n", + " }\n", + "\n", + " # Generate reports for the sub_a and mtn indicators\n", + " sub_a_reports = [sub_a.get_reports(parse_result(dict_results[year][\"sub_a\"], single=True), year, **details) for year in single_years]\n", + " mtn_reports = [mntn.get_report(parse_result(dict_results[year][\"sub_a\"], single=True), year, **details) for year in single_years]\n", + "\n", + " # Concatenate the mtn reports\n", + " mtn_reports_df = pd.concat(mtn_reports)\n", + "\n", + " # Concatenate the sub a reports\n", + " er_mtn_grnvi_df = pd.concat([report[0] for report in sub_a_reports])\n", + " er_mtn_grncov_df = pd.concat([report[1] for report in sub_a_reports])\n", + "\n", + " # Define the output report file path\n", + " report_file_path = os.path.join(drive_home, excel_reports_folder, aoi_name_clean + \".xlsx\")\n", + "\n", + " # Create the Excel file with the reports\n", + " with pd.ExcelWriter(report_file_path) as writer:\n", + " mtn_reports_df.to_excel(writer, sheet_name=\"Table1_ER_MTN_TOTL\", index=False)\n", + " er_mtn_grncov_df.to_excel(writer, sheet_name=\"Table2_ER_MTN_GRNCOV\", index=False)\n", + " er_mtn_grnvi_df.to_excel(writer, sheet_name=\"Table3_ER_MTN_GRNCVI\", index=False)\n", + "\n", + " # Adjust column widths and alignment for each sheet\n", + " for sheetname in writer.sheets:\n", + " worksheet = writer.sheets[sheetname]\n", + " for col in worksheet.columns:\n", + " max_length = max(len(str(cell.value)) for cell in col)\n", + " column = col[0]\n", + " adjusted_width = max(max_length, len(str(column.value))) + 4\n", + " worksheet.column_dimensions[get_column_letter(column.column)].width = adjusted_width\n", + "\n", + " # Align \"obs_value\" column to the right\n", + " if \"OBS\" in column.value:\n", + " for cell in col:\n", + " cell.alignment = Alignment(horizontal=\"right\")\n", + "\n", + " except Exception as e:\n", + " # If an error occurs, catch the exception and handle it\n", + " message = f\"process {counter}, {stats_csv_file}, Error: {e}\"\n", + "\n", + " # Get the current time\n", + " current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')\n", + "\n", + " # Write the error message and file name to the error log file\n", + " error_info = pd.DataFrame([[stats_csv_file, str(e), current_time]], columns=['File Name', 'Error Message', 'Time'])\n", + "\n", + " mode = 'w' if not os.path.exists(error_log_file_path) else 'a'\n", + " header = False if os.path.exists(error_log_file_path) else True\n", + "\n", + " # Append or write to the error log file\n", + " error_info.to_csv(error_log_file_path, mode=mode, header=header, index=False)\n", + "\n", + " print(message)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qTqI3Ag08k5b" + }, + "source": [ + "### 10) Combine excel report files into one" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sVpLtG4Z7Jbe" + }, + "source": [ + "Make a list of files to combine" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sTyhwne7rr9D" + }, + "outputs": [], + "source": [ + "# Directory path where Excel reports are stored\n", + "directory_path = os.path.join(drive_home, excel_reports_folder)\n", + "\n", + "# List files in the directory with '.xlsx' extension\n", + "files = [file for file in os.listdir(directory_path) if file.endswith('.xlsx')]\n", + "\n", + "# Create a list of full file paths\n", + "full_file_paths = [os.path.join(directory_path, file) for file in files]\n", + "\n", + "# Print the number of Excel files found in the folder\n", + "print(f\"Number of Excel files in folder: {len(full_file_paths)}\")\n", + "\n", + "# folder to store outputs in google drive\n", + "create_folder_if_not_exists(final_report_folder)\n", + "\n", + "# File path for the combined final report\n", + "reports_combined_file_path = os.path.join(drive_home, final_report_folder, final_report_name)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "C77flAQu7xWX" + }, + "source": [ + "##### Run function to combine into a single report" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FkS1ErA9AYj6" + }, + "outputs": [], + "source": [ + "append_excel_files(file_paths=full_file_paths,num_sheets=3,output_file_path=reports_combined_file_path)\n", + "\n", + "print (f\"\\n Complete! Output file for SDG 15.4.2 Sub-indicator A here: {reports_combined_file_path}\")" + ] + } + ], + "metadata": { + "colab": { + "include_colab_link": true, + "provenance": [] + }, + "kernelspec": { + "display_name": "(test) test-sepal_mgci", + "language": "python", + "name": "test-sepal_mgci" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/SDG_15_4_2_Sub_B_Default_values.ipynb b/SDG_15_4_2_Sub_B_Default_values.ipynb new file mode 100644 index 0000000..5e834e8 --- /dev/null +++ b/SDG_15_4_2_Sub_B_Default_values.ipynb @@ -0,0 +1,682 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "view-in-github" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "82FWR5yMh0HV" + }, + "source": [ + "# **SDG 15.4.2 Sub-indicator B: Calculate Global Default Values**\n", + "\n", + "* This script allows batch processing for this indicator for all countries.\n", + "\n", + "* Output is a combined excel file on your Google Drive.\n", + "\n", + "* Runs on the cloud using [Google Colab](https://research.google.com/colaboratory/faq.html)\n", + "\n", + "* Requires: [Google Earth Engine](https://earthengine.google.com/) (GEE) account and project and access to Google Drive\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YdoCOI_1yWY0" + }, + "source": [ + "### 1) Install required packages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Kg3K1EPJu_f5" + }, + "outputs": [], + "source": [ + "# to automatically reload modules.\n", + "%load_ext autoreload\n", + "\n", + "# Set to reload all modules before executing code.\n", + "%autoreload 2\n", + "\n", + "# Function to install a package if it's not already installed\n", + "def install_if_not_exists(package_name):\n", + " try:\n", + " __import__(package_name)\n", + " print(f\"{package_name} is already installed.\")\n", + " except ImportError:\n", + " !pip install -q {package_name}\n", + " print(f\"{package_name} has been installed.\")\n", + "\n", + "# List of packages to install if not already installed\n", + "packages_to_install = ['ipyvuetify','ee', 'unidecode', 'google-api-python-client',\n", + " 'google-auth-httplib2', 'google-auth-oauthlib']\n", + "\n", + "# Install necessary packages\n", + "for package in packages_to_install:\n", + " install_if_not_exists(package)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qGfLFLEkwS1n" + }, + "source": [ + "### 2) Access GitHub repository\n", + "Clones repository for SDG 15.4.2 into colab.\n", + "Provides functions and lookup tables etc." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "TNohZrOTvNqT" + }, + "outputs": [], + "source": [ + "# Change the current working directory to \"/content\" for cloning the repo into.\n", + "%cd \"/content\"\n", + "\n", + "# Clone the GitHub repository \"sepal_mgci\" into the current directory.\n", + "# NB 'fatal' error on reruns are typically just saying it already exists\n", + "!git clone https://github.com/sepal-contrib/sepal_mgci\n", + "!cd sepal_mgci && git pull \n", + "\n", + "# Change working directory to the cloned sepal_mgci github repository\n", + "%cd \"/content/sepal_mgci\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FAzzmzMHzFmS" + }, + "source": [ + "### 3) Setup Google Earth Engine\n", + "Launches access request pop up window" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gHI-M8y_qwaP" + }, + "outputs": [], + "source": [ + "# Google Earth Engine project\n", + "gee_project_name = \"ee-andyarnellgee\" # \"insert cloud project here\" # a registered cloud project (if unsure of name see pic here: https://developers.google.com/earth-engine/cloud/assets)\n", + "\n", + "import ee # google earth engine\n", + "\n", + "ee.Authenticate()\n", + "\n", + "ee.Initialize(project=gee_project_name) # NB gee project name is defined in parameters section" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "aUVGL6zwLpX5" + }, + "source": [ + "### 4) Setup Google Drive\n", + "Launches access request pop up window" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NorUzAJ8Kj0Z" + }, + "outputs": [], + "source": [ + "# for accessing google drive\n", + "from google.colab import auth, drive\n", + "from googleapiclient.discovery import build\n", + "\n", + "drive.mount('/content/drive')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "L5jsWgqWyiCP" + }, + "source": [ + "### 5) Import remaining packages\n", + "- imports required packages and functions listed here: sepal_mgci/component/scripts/colab_imports.py\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2Wquh4ZQbgfZ" + }, + "outputs": [], + "source": [ + "from component.scripts.colab_imports import" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Bi0XpRRDo6V1" + }, + "source": [ + "### 6) Set parameters\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5aIr_OxkG-Ky" + }, + "source": [ + "Input parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "t9C1qT5bGoqt" + }, + "outputs": [], + "source": [ + "\n", + "# Admin boundaries asset\n", + "admin_asset_id = \"FAO/GAUL/2015/level0\" # administrative units feature collection\n", + "\n", + "admin_asset_property_name = \"ADM0_NAME\" # property/column name for selecting admin boundaries (e.g. ISO3 code or country name)\n", + "\n", + "\n", + "# Land cover assets\n", + "\n", + "# For Sub-indicator B (sub_b), we need to set the following structure\n", + "sub_b_year = {\n", + " \"baseline\": {\n", + " \"base\": {\n", + " \"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2000\",\n", + " \"year\": 2000,\n", + " },\n", + " \"report\": {\n", + " \"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2015\",\n", + " \"year\": 2015,\n", + " },\n", + " },\n", + " # And the reporting year\n", + " 2: {\"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2018\", \"year\": 2018},\n", + "}\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "naz3Qe4JHFER" + }, + "source": [ + "Output parameters\n", + "- NB if no CSVs are being created, check that 'export = True' below.\n", + "\n", + "---\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mSvTM29TnndR" + }, + "outputs": [], + "source": [ + "final_report_folder = \"sdg_15_4_2_B_combined_report\" # folder name in Google Drive for final output (if doesnt exist creates one)\n", + "\n", + "final_report_name = \"sdg_15_4_2_B_default_global.xlsx\" # file name for final excel output\n", + "\n", + "# export GEE tasks or not\n", + "export = True # default: True. Set to False if debugging or limiting accidental re-exporting of tasks\n", + "\n", + "# prints more messages\n", + "debug = False # default: False. Set to True if debugging code" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iV8-wbddOJuT" + }, + "source": [ + "Temporary output parameters\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "TCBNkALuOL1N" + }, + "outputs": [], + "source": [ + "stats_csv_folder = \"sdg_15_4_2_B_csvs\" # for storing stats tables exported from GEE for each admin boundary/AOI\n", + "\n", + "excel_reports_folder = \"sdg_15_4_2_B_reports\" # for storing formatted excel tables for each admin boundary/AOI\n", + "\n", + "drive_home =\"/content/drive/MyDrive/\" # Google Drive location. Don't change unless you know this is incorrect\n", + "\n", + "error_log_file_path = drive_home + excel_reports_folder + \"/\"+\"1_error_log\" +\".csv\" # for storing errors\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dVfL5w0eBi4I" + }, + "source": [ + "### 7) Setup inputs for processing\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qi5zNGTWN3df" + }, + "source": [ + "Create list of boundaries to process" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BiuBEJwPue2v" + }, + "outputs": [], + "source": [ + "# admin boundary feature collection\n", + "admin_boundaries = ee.FeatureCollection(admin_asset_id)\n", + "\n", + "# list to process\n", + "list_of_countries = admin_boundaries.aggregate_array(admin_asset_property_name).getInfo()\n", + "\n", + "print (\"Length of admin boundaries to process\", len(list_of_countries))\n", + "\n", + "list_of_countries = list(set(list_of_countries)) # remove dupicates\n", + "\n", + "print (\"Length of distinct admin boundaries to process\", (len(set(list_of_countries))))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2G1q9TSiUsc1" + }, + "source": [ + "Read the default land cover remapping table and convert it to a dictionary" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PRSEqq5bu_f7" + }, + "outputs": [], + "source": [ + "default_map_matrix = map_matrix_to_dict(LC_MAP_MATRIX) # LC_MAP_MATRIX stores the path to the remapping table" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "P_qX5MSJZ9gs" + }, + "source": [ + "Set the default transition matrix file path." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "REH6-a1xZ9VI" + }, + "outputs": [], + "source": [ + "default_transition_matrix_path = TRANSITION_MATRIX_FILE # location of the CSV table defining which land cover transitions are “degradation” etc.\n", + "print(default_transition_matrix_path) # view the file path" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4-Ut_-S35Yg8" + }, + "source": [ + "Select years of land cover to process" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "id": "PwqJFWR4u_f7" + }, + "outputs": [], + "source": [ + "# extracts the years from the b_years dictionary\n", + "years = get_b_years(sub_b_year)\n", + "years" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iNxHtR984cNk" + }, + "source": [ + "### 8) Calculate area statistics by country\n", + "* Runs for each country and each mountain biobelt\n", + "* Gets area of land cover reclassified into the 10 SEAM classes\n", + "* Repeat for each year specified\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "VftKLuY4u_f7" + }, + "outputs": [], + "source": [ + "# you can monitor your GEE tasks here : https://code.earthengine.google.com/tasks\n", + "\n", + "create_folder_if_not_exists(stats_csv_folder) # to store outputs in google drive\n", + "\n", + "counter=0 # starting place of counter used to keep track of number of tasks that are being run\n", + "\n", + "for aoi_name in list_of_countries:\n", + "\n", + " aoi = admin_boundaries.filter(ee.Filter.eq(admin_asset_property_name,aoi_name))#.first()\n", + "\n", + " process = ee.FeatureCollection([\n", + " ee.Feature(\n", + " None,\n", + " reduce_regions(\n", + " aoi,\n", + " remap_matrix=default_map_matrix,\n", + " rsa=False,\n", + " dem=DEM_DEFAULT, #default digital elevation model (DEM). Relevant for the real surface area (RSA) implementation.\n", + " lc_years= year,\n", + " transition_matrix=default_transition_matrix_path # a matrix of transitions between land cover classes. Used to assess if change between two inputs is to a different (e.g. degraded) state\n", + " )\n", + " ).set(\"process_id\", \"_\".join([str(y[\"year\"]) for y in year]))\n", + " for year in years # creates GEE images for each year listed and counts areas under different transitions Subindicator B. Images to run are in the 'b_years\" dictionary (above)\n", + " ])\n", + " #make name acceptable for running tasks (i.e., removes special characters)\n", + " task_name = str(sanitize_description(unidecode(aoi_name)))\n", + "\n", + " task = ee.batch.Export.table.toDrive(\n", + " **{ #asterisks unpack dictionary into keyword arguments format\n", + " \"collection\": process,\n", + " \"description\": task_name,\n", + " \"fileFormat\": \"CSV\",\n", + " \"folder\":stats_csv_folder\n", + " }\n", + " )\n", + "\n", + " counter+=1\n", + "\n", + " print (f\"\\r process {counter}/{len(list_of_countries)} {aoi_name} \", end=\"\") #print in place (remove \\r and end=\"\" for verbose version)\n", + "\n", + " if export:\n", + " task.start()\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vM0SIvJtu_f8" + }, + "source": [ + "### 9) Read and translate results into report tables for each country" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3x7jwkZJWwE-" + }, + "source": [ + "#####NOTE: you will need to wait until results files for each country have been created in your google drive (from previous step).\n", + "- see here to monitor the tasks https://code.earthengine.google.com/tasks\n", + "- once tasks are complete, you can run the cell below\n", + "\n", + "This cell formats individual excel reports for each country.\n", + "See Error_log.csv for missing files/errors" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qkpDfHqQu_f9" + }, + "outputs": [], + "source": [ + "# Initialize the counter\n", + "counter = 0\n", + "\n", + "# to store outputs in google drive\n", + "create_folder_if_not_exists(excel_reports_folder)\n", + "\n", + "# Loop over each AOI name in the list of countries\n", + "for aoi_name in list_of_countries:\n", + " counter += 1\n", + "\n", + " # Clean the AOI name\n", + " aoi_name_clean = str(sanitize_description(unidecode(aoi_name)))\n", + "\n", + " # Construct the file path for the stats CSV file\n", + " stats_csv_file = aoi_name_clean + \".csv\"\n", + " stats_csv_file_path = os.path.join(drive_home, stats_csv_folder, stats_csv_file)\n", + "\n", + " message = f\"Process {counter}, {stats_csv_file}\"\n", + "\n", + " try:\n", + " # Read the results from the CSV file and parse it to a dictionary\n", + " dict_results = read_from_csv(stats_csv_file_path)\n", + "\n", + " # details = {\n", + " # \"geo_area_name\": aoi_name,\n", + " # \"ref_area\": \" \",\n", + " # \"source_detail\": \" \",\n", + " # }\n", + "\n", + " sub_b_reports = []\n", + " reporting_years_sub_b = get_reporting_years(sub_b_year, \"sub_b\")\n", + " _, sub_b_years = get_sub_b_items(reporting_years_sub_b)\n", + "\n", + " for year in sub_b_years:\n", + " print(f\"Reporting {year} for sub_b\")\n", + " # Get year label for the report\n", + " parsed_df = parse_to_year(dict_results, year)\n", + " sub_b_reports.append(\n", + " sub_b.get_reports(\n", + " parsed_df,\n", + " year,\n", + " geo_area_name = aoi_name,\n", + " ref_area = \"\",\n", + " source_detail = \"\",\n", + " transition_matrix = default_transition_matrix_path\n", + " ))\n", + " # sub b reports\n", + " er_mtn_dgrp_df = pd.concat([report[0] for report in sub_b_reports])\n", + " er_mtn_dgda_df = pd.concat([report[1] for report in sub_b_reports])\n", + "\n", + "\n", + "\n", + "\n", + " # Define the output report file path\n", + " report_file_path = os.path.join(drive_home, excel_reports_folder, aoi_name_clean + \".xlsx\")\n", + "\n", + " # This will create the excel file with the reports\n", + " with pd.ExcelWriter(report_file_path) as writer:\n", + " er_mtn_dgda_df.to_excel(writer, sheet_name=\"Table4_ER_MTN_DGRDA\", index=False)\n", + " er_mtn_dgrp_df.to_excel(writer, sheet_name=\"Table5_ER_MTN_DGRDP\", index=False)\n", + "\n", + " for sheetname in writer.sheets:\n", + " worksheet = writer.sheets[sheetname]\n", + " for col in worksheet.columns:\n", + " max_length = 0\n", + " column = col[0]\n", + " for cell in col:\n", + " try:\n", + " if len(str(cell.value)) > max_length:\n", + " max_length = len(cell.value)\n", + " except:\n", + " pass\n", + " adjusted_width = max(max_length, len(str(column.value))) + 4\n", + " worksheet.column_dimensions[get_column_letter(column.column)].width = (\n", + " adjusted_width\n", + " )\n", + "\n", + " # Align \"obs_value\" column to the right\n", + " if \"OBS\" in column.value:\n", + " for cell in col:\n", + " cell.alignment = Alignment(horizontal=\"right\")\n", + "\n", + "\n", + " except Exception as e:\n", + " # If an error occurs, catch the exception and handle it\n", + " message = f\"process {counter}, {stats_csv_file}, Error: {e}\"\n", + "\n", + " # Get the current time\n", + " current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')\n", + "\n", + " # Write the error message and file name to the error log file\n", + " error_info = pd.DataFrame([[stats_csv_file, str(e), current_time]], columns=['File Name', 'Error Message', 'Time'])\n", + "\n", + " mode = 'w' if not os.path.exists(error_log_file_path) else 'a'\n", + " header = False if os.path.exists(error_log_file_path) else True\n", + "\n", + " # Append or write to the error log file\n", + " error_info.to_csv(error_log_file_path, mode=mode, header=header, index=False)\n", + "\n", + " print(message)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qTqI3Ag08k5b" + }, + "source": [ + "### 10) Combine excel report files into one" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sVpLtG4Z7Jbe" + }, + "source": [ + "Make a list of files to combine\n", + "- Note: only combines excel files that are present in your google drive (from Step 9)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sTyhwne7rr9D" + }, + "outputs": [], + "source": [ + "# Directory path where Excel reports are stored\n", + "directory_path = os.path.join(drive_home, excel_reports_folder)\n", + "\n", + "# List files in the directory with '.xlsx' extension\n", + "files = [file for file in os.listdir(directory_path) if file.endswith('.xlsx')]\n", + "\n", + "# Create a list of full file paths\n", + "full_file_paths = [os.path.join(directory_path, file) for file in files]\n", + "\n", + "# Print the number of Excel files found in the folder\n", + "print(f\"Number of Excel files in folder: {len(full_file_paths)}\")\n", + "\n", + "# folder to store outputs in google drive\n", + "create_folder_if_not_exists(final_report_folder)\n", + "\n", + "# File path for the combined final report\n", + "reports_combined_file_path = os.path.join(drive_home, final_report_folder, final_report_name)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "C77flAQu7xWX" + }, + "source": [ + "##### Run function to combine into a single report" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FkS1ErA9AYj6" + }, + "outputs": [], + "source": [ + "append_excel_files(file_paths=full_file_paths,num_sheets=3,output_file_path=reports_combined_file_path)\n", + "\n", + "print (f\"\\n Complete! Output file for SDG 15.4.2 Component B here: {reports_combined_file_path}\")" + ] + } + ], + "metadata": { + "colab": { + "include_colab_link": true, + "provenance": [] + }, + "kernelspec": { + "display_name": "(test) test-sepal_mgci", + "language": "python", + "name": "test-sepal_mgci" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/component/scripts/plots.py b/component/scripts/plots.py index 15fede2..b633d31 100644 --- a/component/scripts/plots.py +++ b/component/scripts/plots.py @@ -1,12 +1,9 @@ from typing import Tuple -from component.parameter.module_parameter import transition_degradation_matrix -from collections import defaultdict -import matplotlib.pyplot as plt import numpy as np import pandas as pd -from ipecharts.option import Option, Legend, Tooltip +from ipecharts.option import Option, Legend, Tooltip, XAxis, YAxis from ipecharts.option.series import Sankey from ipecharts.echarts import EChartsWidget @@ -18,7 +15,7 @@ def get_sankey_chart(): sankey_data.links = [] option = Option(series=[sankey_data], tooltip=Tooltip(), legend=Legend()) - chart = EChartsWidget(option=option) + chart = EChartsWidget(option=option, style={"height": "700px", "width": "80%"}) return sankey_data, chart @@ -85,267 +82,55 @@ def get_nodes_and_links( return biobelt_dict -def get_pyecharts_sankey(df: pd.DataFrame, lc_classes_path: str): - """Generate a Sankey diagram using Pyecharts +from ipecharts.option.series import Bar - Args: - df (pd.DataFrame): A DataFrame with columns 'from', 'to', and 'sum' - lc_classes_path (str): Path to the land cover classes CSV file - Returns: - EChartsWidget: A Pyecharts EChartsWidget object - """ +def get_series_data(df): + """Generate series data for a bar chart.""" - from ipecharts.option import Option, Legend, Tooltip - from ipecharts.option.series import Sankey - from ipecharts.echarts import EChartsWidget, EChartsRawWidget - import numpy as np - - s = Sankey(smooth=True, areaStyle={}) - s.data = nodes - s.links = links - - option = Option(series=[s], tooltip=Tooltip(), legend=Legend()) - return EChartsWidget(option=option) - - -def sankey(df, colorDict=None, aspect=4, rightColor=False, fontsize=14): - """ - Sankey Diagram for showing land cover transitions - Inputs: - df = pandas dataframe with first column as left, second column as right and third colum as leftWeight - colorDict = Dictionary of colors to use for each label - {'label':'color'} - aspect = vertical extent of the diagram in units of horizontal extent - rightColor = If true, each strip in the diagram will be be colored - according to its left label - Ouput: fig, ax - """ - left = df[df.columns[0]] - right = df[df.columns[1]] - leftWeight = df["sum"] - rightWeight = [] - - leftLabels = [] - rightLabels = [] - # Check weights - if len(leftWeight) == 0: - leftWeight = np.ones(len(left)) - - if len(rightWeight) == 0: - rightWeight = leftWeight - - fig, ax = plt.subplots(figsize=(4, 6)) - - # Create Dataframe - if isinstance(left, pd.Series): - left = left.reset_index(drop=True) - if isinstance(right, pd.Series): - right = right.reset_index(drop=True) - dataFrame = pd.DataFrame( - { - "left": left, - "right": right, - "leftWeight": leftWeight, - "rightWeight": rightWeight, - }, - index=range(len(left)), - ) - - if len(dataFrame[(dataFrame.left.isnull()) | (dataFrame.right.isnull())]): - raise ValueError("Sankey graph does not support null values.") - - # Identify all labels that appear 'left' or 'right' - - # Identify left labels - if len(leftLabels) == 0: - leftLabels = pd.Series(dataFrame.left.unique()).unique() - - # Identify right labels - if len(rightLabels) == 0: - rightLabels = pd.Series(dataFrame.right.unique()).unique() - - # If no colorDict given - if colorDict is None: - raise ValueError("specify a colour palette") - - # Determine widths of individual strips - ns_l = defaultdict() - ns_r = defaultdict() - for leftLabel in leftLabels: - leftDict = {} - rightDict = {} - for rightLabel in rightLabels: - leftDict[rightLabel] = dataFrame[ - (dataFrame.left == leftLabel) & (dataFrame.right == rightLabel) - ].leftWeight.sum() - rightDict[rightLabel] = dataFrame[ - (dataFrame.left == leftLabel) & (dataFrame.right == rightLabel) - ].rightWeight.sum() - ns_l[leftLabel] = leftDict - ns_r[leftLabel] = rightDict - - # Determine positions of left label patches and total widths - leftWidths = defaultdict() - for i, leftLabel in enumerate(leftLabels): - myD = {} - myD["left"] = dataFrame[dataFrame.left == leftLabel].leftWeight.sum() - if i == 0: - myD["bottom"] = 0 - myD["top"] = myD["left"] - topEdge = myD["top"] - else: - myD["bottom"] = ( - leftWidths[leftLabels[i - 1]]["top"] + 0.02 * dataFrame.leftWeight.sum() - ) - myD["top"] = myD["bottom"] + myD["left"] - topEdge = myD["top"] - leftWidths[leftLabel] = myD - - # Determine positions of right label patches and total widths - rightWidths = defaultdict() - for i, rightLabel in enumerate(rightLabels): - myD = {} - myD["right"] = dataFrame[dataFrame.right == rightLabel].rightWeight.sum() - if i == 0: - myD["bottom"] = 0 - myD["top"] = myD["right"] - topEdge = myD["top"] - else: - myD["bottom"] = ( - rightWidths[rightLabels[i - 1]]["top"] - + 0.02 * dataFrame.rightWeight.sum() - ) - myD["top"] = myD["bottom"] + myD["right"] - topEdge = myD["top"] - rightWidths[rightLabel] = myD - - # Total vertical extent of diagram - xMax = topEdge / aspect - - # Draw vertical bars on left and right of each label's section & print label - for leftLabel in leftLabels: - ax.fill_between( - [-0.02 * xMax, 0], - 2 * [leftWidths[leftLabel]["bottom"]], - 2 * [leftWidths[leftLabel]["bottom"] + leftWidths[leftLabel]["left"]], - color=colorDict[leftLabel], - alpha=0.99, - ) - ax.text( - -0.05 * xMax, - leftWidths[leftLabel]["bottom"] + 0.5 * leftWidths[leftLabel]["left"], - leftLabel, - {"ha": "right", "va": "center"}, - fontsize=fontsize, - ) - for rightLabel in rightLabels: - ax.fill_between( - [xMax, 1.02 * xMax], - 2 * [rightWidths[rightLabel]["bottom"]], - 2 * [rightWidths[rightLabel]["bottom"] + rightWidths[rightLabel]["right"]], - color=colorDict[rightLabel], - alpha=0.99, + series_data = [] + for col in df.columns: + if col == "year": + continue + series_data.append( + { + "name": col, + "data": df[col].tolist(), + "itemStyle": {"color": "#5f8b95"}, + } ) - ax.text( - 1.05 * xMax, - rightWidths[rightLabel]["bottom"] + 0.5 * rightWidths[rightLabel]["right"], - rightLabel, - {"ha": "left", "va": "center"}, - fontsize=fontsize, + return series_data + + +def get_bars(series_data): + """Create a list of bar series from the series data.""" + bars = [] + for series in series_data: + bars.append( + Bar( + **{ + "type": "bar", + "name": series["name"], + "stack": True, + "data": series["data"], + "itemStyle": series["itemStyle"], + } + ) ) - - # Plot strips - for leftLabel in leftLabels: - for rightLabel in rightLabels: - labelColor = leftLabel - if rightColor: - labelColor = rightLabel - if ( - len( - dataFrame[ - (dataFrame.left == leftLabel) & (dataFrame.right == rightLabel) - ] - ) - > 0 - ): - # Create array of y values for each strip, half at left value, - # half at right, convolve - ys_d = np.array( - 50 * [leftWidths[leftLabel]["bottom"]] - + 50 * [rightWidths[rightLabel]["bottom"]] - ) - ys_d = np.convolve(ys_d, 0.05 * np.ones(20), mode="valid") - ys_d = np.convolve(ys_d, 0.05 * np.ones(20), mode="valid") - ys_u = np.array( - 50 * [leftWidths[leftLabel]["bottom"] + ns_l[leftLabel][rightLabel]] - + 50 - * [rightWidths[rightLabel]["bottom"] + ns_r[leftLabel][rightLabel]] - ) - ys_u = np.convolve(ys_u, 0.05 * np.ones(20), mode="valid") - ys_u = np.convolve(ys_u, 0.05 * np.ones(20), mode="valid") - - # Update bottom edges at each label so next strip starts at the right place - leftWidths[leftLabel]["bottom"] += ns_l[leftLabel][rightLabel] - rightWidths[rightLabel]["bottom"] += ns_r[leftLabel][rightLabel] - ax.fill_between( - np.linspace(0, xMax, len(ys_d)), - ys_d, - ys_u, - alpha=0.65, - color=colorDict[labelColor], - ) - ax.axis("off") - ax.set_title( - f"Land cover transitions from {df.columns[0]} to {df.columns[1]}", - fontweight="bold", - ) - left, width = -0.001, 0.5 - bottom, height = 0.25, 0.5 - right = left + width - top = bottom + height - - ax.text( - left, - 0.5 * (bottom + top), - df.columns[0], - horizontalalignment="right", - verticalalignment="center", - rotation="vertical", - alpha=0.15, - fontsize=50, - transform=ax.transAxes, - ) - ax.text( - 1.100, - 0.5 * (bottom + top), - df.columns[1], - horizontalalignment="right", - verticalalignment="center", - rotation="vertical", - alpha=0.15, - fontsize=50, - transform=ax.transAxes, - ) - - return (fig, ax) + return bars -def plot_transition_matrix(): - """Show transition degradation matrix""" +def get_chart(df): - import matplotlib.colors as colors - import seaborn as sns + landcov_names, series_data = get_series_data(df) + bars = get_bars(series_data) - df = transition_degradation_matrix - # Pivot the DataFrame to create a transition matrix - transition_matrix = df.pivot(index="from", columns="to", values="impact_code") - - # Create a custom colormap - cmap = colors.LinearSegmentedColormap.from_list("", ["red", "gray", "green"]) - - # remove legend - # Create a colored transition matrix using seaborn - plt.figure(figsize=(6, 4)) - sns.heatmap(transition_matrix, annot=True, cmap=cmap, center=0, cbar=False) - plt.show() + option = Option( + backgroundColor="#1e1e1e00", + legend=Legend(data=landcov_names), + yAxis=YAxis(type="category", data=landcov_names), + xAxis=XAxis(type="value"), + series=bars, + tooltip=Tooltip(), + ) + return EChartsWidget(option=option) diff --git a/component/tile/dashboard_tile.py b/component/tile/dashboard_tile.py index 3f77467..c05844f 100644 --- a/component/tile/dashboard_tile.py +++ b/component/tile/dashboard_tile.py @@ -102,9 +102,7 @@ def __init__(self, model, *args, **kwargs): # Observe reporting_years_{indicator} from model to update the year_select self.model.observe(self.set_years, f"reporting_years_sub_a") - self.btn.on_event("click", self.render_dashboard) - self.set_years({"new": self.model.reporting_years_sub_a}) def set_years(self, change): @@ -187,6 +185,7 @@ def set_belt_items(self, change): {"baseline": [year1, year2]} or, {"report": [base_start, report_year]} """ + print(change["new"]) look_up_year = change["new"] diff --git a/no_ui.ipynb b/no_ui.ipynb index a0c87a8..9277813 100644 --- a/no_ui.ipynb +++ b/no_ui.ipynb @@ -411,12 +411,14 @@ { "cell_type": "code", "execution_count": 14, + "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "780412feee074a0797be3c743e1ad05a", + "model_id": "807fc495dd9540329638626300163a73", "version_major": 2, "version_minor": 0 }, @@ -425,6 +427,7 @@ ] }, "execution_count": 14, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -433,20 +436,6 @@ "dashboard_tile" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": null, @@ -471,7 +460,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/nox_ui.ipynb b/nox_ui.ipynb index ee4bc4f..0e75c7c 100644 --- a/nox_ui.ipynb +++ b/nox_ui.ipynb @@ -2,12 +2,200 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "6320d7f9-989e-4e60-a809-0453c05affc4", "metadata": { "tags": [] }, "outputs": [ + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "/*******************************************************************************\n", + " * remove any links from fontawesome 5 created by jupyter in favor of\n", + " * fontawesome 6. to be removed when Jupyter updates it\n", + " */\n", + "\n", + "function remove_fa5() {\n", + " let links = document.querySelectorAll(\n", + " \"link[href^='https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@^5']\"\n", + " );\n", + "\n", + " links.forEach((link) => link.remove());\n", + "}\n", + "\n", + "if (document.readyState != \"loading\") remove_fa5();\n", + "else document.addEventListener(\"DOMContentLoaded\", remove_fa5);\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a4d8cc9220e0470cb75de87dff2d025f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "ResizeTrigger()" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, { "data": { "text/html": [ @@ -203,9 +391,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.12.4" } }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/requirements.txt b/requirements.txt index 35516af..bde4083 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,4 +19,4 @@ pygaul seaborn ipyvuetify -ipecharts>=1.0.6 \ No newline at end of file +ipecharts==1.0.7 \ No newline at end of file diff --git a/tests_ui/data/model_results.json b/tests_ui/data/model_results.json new file mode 100644 index 0000000..25d5cdf --- /dev/null +++ b/tests_ui/data/model_results.json @@ -0,0 +1,130 @@ +{"mgcimodel.results":{"2000": {"sub_a": [{"biobelt": 2, + "groups": [{"lc": 3, "sum": 0.7551212500000001}, + {"lc": 4, "sum": 14.158790898437498}, + {"lc": 5, "sum": 5.946592710937501}, + {"lc": 6, "sum": 0.37756365625}]}, + {"biobelt": 3, + "groups": [{"lc": 1, "sum": 5.1936178203125}, + {"lc": 2, "sum": 471.3010137594975}, + {"lc": 3, "sum": 8.02693709375}, + {"lc": 4, "sum": 2969.9735879000873}, + {"lc": 5, "sum": 304.7826624218743}, + {"lc": 6, "sum": 0.2828043515625}]}, + {"biobelt": 4, + "groups": [{"lc": 1, "sum": 168.3620734921871}, + {"lc": 10, "sum": 72.22220274218756}, + {"lc": 2, "sum": 3730.0379216579613}, + {"lc": 3, "sum": 54.574133561213195}, + {"lc": 4, "sum": 29961.075690758567}, + {"lc": 5, "sum": 2908.4876023248403}, + {"lc": 6, "sum": 23.637147807444848}, + {"lc": 8, "sum": 0.4717225078125}]}]}, + "2000_2015_2018": {"baseline_degradation": [{"biobelt": 2, + "groups": [{"lc": 2, "sum": 21.238068515625017}]}, + {"biobelt": 3, + "groups": [{"lc": 1, "sum": 20.665385171875002}, + {"lc": 2, "sum": 3648.1207807305054}, + {"lc": 3, "sum": 90.77445744469966}]}, + {"biobelt": 4, + "groups": [{"lc": 1, "sum": 1135.9250689484684}, + {"lc": 2, "sum": 34925.06165119156}, + {"lc": 3, "sum": 857.8817747122545}]}], + "baseline_transition": [{"biobelt": 2, + "groups": [{"lc": 303, "sum": 0.7551212500000001}, + {"lc": 404, "sum": 14.158790898437498}, + {"lc": 505, "sum": 5.946592710937501}, + {"lc": 606, "sum": 0.37756365625}]}, + {"biobelt": 3, + "groups": [{"lc": 101, "sum": 5.193617820312501}, + {"lc": 201, "sum": 0.094475375}, + {"lc": 202, "sum": 417.8766799007347}, + {"lc": 204, "sum": 53.32985848376223}, + {"lc": 301, "sum": 0.188900203125}, + {"lc": 303, "sum": 7.083183734375001}, + {"lc": 304, "sum": 0.75485315625}, + {"lc": 401, "sum": 0.3777951328125}, + {"lc": 402, "sum": 9.72162321875}, + {"lc": 403, "sum": 0.094531125}, + {"lc": 404, "sum": 2950.157769634462}, + {"lc": 405, "sum": 9.6218687890625}, + {"lc": 501, "sum": 0.566191328125}, + {"lc": 504, "sum": 36.689745804687504}, + {"lc": 505, "sum": 267.5267252890623}, + {"lc": 606, "sum": 0.2828043515625}]}, + {"biobelt": 4, + "groups": [{"lc": 1001, "sum": 0.472149234375}, + {"lc": 1004, "sum": 0.9442580312500001}, + {"lc": 1006, "sum": 0.4721610625}, + {"lc": 1008, "sum": 2.5495191562499993}, + {"lc": 101, "sum": 168.3620734921871}, + {"lc": 1010, "sum": 67.78411525781249}, + {"lc": 201, "sum": 21.711948324969367}, + {"lc": 202, "sum": 3146.0770858570377}, + {"lc": 204, "sum": 562.1545618118869}, + {"lc": 205, "sum": 0.0943256640625}, + {"lc": 301, "sum": 20.490742671874987}, + {"lc": 303, "sum": 33.42306504558823}, + {"lc": 304, "sum": 0.6603258437499999}, + {"lc": 401, "sum": 20.302875874999994}, + {"lc": 402, "sum": 487.86584193982833}, + {"lc": 403, "sum": 5.003511789062501}, + {"lc": 404, "sum": 28876.32492851002}, + {"lc": 405, "sum": 568.4645700508577}, + {"lc": 406, "sum": 3.1139625937499997}, + {"lc": 501, "sum": 4.0603895078125}, + {"lc": 504, "sum": 294.8781569472426}, + {"lc": 505, "sum": 2609.5490558697857}, + {"lc": 601, "sum": 2.4548979296875}, + {"lc": 604, "sum": 1.132203375}, + {"lc": 606, "sum": 20.050046502757354}, + {"lc": 804, "sum": 0.0944044453125}, + {"lc": 808, "sum": 0.3773180625}]}], + "final_degradation": [{"biobelt": 2, + "groups": [{"lc": 2, "sum": 21.238068515625017}]}, + {"biobelt": 3, + "groups": [{"lc": 1, "sum": 27.1817221796875}, + {"lc": 2, "sum": 3728.039329183025}, + {"lc": 3, "sum": 4.3395719843750005}]}, + {"biobelt": 4, + "groups": [{"lc": 1, "sum": 1545.648000734526}, + {"lc": 2, "sum": 35322.996898}, + {"lc": 3, "sum": 50.2235961177696}]}], + "report_transition": [{"biobelt": 2, + "groups": [{"lc": 303, "sum": 0.7551212500000001}, + {"lc": 404, "sum": 14.158790898437498}, + {"lc": 505, "sum": 5.946592710937501}, + {"lc": 606, "sum": 0.37756365625}]}, + {"biobelt": 3, + "groups": [{"lc": 101, "sum": 6.420979859375002}, + {"lc": 202, "sum": 424.57879383823496}, + {"lc": 204, "sum": 3.01950928125}, + {"lc": 303, "sum": 7.177714859375}, + {"lc": 401, "sum": 0.0945545703125}, + {"lc": 402, "sum": 5.4792472265624985}, + {"lc": 404, "sum": 3033.377625196347}, + {"lc": 405, "sum": 1.9808000859375001}, + {"lc": 504, "sum": 2.358327578125}, + {"lc": 505, "sum": 274.7902664999992}, + {"lc": 606, "sum": 0.2828043515625}]}, + {"biobelt": 4, + "groups": [{"lc": 101, "sum": 237.8550770359069}, + {"lc": 1010, "sum": 67.78411525781249}, + {"lc": 201, "sum": 0.094402453125}, + {"lc": 202, "sum": 3611.7380999447223}, + {"lc": 204, "sum": 22.110425399019608}, + {"lc": 301, "sum": 0.1888538125}, + {"lc": 303, "sum": 37.95441260808824}, + {"lc": 304, "sum": 0.2833104140625}, + {"lc": 401, "sum": 0.094376203125}, + {"lc": 402, "sum": 206.30337532542885}, + {"lc": 403, "sum": 0.4724664296875}, + {"lc": 404, "sum": 29315.896854881586}, + {"lc": 405, "sum": 211.81753143719362}, + {"lc": 406, "sum": 1.6042346875}, + {"lc": 501, "sum": 0.188779140625}, + {"lc": 504, "sum": 37.36106989062498}, + {"lc": 505, "sum": 3140.558102553451}, + {"lc": 601, "sum": 0.1887590234375}, + {"lc": 606, "sum": 23.447411135569855}, + {"lc": 808, "sum": 2.9268372187499994}]}]}} +} \ No newline at end of file diff --git a/ui.ipynb b/ui.ipynb index 2266f74..eabf15b 100644 --- a/ui.ipynb +++ b/ui.ipynb @@ -189,9 +189,9 @@ ], "metadata": { "kernelspec": { - "display_name": " (venv) sepal_mgci", + "display_name": "(test) test-sepal_mgci", "language": "python", - "name": "venv-sepal_mgci" + "name": "test-sepal_mgci" }, "language_info": { "codemirror_mode": {