From 14f8c1530265f85e73062f491ee2e2475a8bc45f Mon Sep 17 00:00:00 2001 From: dfguerrerom Date: Tue, 24 Sep 2024 14:11:14 +0200 Subject: [PATCH 1/5] refactor: remove legacy graphs --- component/scripts/plots.py | 242 ------------------------------------- 1 file changed, 242 deletions(-) diff --git a/component/scripts/plots.py b/component/scripts/plots.py index 15fede2..84a2046 100644 --- a/component/scripts/plots.py +++ b/component/scripts/plots.py @@ -107,245 +107,3 @@ def get_pyecharts_sankey(df: pd.DataFrame, lc_classes_path: str): option = Option(series=[s], tooltip=Tooltip(), legend=Legend()) return EChartsWidget(option=option) - - -def sankey(df, colorDict=None, aspect=4, rightColor=False, fontsize=14): - """ - Sankey Diagram for showing land cover transitions - Inputs: - df = pandas dataframe with first column as left, second column as right and third colum as leftWeight - colorDict = Dictionary of colors to use for each label - {'label':'color'} - aspect = vertical extent of the diagram in units of horizontal extent - rightColor = If true, each strip in the diagram will be be colored - according to its left label - Ouput: fig, ax - """ - left = df[df.columns[0]] - right = df[df.columns[1]] - leftWeight = df["sum"] - rightWeight = [] - - leftLabels = [] - rightLabels = [] - # Check weights - if len(leftWeight) == 0: - leftWeight = np.ones(len(left)) - - if len(rightWeight) == 0: - rightWeight = leftWeight - - fig, ax = plt.subplots(figsize=(4, 6)) - - # Create Dataframe - if isinstance(left, pd.Series): - left = left.reset_index(drop=True) - if isinstance(right, pd.Series): - right = right.reset_index(drop=True) - dataFrame = pd.DataFrame( - { - "left": left, - "right": right, - "leftWeight": leftWeight, - "rightWeight": rightWeight, - }, - index=range(len(left)), - ) - - if len(dataFrame[(dataFrame.left.isnull()) | (dataFrame.right.isnull())]): - raise ValueError("Sankey graph does not support null values.") - - # Identify all labels that appear 'left' or 'right' - - # Identify left labels - if len(leftLabels) == 0: - leftLabels = pd.Series(dataFrame.left.unique()).unique() - - # Identify right labels - if len(rightLabels) == 0: - rightLabels = pd.Series(dataFrame.right.unique()).unique() - - # If no colorDict given - if colorDict is None: - raise ValueError("specify a colour palette") - - # Determine widths of individual strips - ns_l = defaultdict() - ns_r = defaultdict() - for leftLabel in leftLabels: - leftDict = {} - rightDict = {} - for rightLabel in rightLabels: - leftDict[rightLabel] = dataFrame[ - (dataFrame.left == leftLabel) & (dataFrame.right == rightLabel) - ].leftWeight.sum() - rightDict[rightLabel] = dataFrame[ - (dataFrame.left == leftLabel) & (dataFrame.right == rightLabel) - ].rightWeight.sum() - ns_l[leftLabel] = leftDict - ns_r[leftLabel] = rightDict - - # Determine positions of left label patches and total widths - leftWidths = defaultdict() - for i, leftLabel in enumerate(leftLabels): - myD = {} - myD["left"] = dataFrame[dataFrame.left == leftLabel].leftWeight.sum() - if i == 0: - myD["bottom"] = 0 - myD["top"] = myD["left"] - topEdge = myD["top"] - else: - myD["bottom"] = ( - leftWidths[leftLabels[i - 1]]["top"] + 0.02 * dataFrame.leftWeight.sum() - ) - myD["top"] = myD["bottom"] + myD["left"] - topEdge = myD["top"] - leftWidths[leftLabel] = myD - - # Determine positions of right label patches and total widths - rightWidths = defaultdict() - for i, rightLabel in enumerate(rightLabels): - myD = {} - myD["right"] = dataFrame[dataFrame.right == rightLabel].rightWeight.sum() - if i == 0: - myD["bottom"] = 0 - myD["top"] = myD["right"] - topEdge = myD["top"] - else: - myD["bottom"] = ( - rightWidths[rightLabels[i - 1]]["top"] - + 0.02 * dataFrame.rightWeight.sum() - ) - myD["top"] = myD["bottom"] + myD["right"] - topEdge = myD["top"] - rightWidths[rightLabel] = myD - - # Total vertical extent of diagram - xMax = topEdge / aspect - - # Draw vertical bars on left and right of each label's section & print label - for leftLabel in leftLabels: - ax.fill_between( - [-0.02 * xMax, 0], - 2 * [leftWidths[leftLabel]["bottom"]], - 2 * [leftWidths[leftLabel]["bottom"] + leftWidths[leftLabel]["left"]], - color=colorDict[leftLabel], - alpha=0.99, - ) - ax.text( - -0.05 * xMax, - leftWidths[leftLabel]["bottom"] + 0.5 * leftWidths[leftLabel]["left"], - leftLabel, - {"ha": "right", "va": "center"}, - fontsize=fontsize, - ) - for rightLabel in rightLabels: - ax.fill_between( - [xMax, 1.02 * xMax], - 2 * [rightWidths[rightLabel]["bottom"]], - 2 * [rightWidths[rightLabel]["bottom"] + rightWidths[rightLabel]["right"]], - color=colorDict[rightLabel], - alpha=0.99, - ) - ax.text( - 1.05 * xMax, - rightWidths[rightLabel]["bottom"] + 0.5 * rightWidths[rightLabel]["right"], - rightLabel, - {"ha": "left", "va": "center"}, - fontsize=fontsize, - ) - - # Plot strips - for leftLabel in leftLabels: - for rightLabel in rightLabels: - labelColor = leftLabel - if rightColor: - labelColor = rightLabel - if ( - len( - dataFrame[ - (dataFrame.left == leftLabel) & (dataFrame.right == rightLabel) - ] - ) - > 0 - ): - # Create array of y values for each strip, half at left value, - # half at right, convolve - ys_d = np.array( - 50 * [leftWidths[leftLabel]["bottom"]] - + 50 * [rightWidths[rightLabel]["bottom"]] - ) - ys_d = np.convolve(ys_d, 0.05 * np.ones(20), mode="valid") - ys_d = np.convolve(ys_d, 0.05 * np.ones(20), mode="valid") - ys_u = np.array( - 50 * [leftWidths[leftLabel]["bottom"] + ns_l[leftLabel][rightLabel]] - + 50 - * [rightWidths[rightLabel]["bottom"] + ns_r[leftLabel][rightLabel]] - ) - ys_u = np.convolve(ys_u, 0.05 * np.ones(20), mode="valid") - ys_u = np.convolve(ys_u, 0.05 * np.ones(20), mode="valid") - - # Update bottom edges at each label so next strip starts at the right place - leftWidths[leftLabel]["bottom"] += ns_l[leftLabel][rightLabel] - rightWidths[rightLabel]["bottom"] += ns_r[leftLabel][rightLabel] - ax.fill_between( - np.linspace(0, xMax, len(ys_d)), - ys_d, - ys_u, - alpha=0.65, - color=colorDict[labelColor], - ) - ax.axis("off") - ax.set_title( - f"Land cover transitions from {df.columns[0]} to {df.columns[1]}", - fontweight="bold", - ) - left, width = -0.001, 0.5 - bottom, height = 0.25, 0.5 - right = left + width - top = bottom + height - - ax.text( - left, - 0.5 * (bottom + top), - df.columns[0], - horizontalalignment="right", - verticalalignment="center", - rotation="vertical", - alpha=0.15, - fontsize=50, - transform=ax.transAxes, - ) - ax.text( - 1.100, - 0.5 * (bottom + top), - df.columns[1], - horizontalalignment="right", - verticalalignment="center", - rotation="vertical", - alpha=0.15, - fontsize=50, - transform=ax.transAxes, - ) - - return (fig, ax) - - -def plot_transition_matrix(): - """Show transition degradation matrix""" - - import matplotlib.colors as colors - import seaborn as sns - - df = transition_degradation_matrix - # Pivot the DataFrame to create a transition matrix - transition_matrix = df.pivot(index="from", columns="to", values="impact_code") - - # Create a custom colormap - cmap = colors.LinearSegmentedColormap.from_list("", ["red", "gray", "green"]) - - # remove legend - # Create a colored transition matrix using seaborn - plt.figure(figsize=(6, 4)) - sns.heatmap(transition_matrix, annot=True, cmap=cmap, center=0, cbar=False) - plt.show() From 5fa3125c11be890d76e509d52127d908e09b5437 Mon Sep 17 00:00:00 2001 From: dfguerrerom Date: Tue, 24 Sep 2024 14:11:34 +0200 Subject: [PATCH 2/5] refactor: remove legacy graphs --- component/scripts/plots.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/component/scripts/plots.py b/component/scripts/plots.py index 84a2046..64a3dcb 100644 --- a/component/scripts/plots.py +++ b/component/scripts/plots.py @@ -1,8 +1,5 @@ from typing import Tuple -from component.parameter.module_parameter import transition_degradation_matrix -from collections import defaultdict -import matplotlib.pyplot as plt import numpy as np import pandas as pd From c11ddac3e9c69e202294a42abb683c7aad5701ab Mon Sep 17 00:00:00 2001 From: dfguerrerom Date: Wed, 25 Sep 2024 13:47:44 +0200 Subject: [PATCH 3/5] test --- SDG_15_4_2_Sub_A_Default_values _SEPAL.ipynb | 759 +++++++++++++++++++ component/scripts/plots.py | 68 +- component/tile/dashboard_tile.py | 10 +- no_ui.ipynb | 29 +- nox_ui.ipynb | 194 ++++- requirements.txt | 4 +- tests_ui/data/model_results.json | 130 ++++ ui.ipynb | 4 +- 8 files changed, 1152 insertions(+), 46 deletions(-) create mode 100644 SDG_15_4_2_Sub_A_Default_values _SEPAL.ipynb create mode 100644 tests_ui/data/model_results.json diff --git a/SDG_15_4_2_Sub_A_Default_values _SEPAL.ipynb b/SDG_15_4_2_Sub_A_Default_values _SEPAL.ipynb new file mode 100644 index 0000000..8e79479 --- /dev/null +++ b/SDG_15_4_2_Sub_A_Default_values _SEPAL.ipynb @@ -0,0 +1,759 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "source": [ + "# **SDG 15.4.2 Sub-indicator A: Calculate Global Default Values**\n", + "\n", + "* This script allows batch processing for this indicator for all countries.\n", + "\n", + "* Output is a combined excel file on your Google Drive.\n", + "\n", + "* Runs on the cloud using [Google Colab](https://research.google.com/colaboratory/faq.html)\n", + "\n", + "* Requires: [Google Earth Engine](https://earthengine.google.com/) (GEE) account and project and access to Google Drive\n" + ], + "metadata": { + "id": "82FWR5yMh0HV" + } + }, + { + "cell_type": "markdown", + "source": [ + "### 1) Install required packages" + ], + "metadata": { + "id": "YdoCOI_1yWY0" + } + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "Kg3K1EPJu_f5", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 208 + }, + "outputId": "460f7ba2-8668-4c4d-824b-6d5068884129" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.1/6.1 MB\u001b[0m \u001b[31m34.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.7/2.7 MB\u001b[0m \u001b[31m54.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m38.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hipyvuetify has been installed.\n", + "ee is already installed.\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m235.5/235.5 kB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hunidecode has been installed.\n", + "google-api-python-client has been installed.\n", + "google-auth-httplib2 has been installed.\n", + "google-auth-oauthlib has been installed.\n", + "geemap is already installed.\n" + ] + } + ], + "source": [ + "# to automatically reload modules.\n", + "%load_ext autoreload\n", + "\n", + "# Set to reload all modules before executing code.\n", + "%autoreload 2\n", + "\n", + "# Function to install a package if it's not already installed\n", + "def install_if_not_exists(package_name):\n", + " try:\n", + " __import__(package_name)\n", + " print(f\"{package_name} is already installed.\")\n", + " except ImportError:\n", + " !pip install -q {package_name}\n", + " print(f\"{package_name} has been installed.\")\n", + "\n", + "# List of packages to install if not already installed\n", + "packages_to_install = ['ipyvuetify','ee', 'unidecode', 'google-api-python-client',\n", + " 'google-auth-httplib2', 'google-auth-oauthlib','geemap'] #admin_boundaries\n", + "\n", + "# Install necessary packages\n", + "for package in packages_to_install:\n", + " install_if_not_exists(package)" + ] + }, + { + "cell_type": "markdown", + "source": [ + "### 2) Access GitHub repository\n", + "Clones repository for SDG 15.4.2 into colab.\n", + "Provides functions and lookup tables etc." + ], + "metadata": { + "id": "qGfLFLEkwS1n" + } + }, + { + "cell_type": "code", + "source": [ + "# Change the current working directory to \"/content\" for cloning the repo into.\n", + "%cd \"/content\"\n", + "\n", + "# Clone the GitHub repository \"sepalz_mgci\" into the current directory.\n", + "# NB 'fatal' error on reruns are typically just saying it already exists\n", + "!git clone https://github.com/sepal-contrib/sepal_mgci\n", + "\n", + "# Change working directory to the cloned sepal_mgci github repository\n", + "%cd \"/content/sepal_mgci\"" + ], + "metadata": { + "id": "TNohZrOTvNqT", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 173 + }, + "outputId": "1ad0a56a-2b6e-4966-ab95-a55a1a612aae" + }, + "execution_count": 2, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "\n", + " \n", + " " + ] + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "/content\n", + "Cloning into 'sepal_mgci'...\n", + "remote: Enumerating objects: 3071, done.\u001b[K\n", + "remote: Counting objects: 100% (1009/1009), done.\u001b[K\n", + "remote: Compressing objects: 100% (426/426), done.\u001b[K\n", + "remote: Total 3071 (delta 663), reused 807 (delta 582), pack-reused 2062\u001b[K\n", + "Receiving objects: 100% (3071/3071), 5.02 MiB | 17.37 MiB/s, done.\n", + "Resolving deltas: 100% (1939/1939), done.\n", + "/content/sepal_mgci\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "### 3) Setup Google Earth Engine\n", + "Launches access request pop up window" + ], + "metadata": { + "id": "Lbg9P7frZRy1" + } + }, + { + "cell_type": "code", + "source": [ + "# Google Earth Engine project\n", + "gee_project_name = \"ee-andyarnellgee\" # \"insert cloud project here\" # a registered cloud project (if unsure of name see pic here: https://developers.google.com/earth-engine/cloud/assets)\n", + "\n", + "import ee # google earth engine\n", + "\n", + "ee.Authenticate()\n", + "\n", + "ee.Initialize(project=gee_project_name) # NB gee project name is defined in parameters section" + ], + "metadata": { + "id": "SLLcPONnZRy2" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### 4) Setup Google Drive\n", + "Launches access request pop up window" + ], + "metadata": { + "id": "cxXsgaXEZRy3" + } + }, + { + "cell_type": "code", + "source": [ + "# for accessing google drive\n", + "from google.colab import auth, drive\n", + "from googleapiclient.discovery import build\n", + "\n", + "drive.mount('/content/drive')" + ], + "metadata": { + "id": "e2NxIhcgZRy3" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### 5) Import remaining packages\n", + "- imports required packages and functions listed in 'colab_imports.py' script (found here: sepal_mgci/component/scripts)\n", + "\n" + ], + "metadata": { + "id": "MsQu9VfqZRy3" + } + }, + { + "cell_type": "code", + "source": [ + "from component.scripts.colab_imports import *" + ], + "metadata": { + "id": "NQ9flKZGZRy4" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### 6) Set parameters\n" + ], + "metadata": { + "id": "Bi0XpRRDo6V1" + } + }, + { + "cell_type": "markdown", + "source": [ + "Input parameters" + ], + "metadata": { + "id": "5aIr_OxkG-Ky" + } + }, + { + "cell_type": "code", + "source": [ + "# Google Earth Engine project\n", + "gee_project_name = \"ee-andyarnellgee\" # \"insert cloud project here\" # a registered cloud project (if unsure of name see pic here: https://developers.google.com/earth-engine/cloud/assets)\n", + "\n", + "\n", + "# Admin boundaries asset\n", + "admin_asset_id = \"FAO/GAUL/2015/level0\" # administrative units feature collection\n", + "\n", + "admin_asset_property_name = \"ADM0_NAME\" # property/column name for selecting admin boundaries (e.g. ISO3 code or country name)\n", + "\n", + "\n", + "# Land cover assets\n", + "\n", + "# For Sub-indicator A (sub_a), we need to set the following structure.\n", + "a_years = {\n", + " 1: {\"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2000\", \"year\": 2000}, # baseline\n", + " 2: {\"year\": 2003, \"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2003\"}, # subsequent reporting years...\n", + " 3: {\"year\": 2007, \"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2007\"},\n", + " 4: {\"year\": 2010, \"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2010\"},\n", + "}\n", + "\n" + ], + "metadata": { + "id": "t9C1qT5bGoqt" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "Output parameters\n", + "\n", + "---\n", + "\n" + ], + "metadata": { + "id": "naz3Qe4JHFER" + } + }, + { + "cell_type": "code", + "source": [ + "final_report_folder = \"sdg_15_4_2_A_combined_report\" # folder name in Google Drive for final output (if doesnt exist creates one)\n", + "\n", + "final_report_name = \"sdg_15_4_2_A_default_global.xlsx\" # file name for final excel output\n", + "\n", + "# export GEE tasks or not\n", + "export = False # default: True. Set to False if debugging or limiting accidental re-exporting of tasks\n", + "\n", + "# prints more messages\n", + "debug = False # default: False. Set to True if debugging code" + ], + "metadata": { + "id": "mSvTM29TnndR" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "Temporary output parameters\n" + ], + "metadata": { + "id": "iV8-wbddOJuT" + } + }, + { + "cell_type": "code", + "source": [ + "stats_csv_folder = \"sdg_15_4_2_A_csvs\" # for storing stats tables exported from GEE for each admin boundary/AOI\n", + "\n", + "excel_reports_folder = \"sdg_15_4_2_A_reports\" # for storing formatted excel tables for each admin boundary/AOI\n", + "\n", + "drive_home =\"/content/drive/MyDrive/\" # Google Drive location. Don't change unless you know this is incorrect\n", + "\n", + "error_log_file_path = drive_home + excel_reports_folder + \"/\"+\"1_error_log\" +\".csv\" # for storing errors\n" + ], + "metadata": { + "id": "TCBNkALuOL1N" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "c_8UzlUnu_f6" + }, + "source": [ + "### 7) Setup inputs for processing" + ] + }, + { + "cell_type": "markdown", + "source": [ + "Create list of boundaries to process" + ], + "metadata": { + "id": "qi5zNGTWN3df" + } + }, + { + "cell_type": "code", + "source": [ + "# admin boundary feature collection\n", + "admin_boundaries = ee.FeatureCollection(admin_asset_id)\n", + "\n", + "# list to process\n", + "list_of_countries = admin_boundaries.aggregate_array(admin_asset_property_name).getInfo()\n", + "\n", + "print (\"Length of admin boundaries to process\", len(list_of_countries))\n", + "\n", + "list_of_countries = list(set(list_of_countries)) # remove dupicates\n", + "\n", + "print (\"Length of distinct admin boundaries to process\", (len(set(list_of_countries))))\n" + ], + "metadata": { + "id": "BiuBEJwPue2v" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "Read the default land cover remapping table and convert it to a dictionary" + ], + "metadata": { + "id": "2G1q9TSiUsc1" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PRSEqq5bu_f7" + }, + "outputs": [], + "source": [ + "default_map_matrix = map_matrix_to_dict(LC_MAP_MATRIX)" + ] + }, + { + "cell_type": "markdown", + "source": [ + "Select years of land cover to process" + ], + "metadata": { + "id": "4-Ut_-S35Yg8" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PwqJFWR4u_f7", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 17 + }, + "outputId": "09d3ab98-6b20-4212-b5ed-f3386c3a32bf" + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "\n", + " \n", + " " + ] + }, + "metadata": {} + } + ], + "source": [ + "# extracts the years from the a_years dictionary (as defined in parameters)\n", + "single_years = [y[\"year\"] for y in a_years.values()]" + ] + }, + { + "cell_type": "markdown", + "source": [ + "### 8) Calculate area statistics by country\n", + "* Runs for each country and each mountain biobelt\n", + "* Gets area of land cover reclassified into the 10 SEAM classes\n", + "* Repeat for each year specified\n" + ], + "metadata": { + "id": "iNxHtR984cNk" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "VftKLuY4u_f7" + }, + "outputs": [], + "source": [ + "# you can monitor your GEE tasks here : https://code.earthengine.google.com/tasks\n", + "create_folder_if_not_exists(stats_csv_folder) # to store outputs in google drive\n", + "\n", + "counter=0 # starting place of counter used to keep track of number of tasks that are being run\n", + "\n", + "for aoi_name in list_of_countries:\n", + "\n", + " aoi = admin_boundaries.filter(ee.Filter.eq(admin_asset_property_name,aoi_name))#.first()\n", + "\n", + " # gets areas of landcover in each mountain belt in each country\n", + " # uses reduce_regions function imported from the cloned sepal_mgci git hub repository (see Imports section)\n", + " # pixels counted at native resolution (scale) of input land cover (or DEM if RSA implementation)\n", + " process = ee.FeatureCollection([\n", + " ee.Feature(\n", + " None,\n", + " reduce_regions(\n", + " aoi,\n", + " remap_matrix=default_map_matrix,\n", + " rsa=False,\n", + " # dem=param.DEM_DEFAULT,\n", + " dem=DEM_DEFAULT, #default digital elevation model (DEM). Relevant for the real surface area (RSA) implementation.\n", + " lc_years= year,\n", + " transition_matrix=False\n", + " )\n", + " ).set(\"process_id\", year[0][\"year\"])\n", + " for year in get_a_years(a_years) # creates GEE images and runs stats on each. Images to run are in the 'a_years\" dictionary (above)\n", + " ])\n", + "\n", + " #make name acceptable for running tasks (i.e., removes special characters)\n", + " task_name = str(sanitize_description(unidecode(aoi_name)))\n", + "\n", + "\n", + " task = ee.batch.Export.table.toDrive(\n", + " **{ #asterisks unpack dictionary into keyword arguments format\n", + " \"collection\": process,\n", + " \"description\": task_name,\n", + " \"fileFormat\": \"CSV\",\n", + " \"folder\":stats_csv_folder,\n", + " \"selectors\": [\n", + " \"process_id\",\n", + " \"sub_a\",\n", + " ],\n", + " }\n", + " )\n", + "\n", + " counter+=1\n", + "\n", + " print (f\"\\r process {counter}/{len(list_of_countries)} {aoi_name} \", end=\"\") #print in place (remove \\r and end=\"\" for verbose version)\n", + "\n", + " if export:\n", + " task.start()\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vM0SIvJtu_f8" + }, + "source": [ + "### 9) Read and translate results into report tables" + ] + }, + { + "cell_type": "markdown", + "source": [ + "#####NOTE: you will need to wait until results files for each country have been created in your google drive (from previous step).\n", + "- see here to monitor the tasks https://code.earthengine.google.com/tasks\n", + "- once tasks are complete, you can run the cell below\n", + "\n", + "This cell formats individual excel reports for each country.\n", + "See Error_log.csv for missing files/errors" + ], + "metadata": { + "id": "3x7jwkZJWwE-" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qkpDfHqQu_f9" + }, + "outputs": [], + "source": [ + "# Initialize the counter\n", + "counter = 0\n", + "\n", + "# to store outputs in google drive\n", + "create_folder_if_not_exists(excel_reports_folder)\n", + "\n", + "# Loop over each AOI name in the list of countries\n", + "for aoi_name in list_of_countries:\n", + " counter += 1\n", + "\n", + " # Clean the AOI name\n", + " aoi_name_clean = str(sanitize_description(unidecode(aoi_name)))\n", + "\n", + " # Construct the file path for the stats CSV file\n", + " stats_csv_file = aoi_name_clean + \".csv\"\n", + " stats_csv_file_path = os.path.join(drive_home, stats_csv_folder, stats_csv_file)\n", + "\n", + " message = f\"Process {counter}, {stats_csv_file}\"\n", + "\n", + " try:\n", + " # Read the results from the CSV file and parse it to a dictionary\n", + " dict_results = read_from_csv(stats_csv_file_path)\n", + "\n", + " details = {\n", + " \"geo_area_name\": aoi_name,\n", + " \"ref_area\": \" \",\n", + " \"source_detail\": \" \",\n", + " }\n", + "\n", + " # Generate reports for the sub_a and mtn indicators\n", + " sub_a_reports = [sub_a.get_reports(parse_result(dict_results[year][\"sub_a\"], single=True), year, **details) for year in single_years]\n", + " mtn_reports = [mntn.get_report(parse_result(dict_results[year][\"sub_a\"], single=True), year, **details) for year in single_years]\n", + "\n", + " # Concatenate the mtn reports\n", + " mtn_reports_df = pd.concat(mtn_reports)\n", + "\n", + " # Concatenate the sub a reports\n", + " er_mtn_grnvi_df = pd.concat([report[0] for report in sub_a_reports])\n", + " er_mtn_grncov_df = pd.concat([report[1] for report in sub_a_reports])\n", + "\n", + " # Define the output report file path\n", + " report_file_path = os.path.join(drive_home, excel_reports_folder, aoi_name_clean + \".xlsx\")\n", + "\n", + " # Create the Excel file with the reports\n", + " with pd.ExcelWriter(report_file_path) as writer:\n", + " mtn_reports_df.to_excel(writer, sheet_name=\"Table1_ER_MTN_TOTL\", index=False)\n", + " er_mtn_grncov_df.to_excel(writer, sheet_name=\"Table2_ER_MTN_GRNCOV\", index=False)\n", + " er_mtn_grnvi_df.to_excel(writer, sheet_name=\"Table3_ER_MTN_GRNCVI\", index=False)\n", + "\n", + " # Adjust column widths and alignment for each sheet\n", + " for sheetname in writer.sheets:\n", + " worksheet = writer.sheets[sheetname]\n", + " for col in worksheet.columns:\n", + " max_length = max(len(str(cell.value)) for cell in col)\n", + " column = col[0]\n", + " adjusted_width = max(max_length, len(str(column.value))) + 4\n", + " worksheet.column_dimensions[get_column_letter(column.column)].width = adjusted_width\n", + "\n", + " # Align \"obs_value\" column to the right\n", + " if \"OBS\" in column.value:\n", + " for cell in col:\n", + " cell.alignment = Alignment(horizontal=\"right\")\n", + "\n", + " except Exception as e:\n", + " # If an error occurs, catch the exception and handle it\n", + " message = f\"process {counter}, {stats_csv_file}, Error: {e}\"\n", + "\n", + " # Get the current time\n", + " current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')\n", + "\n", + " # Write the error message and file name to the error log file\n", + " error_info = pd.DataFrame([[stats_csv_file, str(e), current_time]], columns=['File Name', 'Error Message', 'Time'])\n", + "\n", + " mode = 'w' if not os.path.exists(error_log_file_path) else 'a'\n", + " header = False if os.path.exists(error_log_file_path) else True\n", + "\n", + " # Append or write to the error log file\n", + " error_info.to_csv(error_log_file_path, mode=mode, header=header, index=False)\n", + "\n", + " print(message)\n" + ] + }, + { + "cell_type": "markdown", + "source": [ + "### 10) Combine excel report files into one" + ], + "metadata": { + "id": "qTqI3Ag08k5b" + } + }, + { + "cell_type": "markdown", + "source": [ + "Make a list of files to combine" + ], + "metadata": { + "id": "sVpLtG4Z7Jbe" + } + }, + { + "cell_type": "code", + "source": [ + "# Directory path where Excel reports are stored\n", + "directory_path = os.path.join(drive_home, excel_reports_folder)\n", + "\n", + "# List files in the directory with '.xlsx' extension\n", + "files = [file for file in os.listdir(directory_path) if file.endswith('.xlsx')]\n", + "\n", + "# Create a list of full file paths\n", + "full_file_paths = [os.path.join(directory_path, file) for file in files]\n", + "\n", + "# Print the number of Excel files found in the folder\n", + "print(f\"Number of Excel files in folder: {len(full_file_paths)}\")\n", + "\n", + "# folder to store outputs in google drive\n", + "create_folder_if_not_exists(final_report_folder)\n", + "\n", + "# File path for the combined final report\n", + "reports_combined_file_path = os.path.join(drive_home, final_report_folder, final_report_name)\n" + ], + "metadata": { + "id": "sTyhwne7rr9D" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "##### Run function to combine into a single report" + ], + "metadata": { + "id": "C77flAQu7xWX" + } + }, + { + "cell_type": "code", + "source": [ + "append_excel_files(file_paths=full_file_paths,num_sheets=3,output_file_path=reports_combined_file_path)\n", + "\n", + "print (f\"\\n Complete! Output file for SDG 15.4.2 Sub-indicator A here: {reports_combined_file_path}\")" + ], + "metadata": { + "id": "FkS1ErA9AYj6" + }, + "execution_count": null, + "outputs": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "(test) test-sepal_mgci", + "language": "python", + "name": "test-sepal_mgci" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + }, + "colab": { + "provenance": [], + "include_colab_link": true + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/component/scripts/plots.py b/component/scripts/plots.py index 64a3dcb..b633d31 100644 --- a/component/scripts/plots.py +++ b/component/scripts/plots.py @@ -3,7 +3,7 @@ import numpy as np import pandas as pd -from ipecharts.option import Option, Legend, Tooltip +from ipecharts.option import Option, Legend, Tooltip, XAxis, YAxis from ipecharts.option.series import Sankey from ipecharts.echarts import EChartsWidget @@ -15,7 +15,7 @@ def get_sankey_chart(): sankey_data.links = [] option = Option(series=[sankey_data], tooltip=Tooltip(), legend=Legend()) - chart = EChartsWidget(option=option) + chart = EChartsWidget(option=option, style={"height": "700px", "width": "80%"}) return sankey_data, chart @@ -82,25 +82,55 @@ def get_nodes_and_links( return biobelt_dict -def get_pyecharts_sankey(df: pd.DataFrame, lc_classes_path: str): - """Generate a Sankey diagram using Pyecharts - - Args: - df (pd.DataFrame): A DataFrame with columns 'from', 'to', and 'sum' - lc_classes_path (str): Path to the land cover classes CSV file +from ipecharts.option.series import Bar + + +def get_series_data(df): + """Generate series data for a bar chart.""" + + series_data = [] + for col in df.columns: + if col == "year": + continue + series_data.append( + { + "name": col, + "data": df[col].tolist(), + "itemStyle": {"color": "#5f8b95"}, + } + ) + return series_data + + +def get_bars(series_data): + """Create a list of bar series from the series data.""" + bars = [] + for series in series_data: + bars.append( + Bar( + **{ + "type": "bar", + "name": series["name"], + "stack": True, + "data": series["data"], + "itemStyle": series["itemStyle"], + } + ) + ) + return bars - Returns: - EChartsWidget: A Pyecharts EChartsWidget object - """ - from ipecharts.option import Option, Legend, Tooltip - from ipecharts.option.series import Sankey - from ipecharts.echarts import EChartsWidget, EChartsRawWidget - import numpy as np +def get_chart(df): - s = Sankey(smooth=True, areaStyle={}) - s.data = nodes - s.links = links + landcov_names, series_data = get_series_data(df) + bars = get_bars(series_data) - option = Option(series=[s], tooltip=Tooltip(), legend=Legend()) + option = Option( + backgroundColor="#1e1e1e00", + legend=Legend(data=landcov_names), + yAxis=YAxis(type="category", data=landcov_names), + xAxis=XAxis(type="value"), + series=bars, + tooltip=Tooltip(), + ) return EChartsWidget(option=option) diff --git a/component/tile/dashboard_tile.py b/component/tile/dashboard_tile.py index 91ce8a9..b6b2a57 100644 --- a/component/tile/dashboard_tile.py +++ b/component/tile/dashboard_tile.py @@ -1,12 +1,7 @@ -from IPython.display import display import ipyvuetify as v import pandas as pd import sepal_ui.scripts.utils as su import sepal_ui.sepalwidgets as sw -from ipywidgets import Output -from matplotlib import pyplot as plt -from traitlets import link -from component.model.model import MgciModel import component.parameter.module_parameter as param import component.scripts as cs @@ -15,9 +10,7 @@ from component.message import cm from component.scripts.plots import ( get_nodes_and_links, - get_pyecharts_sankey, get_sankey_chart, - sankey, ) from component.scripts.report_scripts import get_belt_desc from component.widget.map import MapView @@ -111,9 +104,7 @@ def __init__(self, model, *args, **kwargs): # Observe reporting_years_{indicator} from model to update the year_select self.model.observe(self.set_years, f"reporting_years_sub_a") - self.btn.on_event("click", self.render_dashboard) - self.set_years({"new": self.model.reporting_years_sub_a}) def set_years(self, change): @@ -196,6 +187,7 @@ def set_belt_items(self, change): {"baseline": [year1, year2]} or, {"report": [base_start, report_year]} """ + print(change["new"]) look_up_year = change["new"] diff --git a/no_ui.ipynb b/no_ui.ipynb index 8bfa113..102f452 100644 --- a/no_ui.ipynb +++ b/no_ui.ipynb @@ -194,7 +194,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c04979d2133941229962283967e15395", + "model_id": "de89c5e78b264a3baa3d1bc6296eb739", "version_major": 2, "version_minor": 0 }, @@ -246,7 +246,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "88d29637fde44e0d8d0931fedba691da", + "model_id": "2e2336eee15b40c2874c90bb640c2599", "version_major": 2, "version_minor": 0 }, @@ -271,7 +271,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "8a5490ca6a024959962799265845c901", + "model_id": "1906e5a432214aeead5ef332c879b74a", "version_major": 2, "version_minor": 0 }, @@ -306,7 +306,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e30e19fa0fb4456fb7076628b1af5911", + "model_id": "70acfa54e41f4c49a507c5df3575816e", "version_major": 2, "version_minor": 0 }, @@ -349,7 +349,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3b641d723f9944f0b6884558a737f6d5", + "model_id": "ca898ebd44cb49ee8ab8abf53b141fcf", "version_major": 2, "version_minor": 0 }, @@ -374,7 +374,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "da01010a5f564428b060e8eb979e1e03", + "model_id": "26573a71e68841eca8dfdeb275ec5b88", "version_major": 2, "version_minor": 0 }, @@ -394,7 +394,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -403,13 +403,13 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "88fa8c0d74a049949243343bf4a29862", + "model_id": "807fc495dd9540329638626300163a73", "version_major": 2, "version_minor": 0 }, @@ -417,7 +417,7 @@ "DashboardTile(children=[CardTitle(children=['Results dashboard'], layout=None), CardText(children=[Alert(class…" ] }, - "execution_count": 29, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -425,6 +425,13 @@ "source": [ "dashboard_tile" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -443,7 +450,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/nox_ui.ipynb b/nox_ui.ipynb index ee4bc4f..0e75c7c 100644 --- a/nox_ui.ipynb +++ b/nox_ui.ipynb @@ -2,12 +2,200 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "6320d7f9-989e-4e60-a809-0453c05affc4", "metadata": { "tags": [] }, "outputs": [ + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "/*******************************************************************************\n", + " * remove any links from fontawesome 5 created by jupyter in favor of\n", + " * fontawesome 6. to be removed when Jupyter updates it\n", + " */\n", + "\n", + "function remove_fa5() {\n", + " let links = document.querySelectorAll(\n", + " \"link[href^='https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@^5']\"\n", + " );\n", + "\n", + " links.forEach((link) => link.remove());\n", + "}\n", + "\n", + "if (document.readyState != \"loading\") remove_fa5();\n", + "else document.addEventListener(\"DOMContentLoaded\", remove_fa5);\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a4d8cc9220e0470cb75de87dff2d025f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "ResizeTrigger()" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, { "data": { "text/html": [ @@ -203,9 +391,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.12.4" } }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/requirements.txt b/requirements.txt index 97aa3cd..bde4083 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ wheel voila earthengine-api -sepal_ui==2.20.0 +sepal_ui==2.20.1 rasterio<1.3.11 # Leave it here even if it's already in the sepal_ui requirements. Check https://github.com/openforis/sepal/issues/328 scipy @@ -19,4 +19,4 @@ pygaul seaborn ipyvuetify -ipecharts>=1.0.6 \ No newline at end of file +ipecharts==1.0.7 \ No newline at end of file diff --git a/tests_ui/data/model_results.json b/tests_ui/data/model_results.json new file mode 100644 index 0000000..25d5cdf --- /dev/null +++ b/tests_ui/data/model_results.json @@ -0,0 +1,130 @@ +{"mgcimodel.results":{"2000": {"sub_a": [{"biobelt": 2, + "groups": [{"lc": 3, "sum": 0.7551212500000001}, + {"lc": 4, "sum": 14.158790898437498}, + {"lc": 5, "sum": 5.946592710937501}, + {"lc": 6, "sum": 0.37756365625}]}, + {"biobelt": 3, + "groups": [{"lc": 1, "sum": 5.1936178203125}, + {"lc": 2, "sum": 471.3010137594975}, + {"lc": 3, "sum": 8.02693709375}, + {"lc": 4, "sum": 2969.9735879000873}, + {"lc": 5, "sum": 304.7826624218743}, + {"lc": 6, "sum": 0.2828043515625}]}, + {"biobelt": 4, + "groups": [{"lc": 1, "sum": 168.3620734921871}, + {"lc": 10, "sum": 72.22220274218756}, + {"lc": 2, "sum": 3730.0379216579613}, + {"lc": 3, "sum": 54.574133561213195}, + {"lc": 4, "sum": 29961.075690758567}, + {"lc": 5, "sum": 2908.4876023248403}, + {"lc": 6, "sum": 23.637147807444848}, + {"lc": 8, "sum": 0.4717225078125}]}]}, + "2000_2015_2018": {"baseline_degradation": [{"biobelt": 2, + "groups": [{"lc": 2, "sum": 21.238068515625017}]}, + {"biobelt": 3, + "groups": [{"lc": 1, "sum": 20.665385171875002}, + {"lc": 2, "sum": 3648.1207807305054}, + {"lc": 3, "sum": 90.77445744469966}]}, + {"biobelt": 4, + "groups": [{"lc": 1, "sum": 1135.9250689484684}, + {"lc": 2, "sum": 34925.06165119156}, + {"lc": 3, "sum": 857.8817747122545}]}], + "baseline_transition": [{"biobelt": 2, + "groups": [{"lc": 303, "sum": 0.7551212500000001}, + {"lc": 404, "sum": 14.158790898437498}, + {"lc": 505, "sum": 5.946592710937501}, + {"lc": 606, "sum": 0.37756365625}]}, + {"biobelt": 3, + "groups": [{"lc": 101, "sum": 5.193617820312501}, + {"lc": 201, "sum": 0.094475375}, + {"lc": 202, "sum": 417.8766799007347}, + {"lc": 204, "sum": 53.32985848376223}, + {"lc": 301, "sum": 0.188900203125}, + {"lc": 303, "sum": 7.083183734375001}, + {"lc": 304, "sum": 0.75485315625}, + {"lc": 401, "sum": 0.3777951328125}, + {"lc": 402, "sum": 9.72162321875}, + {"lc": 403, "sum": 0.094531125}, + {"lc": 404, "sum": 2950.157769634462}, + {"lc": 405, "sum": 9.6218687890625}, + {"lc": 501, "sum": 0.566191328125}, + {"lc": 504, "sum": 36.689745804687504}, + {"lc": 505, "sum": 267.5267252890623}, + {"lc": 606, "sum": 0.2828043515625}]}, + {"biobelt": 4, + "groups": [{"lc": 1001, "sum": 0.472149234375}, + {"lc": 1004, "sum": 0.9442580312500001}, + {"lc": 1006, "sum": 0.4721610625}, + {"lc": 1008, "sum": 2.5495191562499993}, + {"lc": 101, "sum": 168.3620734921871}, + {"lc": 1010, "sum": 67.78411525781249}, + {"lc": 201, "sum": 21.711948324969367}, + {"lc": 202, "sum": 3146.0770858570377}, + {"lc": 204, "sum": 562.1545618118869}, + {"lc": 205, "sum": 0.0943256640625}, + {"lc": 301, "sum": 20.490742671874987}, + {"lc": 303, "sum": 33.42306504558823}, + {"lc": 304, "sum": 0.6603258437499999}, + {"lc": 401, "sum": 20.302875874999994}, + {"lc": 402, "sum": 487.86584193982833}, + {"lc": 403, "sum": 5.003511789062501}, + {"lc": 404, "sum": 28876.32492851002}, + {"lc": 405, "sum": 568.4645700508577}, + {"lc": 406, "sum": 3.1139625937499997}, + {"lc": 501, "sum": 4.0603895078125}, + {"lc": 504, "sum": 294.8781569472426}, + {"lc": 505, "sum": 2609.5490558697857}, + {"lc": 601, "sum": 2.4548979296875}, + {"lc": 604, "sum": 1.132203375}, + {"lc": 606, "sum": 20.050046502757354}, + {"lc": 804, "sum": 0.0944044453125}, + {"lc": 808, "sum": 0.3773180625}]}], + "final_degradation": [{"biobelt": 2, + "groups": [{"lc": 2, "sum": 21.238068515625017}]}, + {"biobelt": 3, + "groups": [{"lc": 1, "sum": 27.1817221796875}, + {"lc": 2, "sum": 3728.039329183025}, + {"lc": 3, "sum": 4.3395719843750005}]}, + {"biobelt": 4, + "groups": [{"lc": 1, "sum": 1545.648000734526}, + {"lc": 2, "sum": 35322.996898}, + {"lc": 3, "sum": 50.2235961177696}]}], + "report_transition": [{"biobelt": 2, + "groups": [{"lc": 303, "sum": 0.7551212500000001}, + {"lc": 404, "sum": 14.158790898437498}, + {"lc": 505, "sum": 5.946592710937501}, + {"lc": 606, "sum": 0.37756365625}]}, + {"biobelt": 3, + "groups": [{"lc": 101, "sum": 6.420979859375002}, + {"lc": 202, "sum": 424.57879383823496}, + {"lc": 204, "sum": 3.01950928125}, + {"lc": 303, "sum": 7.177714859375}, + {"lc": 401, "sum": 0.0945545703125}, + {"lc": 402, "sum": 5.4792472265624985}, + {"lc": 404, "sum": 3033.377625196347}, + {"lc": 405, "sum": 1.9808000859375001}, + {"lc": 504, "sum": 2.358327578125}, + {"lc": 505, "sum": 274.7902664999992}, + {"lc": 606, "sum": 0.2828043515625}]}, + {"biobelt": 4, + "groups": [{"lc": 101, "sum": 237.8550770359069}, + {"lc": 1010, "sum": 67.78411525781249}, + {"lc": 201, "sum": 0.094402453125}, + {"lc": 202, "sum": 3611.7380999447223}, + {"lc": 204, "sum": 22.110425399019608}, + {"lc": 301, "sum": 0.1888538125}, + {"lc": 303, "sum": 37.95441260808824}, + {"lc": 304, "sum": 0.2833104140625}, + {"lc": 401, "sum": 0.094376203125}, + {"lc": 402, "sum": 206.30337532542885}, + {"lc": 403, "sum": 0.4724664296875}, + {"lc": 404, "sum": 29315.896854881586}, + {"lc": 405, "sum": 211.81753143719362}, + {"lc": 406, "sum": 1.6042346875}, + {"lc": 501, "sum": 0.188779140625}, + {"lc": 504, "sum": 37.36106989062498}, + {"lc": 505, "sum": 3140.558102553451}, + {"lc": 601, "sum": 0.1887590234375}, + {"lc": 606, "sum": 23.447411135569855}, + {"lc": 808, "sum": 2.9268372187499994}]}]}} +} \ No newline at end of file diff --git a/ui.ipynb b/ui.ipynb index 2266f74..eabf15b 100644 --- a/ui.ipynb +++ b/ui.ipynb @@ -189,9 +189,9 @@ ], "metadata": { "kernelspec": { - "display_name": " (venv) sepal_mgci", + "display_name": "(test) test-sepal_mgci", "language": "python", - "name": "venv-sepal_mgci" + "name": "test-sepal_mgci" }, "language_info": { "codemirror_mode": { From c3cf7041d26016f3316458a9531625bac3423160 Mon Sep 17 00:00:00 2001 From: Daniel Guerrero Date: Tue, 24 Sep 2024 04:34:07 -0500 Subject: [PATCH 4/5] fix: remove proje4ct from imports, update repo if it is already cloned (#68) --- SDG_15_4_2_Sub_A_Default_values.ipynb | 317 +++++++++++++------------- SDG_15_4_2_Sub_B_Default_values.ipynb | 307 ++++++++++++------------- component/scripts/colab_imports.py | 58 +++-- 3 files changed, 352 insertions(+), 330 deletions(-) diff --git a/SDG_15_4_2_Sub_A_Default_values.ipynb b/SDG_15_4_2_Sub_A_Default_values.ipynb index 8e79479..2441e50 100644 --- a/SDG_15_4_2_Sub_A_Default_values.ipynb +++ b/SDG_15_4_2_Sub_A_Default_values.ipynb @@ -3,8 +3,8 @@ { "cell_type": "markdown", "metadata": { - "id": "view-in-github", - "colab_type": "text" + "colab_type": "text", + "id": "view-in-github" }, "source": [ "\"Open" @@ -12,6 +12,9 @@ }, { "cell_type": "markdown", + "metadata": { + "id": "82FWR5yMh0HV" + }, "source": [ "# **SDG 15.4.2 Sub-indicator A: Calculate Global Default Values**\n", "\n", @@ -22,35 +25,32 @@ "* Runs on the cloud using [Google Colab](https://research.google.com/colaboratory/faq.html)\n", "\n", "* Requires: [Google Earth Engine](https://earthengine.google.com/) (GEE) account and project and access to Google Drive\n" - ], - "metadata": { - "id": "82FWR5yMh0HV" - } + ] }, { "cell_type": "markdown", - "source": [ - "### 1) Install required packages" - ], "metadata": { "id": "YdoCOI_1yWY0" - } + }, + "source": [ + "### 1) Install required packages" + ] }, { "cell_type": "code", "execution_count": 1, "metadata": { - "id": "Kg3K1EPJu_f5", "colab": { "base_uri": "https://localhost:8080/", "height": 208 }, + "id": "Kg3K1EPJu_f5", "outputId": "460f7ba2-8668-4c4d-824b-6d5068884129" }, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.1/6.1 MB\u001b[0m \u001b[31m34.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.7/2.7 MB\u001b[0m \u001b[31m54.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", @@ -93,44 +93,29 @@ }, { "cell_type": "markdown", + "metadata": { + "id": "qGfLFLEkwS1n" + }, "source": [ "### 2) Access GitHub repository\n", "Clones repository for SDG 15.4.2 into colab.\n", "Provides functions and lookup tables etc." - ], - "metadata": { - "id": "qGfLFLEkwS1n" - } + ] }, { "cell_type": "code", - "source": [ - "# Change the current working directory to \"/content\" for cloning the repo into.\n", - "%cd \"/content\"\n", - "\n", - "# Clone the GitHub repository \"sepalz_mgci\" into the current directory.\n", - "# NB 'fatal' error on reruns are typically just saying it already exists\n", - "!git clone https://github.com/sepal-contrib/sepal_mgci\n", - "\n", - "# Change working directory to the cloned sepal_mgci github repository\n", - "%cd \"/content/sepal_mgci\"" - ], + "execution_count": 2, "metadata": { - "id": "TNohZrOTvNqT", "colab": { "base_uri": "https://localhost:8080/", "height": 173 }, + "id": "TNohZrOTvNqT", "outputId": "1ad0a56a-2b6e-4966-ab95-a55a1a612aae" }, - "execution_count": 2, "outputs": [ { - "output_type": "display_data", "data": { - "text/plain": [ - "" - ], "text/html": [ "\n", " \n", " " + ], + "text/plain": [ + "" ] }, - "metadata": {} + "metadata": {}, + "output_type": "display_data" }, { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "/content\n", "Cloning into 'sepal_mgci'...\n", @@ -174,20 +163,37 @@ "/content/sepal_mgci\n" ] } + ], + "source": [ + "# Change the current working directory to \"/content\" for cloning the repo into.\n", + "%cd \"/content\"\n", + "\n", + "# Clone the GitHub repository \"sepalz_mgci\" into the current directory.\n", + "# NB 'fatal' error on reruns are typically just saying it already exists\n", + "!git clone https://github.com/sepal-contrib/sepal_mgci\n", + "!cd sepal_mgci && git pull\n", + "\n", + "# Change working directory to the cloned sepal_mgci github repository\n", + "%cd \"/content/sepal_mgci\"" ] }, { "cell_type": "markdown", + "metadata": { + "id": "Lbg9P7frZRy1" + }, "source": [ "### 3) Setup Google Earth Engine\n", "Launches access request pop up window" - ], - "metadata": { - "id": "Lbg9P7frZRy1" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "SLLcPONnZRy2" + }, + "outputs": [], "source": [ "# Google Earth Engine project\n", "gee_project_name = \"ee-andyarnellgee\" # \"insert cloud project here\" # a registered cloud project (if unsure of name see pic here: https://developers.google.com/earth-engine/cloud/assets)\n", @@ -197,80 +203,80 @@ "ee.Authenticate()\n", "\n", "ee.Initialize(project=gee_project_name) # NB gee project name is defined in parameters section" - ], - "metadata": { - "id": "SLLcPONnZRy2" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", + "metadata": { + "id": "cxXsgaXEZRy3" + }, "source": [ "### 4) Setup Google Drive\n", "Launches access request pop up window" - ], - "metadata": { - "id": "cxXsgaXEZRy3" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "e2NxIhcgZRy3" + }, + "outputs": [], "source": [ "# for accessing google drive\n", "from google.colab import auth, drive\n", "from googleapiclient.discovery import build\n", "\n", "drive.mount('/content/drive')" - ], - "metadata": { - "id": "e2NxIhcgZRy3" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", + "metadata": { + "id": "MsQu9VfqZRy3" + }, "source": [ "### 5) Import remaining packages\n", "- imports required packages and functions listed in 'colab_imports.py' script (found here: sepal_mgci/component/scripts)\n", "\n" - ], - "metadata": { - "id": "MsQu9VfqZRy3" - } + ] }, { "cell_type": "code", - "source": [ - "from component.scripts.colab_imports import *" - ], + "execution_count": null, "metadata": { "id": "NQ9flKZGZRy4" }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "from component.scripts.colab_imports import *" + ] }, { "cell_type": "markdown", - "source": [ - "### 6) Set parameters\n" - ], "metadata": { "id": "Bi0XpRRDo6V1" - } + }, + "source": [ + "### 6) Set parameters\n" + ] }, { "cell_type": "markdown", - "source": [ - "Input parameters" - ], "metadata": { "id": "5aIr_OxkG-Ky" - } + }, + "source": [ + "Input parameters" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "t9C1qT5bGoqt" + }, + "outputs": [], "source": [ "# Google Earth Engine project\n", "gee_project_name = \"ee-andyarnellgee\" # \"insert cloud project here\" # a registered cloud project (if unsure of name see pic here: https://developers.google.com/earth-engine/cloud/assets)\n", @@ -292,27 +298,27 @@ " 4: {\"year\": 2010, \"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2010\"},\n", "}\n", "\n" - ], - "metadata": { - "id": "t9C1qT5bGoqt" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", + "metadata": { + "id": "naz3Qe4JHFER" + }, "source": [ "Output parameters\n", "\n", "---\n", "\n" - ], - "metadata": { - "id": "naz3Qe4JHFER" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mSvTM29TnndR" + }, + "outputs": [], "source": [ "final_report_folder = \"sdg_15_4_2_A_combined_report\" # folder name in Google Drive for final output (if doesnt exist creates one)\n", "\n", @@ -323,24 +329,24 @@ "\n", "# prints more messages\n", "debug = False # default: False. Set to True if debugging code" - ], - "metadata": { - "id": "mSvTM29TnndR" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", - "source": [ - "Temporary output parameters\n" - ], "metadata": { "id": "iV8-wbddOJuT" - } + }, + "source": [ + "Temporary output parameters\n" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "TCBNkALuOL1N" + }, + "outputs": [], "source": [ "stats_csv_folder = \"sdg_15_4_2_A_csvs\" # for storing stats tables exported from GEE for each admin boundary/AOI\n", "\n", @@ -349,12 +355,7 @@ "drive_home =\"/content/drive/MyDrive/\" # Google Drive location. Don't change unless you know this is incorrect\n", "\n", "error_log_file_path = drive_home + excel_reports_folder + \"/\"+\"1_error_log\" +\".csv\" # for storing errors\n" - ], - "metadata": { - "id": "TCBNkALuOL1N" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -367,15 +368,20 @@ }, { "cell_type": "markdown", - "source": [ - "Create list of boundaries to process" - ], "metadata": { "id": "qi5zNGTWN3df" - } + }, + "source": [ + "Create list of boundaries to process" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BiuBEJwPue2v" + }, + "outputs": [], "source": [ "# admin boundary feature collection\n", "admin_boundaries = ee.FeatureCollection(admin_asset_id)\n", @@ -388,21 +394,16 @@ "list_of_countries = list(set(list_of_countries)) # remove dupicates\n", "\n", "print (\"Length of distinct admin boundaries to process\", (len(set(list_of_countries))))\n" - ], - "metadata": { - "id": "BiuBEJwPue2v" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", - "source": [ - "Read the default land cover remapping table and convert it to a dictionary" - ], "metadata": { "id": "2G1q9TSiUsc1" - } + }, + "source": [ + "Read the default land cover remapping table and convert it to a dictionary" + ] }, { "cell_type": "code", @@ -417,31 +418,27 @@ }, { "cell_type": "markdown", - "source": [ - "Select years of land cover to process" - ], "metadata": { "id": "4-Ut_-S35Yg8" - } + }, + "source": [ + "Select years of land cover to process" + ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "PwqJFWR4u_f7", "colab": { "base_uri": "https://localhost:8080/", "height": 17 }, + "id": "PwqJFWR4u_f7", "outputId": "09d3ab98-6b20-4212-b5ed-f3386c3a32bf" }, "outputs": [ { - "output_type": "display_data", "data": { - "text/plain": [ - "" - ], "text/html": [ "\n", " \n", " " + ], + "text/plain": [ + "" ] }, - "metadata": {} + "metadata": {}, + "output_type": "display_data" } ], "source": [ @@ -478,15 +479,15 @@ }, { "cell_type": "markdown", + "metadata": { + "id": "iNxHtR984cNk" + }, "source": [ "### 8) Calculate area statistics by country\n", "* Runs for each country and each mountain biobelt\n", "* Gets area of land cover reclassified into the 10 SEAM classes\n", "* Repeat for each year specified\n" - ], - "metadata": { - "id": "iNxHtR984cNk" - } + ] }, { "cell_type": "code", @@ -561,6 +562,9 @@ }, { "cell_type": "markdown", + "metadata": { + "id": "3x7jwkZJWwE-" + }, "source": [ "#####NOTE: you will need to wait until results files for each country have been created in your google drive (from previous step).\n", "- see here to monitor the tasks https://code.earthengine.google.com/tasks\n", @@ -568,10 +572,7 @@ "\n", "This cell formats individual excel reports for each country.\n", "See Error_log.csv for missing files/errors" - ], - "metadata": { - "id": "3x7jwkZJWwE-" - } + ] }, { "cell_type": "code", @@ -665,24 +666,29 @@ }, { "cell_type": "markdown", - "source": [ - "### 10) Combine excel report files into one" - ], "metadata": { "id": "qTqI3Ag08k5b" - } + }, + "source": [ + "### 10) Combine excel report files into one" + ] }, { "cell_type": "markdown", - "source": [ - "Make a list of files to combine" - ], "metadata": { "id": "sVpLtG4Z7Jbe" - } + }, + "source": [ + "Make a list of files to combine" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sTyhwne7rr9D" + }, + "outputs": [], "source": [ "# Directory path where Excel reports are stored\n", "directory_path = os.path.join(drive_home, excel_reports_folder)\n", @@ -701,37 +707,36 @@ "\n", "# File path for the combined final report\n", "reports_combined_file_path = os.path.join(drive_home, final_report_folder, final_report_name)\n" - ], - "metadata": { - "id": "sTyhwne7rr9D" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", - "source": [ - "##### Run function to combine into a single report" - ], "metadata": { "id": "C77flAQu7xWX" - } + }, + "source": [ + "##### Run function to combine into a single report" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FkS1ErA9AYj6" + }, + "outputs": [], "source": [ "append_excel_files(file_paths=full_file_paths,num_sheets=3,output_file_path=reports_combined_file_path)\n", "\n", "print (f\"\\n Complete! Output file for SDG 15.4.2 Sub-indicator A here: {reports_combined_file_path}\")" - ], - "metadata": { - "id": "FkS1ErA9AYj6" - }, - "execution_count": null, - "outputs": [] + ] } ], "metadata": { + "colab": { + "include_colab_link": true, + "provenance": [] + }, "kernelspec": { "display_name": "(test) test-sepal_mgci", "language": "python", @@ -748,10 +753,6 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" - }, - "colab": { - "provenance": [], - "include_colab_link": true } }, "nbformat": 4, diff --git a/SDG_15_4_2_Sub_B_Default_values.ipynb b/SDG_15_4_2_Sub_B_Default_values.ipynb index 72b988d..5e834e8 100644 --- a/SDG_15_4_2_Sub_B_Default_values.ipynb +++ b/SDG_15_4_2_Sub_B_Default_values.ipynb @@ -3,8 +3,8 @@ { "cell_type": "markdown", "metadata": { - "id": "view-in-github", - "colab_type": "text" + "colab_type": "text", + "id": "view-in-github" }, "source": [ "\"Open" @@ -12,6 +12,9 @@ }, { "cell_type": "markdown", + "metadata": { + "id": "82FWR5yMh0HV" + }, "source": [ "# **SDG 15.4.2 Sub-indicator B: Calculate Global Default Values**\n", "\n", @@ -22,19 +25,16 @@ "* Runs on the cloud using [Google Colab](https://research.google.com/colaboratory/faq.html)\n", "\n", "* Requires: [Google Earth Engine](https://earthengine.google.com/) (GEE) account and project and access to Google Drive\n" - ], - "metadata": { - "id": "82FWR5yMh0HV" - } + ] }, { "cell_type": "markdown", - "source": [ - "### 1) Install required packages" - ], "metadata": { "id": "YdoCOI_1yWY0" - } + }, + "source": [ + "### 1) Install required packages" + ] }, { "cell_type": "code", @@ -70,17 +70,22 @@ }, { "cell_type": "markdown", + "metadata": { + "id": "qGfLFLEkwS1n" + }, "source": [ "### 2) Access GitHub repository\n", "Clones repository for SDG 15.4.2 into colab.\n", "Provides functions and lookup tables etc." - ], - "metadata": { - "id": "qGfLFLEkwS1n" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "TNohZrOTvNqT" + }, + "outputs": [], "source": [ "# Change the current working directory to \"/content\" for cloning the repo into.\n", "%cd \"/content\"\n", @@ -88,28 +93,29 @@ "# Clone the GitHub repository \"sepal_mgci\" into the current directory.\n", "# NB 'fatal' error on reruns are typically just saying it already exists\n", "!git clone https://github.com/sepal-contrib/sepal_mgci\n", + "!cd sepal_mgci && git pull \n", "\n", "# Change working directory to the cloned sepal_mgci github repository\n", "%cd \"/content/sepal_mgci\"" - ], - "metadata": { - "id": "TNohZrOTvNqT" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", + "metadata": { + "id": "FAzzmzMHzFmS" + }, "source": [ "### 3) Setup Google Earth Engine\n", "Launches access request pop up window" - ], - "metadata": { - "id": "FAzzmzMHzFmS" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gHI-M8y_qwaP" + }, + "outputs": [], "source": [ "# Google Earth Engine project\n", "gee_project_name = \"ee-andyarnellgee\" # \"insert cloud project here\" # a registered cloud project (if unsure of name see pic here: https://developers.google.com/earth-engine/cloud/assets)\n", @@ -119,80 +125,80 @@ "ee.Authenticate()\n", "\n", "ee.Initialize(project=gee_project_name) # NB gee project name is defined in parameters section" - ], - "metadata": { - "id": "gHI-M8y_qwaP" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", + "metadata": { + "id": "aUVGL6zwLpX5" + }, "source": [ "### 4) Setup Google Drive\n", "Launches access request pop up window" - ], - "metadata": { - "id": "aUVGL6zwLpX5" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NorUzAJ8Kj0Z" + }, + "outputs": [], "source": [ "# for accessing google drive\n", "from google.colab import auth, drive\n", "from googleapiclient.discovery import build\n", "\n", "drive.mount('/content/drive')" - ], - "metadata": { - "id": "NorUzAJ8Kj0Z" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", + "metadata": { + "id": "L5jsWgqWyiCP" + }, "source": [ "### 5) Import remaining packages\n", "- imports required packages and functions listed here: sepal_mgci/component/scripts/colab_imports.py\n", "\n" - ], - "metadata": { - "id": "L5jsWgqWyiCP" - } + ] }, { "cell_type": "code", - "source": [ - "from component.scripts.colab_imports import" - ], + "execution_count": null, "metadata": { "id": "2Wquh4ZQbgfZ" }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "from component.scripts.colab_imports import" + ] }, { "cell_type": "markdown", - "source": [ - "### 6) Set parameters\n" - ], "metadata": { "id": "Bi0XpRRDo6V1" - } + }, + "source": [ + "### 6) Set parameters\n" + ] }, { "cell_type": "markdown", - "source": [ - "Input parameters" - ], "metadata": { "id": "5aIr_OxkG-Ky" - } + }, + "source": [ + "Input parameters" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "t9C1qT5bGoqt" + }, + "outputs": [], "source": [ "\n", "# Admin boundaries asset\n", @@ -218,28 +224,28 @@ " # And the reporting year\n", " 2: {\"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2018\", \"year\": 2018},\n", "}\n" - ], - "metadata": { - "id": "t9C1qT5bGoqt" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", + "metadata": { + "id": "naz3Qe4JHFER" + }, "source": [ "Output parameters\n", "- NB if no CSVs are being created, check that 'export = True' below.\n", "\n", "---\n", "\n" - ], - "metadata": { - "id": "naz3Qe4JHFER" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mSvTM29TnndR" + }, + "outputs": [], "source": [ "final_report_folder = \"sdg_15_4_2_B_combined_report\" # folder name in Google Drive for final output (if doesnt exist creates one)\n", "\n", @@ -250,24 +256,24 @@ "\n", "# prints more messages\n", "debug = False # default: False. Set to True if debugging code" - ], - "metadata": { - "id": "mSvTM29TnndR" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", - "source": [ - "Temporary output parameters\n" - ], "metadata": { "id": "iV8-wbddOJuT" - } + }, + "source": [ + "Temporary output parameters\n" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "TCBNkALuOL1N" + }, + "outputs": [], "source": [ "stats_csv_folder = \"sdg_15_4_2_B_csvs\" # for storing stats tables exported from GEE for each admin boundary/AOI\n", "\n", @@ -276,33 +282,33 @@ "drive_home =\"/content/drive/MyDrive/\" # Google Drive location. Don't change unless you know this is incorrect\n", "\n", "error_log_file_path = drive_home + excel_reports_folder + \"/\"+\"1_error_log\" +\".csv\" # for storing errors\n" - ], - "metadata": { - "id": "TCBNkALuOL1N" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", - "source": [ - "### 7) Setup inputs for processing\n" - ], "metadata": { "id": "dVfL5w0eBi4I" - } + }, + "source": [ + "### 7) Setup inputs for processing\n" + ] }, { "cell_type": "markdown", - "source": [ - "Create list of boundaries to process" - ], "metadata": { "id": "qi5zNGTWN3df" - } + }, + "source": [ + "Create list of boundaries to process" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BiuBEJwPue2v" + }, + "outputs": [], "source": [ "# admin boundary feature collection\n", "admin_boundaries = ee.FeatureCollection(admin_asset_id)\n", @@ -315,21 +321,16 @@ "list_of_countries = list(set(list_of_countries)) # remove dupicates\n", "\n", "print (\"Length of distinct admin boundaries to process\", (len(set(list_of_countries))))\n" - ], - "metadata": { - "id": "BiuBEJwPue2v" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", - "source": [ - "Read the default land cover remapping table and convert it to a dictionary" - ], "metadata": { "id": "2G1q9TSiUsc1" - } + }, + "source": [ + "Read the default land cover remapping table and convert it to a dictionary" + ] }, { "cell_type": "code", @@ -344,40 +345,40 @@ }, { "cell_type": "markdown", - "source": [ - "Set the default transition matrix file path." - ], "metadata": { "id": "P_qX5MSJZ9gs" - } + }, + "source": [ + "Set the default transition matrix file path." + ] }, { "cell_type": "code", - "source": [ - "default_transition_matrix_path = TRANSITION_MATRIX_FILE # location of the CSV table defining which land cover transitions are “degradation” etc.\n", - "print(default_transition_matrix_path) # view the file path" - ], + "execution_count": null, "metadata": { "id": "REH6-a1xZ9VI" }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "default_transition_matrix_path = TRANSITION_MATRIX_FILE # location of the CSV table defining which land cover transitions are “degradation” etc.\n", + "print(default_transition_matrix_path) # view the file path" + ] }, { "cell_type": "markdown", - "source": [ - "Select years of land cover to process" - ], "metadata": { "id": "4-Ut_-S35Yg8" - } + }, + "source": [ + "Select years of land cover to process" + ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "PwqJFWR4u_f7", - "collapsed": true + "collapsed": true, + "id": "PwqJFWR4u_f7" }, "outputs": [], "source": [ @@ -388,15 +389,15 @@ }, { "cell_type": "markdown", + "metadata": { + "id": "iNxHtR984cNk" + }, "source": [ "### 8) Calculate area statistics by country\n", "* Runs for each country and each mountain biobelt\n", "* Gets area of land cover reclassified into the 10 SEAM classes\n", "* Repeat for each year specified\n" - ], - "metadata": { - "id": "iNxHtR984cNk" - } + ] }, { "cell_type": "code", @@ -462,6 +463,9 @@ }, { "cell_type": "markdown", + "metadata": { + "id": "3x7jwkZJWwE-" + }, "source": [ "#####NOTE: you will need to wait until results files for each country have been created in your google drive (from previous step).\n", "- see here to monitor the tasks https://code.earthengine.google.com/tasks\n", @@ -469,10 +473,7 @@ "\n", "This cell formats individual excel reports for each country.\n", "See Error_log.csv for missing files/errors" - ], - "metadata": { - "id": "3x7jwkZJWwE-" - } + ] }, { "cell_type": "code", @@ -586,25 +587,30 @@ }, { "cell_type": "markdown", - "source": [ - "### 10) Combine excel report files into one" - ], "metadata": { "id": "qTqI3Ag08k5b" - } + }, + "source": [ + "### 10) Combine excel report files into one" + ] }, { "cell_type": "markdown", + "metadata": { + "id": "sVpLtG4Z7Jbe" + }, "source": [ "Make a list of files to combine\n", "- Note: only combines excel files that are present in your google drive (from Step 9)." - ], - "metadata": { - "id": "sVpLtG4Z7Jbe" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sTyhwne7rr9D" + }, + "outputs": [], "source": [ "# Directory path where Excel reports are stored\n", "directory_path = os.path.join(drive_home, excel_reports_folder)\n", @@ -623,37 +629,36 @@ "\n", "# File path for the combined final report\n", "reports_combined_file_path = os.path.join(drive_home, final_report_folder, final_report_name)\n" - ], - "metadata": { - "id": "sTyhwne7rr9D" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", - "source": [ - "##### Run function to combine into a single report" - ], "metadata": { "id": "C77flAQu7xWX" - } + }, + "source": [ + "##### Run function to combine into a single report" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FkS1ErA9AYj6" + }, + "outputs": [], "source": [ "append_excel_files(file_paths=full_file_paths,num_sheets=3,output_file_path=reports_combined_file_path)\n", "\n", "print (f\"\\n Complete! Output file for SDG 15.4.2 Component B here: {reports_combined_file_path}\")" - ], - "metadata": { - "id": "FkS1ErA9AYj6" - }, - "execution_count": null, - "outputs": [] + ] } ], "metadata": { + "colab": { + "include_colab_link": true, + "provenance": [] + }, "kernelspec": { "display_name": "(test) test-sepal_mgci", "language": "python", @@ -670,10 +675,6 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" - }, - "colab": { - "provenance": [], - "include_colab_link": true } }, "nbformat": 4, diff --git a/component/scripts/colab_imports.py b/component/scripts/colab_imports.py index 14ae790..dae5ac6 100644 --- a/component/scripts/colab_imports.py +++ b/component/scripts/colab_imports.py @@ -1,15 +1,13 @@ import os -import ee # google earth engine +import ee # google earth engine -ee.Authenticate() - -ee.Initialize(project="ee-andyarnellgee") # NB gee project name is defined in parameters section - -from datetime import datetime # for time stamping error log -import pandas as pd # pandas library for tabular data manipulation -import re # for manipulating strings -from unidecode import unidecode # converting symbols in country names to ascii compliant (required for naming GEE tasks) +from datetime import datetime # for time stamping error log +import pandas as pd # pandas library for tabular data manipulation +import re # for manipulating strings +from unidecode import ( + unidecode, +) # converting symbols in country names to ascii compliant (required for naming GEE tasks) # formatting excel report file from openpyxl.utils import get_column_letter @@ -27,17 +25,39 @@ LC_MAP_MATRIX = Path(__file__).parents[1] / "parameter/lc_map_matrix.csv" -TRANSITION_MATRIX_FILE =Path(__file__).parents[1] / "parameter/transition_matrix.csv" +TRANSITION_MATRIX_FILE = Path(__file__).parents[1] / "parameter/transition_matrix.csv" # # # # Import scripts and modules from cloned GitHub repository (i.e., functions for indicator calculation and formatting) -from component.scripts.gee import reduce_regions # for running summary statistics in GEE - -from component.scripts.scripts import (get_a_years, map_matrix_to_dict, parse_result, read_from_csv, - get_b_years, map_matrix_to_dict, get_sub_b_items, get_reporting_years, parse_to_year)# parameter prep and reformatting -from component.scripts import sub_a, sub_b, mountain_area as mntn ###TO DO: ADD DESCRIPTIONS - -from component.scripts.colab_combining_files import sanitize_description,append_excel_files - -from component.scripts.colab_drive_folders import folder_exists,create_folder, create_folder_if_not_exists +from component.scripts.gee import ( + reduce_regions, +) # for running summary statistics in GEE + +from component.scripts.scripts import ( + get_a_years, + map_matrix_to_dict, + parse_result, + read_from_csv, + get_b_years, + map_matrix_to_dict, + get_sub_b_items, + get_reporting_years, + parse_to_year, +) # parameter prep and reformatting +from component.scripts import ( + sub_a, + sub_b, + mountain_area as mntn, +) ###TO DO: ADD DESCRIPTIONS + +from component.scripts.colab_combining_files import ( + sanitize_description, + append_excel_files, +) + +from component.scripts.colab_drive_folders import ( + folder_exists, + create_folder, + create_folder_if_not_exists, +) print("Imports complete") From 59a547a1007054856b58720e48999b50a67af9aa Mon Sep 17 00:00:00 2001 From: dfguerrerom Date: Wed, 25 Sep 2024 14:58:16 +0200 Subject: [PATCH 5/5] add sepal --- .gitignore | 1 + SDG_15_4_2_Sub_A_Default_values_sepal.ipynb | 750 ++++++++++++++++++++ component/scripts/gee.py | 2 - 3 files changed, 751 insertions(+), 2 deletions(-) create mode 100644 SDG_15_4_2_Sub_A_Default_values_sepal.ipynb diff --git a/.gitignore b/.gitignore index e8d946f..7ae0edc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Byte-compiled / optimized / DLL files __pycache__/ +content/* *.py[cod] *$py.class *Untitled* diff --git a/SDG_15_4_2_Sub_A_Default_values_sepal.ipynb b/SDG_15_4_2_Sub_A_Default_values_sepal.ipynb new file mode 100644 index 0000000..e1f1591 --- /dev/null +++ b/SDG_15_4_2_Sub_A_Default_values_sepal.ipynb @@ -0,0 +1,750 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "view-in-github" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "82FWR5yMh0HV" + }, + "source": [ + "# **SDG 15.4.2 Sub-indicator A: Calculate Global Default Values**\n", + "\n", + "* This script allows batch processing for this indicator for all countries.\n", + "\n", + "* Output is a combined excel file on your Google Drive.\n", + "\n", + "* Runs on the cloud using [Google Colab](https://research.google.com/colaboratory/faq.html)\n", + "\n", + "* Requires: [Google Earth Engine](https://earthengine.google.com/) (GEE) account and project and access to Google Drive\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import ee\n", + "ee.Initialize()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Create a base directory\n", + "\n", + "base_dir = Path(\"content/sepal_mgci\")\n", + "base_dir.mkdir(parents=True, exist_ok=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MsQu9VfqZRy3" + }, + "source": [ + "### 5) Import remaining packages\n", + "- imports required packages and functions listed in 'colab_imports.py' script (found here: sepal_mgci/component/scripts)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "NQ9flKZGZRy4" + }, + "outputs": [ + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": "/*******************************************************************************\n * remove any links from fontawesome 5 created by jupyter in favor of\n * fontawesome 6. to be removed when Jupyter updates it\n */\n\nfunction remove_fa5() {\n let links = document.querySelectorAll(\n \"link[href^='https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@^5']\"\n );\n\n links.forEach((link) => link.remove());\n}\n\nif (document.readyState != \"loading\") remove_fa5();\nelse document.addEventListener(\"DOMContentLoaded\", remove_fa5);\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import os\n", + "\n", + "from datetime import datetime # for time stamping error log\n", + "import pandas as pd # pandas library for tabular data manipulation\n", + "from unidecode import (\n", + " unidecode,\n", + ") # converting symbols in country names to ascii compliant (required for naming GEE tasks)\n", + "\n", + "# formatting excel report file\n", + "from openpyxl.utils import get_column_letter\n", + "from openpyxl.styles import Alignment\n", + "\n", + "from pathlib import Path\n", + "\n", + "# # # # Import scripts and modules from cloned GitHub repository (i.e., functions for indicator calculation and formatting)\n", + "from component.scripts.gee import (\n", + " reduce_regions,\n", + ") # for running summary statistics in GEE\n", + "\n", + "from component.scripts.scripts import (\n", + " get_a_years,\n", + " map_matrix_to_dict,\n", + " parse_result,\n", + " read_from_csv,\n", + " map_matrix_to_dict,\n", + ") # parameter prep and reformatting\n", + "from component.scripts import (\n", + " sub_a,\n", + " mountain_area as mntn,\n", + ") ###TO DO: ADD DESCRIPTIONS\n", + "\n", + "from component.scripts.colab_combining_files import (\n", + " sanitize_description,\n", + " append_excel_files,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Bi0XpRRDo6V1" + }, + "source": [ + "### 6) Set parameters\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5aIr_OxkG-Ky" + }, + "source": [ + "Input parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "DEM_DEFAULT = \"CGIAR/SRTM90_V4\"\n", + "\n", + "# Define the translation matrix between ESA and MGCI LC classes\n", + "\n", + "LC_MAP_MATRIX = Path(\"content/corine_lc_map_matrix2.csv\")\n", + "TRANSITION_MATRIX_FILE = Path(\"component/parameter/transition_matrix.csv\")\n", + "\n", + "# Check they both exist\n", + "assert LC_MAP_MATRIX.exists()\n", + "assert TRANSITION_MATRIX_FILE.exists()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "t9C1qT5bGoqt" + }, + "outputs": [], + "source": [ + "# Admin boundaries asset\n", + "admin_asset_id = \"FAO/GAUL/2015/level0\" # administrative units feature collection\n", + "\n", + "admin_asset_property_name = \"ADM0_NAME\" # property/column name for selecting admin boundaries (e.g. ISO3 code or country name)\n", + "\n", + "\n", + "# Land cover assets\n", + "\n", + "# For Sub-indicator A (sub_a), we need to set the following structure.\n", + "a_years = {\n", + " 1: {\"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2000\", \"year\": 2000}, # baseline\n", + " 2: {\"year\": 2003, \"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2003\"}, # subsequent reporting years...\n", + " 3: {\"year\": 2007, \"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2007\"},\n", + " 4: {\"year\": 2010, \"asset\": \"users/amitghosh/sdg_module/esa/cci_landcover/2010\"},\n", + "}\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "naz3Qe4JHFER" + }, + "source": [ + "Output parameters\n", + "\n", + "---\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mSvTM29TnndR" + }, + "outputs": [], + "source": [ + "final_report_folder = \"sdg_15_4_2_A_combined_report\" # folder name in Google Drive for final output (if doesnt exist creates one)\n", + "\n", + "final_report_name = \"sdg_15_4_2_A_default_global.xlsx\" # file name for final excel output\n", + "\n", + "# export GEE tasks or not\n", + "export = False # default: True. Set to False if debugging or limiting accidental re-exporting of tasks\n", + "\n", + "# prints more messages\n", + "debug = False # default: False. Set to True if debugging code" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iV8-wbddOJuT" + }, + "source": [ + "Temporary output parameters\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "TCBNkALuOL1N" + }, + "outputs": [], + "source": [ + "stats_csv_folder = \"sdg_15_4_2_A_csvs\" # for storing stats tables exported from GEE for each admin boundary/AOI\n", + "\n", + "excel_reports_folder = \"sdg_15_4_2_A_reports\" # for storing formatted excel tables for each admin boundary/AOI\n", + "\n", + "drive_home =\"/content/drive/MyDrive/\" # Google Drive location. Don't change unless you know this is incorrect\n", + "\n", + "error_log_file_path = drive_home + excel_reports_folder + \"/\"+\"1_error_log\" +\".csv\" # for storing errors\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "c_8UzlUnu_f6" + }, + "source": [ + "### 7) Setup inputs for processing" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qi5zNGTWN3df" + }, + "source": [ + "Create list of boundaries to process" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BiuBEJwPue2v" + }, + "outputs": [], + "source": [ + "# admin boundary feature collection\n", + "admin_boundaries = ee.FeatureCollection(admin_asset_id)\n", + "\n", + "# list to process\n", + "list_of_countries = admin_boundaries.aggregate_array(admin_asset_property_name).getInfo()\n", + "\n", + "print (\"Length of admin boundaries to process\", len(list_of_countries))\n", + "\n", + "list_of_countries = list(set(list_of_countries)) # remove dupicates\n", + "\n", + "print (\"Length of distinct admin boundaries to process\", (len(set(list_of_countries))))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2G1q9TSiUsc1" + }, + "source": [ + "Read the default land cover remapping table and convert it to a dictionary" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PRSEqq5bu_f7" + }, + "outputs": [], + "source": [ + "default_map_matrix = map_matrix_to_dict(LC_MAP_MATRIX)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4-Ut_-S35Yg8" + }, + "source": [ + "Select years of land cover to process" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 17 + }, + "id": "PwqJFWR4u_f7", + "outputId": "09d3ab98-6b20-4212-b5ed-f3386c3a32bf" + }, + "outputs": [], + "source": [ + "# extracts the years from the a_years dictionary (as defined in parameters)\n", + "single_years = [y[\"year\"] for y in a_years.values()]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iNxHtR984cNk" + }, + "source": [ + "### 8) Calculate area statistics by country\n", + "* Runs for each country and each mountain biobelt\n", + "* Gets area of land cover reclassified into the 10 SEAM classes\n", + "* Repeat for each year specified\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "VftKLuY4u_f7" + }, + "outputs": [], + "source": [ + "# you can monitor your GEE tasks here : https://code.earthengine.google.com/tasks\n", + "create_folder_if_not_exists(stats_csv_folder) # to store outputs in google drive\n", + "\n", + "counter=0 # starting place of counter used to keep track of number of tasks that are being run\n", + "\n", + "for aoi_name in list_of_countries:\n", + "\n", + " aoi = admin_boundaries.filter(ee.Filter.eq(admin_asset_property_name,aoi_name))#.first()\n", + "\n", + " # gets areas of landcover in each mountain belt in each country\n", + " # uses reduce_regions function imported from the cloned sepal_mgci git hub repository (see Imports section)\n", + " # pixels counted at native resolution (scale) of input land cover (or DEM if RSA implementation)\n", + " process = ee.FeatureCollection([\n", + " ee.Feature(\n", + " None,\n", + " reduce_regions(\n", + " aoi,\n", + " remap_matrix=default_map_matrix,14 test sepal_mgci /home/dguerrero/module-venv/sepal_mgci\n", + " dem=DEM_DEFAULT, #default digital elevation model (DEM). Relevant for the real surface area (RSA) implementation.\n", + " lc_years= year,\n", + " transition_matrix=False\n", + " )\n", + " ).set(\"process_id\", year[0][\"year\"])\n", + " for year in get_a_years(a_years) # creates GEE images and runs stats on each. Images to run are in the 'a_years\" dictionary (above)\n", + " ])\n", + "\n", + " #make name acceptable for running tasks (i.e., removes special characters)\n", + " task_name = str(sanitize_description(unidecode(aoi_name)))\n", + "\n", + "\n", + " task = ee.batch.Export.table.toDrive(\n", + " **{ #asterisks unpack dictionary into keyword arguments format\n", + " \"collection\": process,\n", + " \"description\": task_name,\n", + " \"fileFormat\": \"CSV\",\n", + " \"folder\":stats_csv_folder,\n", + " \"selectors\": [\n", + " \"process_id\",\n", + " \"sub_a\",\n", + " ],\n", + " }\n", + " )\n", + "\n", + " counter+=1\n", + "\n", + " print (f\"\\r process {counter}/{len(list_of_countries)} {aoi_name} \", end=\"\") #print in place (remove \\r and end=\"\" for verbose version)\n", + "\n", + " if export:\n", + " task.start()\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vM0SIvJtu_f8" + }, + "source": [ + "### 9) Read and translate results into report tables" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3x7jwkZJWwE-" + }, + "source": [ + "#####NOTE: you will need to wait until results files for each country have been created in your google drive (from previous step).\n", + "- see here to monitor the tasks https://code.earthengine.google.com/tasks\n", + "- once tasks are complete, you can run the cell below\n", + "\n", + "This cell formats individual excel reports for each country.\n", + "See Error_log.csv for missing files/errors" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qkpDfHqQu_f9" + }, + "outputs": [], + "source": [ + "# Initialize the counter\n", + "counter = 0\n", + "\n", + "# to store outputs in google drive\n", + "create_folder_if_not_exists(excel_reports_folder)\n", + "\n", + "# Loop over each AOI name in the list of countries\n", + "for aoi_name in list_of_countries:\n", + " counter += 1\n", + "\n", + " # Clean the AOI name\n", + " aoi_name_clean = str(sanitize_description(unidecode(aoi_name)))\n", + "\n", + " # Construct the file path for the stats CSV file\n", + " stats_csv_file = aoi_name_clean + \".csv\"\n", + " stats_csv_file_path = os.path.join(drive_home, stats_csv_folder, stats_csv_file)\n", + "\n", + " message = f\"Process {counter}, {stats_csv_file}\"\n", + "\n", + " try:\n", + " # Read the results from the CSV file and parse it to a dictionary\n", + " dict_results = read_from_csv(stats_csv_file_path)\n", + "\n", + " details = {\n", + " \"geo_area_name\": aoi_name,\n", + " \"ref_area\": \" \",\n", + " \"source_detail\": \" \",\n", + " }\n", + "\n", + " # Generate reports for the sub_a and mtn indicators\n", + " sub_a_reports = [sub_a.get_reports(parse_result(dict_results[year][\"sub_a\"], single=True), year, **details) for year in single_years]\n", + " mtn_reports = [mntn.get_report(parse_result(dict_results[year][\"sub_a\"], single=True), year, **details) for year in single_years]\n", + "\n", + " # Concatenate the mtn reports\n", + " mtn_reports_df = pd.concat(mtn_reports)\n", + "\n", + " # Concatenate the sub a reports\n", + " er_mtn_grnvi_df = pd.concat([report[0] for report in sub_a_reports])\n", + " er_mtn_grncov_df = pd.concat([report[1] for report in sub_a_reports])\n", + "\n", + " # Define the output report file path\n", + " report_file_path = os.path.join(drive_home, excel_reports_folder, aoi_name_clean + \".xlsx\")\n", + "\n", + " # Create the Excel file with the reports\n", + " with pd.ExcelWriter(report_file_path) as writer:\n", + " mtn_reports_df.to_excel(writer, sheet_name=\"Table1_ER_MTN_TOTL\", index=False)\n", + " er_mtn_grncov_df.to_excel(writer, sheet_name=\"Table2_ER_MTN_GRNCOV\", index=False)\n", + " er_mtn_grnvi_df.to_excel(writer, sheet_name=\"Table3_ER_MTN_GRNCVI\", index=False)\n", + "\n", + " # Adjust column widths and alignment for each sheet\n", + " for sheetname in writer.sheets:\n", + " worksheet = writer.sheets[sheetname]\n", + " for col in worksheet.columns:\n", + " max_length = max(len(str(cell.value)) for cell in col)\n", + " column = col[0]\n", + " adjusted_width = max(max_length, len(str(column.value))) + 4\n", + " worksheet.column_dimensions[get_column_letter(column.column)].width = adjusted_width\n", + "\n", + " # Align \"obs_value\" column to the right\n", + " if \"OBS\" in column.value:\n", + " for cell in col:\n", + " cell.alignment = Alignment(horizontal=\"right\")\n", + "\n", + " except Exception as e:\n", + " # If an error occurs, catch the exception and handle it\n", + " message = f\"process {counter}, {stats_csv_file}, Error: {e}\"\n", + "\n", + " # Get the current time\n", + " current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')\n", + "\n", + " # Write the error message and file name to the error log file\n", + " error_info = pd.DataFrame([[stats_csv_file, str(e), current_time]], columns=['File Name', 'Error Message', 'Time'])\n", + "\n", + " mode = 'w' if not os.path.exists(error_log_file_path) else 'a'\n", + " header = False if os.path.exists(error_log_file_path) else True\n", + "\n", + " # Append or write to the error log file\n", + " error_info.to_csv(error_log_file_path, mode=mode, header=header, index=False)\n", + "\n", + " print(message)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qTqI3Ag08k5b" + }, + "source": [ + "### 10) Combine excel report files into one" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sVpLtG4Z7Jbe" + }, + "source": [ + "Make a list of files to combine" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sTyhwne7rr9D" + }, + "outputs": [], + "source": [ + "# Directory path where Excel reports are stored\n", + "directory_path = os.path.join(drive_home, excel_reports_folder)\n", + "\n", + "# List files in the directory with '.xlsx' extension\n", + "files = [file for file in os.listdir(directory_path) if file.endswith('.xlsx')]\n", + "\n", + "# Create a list of full file paths\n", + "full_file_paths = [os.path.join(directory_path, file) for file in files]\n", + "\n", + "# Print the number of Excel files found in the folder\n", + "print(f\"Number of Excel files in folder: {len(full_file_paths)}\")\n", + "\n", + "# folder to store outputs in google drive\n", + "create_folder_if_not_exists(final_report_folder)\n", + "\n", + "# File path for the combined final report\n", + "reports_combined_file_path = os.path.join(drive_home, final_report_folder, final_report_name)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "C77flAQu7xWX" + }, + "source": [ + "##### Run function to combine into a single report" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FkS1ErA9AYj6" + }, + "outputs": [], + "source": [ + "append_excel_files(file_paths=full_file_paths,num_sheets=3,output_file_path=reports_combined_file_path)\n", + "\n", + "print (f\"\\n Complete! Output file for SDG 15.4.2 Sub-indicator A here: {reports_combined_file_path}\")" + ] + } + ], + "metadata": { + "colab": { + "include_colab_link": true, + "provenance": [] + }, + "kernelspec": { + "display_name": "(test) test-sepal_mgci", + "language": "python", + "name": "test-sepal_mgci" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/component/scripts/gee.py b/component/scripts/gee.py index 9045640..9bdc27e 100644 --- a/component/scripts/gee.py +++ b/component/scripts/gee.py @@ -14,8 +14,6 @@ def no_remap(image: ee.Image, remap_matrix: Optional[dict] = None): """return remapped or raw image if there's a matrix""" - print(f"NO_DATA_VALUE {NO_DATA_VALUE}") - print(f"NO_DATA_VALUE {remap_matrix}") if remap_matrix: from_, to_ = list(zip(*remap_matrix.items()))