Merge branch 'master' into Add-data-North-Uganda-2020

.github/workflows/data-pipeline.yml

@@ -16,11 +16,11 @@ jobs:
       with:
         ref: ${{ github.event.pull_request.head.ref }}
     - name: Setup environment
-      uses: mamba-org/provision-with-micromamba@main
+      uses: mamba-org/setup-micromamba@v1
       with:
         environment-file: environment-dev.yml
         environment-name: landcover-mapping
-        cache-env: true
+        cache-environment: true
     - uses: google-github-actions/auth@v0
       with:
         credentials_json: ${{ secrets.GCP_SA_KEY }}

.github/workflows/test.yml

@@ -15,11 +15,11 @@ jobs:
       with:
         ref: ${{ github.event.pull_request.head.ref }}
     - name: Setup environment
-      uses: mamba-org/provision-with-micromamba@main
+      uses: mamba-org/setup-micromamba@v1
       with:
         environment-file: environment-dev.yml
         environment-name: landcover-mapping
-        cache-env: true
+        cache-environment: true
     - name: Run unit tests
       run: python -m unittest
 
@@ -34,11 +34,11 @@ jobs:
       with:
         ref: ${{ github.event.pull_request.head.ref }}
     - name: Setup environment
-      uses: mamba-org/provision-with-micromamba@main
+      uses: mamba-org/setup-micromamba@v1
       with:
         environment-file: environment-dev.yml
         environment-name: landcover-mapping
-        cache-env: true
+        cache-environment: true
     - uses: google-github-actions/auth@v0
       with:
         credentials_json: ${{ secrets.GCP_SA_KEY }}
@@ -63,11 +63,11 @@ jobs:
       with:
         ref: ${{ github.event.pull_request.head.ref }}
     - name: Setup environment
-      uses: mamba-org/provision-with-micromamba@main
+      uses: mamba-org/setup-micromamba@v1
       with:
         environment-file: environment-dev.yml
         environment-name: landcover-mapping
-        cache-env: true
+        cache-environment: true
     - uses: google-github-actions/auth@v0
       with:
         credentials_json: ${{ secrets.GCP_SA_KEY }}
@@ -99,11 +99,13 @@ jobs:
       with:
         ref: ${{ github.event.pull_request.head.ref }}
     - name: Setup environment
-      uses: mamba-org/provision-with-micromamba@main
+      uses: mamba-org/setup-micromamba@v1
       with:
         environment-file: environment-dev.yml
         environment-name: landcover-mapping
-        cache-env: true
+        cache-environment: true
+    - name: Install seaborn
+      run: pip install seaborn
     - uses: google-github-actions/auth@v0
       with:
         credentials_json: ${{ secrets.GCP_SA_KEY }}
@@ -127,11 +129,11 @@ jobs:
       with:
         ref: ${{ github.event.pull_request.head.ref }}
     - name: Setup environment
-      uses: mamba-org/provision-with-micromamba@main
+      uses: mamba-org/setup-micromamba@v1
       with:
         environment-file: environment-dev.yml
         environment-name: landcover-mapping
-        cache-env: true
+        cache-environment: true
     - uses: google-github-actions/auth@v0
       with:
         credentials_json: ${{ secrets.GCP_SA_KEY }}

.github/workflows/train.yml

@@ -40,11 +40,11 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Setup environment
-      uses: mamba-org/provision-with-micromamba@main
+      uses: mamba-org/setup-micromamba@v1
       with:
         environment-file: environment-dev.yml
         environment-name: landcover-mapping
-        cache-env: true
+        cache-environment: true
 
     - name: Login to Weights and Biases
       run: wandb login $WANDB_API_KEY

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ ci:
     autoupdate_schedule: 'quarterly'
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
     -   id: check-yaml
     -   id: end-of-file-fixer
@@ -12,16 +12,16 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/psf/black
-    rev: 24.3.0
+    rev: 24.8.0
     hooks:
     - id: black
 -   repo: https://github.com/pycqa/flake8
-    rev: 7.0.0
+    rev: 7.1.1
     hooks:
     -   id: flake8
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.9.0
+    rev: v1.11.2
     hooks:
     -   id: mypy
         args: [--no-strict-optional]
-        additional_dependencies: [types-requests, types-python-dateutil]
+        additional_dependencies: [types-requests, types-python-dateutil, types-PyYAML]

data/datasets.dvc

@@ -1,6 +1,6 @@
 outs:
-- md5: b45a782128aebf6786df801a75fbd46d.dir
-  size: 734199009
+- md5: 255771180339afd476bb65d11d0406e5.dir
+  size: 745188620
   nfiles: 62
   path: datasets
   hash: md5

data/report.txt

@@ -287,7 +287,7 @@ eo_data_skipped       82
 
 
 
-Senegal_CEO_2022 (Timesteps: 19)
+Senegal_CEO_2022 (Timesteps: 24)
 ----------------------------------------------------------------------------
 disagreement: 0.1%
 eo_data_complete    1499
@@ -400,38 +400,36 @@ eo_data_complete    1350
 
 
 
-Namibia_field_samples_22_23 (Timesteps: 15)
+Namibia_field_samples_22_23 (Timesteps: 24)
 ----------------------------------------------------------------------------
 eo_data_complete    4811
 ✔ training amount: 4811, positive class: 8.3%
 
 
 
-SudanGedarefDarfurAlJazirah2022 (Timesteps: 19)
+SudanGedarefDarfurAlJazirah2022 (Timesteps: 24)
 ----------------------------------------------------------------------------
-eo_data_complete    1196
-✔ training amount: 483, positive class: 27.5%
-✔ validation amount: 338, positive class: 33.4%
-✔ testing amount: 375, positive class: 34.4%
+eo_data_complete         1062
+eo_data_export_failed     134
+✔ training amount: 431, positive class: 24.6%
+✔ validation amount: 296, positive class: 29.1%
+✔ testing amount: 335, positive class: 31.6%
 
 
 
-SudanGedarefDarfurAlJazirah2023 (Timesteps: 10)
+SudanGedarefDarfurAlJazirah2023 (Timesteps: 19)
 ----------------------------------------------------------------------------
-eo_data_complete         731
-eo_data_export_failed    465
-✔ training amount: 277, positive class: 24.5%
-✔ validation amount: 215, positive class: 28.8%
-✔ testing amount: 239, positive class: 23.4%
+eo_data_export_failed    1196
 
 
 
-Uganda_NorthCEO2022 (Timesteps: 19)
+Uganda_NorthCEO2022 (Timesteps: 24)
 ----------------------------------------------------------------------------
-eo_data_complete    1000
-✔ training amount: 387, positive class: 20.9%
-✔ validation amount: 294, positive class: 23.1%
-✔ testing amount: 319, positive class: 17.6%
+eo_data_complete         989
+eo_data_export_failed     11
+✔ training amount: 399, positive class: 21.3%
+✔ validation amount: 281, positive class: 21.4%
+✔ testing amount: 309, positive class: 19.4%
 
 
 

data/shapefiles/Ethiopia_Tigray_2022_boundary/Ethiopia_Tigray_2022_boundary.cpg

@@ -0,0 +1 @@
+ISO-8859-1

data/shapefiles/Ethiopia_Tigray_2022_boundary/Ethiopia_Tigray_2022_boundary.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]]

datasets.py

@@ -381,7 +381,7 @@ def load_labels(self) -> pd.DataFrame:
         df.rename(columns={"Latitude": LAT, "Longitude": LON}, inplace=True)
         df = df.drop_duplicates(subset=[LAT, LON]).reset_index(drop=True)
         df[CLASS_PROB] = (df["Landcover"] == "crop").astype(int)
-        df[START], df[END] = date(2022, 1, 1), date(2023, 3, 31)
+        df[START], df[END] = date(2022, 1, 1), date(2023, 12, 31)
         df[SUBSET] = "training"
         return df
 
@@ -411,7 +411,7 @@ def load_labels(self) -> pd.DataFrame:
                 "email": join_unique,
             }
         )
-        df[START], df[END] = date(2022, 1, 1), date(2023, 7, 31)
+        df[START], df[END] = date(2022, 1, 1), date(2023, 12, 31)
         df[SUBSET] = train_val_test_split(df.index, 0.3, 0.3)
         return df
 
@@ -439,7 +439,7 @@ def load_labels(self) -> pd.DataFrame:
         )
         # Only keep examples with multiple labelers
         df = df[df["num_labelers"] > 1].copy()
-        df[START], df[END] = date(2023, 1, 1), date(2023, 10, 31)
+        df[START], df[END] = date(2023, 1, 1), date(2024, 7, 31)
         df[SUBSET] = train_val_test_split(df.index, 0.3, 0.3)
         return df
 
@@ -471,7 +471,7 @@ def load_labels(self) -> pd.DataFrame:
                 "email": join_unique,
             }
         )
-        df[START], df[END] = date(2022, 1, 1), date(2023, 7, 31)
+        df[START], df[END] = date(2022, 1, 1), date(2023, 12, 31)
         df[SUBSET] = train_val_test_split(df.index, 0.3, 0.3)
         return df
 

environment-dev.yml

@@ -8,12 +8,15 @@ dependencies:
     - scikit-learn
     - cartopy
     - pytorch=1.7.1
-    - gdal
+    - gdal=3.6.2
     - pip:
           - pre-commit==2.20.0
           - pytorch-lightning==0.7.1 # lots of API changes
           - wandb
-          - openmapflow[data]==0.2.4rc4
+          - openmapflow[data]==0.2.5rc1
+          - rasterio==1.3.6
+          - geopandas==0.9.0
+          - fiona==1.9.6
           - dvc[gs]
           - fsspec==2022.11.0 # https://github.com/iterative/dvc-azure/issues/34
           - einops

maps/Sudan/area-sudan-22sept2024.csv

@@ -0,0 +1,3 @@
+Year,AlJazirah crop area,95% CI,AlJazirah Non-crop area,95% CI,AlQadarif crop area,95% CI,AlQadarif Non-crop area,95% CI,WestDarfur crop area,95% CI,WestDarfur Non-crop area,95% CI,CentralDarfur crop area,95% CI,CentralDarfur Non-crop area,95% CI,SouthDarfur crop area,95% CI,SouthDarfur Non-crop area,95% CI
+2022,1038357.67,211911.48,1405219.82,211911.48,3699444.03,327081.69,2728299.09,327081.69,365424.27,155661.61,1911921.47,155661.61,557058.29,185579.4,3165385.83,185579.4,1473342.87,287520.97,6349430.03,287520.97
+2023,1165285.74,209330.3,1278291.75,209330.3,3710186.82,337116.99,2717556.29,337116.99,351607.46,150545.59,1925738.28,150545.59,562417.16,178617.87,3160026.96,178617.87,1261626.04,291377.41,6561146.86,291377.41

maps/Sudan/config-area-sudan.yml

@@ -0,0 +1,63 @@
+rois:
+
+  - roi_name: AlJazirah
+    roi_boundary_path: '../data/shapefiles/aljazirah_boundary.zip'
+    samples_and_cropmap:
+      - year: 2022
+        set1_labels_path: '../maps/Sudan/sudan_2022_updated22sept2024.csv'
+        set2_labels_path: '../maps/Sudan/sudan_2022_updated22sept2024.csv'
+        crop_mask_path: '../maps/Sudan/rasters/Sudan_dw_AlJazirah_2022_dw_may_march_cropmask-32637-cropped.tif'
+      - year: 2023
+        set1_labels_path: '../maps/Sudan/sudan_2023_updated22sept2024.csv'
+        set2_labels_path: '../maps/Sudan/sudan_2023_updated22sept2024.csv'
+        crop_mask_path: '../maps/Sudan/rasters/Sudan_dw_AlJazirah_2023_dw_may_march_cropmask-32637-cropped.tif'
+
+  - roi_name: AlQadarif
+    roi_boundary_path: '../data/shapefiles/alqadarif_boundary.zip'
+    samples_and_cropmap:
+      - year: 2022
+        set1_labels_path: '../maps/Sudan/sudan_2022_updated22sept2024.csv'
+        set2_labels_path: '../maps/Sudan/sudan_2022_updated22sept2024.csv'
+        crop_mask_path: '../maps/Sudan/rasters/Sudan_dw_AlQadarif_2022_dw_may_march_cropmask-32637-cropped.tif'
+      - year: 2023
+        set1_labels_path: '../maps/Sudan/sudan_2023_updated22sept2024.csv'
+        set2_labels_path: '../maps/Sudan/sudan_2023_updated22sept2024.csv'
+        crop_mask_path: '../maps/Sudan/rasters/Sudan_dw_AlQadarif_2023_dw_may_march_cropmask-32637-cropped.tif'
+
+  - roi_name: WestDarfur
+    roi_boundary_path: '../data/shapefiles/westdarfur_boundary.zip'
+    samples_and_cropmap:
+      - year: 2022
+        set1_labels_path: '../maps/Sudan/sudan_2022_updated22sept2024.csv'
+        set2_labels_path: '../maps/Sudan/sudan_2022_updated22sept2024.csv'
+        crop_mask_path: '../maps/Sudan/rasters/Sudan_dw_WestDarfur_2022_dw_may_march_cropmask-32635-cropped.tif'
+      - year: 2023
+        set1_labels_path: '../maps/Sudan/sudan_2023_updated22sept2024.csv'
+        set2_labels_path: '../maps/Sudan/sudan_2023_updated22sept2024.csv'
+        crop_mask_path: '../maps/Sudan/rasters/Sudan_dw_WestDarfur_2023_dw_may_march_cropmask-32635-cropped.tif'
+
+  - roi_name: CentralDarfur
+    roi_boundary_path: '../data/shapefiles/centraldarfur_boundary.zip'
+    samples_and_cropmap:
+      - year: 2022
+        set1_labels_path: '../maps/Sudan/sudan_2022_updated22sept2024.csv'
+        set2_labels_path: '../maps/Sudan/sudan_2022_updated22sept2024.csv'
+        crop_mask_path: '../maps/Sudan/rasters/Sudan_dw_CentralDarfur_2022_dw_may_march_cropmask-32635-cropped.tif'
+      - year: 2023
+        set1_labels_path: '../maps/Sudan/sudan_2023_updated22sept2024.csv'
+        set2_labels_path: '../maps/Sudan/sudan_2023_updated22sept2024.csv'
+        crop_mask_path: '../maps/Sudan/rasters/Sudan_dw_CentralDarfur_2023_dw_may_march_cropmask-32635-cropped.tif'
+
+  - roi_name: SouthDarfur
+    roi_boundary_path: '../data/shapefiles/southdarfur_boundary.zip'
+    samples_and_cropmap:
+      - year: 2022
+        set1_labels_path: '../maps/Sudan/sudan_2022_updated22sept2024.csv'
+        set2_labels_path: '../maps/Sudan/sudan_2022_updated22sept2024.csv'
+        crop_mask_path: '../maps/Sudan/rasters/Sudan_dw_SouthDarfur_2022_dw_may_march_cropmask-32635-cropped.tif'
+      - year: 2023
+        set1_labels_path: '../maps/Sudan/sudan_2023_updated22sept2024.csv'
+        set2_labels_path: '../maps/Sudan/sudan_2023_updated22sept2024.csv'
+        crop_mask_path: '../maps/Sudan/rasters/Sudan_dw_SouthDarfur_2023_dw_may_march_cropmask-32635-cropped.tif'
+
+output_path: '../maps/Sudan/area-sudan-22sept2024.csv'