From 2b09d5ea95a53a30d9e117da5b1645d9dcb7ba0c Mon Sep 17 00:00:00 2001 From: Jiahe Date: Sun, 30 Jul 2023 11:53:42 -0700 Subject: [PATCH 1/4] Add windows support to dataloader --- methanemapper/datasets/hyper_dataloader.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/methanemapper/datasets/hyper_dataloader.py b/methanemapper/datasets/hyper_dataloader.py index 9f0d21f..bf17fad 100644 --- a/methanemapper/datasets/hyper_dataloader.py +++ b/methanemapper/datasets/hyper_dataloader.py @@ -2,6 +2,7 @@ import json from pathlib import Path +import os import numpy as np import torch @@ -103,8 +104,8 @@ def __init__(self, img_folder, ann_file, stats_file): if not stats_file == None: print("loading mean and std for each band...") - self.mean = np.load(f"{stats_file}/dataset_mean.npy") - self.std = np.load(f"{stats_file}/dataset_std.npy") + self.mean = np.load(os.path.join(f"{stats_file}", f"dataset_mean.npy")) + self.std = np.load(os.path.join(f"{stats_file}", f"dataset_std.npy")) def createList(self): # create list of all images and annotations @@ -138,10 +139,10 @@ def createPaths(self): _iid = self.img_id[_ann_key] _pid = self.patch_id[_ann_key] - _rgb = glob.glob(f"{self.img_dir}/rgb_tiles/{_iid}_*/*_{_pid}.npy")[0] - _tmp = _rgb.split("/") - _mf = f"{self.img_dir}/mf_tiles/{_tmp[-2]}/{_tmp[-1]}" - _raw = f"{self.img_dir}/rdata_tiles/{_tmp[-2]}/{_tmp[-1]}" + _rgb = glob.glob(os.path.join(f"{self.img_dir}", f"rgb_tiles", f"{_iid}_*", f"*_{_pid}.npy"))[0] + _tmp = _rgb.split(os.sep) + _mf = os.path.join(f"{self.img_dir}", f"mf_tiles", f"{_tmp[-2]}", f"{_tmp[-1]}") + _raw = os.path.join(f"{self.img_dir}", f"rdata_tiles", f"{_tmp[-2]}", f"{_tmp[-1]}") rgb_paths.append(_rgb), mf_paths.append(_mf), raw_paths.append(_raw) assert len(rgb_paths) == len(self.anns), "Number of annotations are different from images" From 2dbf770f3bbeb14769b1259c0ab2de2326848c47 Mon Sep 17 00:00:00 2001 From: Jiahe Date: Sun, 30 Jul 2023 11:59:24 -0700 Subject: [PATCH 2/4] Add Windows Batch files --- methanemapper/plume_box_run.bat | 3 +++ methanemapper/plume_mask_run.bat | 1 + 2 files changed, 4 insertions(+) create mode 100644 methanemapper/plume_box_run.bat create mode 100644 methanemapper/plume_mask_run.bat diff --git a/methanemapper/plume_box_run.bat b/methanemapper/plume_box_run.bat new file mode 100644 index 0000000..6ab64be --- /dev/null +++ b/methanemapper/plume_box_run.bat @@ -0,0 +1,3 @@ +python hyper_main.py --hyper_path ./data --dataset_file hyper_seg --pretrained True --output_dir ./exps/box_model --batch_size 12 --num_workers 8 +::CUDA_VISIBLE_DEVICES=2 python hyper_main.py --masks --hyper_path ./data --dataset_file hyper_seg --output_dir ./exps/segm_model --batch_size 12 --num_workers 6 --frozen_weights ./exps/box_model/mAPfix_checkpoint.pth --epoch 100 --lr_drop 15 --use_wandb True +::CUDA_VISIBLE_DEVICES=2 python hyper_main.py --masks --hyper_path ./data --dataset_file hyper_seg --output_dir ./exps/segm_model --batch_size 12 --num_workers 6 --frozen_weights ./exps/box_model/mAPfix_checkpoint.pth --epoch 100 --lr_drop 15 diff --git a/methanemapper/plume_mask_run.bat b/methanemapper/plume_mask_run.bat new file mode 100644 index 0000000..e0ec94e --- /dev/null +++ b/methanemapper/plume_mask_run.bat @@ -0,0 +1 @@ +python hyper_main.py --masks --hyper_path ./data --dataset_file hyper_seg --output_dir ./exps/segm_model --batch_size 12 --num_workers 8 --frozen_weights ./exps/box_model/checkpoint.pth --epoch 100 --lr_drop 15 --use_wandb True From 5ad9f69957e5e4123c9ad6a9bd04b9dd56fac43d Mon Sep 17 00:00:00 2001 From: Jiahe Date: Sun, 30 Jul 2023 13:21:07 -0700 Subject: [PATCH 3/4] Instructions for training on Windows. --- methanemapper/README.md | 61 +++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/methanemapper/README.md b/methanemapper/README.md index 1af850c..a12144b 100644 --- a/methanemapper/README.md +++ b/methanemapper/README.md @@ -3,7 +3,7 @@ MethaneMapper repository structure is similar to DETR repository from META. The -There are no extra compiled components in MethaneMapper and package dependencies are minimal mentioned in the [requirements.txt](https://github.com/UCSB-VRL/MethaneMapper-Spectral-Absorption-aware-Hyperspectral-Transformer-for-Methane-Detection/blob/main/requirements.txt). Codebase is very simple to use. +There are no extra compiled components in MethaneMapper and package dependencies are minimal mentioned in the [requirements.txt](https://github.com/UCSB-VRL/MethaneMapper-Spectral-Absorption-aware-Hyperspectral-Transformer-for-Methane-Detection/blob/main/requirements.txt). Codebase is very simple to use. #### Steps: 1. Clone the repository @@ -29,28 +29,55 @@ Directory structure from download : |_ rdata_tiles #raw 90 bands from hyperspectral image |_ rgb_tiles #reconstructed RGB image from hyperspectral image ``` -No create soft links to the "./data/train_dataset" folder as follows to avoud duplication of dataset (~size 10 TB): +Now create soft links to the "./data/train_dataset" folder as follows to avoud duplication of dataset (~size 10 TB): 1. matched_filter output tiles to mf_tiles folder + ###### Linux/MacOS + ``` + ln -s ./training_dataset2020/mf_tiles methanemapper/data/train/mf_tiles + ln -s ./training_dataset2020/mf_tiles methanemapper/data/val/mf_tiles + ``` + ###### Windows + ``` + mklink /j methanemapper\data\train\mf_tiles .\training_dataset2020\mf_tiles + mklink /j methanemapper\data\val\mf_tiles .\training_dataset2020\mf_tiles + ``` +2. Link the reconstructed RGB image tiles to rgb_tiles folder + ###### Linux/MacOS + ``` + ln -s ./training_dataset2020/rgb_tiles methanemapper/data/train/rgb_tiles + ln -s ./training_dataset2020/rgb_tiles methanemapper/data/val/rgb_tiles + ``` + ###### Windows + ``` + mklink /j methanemapper\data\train\rgb_tiles .\training_dataset2020\rgb_tiles + mklink /j methanemapper\data\val\rgb_tiles .\training_dataset2020\rgb_tiles + ``` +3. Link the raw 90 bands tiles to rdata_tiles folder + ###### Linux/MacOS + ``` + ln -s ./training_dataset2020/rdata_tiles methanemapper/data/train/rdata_tiles + ln -s ./training_dataset2020/rdata_tiles methanemapper/data/val/rdata_tiles + ``` + ###### Windows + ``` + mklink /j methanemapper\data\train\rdata_tiles .\training_dataset2020\rdata_tiles + mklink /j methanemapper\data\val\rdata_tiles .\training_dataset2020\rdata_tiles + ``` +#### Training methane plume bounding box detector +###### Linux/MacOS ``` -ln -s ./training_dataset2020/mf_tiles methanemapper/data/train_dataset/train/mf_tiles -ln -s ./training_dataset2020/mf_tiles methanemapper/data/train_dataset/val/mf_tiles -``` -2. Link the raw 90 bands tiles to rgb_tiles folder -``` -ln -s ./training_dataset2020/rgb_tiles methanemapper/data/train_dataset/train/rgb_tiles -ln -s ./training_dataset2020/rgb_tiles methanemapper/data/train_dataset/val/rgb_tiles +./plume_box_run.sh ``` -3. Link the reconstructed RGB image tiles to rdata_tiles folder +###### Windows ``` -ln -s ./training_dataset2020/rdata_tiles methanemapper/data/train_dataset/train/rdata_tiles -ln -s ./training_dataset2020/rdata_tiles methanemapper/data/train_dataset/val/rdata_tiles +plume_box_run.bat ``` - -#### Training methane plume bounding box detector +#### Fine-tuning plume segmentation mask detector +###### Linux/MacOS ``` -./plume_box_run.sh +./plume_mask_run.sh ``` -#### Fine-tuning plume segmentation mask detector +###### Windows ``` -./plume_mask_run.sh +plume_mask_run.bat ``` From 421578f1a3117ff654c4649a63e854478dbebcf2 Mon Sep 17 00:00:00 2001 From: Jiahe Date: Thu, 10 Aug 2023 17:57:10 -0700 Subject: [PATCH 4/4] 128*128px tile support --- methanemapper/datasets/hyper_dataloader.py | 51 ++++++++++++++++------ methanemapper/hyper_main.py | 1 + 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/methanemapper/datasets/hyper_dataloader.py b/methanemapper/datasets/hyper_dataloader.py index bf17fad..dcaf0e0 100644 --- a/methanemapper/datasets/hyper_dataloader.py +++ b/methanemapper/datasets/hyper_dataloader.py @@ -14,20 +14,21 @@ class HyperSegment(tdata.Dataset): - def __init__(self, img_folder, ann_file, stats_file, return_masks): - self.data_items = LoadItems(img_folder, ann_file, stats_file) + def __init__(self, img_folder, ann_file, stats_file, args): + self.data_items = LoadItems(img_folder, ann_file, stats_file, args) self._transform_rgb = makeHyperTransform(img_type="rgb") self._transform_mf = makeHyperTransform(img_type="mf") self._transform_raw = makeHyperTransform(img_type="raw", stats=self.data_items) - self.prepare = ConvertHyperToMask(return_masks) + self.prepare = ConvertHyperToMask(args.masks) self.target_keys = list(self.data_items.anns.keys()) + self.small_tiles = args.small_tiles def __getitem__(self, idx): rgb_path = self.data_items.rgb_paths[idx] mf_path = self.data_items.mf_paths[idx] raw_path = self.data_items.raw_paths[idx] target = self.data_items.anns[self.target_keys[idx]] - rgb_img, mf_img, raw_img, target = self.prepare(rgb_path, mf_path, raw_path, target) + rgb_img, mf_img, raw_img, target = self.prepare(rgb_path, mf_path, raw_path, target, self.small_tiles) if self._transform_rgb is not None: rgb_img, target = self._transform_rgb(rgb_img, target) @@ -89,7 +90,7 @@ def makeHyperTransform(img_type, stats=None): class LoadItems: - def __init__(self, img_folder, ann_file, stats_file): + def __init__(self, img_folder, ann_file, stats_file, args): # load dataset self.anns, self.imgs = dict(), dict() self.img_dir = img_folder @@ -100,7 +101,7 @@ def __init__(self, img_folder, ann_file, stats_file): assert type(dataset) == dict, "annotation file format {} not supported".format(type(dataset)) self.dataset = dataset self.createList() - self.createPaths() + self.createPaths(args) if not stats_file == None: print("loading mean and std for each band...") @@ -130,7 +131,7 @@ def createList(self): self.img_id = img_id self.patch_id = patch_id - def createPaths(self): + def createPaths(self, args): # create a list of all rgb, mf, raw images print("creating image paths list") rgb_paths, mf_paths, raw_paths = [], [], [] @@ -139,7 +140,11 @@ def createPaths(self): _iid = self.img_id[_ann_key] _pid = self.patch_id[_ann_key] - _rgb = glob.glob(os.path.join(f"{self.img_dir}", f"rgb_tiles", f"{_iid}_*", f"*_{_pid}.npy"))[0] + _rgb = glob.glob( + os.path.join( + f"{self.img_dir}", f"rgb_tiles", f"{_iid}_*", f"*_{_pid}." + ("npz" if args.small_tiles else "npy") + ) + )[0] _tmp = _rgb.split(os.sep) _mf = os.path.join(f"{self.img_dir}", f"mf_tiles", f"{_tmp[-2]}", f"{_tmp[-1]}") _raw = os.path.join(f"{self.img_dir}", f"rdata_tiles", f"{_tmp[-2]}", f"{_tmp[-1]}") @@ -157,11 +162,31 @@ class ConvertHyperToMask(object): def __init__(self, return_masks=False): self.return_masks = return_masks - def __call__(self, rgb_path, mf_path, raw_path, target): + def __call__(self, rgb_path, mf_path, raw_path, target, small_tiles): # dim order according to coco : ch, h, w - rgb_img = np.load(rgb_path).transpose((2, 0, 1)) - mf_img = np.expand_dims(np.load(mf_path), axis=2).transpose((2, 0, 1)) - raw_img = np.load(raw_path).transpose((2, 0, 1)) + def load(path): + if small_tiles: + i = -5 + while path[i:-4].isdigit(): + i = i - 1 + y = int(path[i + 1 : -4]) + j = i - 1 + while path[j:i].isdigit(): + j = j - 1 + x = int(path[j + 1 : i]) + arr00 = np.load(path)["arr_0"] + arr01 = np.load(path[:j] + "_" + str(x) + "_" + str(y + 1) + ".npz")["arr_0"] + arr10 = np.load(path[:j] + "_" + str(x + 1) + "_" + str(y) + ".npz")["arr_0"] + arr11 = np.load(path[:j] + "_" + str(x + 1) + "_" + str(y + 1) + ".npz")["arr_0"] + return np.concatenate( + (np.concatenate((arr00, arr10), axis=0), np.concatenate((arr01, arr11), axis=0)), axis=1 + ) + else: + return np.load(path) + + rgb_img = load(rgb_path).transpose((2, 0, 1)) + mf_img = np.expand_dims(load(mf_path), axis=2).transpose((2, 0, 1)) + raw_img = load(raw_path).transpose((2, 0, 1)) _, h, w = rgb_img.shape @@ -238,6 +263,6 @@ def buildHyperSeg(image_set, args): } img_folder, ann_file, stats_file = PATHS[image_set] - dataset = HyperSegment(img_folder, ann_file, stats_file, return_masks=args.masks) + dataset = HyperSegment(img_folder, ann_file, stats_file, args) return dataset diff --git a/methanemapper/hyper_main.py b/methanemapper/hyper_main.py index 02d9f1b..6d42e45 100644 --- a/methanemapper/hyper_main.py +++ b/methanemapper/hyper_main.py @@ -98,6 +98,7 @@ def get_args_parser(): parser.add_argument("--dataset_file", default="coco") parser.add_argument("--hyper_path", type=str) parser.add_argument("--remove_difficult", action="store_true") + parser.add_argument("--small_tiles", default=False, type=bool) parser.add_argument("--device", default="cuda", help="device to use for training / testing") parser.add_argument("--seed", default=42, type=int)