From 9fdacb61f3172b218edd146ecb44cc0442463862 Mon Sep 17 00:00:00 2001 From: Michiaki Tatsubori Date: Tue, 23 Jan 2024 20:27:00 +0900 Subject: [PATCH 1/2] Skip downloading an entry if it is not in the given pre-sampled list. --- experiments/ssl4eo/download_ssl4eo.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/experiments/ssl4eo/download_ssl4eo.py b/experiments/ssl4eo/download_ssl4eo.py index ae312e63aca..e630b9c20c2 100755 --- a/experiments/ssl4eo/download_ssl4eo.py +++ b/experiments/ssl4eo/download_ssl4eo.py @@ -475,6 +475,9 @@ def update(self, delta: int = 1) -> int: def worker(idx: int) -> None: if idx in ext_coords.keys(): return + # Skip when not in pre-sampled coords like sampled_locations.csv + if idx not in match_coords.keys(): + return worker_start = time.time() patches, center_coord = get_random_patches_match( From 4740a4cf08e8cb7d5e0692244b01c9bdb1952494 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 29 Feb 2024 15:42:33 +0100 Subject: [PATCH 2/2] Add warning message when skipping --- experiments/ssl4eo/download_ssl4eo.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/experiments/ssl4eo/download_ssl4eo.py b/experiments/ssl4eo/download_ssl4eo.py index e630b9c20c2..7389dbf24f0 100755 --- a/experiments/ssl4eo/download_ssl4eo.py +++ b/experiments/ssl4eo/download_ssl4eo.py @@ -52,6 +52,7 @@ import json import os import time +import warnings from collections import defaultdict from datetime import date, timedelta from multiprocessing.dummy import Lock, Pool @@ -473,10 +474,13 @@ def update(self, delta: int = 1) -> int: counter = Counter() def worker(idx: int) -> None: + # Skip if idx has already been downloaded if idx in ext_coords.keys(): return - # Skip when not in pre-sampled coords like sampled_locations.csv + + # Skip if idx is not in pre-sampled coordinates if idx not in match_coords.keys(): + warnings.warn(f"{idx} not found in {args.match_file}, skipping.") return worker_start = time.time()