Option to ignore SSL certificate

img2dataset/downloader.py

@@ -4,6 +4,7 @@
 from threading import Semaphore
 import urllib.request
 import io
+import ssl
 import math
 import exifread
 import json
@@ -43,7 +44,10 @@ def download_image(row, timeout, user_agent_token, disallowed_header_directives)
         user_agent_string += f" (compatible; {user_agent_token}; +https://github.com/rom1504/img2dataset)"
     try:
         request = urllib.request.Request(url, data=None, headers={"User-Agent": user_agent_string})
-        with urllib.request.urlopen(request, timeout=timeout) as r:
+        ctx = ssl.create_default_context()
+        ctx.check_hostname = False
+        ctx.verify_mode = ssl.CERT_NONE
+        with urllib.request.urlopen(request, context=ctx, timeout=timeout) as r:
             if disallowed_header_directives and is_disallowed(
                 r.headers,
                 user_agent_token,
@@ -97,6 +101,7 @@ def __init__(
         user_agent_token,
         disallowed_header_directives,
         blurring_bbox_col=None,
+        ignore_ssl_certificate=False,
     ) -> None:
         self.sample_writer_class = sample_writer_class
         self.resizer = resizer
@@ -119,6 +124,7 @@ def __init__(
             else {directive.strip().lower() for directive in disallowed_header_directives}
         )
         self.blurring_bbox_col = blurring_bbox_col
+        self.ignore_ssl_certificate = ignore_ssl_certificate
 
     def __call__(
         self,

img2dataset/main.py

@@ -108,6 +108,7 @@ def download(
     max_shard_retry: int = 1,
     user_agent_token: Optional[str] = None,
     disallowed_header_directives: Optional[List[str]] = None,
+    ignore_ssl_certificate: bool = False,
 ):
     """Download is the main entry point of img2dataset, it uses multiple processes and download multiple files"""
     if disallowed_header_directives is None:
@@ -247,6 +248,7 @@ def signal_handler(signal_arg, frame):  # pylint: disable=unused-argument
         user_agent_token=user_agent_token,
         disallowed_header_directives=disallowed_header_directives,
         blurring_bbox_col=bbox_col,
+        ignore_ssl_certificate=ignore_ssl_certificate,
     )
 
     print("Starting the downloading of this file")