Skip to content

Commit

Permalink
Run downloads in parallel
Browse files Browse the repository at this point in the history
  • Loading branch information
MartinThoma committed Dec 3, 2023
1 parent 2a42b0b commit 1f5ed08
Showing 1 changed file with 8 additions and 2 deletions.
10 changes: 8 additions & 2 deletions tests/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import concurrent.futures
import csv
import ssl
import urllib.request
Expand Down Expand Up @@ -130,8 +131,13 @@ def download_test_pdfs():
This is especially important to avoid pytest timeouts.
"""
pdfs = read_csv_to_list_of_dicts(Path(__file__).parent / "example_files.csv")
for pdf in pdfs:
get_data_from_url(pdf["url"], name=pdf["local_filename"])

with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
futures = [
executor.submit(get_data_from_url, pdf["url"], name=pdf["local_filename"])
for pdf in pdfs
]
concurrent.futures.wait(futures)


def test_csv_consistency():
Expand Down

0 comments on commit 1f5ed08

Please sign in to comment.