Skip to content

Commit

Permalink
fix(example): move python code into __main__ (#350)
Browse files Browse the repository at this point in the history
Co-authored-by: songjunjie <[email protected]>
  • Loading branch information
Adenialzz and songjunjie authored Jan 6, 2024
1 parent 9871ecd commit a925735
Showing 1 changed file with 19 additions and 18 deletions.
37 changes: 19 additions & 18 deletions examples/simple_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,25 @@
import shutil
import os

output_dir = os.path.abspath("bench")
if __name__ == '__main__':
output_dir = os.path.abspath("bench")

if os.path.exists(output_dir):
shutil.rmtree(output_dir)
if os.path.exists(output_dir):
shutil.rmtree(output_dir)

download(
processes_count=16,
thread_count=32,
url_list="../tests/test_files/test_10000.parquet",
image_size=256,
output_folder=output_dir,
output_format="files",
input_format="parquet",
url_col="URL",
caption_col="TEXT",
enable_wandb=True,
number_sample_per_shard=1000,
distributor="multiprocessing",
)
download(
processes_count=16,
thread_count=32,
url_list="../tests/test_files/test_10000.parquet",
image_size=256,
output_folder=output_dir,
output_format="files",
input_format="parquet",
url_col="URL",
caption_col="TEXT",
enable_wandb=True,
number_sample_per_shard=1000,
distributor="multiprocessing",
)

# rm -rf bench
# rm -rf bench

0 comments on commit a925735

Please sign in to comment.