add progress bar in terminal

extra_data/tests/utils/gendata.py

@@ -4,6 +4,20 @@
 
 import h5py
 
+from ..utils import progress_bar
+
+
+def progress(processed, total, *, show=True):
+    """Show progress information"""
+    if not show:
+        return
+
+    pbar = progress_bar(processed, total)
+    if sys.stderr.isatty():
+        # "\x1b[2K": delete whole line, "\x1b[1A": move up cursor
+        print("\x1b[2K\x1b[1A\x1b[2K\x1b[1A", file=sys.stderr)
+    print(pbar, file=sys.stderr)
+
 
 def clone_file_structure(h5file: Path, output: Path) -> None:
     clone = h5py.File(output / h5file.name, "w")
@@ -17,17 +31,17 @@ def visitor(name, obj):
                 or name.startswith("CONTROL")
                 or name.startswith("RUN")
             ):
-                clone.create_dataset_like(obj)
+                clone.create_dataset_like(name, obj)
             else:
-                clone.create_dataset_like(obj, data=obj[()])
+                clone.create_dataset_like(name, obj, data=obj[()])
 
     original = h5py.File(h5file)
     original.visititems(visitor)
 
 
-def clone(input: Path, output: Path) -> None:
+def clone(input: Path, output: Path, *, term_progress=False) -> None:
     """Clone EuXFEL HDF5 file structure without any of its data.
-    
+
     Clone the input file or files present the input directory.
     The cloned files will be writen to output.
     """
@@ -42,10 +56,12 @@ def clone(input: Path, output: Path) -> None:
         if output == input:
             raise ValueError("Input and output must be different directories.")
         # clone all hdf5 file present in the given directory
-        for file_ in input.glob("*"):
-            if not h5py.is_hdf5(file_):
-                continue
+        h5files = [f for f in input.glob("*") if h5py.is_hdf5(f)]
+
+        progress(0, len(h5files), show=term_progress)
+        for n, file_ in enumerate(h5files, start=1):
             clone_file_structure(file_, output)
+            progress(n, len(h5files), show=term_progress)
     else:
         raise ValueError(f"invalid input: {input}")
 
@@ -62,7 +78,9 @@ def main(argv=None):
     path_in = Path(args.input).expanduser()
     path_out = Path(args.output).expanduser()
 
-    clone(path_in, path_out)
+    print(f"Cloning file(s) structure:\ninput: {path_in}\nOutput: {path_out}\n")
+    clone(path_in, path_out, term_progress=True)
+    print("Done.")
 
 
 if __name__ == "__main__":

extra_data/utils.py

@@ -9,14 +9,23 @@
 """
 
 import os
+from shutil import get_terminal_size
 
 
 def available_cpu_cores():
     # This process may be restricted to a subset of the cores on the machine;
     # sched_getaffinity() tells us which on some Unix flavours (inc Linux)
-    if hasattr(os, 'sched_getaffinity'):
+    if hasattr(os, "sched_getaffinity"):
         return len(os.sched_getaffinity(0))
     else:
         # Fallback, inc on Windows
         ncpu = os.cpu_count() or 2
         return min(ncpu, 8)
+
+
+def progress_bar(done, total, suffix=" "):
+    line = f"Progress: {done}/{total}{suffix}[{{}}]"
+    length = min(get_terminal_size().columns - len(line), 50)
+    filled = int(length * done // total)
+    bar = "#" * filled + " " * (length - filled)
+    return line.format(bar)

extra_data/validation.py

@@ -4,13 +4,12 @@
 import numpy as np
 import os
 import os.path as osp
-from shutil import get_terminal_size
 from signal import signal, SIGINT, SIG_IGN
 import sys
 
 from .reader import H5File, FileAccess
 from .run_files_map import RunFilesMap
-
+from .utils import progress_bar
 
 class ValidationError(Exception):
     def __init__(self, problems):
@@ -212,14 +211,6 @@ def check_index_contiguous(firsts, counts, record):
         ))
 
 
-def progress_bar(done, total, suffix=' '):
-    line = f'Progress: {done}/{total}{suffix}[{{}}]'
-    length = min(get_terminal_size().columns - len(line), 50)
-    filled = int(length * done // total)
-    bar = '#' * filled + ' ' * (length - filled)
-    return line.format(bar)
-
-
 def _check_file(args):
     runpath, filename = args
     filepath = osp.join(runpath, filename)