bz2-size

#!/usr/bin/env python3

try:
    import common
except ImportError:
    import sys
    import os
    from os import path
    home = path.abspath(path.dirname(sys.argv[0]))
    sys.path.append(path.join(home, "src"))

import common
from common import finish, human_size

import multiprocessing
import concurrent.futures
import subprocess
import shlex

def get_size(filename):
    return (filename, int(subprocess.check_output("bzcat " + shlex.quote(filename) + " | wc -c", shell=True)))

def main():
    import argparse
    parser = argparse.ArgumentParser(description="Clusters a directory containing Salento JSON datasets.")
    get_input_files = common.parser_add_input_files(parser)
    get_nprocs = common.parser_add_parallelism(parser)
    args = parser.parse_args()

    class Accum:
        def __init__(self):
            self.total_size = 0
        def __call__(self, elem):
            filename, x = elem
            print(filename, x)
            self.total_size += x

    accum = Accum()

    with finish(concurrent.futures.ThreadPoolExecutor(max_workers=get_nprocs(args)), accum) as executor:
        for x in get_input_files(args, lazy=True, globs=["*.bz2"]):
            executor.submit(get_size, x)

    print(human_size(accum.total_size))


if __name__ == '__main__':
    main()