Tqdm (#21)

* make the script s a function

* add tqdm progress bar

* bump version

* fix pbar bug

* update min version of old deps

* update min version of old deps

* revamp script test

.github/workflows/python-package.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
         os: [ubuntu-latest, macos-latest, windows-latest]
 
     steps:

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [v0.2.7]
+
+- Revamp some configs for maintenance (readthedocs, pyproject.toml)
+- Now uses tqdm for the progress bar
+- Moved the main script to a proper entrypoint instead of an added file
+    - This changes nothing in the way the script is called but just makes it more robust on Windows
+
 ## [v0.2.6] - 2022-11-04
 
 - Change the main script name from **get_distribution** to **autodmri_get_distribution** for consistency with the package name.

autodmri/estimator.py

@@ -7,13 +7,14 @@
 from autodmri.blocks import extract_patches
 
 from joblib import Parallel, delayed
+from tqdm import tqdm
 
 ###########################################
 # These functions are for over dwis
 ###########################################
 
 
-def estimate_from_dwis(data, axis=-2, return_mask=False, exclude_mask=None, ncores=-1, method='moments', verbose=0, fast_median=False):
+def estimate_from_dwis(data, axis=-2, return_mask=False, exclude_mask=None, ncores=-1, method='moments', verbose=False, fast_median=False):
     '''Given the data, splits over each slice to compute parameters of the gamma distribution
 
     input
@@ -32,7 +33,7 @@ def estimate_from_dwis(data, axis=-2, return_mask=False, exclude_mask=None, ncor
 
         method='moments' or method='maxlk' : which algorithm to use to estimate sigma and N
 
-        verbose : print progress for joblib, can be an integer to increase verbosity
+        verbose : bool, Shows a progress bar for parallel processing
 
         fast_median : Computes the median of medians from each volume.
         Useful for large datasets with many volumes (e.g. HCP) since the median requires a full copy of the data and sorting.
@@ -74,7 +75,10 @@ def estimate_from_dwis(data, axis=-2, return_mask=False, exclude_mask=None, ncor
     else:
         exclude_mask = exclude_mask.swapaxes(0, axis)
 
-    output = Parallel(n_jobs=ncores, verbose=verbose)(delayed(_inner)(swapped_data[i], median, exclude_mask[i], method) for i in ranger)
+    if verbose:
+        ranger = tqdm(ranger)
+
+    output = Parallel(n_jobs=ncores)(delayed(_inner)(swapped_data[i], median, exclude_mask[i], method) for i in ranger)
 
     # output is each slice we took along axis, so the mask might be reversed
     sigma = np.zeros(len(output))
@@ -172,7 +176,7 @@ def get_mask(data, N_min, N_max, phi, alpha_prob=0.05):
 ###########################################
 
 
-def estimate_from_nmaps(data, size=5, return_mask=True, method='moments', full=False, ncores=-1, use_rejection=False, verbose=0):
+def estimate_from_nmaps(data, size=5, return_mask=True, method='moments', full=False, ncores=-1, use_rejection=False, verbose=False):
     '''Given the data, estimates parameters of the gamma distribution in small 3D windows.
 
     input
@@ -193,7 +197,7 @@ def estimate_from_nmaps(data, size=5, return_mask=True, method='moments', full=F
 
         use_rejection : if True, iterate to reject voxels in each estimated window, but this is much slower than just using all of the data.
 
-        verbose : print progress for joblib, can be an integer to increase verbosity
+        verbose : bool, Shows a progress bar for parallel processing
 
     output
     -------
@@ -215,8 +219,12 @@ def estimate_from_nmaps(data, size=5, return_mask=True, method='moments', full=F
 
         indexer = list(np.ndindex(reshaped_maps.shape[:reshaped_maps.ndim//2 - 1]))
 
-        output = Parallel(n_jobs=ncores,
-                          verbose=verbose)(delayed(proc_inner)(reshaped_maps[i], median, size, method, use_rejection) for i in indexer)
+        if verbose:
+            ranger = tqdm(indexer)
+        else:
+            ranger = indexer
+
+        output = Parallel(n_jobs=ncores)(delayed(proc_inner)(reshaped_maps[i], median, size, method, use_rejection) for i in ranger)
 
         indexer = (np.index_exp[idx[0]:idx[0] + size, idx[1]:idx[1] + size, idx[2]:idx[2] + size] for idx in indexer)
 
@@ -242,8 +250,12 @@ def estimate_from_nmaps(data, size=5, return_mask=True, method='moments', full=F
         N = np.zeros(reshaped_maps.shape[0], dtype=np.float32)
         mask = np.zeros((reshaped_maps.shape[0], size**3), dtype=bool)
 
-        output = Parallel(n_jobs=ncores,
-                          verbose=verbose)(delayed(proc_inner)(reshaped_maps[i], median, size, method, use_rejection) for i in range(reshaped_maps.shape[0]))
+        ranger = range(reshaped_maps.shape[0])
+
+        if verbose:
+            ranger = tqdm(ranger)
+
+        output = Parallel(n_jobs=ncores)(delayed(proc_inner)(reshaped_maps[i], median, size, method, use_rejection) for i in ranger)
 
         for i, (s, n, m) in enumerate(output):
             sigma[i] = s

scripts/autodmri_get_distribution → autodmri/scripts.py

@@ -1,11 +1,9 @@
-#!/usr/bin/env python
-
 import numpy as np
 import nibabel as nib
 
+import os
 import argparse
 import logging
-import os
 
 from autodmri.estimator import estimate_from_dwis, estimate_from_nmaps
 
@@ -85,7 +83,7 @@ def buildArgsParser():
                    help='If set, overwrites the output text file if it already exists.')
 
     p.add_argument('-v', '--verbose', action='store_true', dest='verbose',
-                   help='If set, print useful information during processing.')
+                   help='If set, shows a progress bar during processing and prints useful information.')
 
     p.add_argument('-l', '--log', dest='logfile', metavar='file',
                    help='Save the logging output to this file. Implies verbose output.')

autodmri/tests/test_script.py

@@ -0,0 +1,17 @@
+import subprocess
+import pytest
+
+from pathlib import Path
+
+cwd = Path(__file__).parents[2] / Path("datasets")
+commands = ['autodmri_get_distribution data_SENSE3_MB3_noisemap.nii.gz sigma_maxlk_nmaps.nii.gz N_maxlk_nmaps.nii.gz mask_maxlk_nmaps.nii.gz -m maxlk --noise_maps',
+            'autodmri_get_distribution data_SENSE3_MB3_noisemap.nii.gz sigma_nmaps.nii.gz N_nmaps.nii.gz mask_nmaps.nii.gz --noise_maps',
+            'autodmri_get_distribution data_SENSE3_MB3_noisemap.nii.gz sigma_nmaps.nii.gz N_nmaps.nii.gz mask_nmaps.nii.gz --noise_maps -f --subsample',
+            'autodmri_get_distribution data_SENSE3_MB3_noisemap.nii.gz sigma_nmaps.nii.gz N_nmaps.nii.gz mask_nmaps.nii.gz --noise_maps -f --fast_median -m maxlk',
+            'autodmri_get_distribution dwi_1_8.nii.gz sigma.nii.gz N.nii.gz mask.nii.gz -v',
+            'autodmri_get_distribution dwi_1_8.nii.gz sigma.nii.gz N.nii.gz mask.nii.gz -m maxlk -f --ncores 4',
+            'autodmri_get_distribution dwi_1_8.nii.gz sigma_maxlk.nii.gz N_maxlk.nii.gz mask_maxlk.nii.gz -m maxlk --size 3 -f -v --axis 0']
+
+@pytest.mark.parametrize('command', commands)
+def test_script(command):
+    subprocess.run([command], shell=True, cwd=cwd, check=True)

docs/conf.py

@@ -60,9 +60,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = 'v0.2.3'
+version = 'v0.2.7'
 # The full version, including alpha/beta/rc tags.
-release = 'v0.2.3'
+release = 'v0.2.7'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/requirements.txt

@@ -1,5 +1,5 @@
 sphinx >= 5.3.0
 sphinx_rtd_theme >= 1.1.1
-readthedocs-sphinx-search >= 0.1.1
+readthedocs-sphinx-search >= 0.3.2
 sphinx-rtd-theme
 myst-parser

example/README.md

@@ -22,7 +22,7 @@ When estimating over noise maps (option `--noise_maps`), the mask instead indica
 
 ### Estimating distributions from DWIs
 
-Here I will use an old invivo dataset I have lying around, which you can download [here](https://github.com/samuelstjean/nlsam_data/raw/master/invivo/dwi_1_8.nii.gz).
+Here I will use an old invivo dataset I have lying around, which you can find in the datasets folder or download [here](https://github.com/samuelstjean/nlsam_data/).
 The data looks like this
 
 ![](dwi_1_8.png)

pyproject.toml

@@ -1,3 +1,31 @@
 [build-system]
 requires = ["setuptools>=61.0"]
 build-backend = "setuptools.build_meta"
+
+[project]
+description = 'Implementation of "Automated characterization of noise distributions in diffusion MRI data".'
+name = "autodmri"
+version = '0.2.7'
+authors = [{ name = "Samuel St-Jean" }]
+readme = "README.md"
+requires-python = ">=3.7"
+license = { text = "MIT" }
+
+dependencies = [
+    'numpy>=1.15',
+    'scipy>=1.0',
+    'tqdm>=4.56',
+    'joblib>=0.12',
+    'nibabel>=2.4',
+]
+
+[project.scripts]
+autodmri_get_distribution = "autodmri.scripts:main"
+
+[project.urls]
+homepage = "https://github.com/samuelstjean/autodmri"
+documentation = "https://autodmri.readthedocs.io/en/latest/"
+changelog = "https://github.com/samuelstjean/autodmri/blob/master/CHANGELOG.md"
+
+[tool.setuptools]
+packages = ["autodmri"]

setup.py

@@ -1,21 +1,3 @@
-from setuptools import setup, find_packages
+from setuptools import setup
 
-with open('README.md', encoding='utf-8') as f:
-    long_description = f.read()
-
-setup(
-    name='autodmri',
-    version='0.2.6',
-    author='Samuel St-Jean',
-    packages=find_packages(),
-    scripts=['scripts/autodmri_get_distribution'],
-    url='https://github.com/samuelstjean/autodmri',
-    license='MIT',
-    description='Implementation of "Automated characterization of noise distributions in diffusion MRI data".',
-    long_description=long_description,
-    long_description_content_type='text/markdown',
-    install_requires=['numpy>=1.10',
-                      'scipy>=0.19',
-                      'joblib>=0.12',
-                      'nibabel>=2.2'],
-)
+setup()