fix up test bugs, support pandas 2.0, pin numpy until we replace deep…

…dish, use only pip for GA

.github/workflows/auto_formatting_tests_and_coverage.yml

@@ -60,22 +60,20 @@ jobs:
         run: |
           conda activate test
           conda env list
-          conda install -y pip pandas">=1.1.0,<1.2" numpy scipy matplotlib seaborn scikit-learn
-          conda install -y -c conda-forge pytest pytest-sugar coveralls black
-          pip install . -r requirements.txt
-          pip install . -r optional-dependencies.txt
+          pip install -r requirements-dev.txt
+          pip install -r optional-dependencies.txt
 
       # Check code formatting
       - name: Check code formatting
         run: |
+          conda activate test
           black --version
           black --check --diff --verbose nltools
 
       # Actually run the tests with coverage
       - name: Run Tests
         run: |
           conda activate test
-          conda env list
           coverage run --source=nltools -m pytest -rs
 
       # Send coverage to coveralls.io but waiting on parallelization to finish
@@ -117,28 +115,10 @@ jobs:
         with:
           python-version: "3.8"
 
-      # - name: Upgrade pip
-      #   run: |
-      #     # install pip=>20.1 to use "pip cache dir"
-      #     python3 -m pip install --upgrade pip
-
-      # - name: Setup pip-cache
-      #   id: pip-cache
-      #   run: echo "::set-output name=dir::$(pip cache dir)"
-
-      # - name: Cache deps
-      #   uses: actions/cache@v2
-      #   with:
-      #     path: ${{ steps.pip-cache.outputs.dir }}
-      #     key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-      #     restore-keys: |
-      #       ${{ runner.os }}-pip-
-
       - name: Install deps
         run: |
-          python3 -m pip install . -r requirements.txt
-          python3 -m pip install . -r requirements-dev.txt
-          python3 -m pip install . -r optional-dependencies.txt
+          python3 -m pip install -r requirements-dev.txt
+          python3 -m pip install -r optional-dependencies.txt
 
       - name: Build docs
         run: |

.gitignore

@@ -55,3 +55,5 @@ docs/backreferences/
 environment.yml
 rep_id.csv
 y.csv
+env
+.vscode/settings.json

nltools/data/adjacency.py

@@ -1225,7 +1225,6 @@ def isc_group(
         }
 
         if method == "permute":
-
             isc_group_differences_null = np.array(
                 Parallel(n_jobs=n_jobs)(
                     delayed(_permute_isc_group)(
@@ -1363,7 +1362,7 @@ def plot_mds(
         if labels_color is None:
             labels_color = ["black"] * len(labels)
         if n_components == 3:
-            for ((x, y, z), label, color) in zip(proj, labels, labels_color):
+            for (x, y, z), label, color in zip(proj, labels, labels_color):
                 ax.text(
                     x,
                     y,
@@ -1373,7 +1372,7 @@ def plot_mds(
                     bbox=dict(facecolor=color, alpha=1, boxstyle="round,pad=0.3"),
                 )
         else:
-            for ((x, y), label, color) in zip(proj, labels, labels_color):
+            for (x, y), label, color in zip(proj, labels, labels_color):
                 ax.text(
                     x,
                     y,

nltools/data/brain_data.py

@@ -881,10 +881,10 @@ def append(self, data, **kwargs):
             out = deepcopy(self)
             out.data = np.vstack([self.data, data.data])
             if out.Y.size:
-                out.Y = self.Y.append(data.Y)
+                out.Y = pd.concat([self.Y, data.Y])
             if self.X.size:
                 if isinstance(self.X, pd.DataFrame):
-                    out.X = self.X.append(data.X, **kwargs)
+                    out.X = pd.concat([self.X, data.X], **kwargs)
                 else:
                     out.X = np.vstack([self.X, data.X])
         return out
@@ -2180,7 +2180,6 @@ def temporal_resample(self, sampling_freq=None, target=None, target_type="hz"):
 
 class Groupby(object):
     def __init__(self, data, mask):
-
         data = check_brain_data(data)
         mask = check_brain_data(mask)
 

nltools/data/design_matrix.py

@@ -53,7 +53,6 @@ class Design_Matrix(DataFrame):
     _metadata = ["sampling_freq", "convolved", "polys", "multi"]
 
     def __init__(self, *args, **kwargs):
-
         sampling_freq = kwargs.pop("sampling_freq", None)
         convolved = kwargs.pop("convolved", [])
         polys = kwargs.pop("polys", [])
@@ -178,7 +177,6 @@ def _horzcat(self, to_append, fill_na):
         return out
 
     def _vertcat(self, df, keep_separate, unique_cols, fill_na, verbose):
-
         """Used by .append(). Append another design matrix row-wise (vert cat).
         Always returns a new design matrix.
 
@@ -793,8 +791,8 @@ def clean(self, fill_na=0, exclude_polys=False, thresh=0.95, verbose=True):
 
         keep = []
         remove = []
-        for i, c in out.iteritems():
-            for j, c2 in out.iteritems():
+        for i, c in out.items():
+            for j, c2 in out.items():
                 if i != j:
                     r = np.abs(pearsonr(c, c2)[0])
                     if (r >= thresh) and (j not in keep) and (j not in remove):

nltools/datasets.py

@@ -71,7 +71,7 @@ def get_collection_image_metadata(collection=None, data_dir=None, limit=10):
         dat = pd.DataFrame(columns=i["results"][0].keys())
         while int(offset) < int(i["count"]):
             for x in i["results"]:
-                dat = dat.append(x, ignore_index=True)
+                dat = pd.concat([dat, x], ignore_index=True)
             offset = offset + limit
             i = api.get_collection_images(
                 collection_id=collection, limit=limit, offset=offset

nltools/stats.py

@@ -448,7 +448,7 @@ def upsample(
                 UserWarning,
             )
         out = pd.DataFrame(columns=numeric_data.columns, index=None)
-        for i, x in numeric_data.iteritems():
+        for i, x in numeric_data.items():
             interpolate = interp1d(orig_spacing, x, kind=method)
             out.loc[:, i] = interpolate(new_spacing)
         return out
@@ -619,8 +619,8 @@ def two_sample_permutation(
 
     stats = {"mean": np.nanmean(data1) - np.nanmean(data2)}
     data = pd.DataFrame(data={"Values": data1, "Group": np.ones(len(data1))})
-    data = data.append(
-        pd.DataFrame(data={"Values": data2, "Group": np.zeros(len(data2))})
+    data = pd.concat(
+        [data, pd.DataFrame(data={"Values": data2, "Group": np.zeros(len(data2))})]
     )
     all_p = Parallel(n_jobs=n_jobs)(
         delayed(_permute_group)(data, random_state=seeds[i]) for i in range(n_permute)
@@ -955,7 +955,6 @@ def _robust_estimator(vals, X, robust_estimator="hc0", nlags=1):
 
         # Now loop over additional lags
         for l in range(1, nlags + 1):
-
             V = np.diag(vals[l:] * vals[:-l])
             meat_1 = np.dot(np.dot(X[l:].T, V), X[:-l])
             meat_2 = np.dot(np.dot(X[:-l].T, V), X[l:])
@@ -1121,7 +1120,6 @@ def regress(X, Y, mode="ols", stats="full", **kwargs):
 
     # Compute standard errors based on regression mode
     if mode == "ols" or mode == "robust":
-
         b = np.dot(np.linalg.pinv(X), Y)
 
         # Return betas and stop other computations if that's all that's requested

nltools/tests/test_adjacency.py

@@ -370,7 +370,7 @@ def test_isc(sim_adjacency_single):
         assert len(stats["null_distribution"]) == n_boot
 
 
-def test_isc_group():
+def test_isc_group_adj():
     n_samples = 100
     diff = 0.2
     data = np.random.multivariate_normal(

nltools/tests/test_design_matrix.py

@@ -132,7 +132,6 @@ def test_append(sim_design_matrix):
 
 
 def test_clean(sim_design_matrix):
-
     # Drop correlated column
     corr_cols = sim_design_matrix.assign(new_col=lambda df: df.iloc[:, 0])
     out = corr_cols.clean(verbose=True)

nltools/tests/test_prefs.py

@@ -4,7 +4,6 @@
 
 
 def test_change_mni_resolution():
-
     # Defaults
     brain = Brain_Data()
     assert brain.mask.affine[1, 1] == 2.0

requirements-dev.txt

@@ -1,5 +1,8 @@
+-r requirements.txt
 pytest
-black
+pytest-sugar
+black==23.3.0
+coveralls
 sphinx
 sphinx_gallery
 sphinx_bootstrap_theme

requirements.txt

@@ -1,11 +1,11 @@
-	nibabel>=3.0.1
-	scikit-learn>=0.21.0
-	nilearn>=0.6.0
-    pandas >= 1.1.0
-	numpy>=1.9
-	seaborn>=0.7.0
-	matplotlib>=2.2.0
-	scipy
-	pynv
-	joblib>=0.15
-	deepdish>=0.3.6
+nibabel>=3.0.1
+scikit-learn>=0.21.0
+nilearn>=0.6.0
+pandas>=1.1.0
+numpy<1.24
+seaborn>=0.7.0
+matplotlib>=2.2.0
+scipy
+pynv
+joblib>=0.15
+deepdish>=0.3.6