diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index 25457d3f4..51aece137 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -10,14 +10,21 @@ jobs: run-tests: needs: assess-file-changes if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} - uses: neurodatawithoutborders/nwbinspector/.github/workflows/testing.yml@dev + uses: neurodatawithoutborders/nwbinspector/.github/workflows/testing.yml@use_existing_data_files run-past-gallery: needs: assess-file-changes if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} - uses: neurodatawithoutborders/nwbinspector/.github/workflows/version_gallery.yml@dev + uses: neurodatawithoutborders/nwbinspector/.github/workflows/version_gallery.yml@use_existing_data_files run-dev-gallery: needs: assess-file-changes if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} - uses: neurodatawithoutborders/nwbinspector/.github/workflows/dev-gallery.yml@dev + uses: neurodatawithoutborders/nwbinspector/.github/workflows/dev-gallery.yml@use_existing_data_files + + update-testing-files: + needs: assess-file-changes + if: ${{ needs.assess-file-changes.outputs.TESTING_CHANGED == 'true' }} + uses: neurodatawithoutborders/nwbinspector/.github/workflows/update-testing-files.yml@use_existing_data_files + secrets: + DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} diff --git a/.github/workflows/dev-gallery.yml b/.github/workflows/dev-gallery.yml index a16d3c97c..c0a987d6f 100644 --- a/.github/workflows/dev-gallery.yml +++ b/.github/workflows/dev-gallery.yml @@ -3,7 +3,7 @@ on: workflow_call jobs: build-and-test: - name: Testing against past PyNWB versions + name: Testing against dev PyNWB version runs-on: ubuntu-latest strategy: fail-fast: false @@ -25,8 +25,10 @@ jobs: pip install -e . pip install git+https://github.com/neurodatawithoutborders/pynwb@dev pip install dandi - - name: Download testing data - run: dandi download 'https://gui-staging.dandiarchive.org/#/dandiset/204919' + - name: Download testing data and set config path + run: | + dandi download "https://gui-staging.dandiarchive.org/#/dandiset/204919" + python -c "from nwbinspector.testing import update_testing_config; update_testing_config(key='LOCAL_PATH', value='./204919/testing_files/')" - name: Uninstall h5py run: pip uninstall -y h5py - name: Install ROS3 diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 35e7c92ed..e8d8c07fe 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -22,16 +22,20 @@ jobs: run: | pip install pytest pip install pytest-cov + - name: Install package run: | pip install -e . pip install dandi - - name: Download testing data - run: dandi download 'https://gui-staging.dandiarchive.org/#/dandiset/204919' + - name: Download testing data and set config path + run: | + dandi download "https://gui-staging.dandiarchive.org/#/dandiset/204919" + python -c "from nwbinspector.testing import update_testing_config; update_testing_config(key='LOCAL_PATH', value='./204919/testing_files/')" - name: Uninstall h5py run: pip uninstall -y h5py - name: Install ROS3 run: conda install -c conda-forge h5py + - name: Run pytest with coverage run: pytest -rsx --cov=./ --cov-report xml:./nwbinspector/nwbinspector/coverage.xml - if: ${{ matrix.python-version == '3.10' && matrix.os == 'ubuntu-latest' }} diff --git a/.github/workflows/update-testing-files.yml b/.github/workflows/update-testing-files.yml new file mode 100644 index 000000000..40be088e2 --- /dev/null +++ b/.github/workflows/update-testing-files.yml @@ -0,0 +1,37 @@ +name: Generate and Upload Testing Files +on: + workflow_dispatch: + workflow_call: + secrets: + DANDI_API_KEY: + required: true + +env: + DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} + +jobs: + build-and-test: + name: Generate and Upload Testing Files + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v2 + - run: git fetch --prune --unshallow --tags + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Install DANDI + run: pip install -e .[dandi] + - name: Update testing configuration path + run: python -c "from nwbinspector.testing import update_testing_config; update_testing_config(key='LOCAL_PATH', value='./testing_files/')" + - name: Generate testing files + run: python -c "from nwbinspector.testing import generate_testing_files; generate_testing_files()" + - name: Upload to DANDI + run: | + dandi download 'https://gui-staging.dandiarchive.org/#/dandiset/204919' --download dandiset.yaml + cd 204919 + mv ../testing_files . + dandi upload -i dandi-staging --validation skip # These are purposefully invalid files diff --git a/.github/workflows/version_gallery.yml b/.github/workflows/version_gallery.yml index 29bb404cf..4cbaf8f59 100644 --- a/.github/workflows/version_gallery.yml +++ b/.github/workflows/version_gallery.yml @@ -28,5 +28,11 @@ jobs: run: | pip install -e . pip install pynwb==${{ matrix.pynwb-version }} + pip install dandi + - name: Download testing data and set config path + run: | + dandi download "https://gui-staging.dandiarchive.org/#/dandiset/204919" + python -c "from nwbinspector.testing import update_testing_config; update_testing_config(key='LOCAL_PATH', value='./204919/testing_files/')" + - name: Run pytest with coverage run: pytest -rsx diff --git a/.gitignore b/.gitignore index 2cce8e938..4bcbdea5b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,10 @@ *.nwb manifest.json +# Testing +tests/testing_config.json +*/testing_files/* + # Python byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/CHANGELOG.md b/CHANGELOG.md index c8bf828db..650c56abd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Upcoming +### Improvements + +* Added PyNWB v2.1.0 specific file generation functions to the `testing` submodule, and altered the tests for `ImageSeries` to use these pre-existing files when available. Also included automated workflow to push the generated files to a DANDI-staging server for public access. [PR #288](https://github.com/NeurodataWithoutBorders/nwbinspector/pull/288) + +### Fixes + +* Fixed relative path detection for cross-platform strings in `check_image_series_external_file_relative` [PR #288](https://github.com/NeurodataWithoutBorders/nwbinspector/pull/288) + + + # v0.4.14 ### Fixes diff --git a/README.md b/README.md index 5c8111319..c11bdecfb 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,7 @@ [![codecov](https://codecov.io/gh/NeurodataWithoutBorders/nwbinspector/branch/dev/graphs/badge.svg?branch=dev)](https://codecov.io/github/NeurodataWithoutBorders/nwbinspector?branch=dev) [![License](https://img.shields.io/pypi/l/pynwb.svg)](https://github.com/NeurodataWithoutBorders/nwbinspector/license.txt) -Inspect NWB files for compliance with [NWB Best Practices](https://www.nwb.org/best-practices/). This inspector is meant as a companion to the pynwb validator, which checks for strict schema compliance. In contrast, this tool attempts to apply some common sense to find components of the file that are technically compliant, but probably incorrect, or suboptimal, or deviate from best practices. This tool is meant simply as a data review aid. It does not catch all best practice violations, and any warnings it does produce should be checked by a knowledgeable reviewer. - -This project is under active development. You may use this as a stand-alone tool, but we do not advise you to code against this project at this time as we do expect the warnings to change as the project develops. +Inspect NWB files for compliance with [NWB Best Practices](https://nwbinspector.readthedocs.io/en/dev/best_practices/best_practices_index.html). This inspector is meant as a companion to the PyNWB validator, which checks for strict schema compliance. In contrast, this tool attempts to apply some common sense to find components of the file that are technically compliant, but possibly incorrect, suboptimal in their representation, or deviate from best practices. This tool is meant simply as a data review aid. It does not catch all best practice violations, and any warnings it does produce should be checked by a knowledgeable reviewer. ## Installation ```bash @@ -23,7 +21,4 @@ nwbinspector path/to/my/data.nwb # supply a path to a directory containing NWB files nwbinspector path/to/my/data/dir/ - -# optional: supply modules to import before reading the file, e.g., for NWB extensions -nwbinspector path/to/my/data.nwb -m my_extension_module1 my_extension_module2 ``` diff --git a/base_test_config.json b/base_test_config.json new file mode 100644 index 000000000..442057399 --- /dev/null +++ b/base_test_config.json @@ -0,0 +1,3 @@ +{ + "LOCAL_PATH": "/shared/catalystneuro/" +} diff --git a/nwbinspector/testing.py b/nwbinspector/testing.py deleted file mode 100644 index 35561911e..000000000 --- a/nwbinspector/testing.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Helper functions for internal use across the testing suite.""" -import os -from distutils.util import strtobool -from typing import Tuple, Optional - -from .tools import check_streaming_enabled -from .utils import is_module_installed - - -def check_streaming_tests_enabled() -> Tuple[bool, Optional[str]]: - """ - General purpose helper for determining if the testing environment can support S3 DANDI streaming. - - Returns the boolean status of the check and, if False, provides a string reason for the failure for the user to - utilize as they please (raise an error or warning with that message, print it, or ignore it). - """ - failure_reason = "" - - environment_skip_flag = os.environ.get("NWBI_SKIP_NETWORK_TESTS", "") - environment_skip_flag_bool = ( - strtobool(os.environ.get("NWBI_SKIP_NETWORK_TESTS", "")) if environment_skip_flag != "" else False - ) - if environment_skip_flag_bool: - failure_reason += "Environmental variable set to skip network tets." - - streaming_enabled, streaming_failure_reason = check_streaming_enabled() - if not streaming_enabled: - failure_reason += streaming_failure_reason - - have_dandi = is_module_installed("dandi") - if not have_dandi: - failure_reason += "The DANDI package is not installed on the system." - - failure_reason = None if failure_reason == "" else failure_reason - return streaming_enabled and not environment_skip_flag_bool and have_dandi, failure_reason diff --git a/setup.py b/setup.py index 8182947d3..f45122fe9 100644 --- a/setup.py +++ b/setup.py @@ -1,25 +1,34 @@ from setuptools import setup, find_packages from pathlib import Path +from shutil import copy root = Path(__file__).parent with open(root / "README.md", "r") as f: long_description = f.read() with open(root / "requirements.txt") as f: install_requires = f.readlines() -with open(root / "nwbinspector" / "version.py") as f: - exec(f.read()) +with open(root / "src" / "nwbinspector" / "version.py") as f: + version = f.read() + +# Instantiate the testing configuration file from the base file `base_test_config.json` +base_test_config = Path.cwd() / "base_test_config.json" +local_test_config = Path.cwd() / "tests" / "testing_config.json" +if not local_test_config.exists(): + copy(src=base_test_config, dst=local_test_config) setup( name="nwbinspector", - version=__version__, + version=version.split('"')[1], description="Tool to inspect NWB files for best practices compliance.", long_description=long_description, long_description_content_type="text/markdown", author="Ryan Ly, Ben Dichter, and Cody Baker.", author_email="rly@lbl.gov, ben.dichter@gmail.com, cody.baker@catalystneuro.com", - packages=find_packages(), - include_package_data=True, - url="https://github.com/NeurodataWithoutBorders/nwbinspector", + url="https://nwbinspector.readthedocs.io/", + keywords="nwb", + packages=find_packages(where="src"), + package_dir={"": "src"}, + include_package_data=True, # Includes files described in MANIFEST.in in the installation install_requires=install_requires, extras_require=dict(dandi=["dandi>=0.39.2"]), entry_points={"console_scripts": ["nwbinspector=nwbinspector.nwbinspector:inspect_all_cli"]}, diff --git a/nwbinspector/__init__.py b/src/nwbinspector/__init__.py similarity index 100% rename from nwbinspector/__init__.py rename to src/nwbinspector/__init__.py diff --git a/nwbinspector/checks/__init__.py b/src/nwbinspector/checks/__init__.py similarity index 100% rename from nwbinspector/checks/__init__.py rename to src/nwbinspector/checks/__init__.py diff --git a/nwbinspector/checks/behavior.py b/src/nwbinspector/checks/behavior.py similarity index 100% rename from nwbinspector/checks/behavior.py rename to src/nwbinspector/checks/behavior.py diff --git a/nwbinspector/checks/ecephys.py b/src/nwbinspector/checks/ecephys.py similarity index 100% rename from nwbinspector/checks/ecephys.py rename to src/nwbinspector/checks/ecephys.py diff --git a/nwbinspector/checks/general.py b/src/nwbinspector/checks/general.py similarity index 100% rename from nwbinspector/checks/general.py rename to src/nwbinspector/checks/general.py diff --git a/nwbinspector/checks/icephys.py b/src/nwbinspector/checks/icephys.py similarity index 100% rename from nwbinspector/checks/icephys.py rename to src/nwbinspector/checks/icephys.py diff --git a/nwbinspector/checks/image_series.py b/src/nwbinspector/checks/image_series.py similarity index 91% rename from nwbinspector/checks/image_series.py rename to src/nwbinspector/checks/image_series.py index 3e58bbfe1..8eb20bf98 100644 --- a/nwbinspector/checks/image_series.py +++ b/src/nwbinspector/checks/image_series.py @@ -1,4 +1,5 @@ """Check functions specific to ImageSeries.""" +import ntpath from pathlib import Path from pynwb.image import ImageSeries @@ -39,7 +40,7 @@ def check_image_series_external_file_relative(image_series: ImageSeries): return for file_path in image_series.external_file: file_path = file_path.decode() if isinstance(file_path, bytes) else file_path - if Path(file_path).is_absolute(): + if ntpath.isabs(file_path): # ntpath required for cross-platform detection yield InspectorMessage( message=( f"The external file '{file_path}' is not a relative path. " @@ -58,6 +59,4 @@ def check_image_series_data_size(image_series: ImageSeries, gb_lower_bound: floa data = image_series.data data_size_gb = data.size * data.dtype.itemsize / 1e9 if data_size_gb > gb_lower_bound: - return InspectorMessage( - message=f"ImageSeries {image_series.name} is too large. Use external mode for storage", - ) + return InspectorMessage(message=f"ImageSeries {image_series.name} is too large. Use external mode for storage") diff --git a/nwbinspector/checks/images.py b/src/nwbinspector/checks/images.py similarity index 100% rename from nwbinspector/checks/images.py rename to src/nwbinspector/checks/images.py diff --git a/nwbinspector/checks/nwb_containers.py b/src/nwbinspector/checks/nwb_containers.py similarity index 100% rename from nwbinspector/checks/nwb_containers.py rename to src/nwbinspector/checks/nwb_containers.py diff --git a/nwbinspector/checks/nwbfile_metadata.py b/src/nwbinspector/checks/nwbfile_metadata.py similarity index 100% rename from nwbinspector/checks/nwbfile_metadata.py rename to src/nwbinspector/checks/nwbfile_metadata.py diff --git a/nwbinspector/checks/ogen.py b/src/nwbinspector/checks/ogen.py similarity index 100% rename from nwbinspector/checks/ogen.py rename to src/nwbinspector/checks/ogen.py diff --git a/nwbinspector/checks/ophys.py b/src/nwbinspector/checks/ophys.py similarity index 100% rename from nwbinspector/checks/ophys.py rename to src/nwbinspector/checks/ophys.py diff --git a/nwbinspector/checks/tables.py b/src/nwbinspector/checks/tables.py similarity index 100% rename from nwbinspector/checks/tables.py rename to src/nwbinspector/checks/tables.py diff --git a/nwbinspector/checks/time_series.py b/src/nwbinspector/checks/time_series.py similarity index 100% rename from nwbinspector/checks/time_series.py rename to src/nwbinspector/checks/time_series.py diff --git a/nwbinspector/config.schema.json b/src/nwbinspector/config.schema.json similarity index 100% rename from nwbinspector/config.schema.json rename to src/nwbinspector/config.schema.json diff --git a/nwbinspector/inspector_tools.py b/src/nwbinspector/inspector_tools.py similarity index 100% rename from nwbinspector/inspector_tools.py rename to src/nwbinspector/inspector_tools.py diff --git a/nwbinspector/internal_configs/dandi.inspector_config.yaml b/src/nwbinspector/internal_configs/dandi.inspector_config.yaml similarity index 100% rename from nwbinspector/internal_configs/dandi.inspector_config.yaml rename to src/nwbinspector/internal_configs/dandi.inspector_config.yaml diff --git a/nwbinspector/nwbinspector.py b/src/nwbinspector/nwbinspector.py similarity index 100% rename from nwbinspector/nwbinspector.py rename to src/nwbinspector/nwbinspector.py diff --git a/nwbinspector/register_checks.py b/src/nwbinspector/register_checks.py similarity index 100% rename from nwbinspector/register_checks.py rename to src/nwbinspector/register_checks.py diff --git a/src/nwbinspector/testing.py b/src/nwbinspector/testing.py new file mode 100644 index 000000000..0a9bf1e45 --- /dev/null +++ b/src/nwbinspector/testing.py @@ -0,0 +1,120 @@ +"""Helper functions for internal use across the testing suite.""" +import os +import json +from distutils.util import strtobool +from pathlib import Path +from typing import Tuple, Optional + +from packaging.version import Version +from pynwb import NWBHDF5IO +from pynwb.image import ImageSeries + +from .tools import check_streaming_enabled, make_minimal_nwbfile +from .utils import is_module_installed, get_package_version + + +def check_streaming_tests_enabled() -> Tuple[bool, Optional[str]]: + """ + General purpose helper for determining if the testing environment can support S3 DANDI streaming. + + Returns the boolean status of the check and, if False, provides a string reason for the failure for the user to + utilize as they please (raise an error or warning with that message, print it, or ignore it). + """ + failure_reason = "" + + environment_skip_flag = os.environ.get("NWBI_SKIP_NETWORK_TESTS", "") + environment_skip_flag_bool = ( + strtobool(os.environ.get("NWBI_SKIP_NETWORK_TESTS", "")) if environment_skip_flag != "" else False + ) + if environment_skip_flag_bool: + failure_reason += "Environmental variable set to skip network tets." + + streaming_enabled, streaming_failure_reason = check_streaming_enabled() + if not streaming_enabled: + failure_reason += streaming_failure_reason + + have_dandi = is_module_installed("dandi") + if not have_dandi: + failure_reason += "The DANDI package is not installed on the system." + + failure_reason = None if failure_reason == "" else failure_reason + return streaming_enabled and not environment_skip_flag_bool and have_dandi, failure_reason + + +def load_testing_config() -> dict: + """Helper function for loading the testing configuration file as a dictionary.""" + test_config_file_path = Path(__file__).parent.parent.parent / "tests" / "testing_config.json" + + # This error would only occur if someone installed a previous version + # directly from GitHub and then updated the branch/commit in-place + if not test_config_file_path.exists(): # pragma: no cover + raise FileNotFoundError( + "The testing configuration file not found at the location '{test_config_file_path}'! " + "Please try reinstalling the package." + ) + + with open(file=test_config_file_path) as file: + test_config = json.load(file) + + return test_config + + +def update_testing_config(key: str, value): + """Update a key/value pair in the testing configuration file through the API.""" + test_config_file_path = Path(__file__).parent.parent.parent / "tests" / "testing_config.json" + + testing_config = load_testing_config() + + if key not in testing_config: + raise KeyError("Updating the testing configuration file via the API is only possible for the pre-defined keys!") + testing_config[key] = value + + with open(file=test_config_file_path, mode="w") as file: + json.dump(testing_config, file) + + +def generate_testing_files(): # pragma: no cover + """Generate a local copy of the NWB files required for all tests.""" + generate_image_series_testing_files() + + +def generate_image_series_testing_files(): # pragma: no cover + """Generate a local copy of the NWB files required for the image series tests.""" + assert get_package_version(name="pynwb") == Version("2.1.0"), "Generating the testing files requires PyNWB v2.1.0!" + + testing_config = load_testing_config() + + local_path = Path(testing_config["LOCAL_PATH"]) + local_path.mkdir(exist_ok=True, parents=True) + + movie_1_file_path = local_path / "temp_movie_1.mov" + nested_folder = local_path / "nested_folder" + nested_folder.mkdir(exist_ok=True) + movie_2_file_path = nested_folder / "temp_movie_2.avi" + for file_path in [movie_1_file_path, movie_2_file_path]: + with open(file=file_path, mode="w") as file: + file.write("Not a movie file, but at least it exists.") + nwbfile = make_minimal_nwbfile() + nwbfile.add_acquisition( + ImageSeries( + name="TestImageSeriesGoodExternalPaths", + rate=1.0, + external_file=[ + "/".join([".", movie_1_file_path.name]), + "/".join([".", nested_folder.name, movie_2_file_path.name]), + ], + ) + ) + nwbfile.add_acquisition( + ImageSeries( + name="TestImageSeriesExternalPathDoesNotExist", + rate=1.0, + external_file=["madeup_file.mp4"], + ) + ) + absolute_file_path = str(Path("madeup_file.mp4").absolute()) + nwbfile.add_acquisition( + ImageSeries(name="TestImageSeriesExternalPathIsNotRelative", rate=1.0, external_file=[absolute_file_path]) + ) + with NWBHDF5IO(path=local_path / "image_series_testing_file.nwb", mode="w") as io: + io.write(nwbfile) diff --git a/nwbinspector/tools.py b/src/nwbinspector/tools.py similarity index 100% rename from nwbinspector/tools.py rename to src/nwbinspector/tools.py diff --git a/nwbinspector/utils.py b/src/nwbinspector/utils.py similarity index 100% rename from nwbinspector/utils.py rename to src/nwbinspector/utils.py diff --git a/nwbinspector/version.py b/src/nwbinspector/version.py similarity index 100% rename from nwbinspector/version.py rename to src/nwbinspector/version.py diff --git a/tests/test_testing_module.py b/tests/test_testing_module.py new file mode 100644 index 000000000..c5c1eec88 --- /dev/null +++ b/tests/test_testing_module.py @@ -0,0 +1,17 @@ +from nwbinspector.testing import load_testing_config, update_testing_config + + +def test_update_testing_config(): + test_key = "LOCAL_PATH" + test_value = "/a/new/path/" + + initial_testing_config = load_testing_config() + try: + update_testing_config(key=test_key, value=test_value) + updated_testing_config = load_testing_config() + finally: + # Restore + update_testing_config(key=test_key, value=initial_testing_config[test_key]) + + assert updated_testing_config[test_key] != initial_testing_config[test_key] + assert updated_testing_config[test_key] == test_value diff --git a/tests/unit_tests/test_image_series.py b/tests/unit_tests/test_image_series.py index 0dfc3ae55..f61c61f40 100644 --- a/tests/unit_tests/test_image_series.py +++ b/tests/unit_tests/test_image_series.py @@ -1,6 +1,4 @@ -from unittest import TestCase -from tempfile import mkdtemp -from shutil import rmtree +import unittest from pathlib import Path import numpy as np @@ -14,70 +12,52 @@ check_image_series_external_file_relative, check_image_series_data_size, ) -from nwbinspector.tools import make_minimal_nwbfile +from nwbinspector.testing import load_testing_config +testing_config = load_testing_config() +testing_file = Path(testing_config["LOCAL_PATH"]) / "image_series_testing_file.nwb" + + +@unittest.skipIf( + not testing_file.exists(), + reason=f"The ImageSeries unit tests were skipped because the required file ({testing_file}) was not found!", +) +class TestExternalFileValid(unittest.TestCase): + @classmethod + def setUpClass(cls): + testing_config = load_testing_config() + cls.testing_file = Path(testing_config["LOCAL_PATH"]) / "image_series_testing_file.nwb" -class TestExternalFileValid(TestCase): def setUp(self): - self.tempdir = Path(mkdtemp()) - self.tempfile = self.tempdir / "tempfile.mov" - self.nested_tempdir_2 = self.tempdir / "nested_dir" - self.nested_tempdir_2.mkdir(parents=True) - self.tempfile2 = self.nested_tempdir_2 / "tempfile2.avi" - for file in [self.tempfile, self.tempfile2]: - with open(file=file, mode="w") as fp: - fp.write("Not a movie file, but at least it exists.") - self.nwbfile = make_minimal_nwbfile() - self.nwbfile.add_acquisition( - ImageSeries( - name="TestImageSeries", - rate=1.0, - external_file=[ - "/".join([".", self.tempfile.name]), - "/".join([".", self.tempfile2.parent.stem, self.tempfile2.name]), - ], - ) - ) - self.nwbfile.add_acquisition( - ImageSeries( - name="TestImageSeriesBad1", - rate=1.0, - external_file=["madeup_file.mp4"], - ) - ) - self.absolute_file_path = str(Path("madeup_file.mp4").absolute()) - self.nwbfile.add_acquisition( - ImageSeries(name="TestImageSeriesBad2", rate=1.0, external_file=[self.absolute_file_path]) - ) - image_module = self.nwbfile.create_processing_module(name="behavior", description="testing imageseries") - image_module.add(ImageSeries(name="TestImageSeries2", rate=1.0, external_file=[self.tempfile, self.tempfile2])) - with NWBHDF5IO(path=self.tempdir / "tempnwbfile.nwb", mode="w") as io: - io.write(self.nwbfile) + self.io = NWBHDF5IO(path=self.testing_file, mode="r") + self.nwbfile = self.io.read() def tearDown(self): - rmtree(self.tempdir) + self.io.close() def test_check_image_series_external_file_valid_pass(self): - with NWBHDF5IO(path=self.tempdir / "tempnwbfile.nwb", mode="r") as io: - nwbfile = io.read() - assert check_image_series_external_file_valid(image_series=nwbfile.acquisition["TestImageSeries"]) is None + assert ( + check_image_series_external_file_valid( + image_series=self.nwbfile.acquisition["TestImageSeriesGoodExternalPaths"] + ) + is None + ) def test_check_image_series_external_file_valid_bytestring_pass(self): - """Can't call the io.write() step in setUp as that decodes the bytes with our version of h5py.""" - nwbfile = make_minimal_nwbfile() - nwbfile.add_acquisition( - ImageSeries( - name="TestImageSeries", - rate=1.0, - external_file=[bytes("/".join([".", self.tempfile.name]), "utf-8")], - ) + """Can't use the NWB file since the call to io.write() decodes the bytes with modern versions of h5py.""" + good_external_path = Path(self.nwbfile.acquisition["TestImageSeriesGoodExternalPaths"].external_file[0]) + image_series = ImageSeries( + name="TestImageSeries", + rate=1.0, + external_file=[bytes("/".join([".", good_external_path.name]), "utf-8")], ) - assert check_image_series_external_file_relative(image_series=nwbfile.acquisition["TestImageSeries"]) is None + assert check_image_series_external_file_relative(image_series=image_series) is None def test_check_image_series_external_file_valid(self): - with NWBHDF5IO(path=self.tempdir / "tempnwbfile.nwb", mode="r") as io: + with NWBHDF5IO(path=self.testing_file, mode="r") as io: nwbfile = io.read() - image_series = nwbfile.acquisition["TestImageSeriesBad1"] + image_series = nwbfile.acquisition["TestImageSeriesExternalPathDoesNotExist"] + assert check_image_series_external_file_valid(image_series=image_series)[0] == InspectorMessage( message=( "The external file 'madeup_file.mp4' does not exist. Please confirm the relative location to the" @@ -86,45 +66,42 @@ def test_check_image_series_external_file_valid(self): importance=Importance.CRITICAL, check_function_name="check_image_series_external_file_valid", object_type="ImageSeries", - object_name="TestImageSeriesBad1", - location="/acquisition/TestImageSeriesBad1", + object_name="TestImageSeriesExternalPathDoesNotExist", + location="/acquisition/TestImageSeriesExternalPathDoesNotExist", ) def test_check_image_series_external_file_relative_pass(self): - with NWBHDF5IO(path=self.tempdir / "tempnwbfile.nwb", mode="r") as io: + with NWBHDF5IO(path=self.testing_file, mode="r") as io: nwbfile = io.read() + assert ( - check_image_series_external_file_relative(image_series=nwbfile.acquisition["TestImageSeries"]) is None + check_image_series_external_file_relative( + image_series=nwbfile.acquisition["TestImageSeriesGoodExternalPaths"] + ) + is None ) - def test_check_image_series_external_file_relative_bytestring_pass(self): - """Can't call the io.write() step in setUp as that decodes the bytes with our version of h5py.""" - image_series = ImageSeries( - name="TestImageSeries", - rate=1.0, - external_file=[bytes("/".join([".", self.tempfile.name]), "utf-8")], - ) - assert check_image_series_external_file_relative(image_series=image_series) is None - - def test_check_image_series_external_file_relative(self): - with NWBHDF5IO(path=self.tempdir / "tempnwbfile.nwb", mode="r") as io: + def test_check_image_series_external_file_relative_trigger(self): + with NWBHDF5IO(path=self.testing_file, mode="r") as io: nwbfile = io.read() - image_series = nwbfile.acquisition["TestImageSeriesBad2"] + image_series = nwbfile.acquisition["TestImageSeriesExternalPathIsNotRelative"] + assert check_image_series_external_file_relative(image_series=image_series)[0] == InspectorMessage( message=( - f"The external file '{self.absolute_file_path}' is not a relative path. " + f"The external file '{image_series.external_file[0]}' is not a relative path. " "Please adjust the absolute path to be relative to the location of the NWBFile." ), importance=Importance.BEST_PRACTICE_VIOLATION, check_function_name="check_image_series_external_file_relative", object_type="ImageSeries", - object_name="TestImageSeriesBad2", - location="/acquisition/TestImageSeriesBad2", + object_name="TestImageSeriesExternalPathIsNotRelative", + location="/acquisition/TestImageSeriesExternalPathIsNotRelative", ) def test_check_image_series_external_file_valid_pass_non_external(): image_series = ImageSeries(name="TestImageSeries", rate=1.0, data=np.zeros(shape=(3, 3, 3, 3)), unit="TestUnit") + assert check_image_series_external_file_valid(image_series=image_series) is None @@ -134,11 +111,11 @@ def test_check_small_image_series_stored_internally(): total_elements = int(gb_size * 1e9 / np.dtype("float").itemsize) // (frame_length * frame_length) data = np.zeros(shape=(total_elements, frame_length, frame_length, 1)) image_series = ImageSeries(name="ImageSeriesLarge", rate=1.0, data=data, unit="TestUnit") + assert check_image_series_data_size(image_series=image_series) is None def test_check_large_image_series_stored_internally(): - gb_size = 0.010 # 10 MB frame_length = 10 total_elements = int(gb_size * 1e9 / np.dtype("float").itemsize) // (frame_length * frame_length)