diff --git a/.github/workflows/testing_pipelines.yml b/.github/workflows/testing_pipelines.yml index 8892eacd0..400bc20b2 100644 --- a/.github/workflows/testing_pipelines.yml +++ b/.github/workflows/testing_pipelines.yml @@ -74,24 +74,29 @@ jobs: - name: Get ephy_testing_data current head hash id: ephys run: echo "::set-output name=HASH_EPHY_DATASET::$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" + - name: Cache ephys dataset - ${{ steps.ephys.outputs.HASH_EPHY_DATASET }} uses: actions/cache@v4 id: cache-ephys-datasets with: path: ./ephy_testing_data key: ephys-datasets-${{ matrix.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} + - name: Get ophys_testing_data current head hash id: ophys run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" + - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} uses: actions/cache@v4 id: cache-ophys-datasets with: path: ./ophys_testing_data key: ophys-datasets-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} + - name: Get behavior_testing_data current head hash id: behavior run: echo "::set-output name=HASH_BEHAVIOR_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/behavior_testing_data.git HEAD | cut -f1)" + - name: Cache behavior dataset - ${{ steps.behavior.outputs.HASH_BEHAVIOR_DATASET }} uses: actions/cache@v4 id: cache-behavior-datasets diff --git a/environments/environment-Linux.yml b/environments/environment-Linux.yml index 00d3773d1..3d37525a5 100644 --- a/environments/environment-Linux.yml +++ b/environments/environment-Linux.yml @@ -7,7 +7,7 @@ dependencies: - nodejs = 18.16.1 - numcodecs = 0.11.0 # install these from conda-forge so that dependent packages get included in the distributable - - jsonschema = 4.18.0 # installs jsonschema-specifications + - jsonschema = 4.18.0 # installs jsonschema-specifications - pip - pip: - chardet == 5.1.0 @@ -15,11 +15,14 @@ dependencies: - flask == 2.3.2 - flask-cors == 4.0.0 - flask_restx == 1.1.0 - - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. + - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. # For stability, NeuroConv is pinned at a commit just prior to breaking SpikeInterface compatibility - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@fa636458aa5c321f1c2c08f6e682b4a52d5a83f3#neuroconv[dandi,compressors,ecephys,ophys,behavior,text] # For stability, pinning SpikeInterface to a version that works with NeuroConv and with tutorial generation - spikeinterface == 0.100.5 - - scikit-learn == 1.4.0 # Tutorial data generation - - tqdm_publisher >= 0.0.1 # Progress bars - - tzlocal >= 5.2 # Frontend timezone handling + - scikit-learn == 1.4.0 # Tutorial data generation + - tqdm_publisher >= 0.0.1 # Progress bars + - tzlocal >= 5.2 # Frontend timezone handling + - ndx-pose == 0.1.1 + - nwbinspector==0.6.1 + - tables diff --git a/environments/environment-MAC-apple-silicon.yml b/environments/environment-MAC-apple-silicon.yml index fa40a3b4f..d193ce791 100644 --- a/environments/environment-MAC-apple-silicon.yml +++ b/environments/environment-MAC-apple-silicon.yml @@ -7,11 +7,11 @@ dependencies: - nodejs = 18.16.1 # Install these from conda-forge so that dependent packages get included in the distributable - numcodecs = 0.11.0 - - lxml = 4.9.3 # PyPI build fails due to x64/arm64 mismatch so install from conda-forge - - pyedflib = 0.1.32 # PyPI build fails due to x64/arm64 mismatch so install from conda-forge - - numpy # May have x64/arm64 mismatch issues so install from conda-forge + - lxml = 4.9.3 # PyPI build fails due to x64/arm64 mismatch so install from conda-forge + - pyedflib = 0.1.32 # PyPI build fails due to x64/arm64 mismatch so install from conda-forge + - numpy # May have x64/arm64 mismatch issues so install from conda-forge - pytables = 3.8 # PyPI build fails on arm64 so install from conda-forge (used by neuroconv deps) - - jsonschema = 4.18.0 # Also installs jsonschema-specifications + - jsonschema = 4.18.0 # Also installs jsonschema-specifications - pip - pip: - setuptools==70.0.0 @@ -21,13 +21,15 @@ dependencies: - flask == 2.3.2 - flask-cors == 4.0.0 - flask_restx == 1.1.0 - - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. + - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. # NOTE: the NeuroConv wheel on PyPI includes sonpy which is not compatible with arm64, so build and install # NeuroConv from GitHub, which will remove the sonpy dependency when building from Mac arm64 # For stability, NeuroConv is pinned at a commit just prior to breaking SpikeInterface compatibility - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@fa636458aa5c321f1c2c08f6e682b4a52d5a83f3#neuroconv[dandi,compressors,ecephys,ophys,behavior,text] # For stability, pinning SpikeInterface to a version that works with NeuroConv and with tutorial generation - spikeinterface == 0.100.5 - - scikit-learn == 1.4.0 # Tutorial data generation - - tqdm_publisher >= 0.0.1 # Progress bars - - tzlocal >= 5.2 # Frontend timezone handling + - scikit-learn == 1.4.0 # Tutorial data generation + - tqdm_publisher >= 0.0.1 # Progress bars + - tzlocal >= 5.2 # Frontend timezone handling + - ndx-pose == 0.1.1 + - nwbinspector==0.6.1 diff --git a/environments/environment-MAC-intel.yml b/environments/environment-MAC-intel.yml index 41de8e5ac..61d38320d 100644 --- a/environments/environment-MAC-intel.yml +++ b/environments/environment-MAC-intel.yml @@ -7,7 +7,7 @@ dependencies: - nodejs = 18.16.1 - numcodecs = 0.11.0 # install these from conda-forge so that dependent packages get included in the distributable - - jsonschema = 4.18.0 # installs jsonschema-specifications + - jsonschema = 4.18.0 # installs jsonschema-specifications - pip - pip: - setuptools==70.0.0 @@ -18,11 +18,14 @@ dependencies: - flask == 2.3.2 - flask-cors == 4.0.0 - flask_restx == 1.1.0 - - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. + - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. # For stability, NeuroConv is pinned at a commit just prior to breaking SpikeInterface compatibility - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@fa636458aa5c321f1c2c08f6e682b4a52d5a83f3#neuroconv[dandi,compressors,ecephys,ophys,behavior,text] # For stability, pinning SpikeInterface to a version that works with NeuroConv and with tutorial generation - spikeinterface == 0.100.5 - - scikit-learn == 1.4.0 # Tutorial data generation - - tqdm_publisher >= 0.0.1 # Progress bars - - tzlocal >= 5.2 # Frontend timezone handling + - scikit-learn == 1.4.0 # Tutorial data generation + - tqdm_publisher >= 0.0.1 # Progress bars + - tzlocal >= 5.2 # Frontend timezone handling + - ndx-pose == 0.1.1 + - nwbinspector==0.6.1 + - tables diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml index 7b04f568e..b7d650661 100644 --- a/environments/environment-Windows.yml +++ b/environments/environment-Windows.yml @@ -12,17 +12,20 @@ dependencies: - pip: - setuptools==70.0.0 - PyInstaller==6.7.0 - - pyinstaller-hooks-contrib == 2024.6 # Fix needed for pydantic v2; otherwise imports pydantic.compiled + - pyinstaller-hooks-contrib == 2024.6 # Fix needed for pydantic v2; otherwise imports pydantic.compiled - chardet == 5.1.0 - configparser == 6.0.0 - flask == 2.3.2 - flask-cors === 3.0.10 - flask_restx == 1.1.0 - - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. + - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. # For stability, NeuroConv is pinned at a commit just prior to breaking SpikeInterface compatibility - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@fa636458aa5c321f1c2c08f6e682b4a52d5a83f3#neuroconv[dandi,compressors,ecephys,ophys,behavior,text] # For stability, pinning SpikeInterface to a version that works with NeuroConv and with tutorial generation - spikeinterface == 0.100.5 - - scikit-learn == 1.4.0 # Tutorial data generation - - tqdm_publisher >= 0.0.1 # Progress bars - - tzlocal >= 5.2 # Frontend timezone handling + - scikit-learn == 1.4.0 # Tutorial data generation + - tqdm_publisher >= 0.0.1 # Progress bars + - tzlocal >= 5.2 # Frontend timezone handling + - ndx-pose == 0.1.1 + - nwbinspector==0.6.1 + - tables diff --git a/src/example_pipelines.yml b/src/example_pipelines.yml index cd5803376..fe8d11541 100644 --- a/src/example_pipelines.yml +++ b/src/example_pipelines.yml @@ -26,80 +26,66 @@ SpikeGLX-Phy: PhySortingInterface: folder_path: ephy_testing_data/phy/phy_example_0 - SpikeGLX_v1_SingleProbe_AP: SpikeGLXRecordingInterface: file_path: ephy_testing_data/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin - SpikeGLXConverter-Phy: SpikeGLXConverterPipe: folder_path: ephy_testing_data/spikeglx/Noise4Sam_g0 PhySortingInterface: folder_path: ephy_testing_data/phy/phy_example_0 - Suite2P_SinglePlane_SingleChannel: metadata: NWBFile: session_start_time: 2024-05-09T00:00 Ophys: ImagingPlane: - - - excitation_lambda: NaN + - excitation_lambda: NaN optical_channel: - - - emission_lambda: NaN + - emission_lambda: NaN interfaces: Suite2pSegmentationInterface: folder_path: ophys_testing_data/segmentation_datasets/suite2p - Caiman_SinglePlane_SingleChannel: metadata: NWBFile: session_start_time: 2024-05-09T00:00 Ophys: ImagingPlane: - - - excitation_lambda: NaN + - excitation_lambda: NaN optical_channel: - - - emission_lambda: NaN + - emission_lambda: NaN interfaces: CaimanSegmentationInterface: file_path: ophys_testing_data/segmentation_datasets/caiman/caiman_analysis.hdf5 - ScanImage_SinglePlane_SingleChannel: - metadata: Ophys: ImagingPlane: - - - excitation_lambda: NaN + - excitation_lambda: NaN optical_channel: - - - emission_lambda: NaN + - emission_lambda: NaN interfaces: ScanImageImagingInterface: file_path: ophys_testing_data/imaging_datasets/ScanImage/sample_scanimage_version_3_8.tiff - TDT: metadata: - NWBFile: - session_start_time: 2024-05-09T00:00 + NWBFile: + session_start_time: 2024-05-09T00:00 interfaces: TdtRecordingInterface: folder_path: ephy_testing_data/tdt/aep_05 gain: 1 - CellExplorer: metadata: NWBFile: @@ -109,7 +95,6 @@ CellExplorer: CellExplorerRecordingInterface: folder_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed - CellExplorerUnits: metadata: NWBFile: @@ -121,7 +106,6 @@ CellExplorerUnits: CellExplorerSortingInterface: file_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed/Peter_MS22_180629_110319_concat_stubbed.spikes.cellinfo.mat - CellExplorerSortingOnly: metadata: NWBFile: @@ -131,24 +115,19 @@ CellExplorerSortingOnly: CellExplorerSortingInterface: file_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed/Peter_MS22_180629_110319_concat_stubbed.spikes.cellinfo.mat - IntanSingleFileRecording: metadata: NWBFile: session_start_time: 2024-05-09T00:00 Ecephys: ElectrodeGroup: - - - location: Test location - - - location: Test location - - - location: Test location + - location: Test location + - location: Test location + - location: Test location interfaces: IntanRecordingInterface: file_path: ephy_testing_data/intan/intan_rhd_test_1.rhd - MEArec: metadata: NWBFile: @@ -158,22 +137,18 @@ MEArec: MEArecRecordingInterface: file_path: ephy_testing_data/mearec/mearec_test_10s.h5 - BrukerTiffMultiChannelSinglePlane: metadata: Ophys: ImagingPlane: - - - excitation_lambda: NaN + - excitation_lambda: NaN optical_channel: - - - emission_lambda: NaN + - emission_lambda: NaN interfaces: BrukerTiffSinglePlaneConverter: folder_path: ophys_testing_data/imaging_datasets/BrukerTif/NCCR32_2023_02_20_Into_the_void_t_series_baseline-000 - DeepLabCut: metadata: NWBFile: @@ -181,9 +156,8 @@ DeepLabCut: interfaces: DeepLabCutInterface: - file_path: behavior_testing_data/DLC/m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5 - config_file_path: behavior_testing_data/DLC/config.yaml - + file_path: behavior_testing_data/DLC/open_field_with_video/m3v1mp4DLC_resnet50_openfieldOct30shuffle1_1000.h5 + config_file_path: behavior_testing_data/DLC/open_field_with_video/config.yaml SLEAP: metadata: