diff --git a/CHANGELOG.md b/CHANGELOG.md index b9452f4..04104e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. +## [0.1.3] - 2022-09-20 + ++ Add - Notebook for Allen Institute workshop + ## [0.1.2] - 2022-09-19 + Update - Increment version to test CI/CD @@ -39,6 +43,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and + Add - Containerization for pytests + Comment - Phase previously designated 0.1.0 -> 0.0.0 +[0.1.3]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.3 [0.1.2]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.2 [0.1.1]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.1 [0.1.0]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.0 diff --git a/notebooks/2022-allen-institute-workshop.ipynb b/notebooks/2022-allen-institute-workshop.ipynb new file mode 100644 index 0000000..259b742 --- /dev/null +++ b/notebooks/2022-allen-institute-workshop.ipynb @@ -0,0 +1,449 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "11e51a9c", + "metadata": {}, + "source": [ + "# Allen Institute Calcium Imaging Workshop\n", + "September 22, 2022\n", + "+ In this notebook, we will show how to interact with a database in Python and how re-run an analysis.\n", + "\n", + "+ Other notebooks in this directory describe the process for running the analysis steps in more detail.\n", + "\n", + "+ This notebook is meant to be run on CodeBook (`https://codebook.datajoint.io`) which contains example data.\n", + "\n", + "+ First run the `01-configure` and `04-automate` notebooks to set up your environment and load example data into the database, respectively." + ] + }, + { + "cell_type": "markdown", + "id": "14a6ba1d", + "metadata": {}, + "source": [ + "## Configuration\n", + "\n", + "Import the relevant packages." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1e0295c-57bf-49d2-9a1e-931d0e53264b", + "metadata": {}, + "outputs": [], + "source": [ + "import datajoint as dj\n", + "import numpy as np\n", + "from matplotlib import pyplot\n", + "import os" + ] + }, + { + "cell_type": "markdown", + "id": "e652916f", + "metadata": {}, + "source": [ + "Enter database credentials. A DataJoint workflow requires a connection to an existing relational database. The connection setup parameters are defined in the `dj.config` python dictionary." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fdc35bd8-2381-4769-9f43-33a7f3fd2332", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "dj.config['custom'] = {'database.prefix': '_allen_ophys_',\n", + " 'imaging_root_data_dir': '/home/inbox/0_1_0a2/'}" + ] + }, + { + "cell_type": "markdown", + "id": "7c545770", + "metadata": {}, + "source": [ + "Import the workflow. The current workflow is composed of multiple database schemas, each of them corresponding to a module within the `workflow_calcium_imaging.pipeline` file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ee08b24f", + "metadata": {}, + "outputs": [], + "source": [ + "from workflow_calcium_imaging.pipeline import lab, subject, session, scan, imaging" + ] + }, + { + "cell_type": "markdown", + "id": "2bd6d86b", + "metadata": {}, + "source": [ + "## Workflow diagram\n", + "\n", + "Plot the workflow diagram. In relational databases, the entities (i.e. rows) in different tables are connected to each other. Visualization of this relationship helps one to write accurate queries. For the calcium imaging workflow, this connection is as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8dd6d3ee", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "dj.Diagram(lab.Lab) + dj.Diagram(subject.Subject) + dj.Diagram(session.Session) + \\\n", + "dj.Diagram(scan) + dj.Diagram(imaging)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1caa7ced-93f6-4182-aa57-61bfbf961d38", + "metadata": {}, + "outputs": [], + "source": [ + "subject.Subject()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a110dca3-9149-40dd-8776-5b61476ccda1", + "metadata": {}, + "outputs": [], + "source": [ + "scan.Scan()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a4ea243", + "metadata": {}, + "outputs": [], + "source": [ + "imaging.Fluorescence()" + ] + }, + { + "cell_type": "markdown", + "id": "5ca53cb7", + "metadata": {}, + "source": [ + "## Fetch data from the database\n", + "\n", + "Fetch a fluorescence trace for a single mask and plot these values." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "082bcda0", + "metadata": {}, + "outputs": [], + "source": [ + "imaging.Fluorescence.Trace()" + ] + }, + { + "cell_type": "markdown", + "id": "0e3ebf0f", + "metadata": {}, + "source": [ + "Restrict the table with specific criteria." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "955af65d", + "metadata": {}, + "outputs": [], + "source": [ + "query_trace = imaging.Fluorescence.Trace & 'subject=\"subject3\"' \\\n", + " & 'session_datetime=\"2022-09-01 19:16:44\"' \\\n", + " & 'mask_id=120'\n", + "query_trace" + ] + }, + { + "cell_type": "markdown", + "id": "509d5335", + "metadata": {}, + "source": [ + "Fetch a fluorescence trace from the database." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "96c1a6cd", + "metadata": {}, + "outputs": [], + "source": [ + "trace = (query_trace).fetch('fluorescence')" + ] + }, + { + "cell_type": "markdown", + "id": "a9bc7039", + "metadata": {}, + "source": [ + "Plot the fluorescence trace." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a9a2cd6a", + "metadata": {}, + "outputs": [], + "source": [ + "sampling_rate = (scan.ScanInfo & 'subject=\"subject3\"' & 'session_datetime=\"2022-09-01 19:16:44\"').fetch1('fps')\n", + "\n", + "pyplot.plot(np.r_[:trace.size] * 1/sampling_rate, trace, 'k')\n", + "\n", + "pyplot.title('Fluorescence trace for mask 120',labelsize=14)\n", + "pyplot.tick_params(labelsize=14)\n", + "pyplot.set_xlabel('Time (s)')\n", + "pyplot.set_ylabel('Activity (a.u.)')" + ] + }, + { + "cell_type": "markdown", + "id": "68559c95-3c4e-4c6a-acbb-c06796a8399c", + "metadata": {}, + "source": [ + "## Run analysis\n", + "\n", + "The workflow has already been run for with a parameter set (paramset_idx=1). Let's re-run Suite2p with a different parameter set, changing the cell diameter to 10 microns." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b01dfad", + "metadata": {}, + "outputs": [], + "source": [ + "dj.Diagram(imaging.Processing)-2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ec4c0bb-c724-4c0a-add7-ee198dd632d0", + "metadata": {}, + "outputs": [], + "source": [ + "imaging.ProcessingTask()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7505c558", + "metadata": {}, + "outputs": [], + "source": [ + "imaging.ProcessingParamSet()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55196834", + "metadata": {}, + "outputs": [], + "source": [ + "params_suite2p = {'look_one_level_down': 0.0,\n", + " 'fast_disk': [],\n", + " 'delete_bin': False,\n", + " 'mesoscan': False,\n", + " 'h5py': [],\n", + " 'h5py_key': 'data',\n", + " 'save_path0': [],\n", + " 'subfolders': [],\n", + " 'nplanes': 1,\n", + " 'nchannels': 1,\n", + " 'functional_chan': 1,\n", + " 'tau': 1.0,\n", + " 'fs': 10.0,\n", + " 'force_sktiff': False,\n", + " 'preclassify': 0.0,\n", + " 'save_mat': False,\n", + " 'combined': True,\n", + " 'aspect': 1.0,\n", + " 'do_bidiphase': False,\n", + " 'bidiphase': 0.0,\n", + " 'do_registration': True,\n", + " 'keep_movie_raw': False,\n", + " 'nimg_init': 300,\n", + " 'batch_size': 500,\n", + " 'maxregshift': 0.1,\n", + " 'align_by_chan': 1,\n", + " 'reg_tif': False,\n", + " 'reg_tif_chan2': False,\n", + " 'subpixel': 10,\n", + " 'smooth_sigma': 1.15,\n", + " 'th_badframes': 1.0,\n", + " 'pad_fft': False,\n", + " 'nonrigid': True,\n", + " 'block_size': [128, 128],\n", + " 'snr_thresh': 1.2,\n", + " 'maxregshiftNR': 5.0,\n", + " '1Preg': False,\n", + " 'spatial_hp': 50.0,\n", + " 'pre_smooth': 2.0,\n", + " 'spatial_taper': 50.0,\n", + " 'roidetect': True,\n", + " 'sparse_mode': False,\n", + " 'diameter': 10,\n", + " 'spatial_scale': 0,\n", + " 'connected': True,\n", + " 'nbinned': 5000,\n", + " 'max_iterations': 20,\n", + " 'threshold_scaling': 1.0,\n", + " 'max_overlap': 0.75,\n", + " 'high_pass': 100.0,\n", + " 'inner_neuropil_radius': 2,\n", + " 'min_neuropil_pixels': 350,\n", + " 'allow_overlap': False,\n", + " 'chan2_thres': 0.65,\n", + " 'baseline': 'maximin',\n", + " 'win_baseline': 60.0,\n", + " 'sig_baseline': 10.0,\n", + " 'prctile_baseline': 8.0,\n", + " 'neucoeff': 0.7,\n", + " 'xrange': np.array([0, 0]),\n", + " 'yrange': np.array([0, 0])}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b42e5074", + "metadata": {}, + "outputs": [], + "source": [ + "imaging.ProcessingParamSet.insert_new_params(processing_method='suite2p', \n", + " paramset_idx=1, \n", + " params=params_suite2p,\n", + " paramset_desc='diameter=10')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41fe465c", + "metadata": {}, + "outputs": [], + "source": [ + "imaging.ProcessingParamSet()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5fb498b3", + "metadata": {}, + "outputs": [], + "source": [ + "os.makedirs('/home/inbox/0_1_0a2/subject3/210107_run00_orientation_8dir/suite2p_1', exist_ok=True)\n", + "\n", + "imaging.ProcessingTask.insert1(dict(subject='subject3', \n", + " session_datetime='2022-09-01 19:16:44', \n", + " scan_id=0,\n", + " paramset_idx=1,\n", + " processing_output_dir='subject3/210107_run00_orientation_8dir/suite2p_1'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec0841aa", + "metadata": {}, + "outputs": [], + "source": [ + "imaging.ProcessingTask()" + ] + }, + { + "cell_type": "markdown", + "id": "637b9ec1", + "metadata": {}, + "source": [ + "Run Suite2p for the new parameter set and save the results to the respective tables." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5d6bc13", + "metadata": {}, + "outputs": [], + "source": [ + "populate_settings = dict(display_progress=True)\n", + "\n", + "imaging.Processing.populate(**populate_settings)\n", + "\n", + "key = (imaging.ProcessingTask & 'subject=\"subject3\"' & 'session_datetime=\"2022-09-01 19:16:44\"').fetch1('KEY')\n", + "\n", + "imaging.Curation().create1_from_processing_task(key)\n", + "\n", + "imaging.MotionCorrection.populate(**populate_settings)\n", + "\n", + "imaging.Segmentation.populate(**populate_settings)\n", + "\n", + "imaging.Fluorescence.populate(**populate_settings)\n", + "\n", + "imaging.Activity.populate(**populate_settings)" + ] + }, + { + "cell_type": "markdown", + "id": "4c8087a9", + "metadata": {}, + "source": [ + "## Summary and next steps\n", + "\n", + "In this notebook we explored how to query and fetch data from the database, and re-run analysis with new parameters. Next, please explore more of the features of the DataJoint Elements in the other notebooks. Once you are ready to begin setting up your pipeline, fork this repository on GitHub and begin adapting it for your projects requirements." + ] + } + ], + "metadata": { + "jupytext": { + "formats": "ipynb,py" + }, + "kernelspec": { + "display_name": "Python 3.7.9 ('workflow-calcium-imaging')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.9" + }, + "vscode": { + "interpreter": { + "hash": "2da0f701b958ef27dbb9ddfd0ee0eb0d75664c3cb2b4bc67fd0780a51e2b3863" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/py_scripts/2022-allen-institute-workshop.py b/notebooks/py_scripts/2022-allen-institute-workshop.py new file mode 100644 index 0000000..4322c41 --- /dev/null +++ b/notebooks/py_scripts/2022-allen-institute-workshop.py @@ -0,0 +1,204 @@ +# --- +# jupyter: +# jupytext: +# formats: ipynb,py +# text_representation: +# extension: .py +# format_name: light +# format_version: '1.5' +# jupytext_version: 1.14.0 +# kernelspec: +# display_name: Python 3.7.9 ('workflow-calcium-imaging') +# language: python +# name: python3 +# --- + +# # Allen Institute Calcium Imaging Workshop +# September 22, 2022 +# + In this notebook, we will show how to interact with a database in Python and how re-run an analysis. +# +# + Other notebooks in this directory describe the process for running the analysis steps in more detail. +# +# + This notebook is meant to be run on CodeBook (`https://codebook.datajoint.io`) which contains example data. +# +# + First run the `01-configure` and `04-automate` notebooks to set up your environment and load example data into the database, respectively. + +# ## Configuration +# +# Import the relevant packages. + +import datajoint as dj +import numpy as np +from matplotlib import pyplot +import os + +# Enter database credentials. A DataJoint workflow requires a connection to an existing relational database. The connection setup parameters are defined in the `dj.config` python dictionary. + +# + tags=[] +dj.config['custom'] = {'database.prefix': '_allen_ophys_', + 'imaging_root_data_dir': '/home/inbox/0_1_0a2/'} +# - + +# Import the workflow. The current workflow is composed of multiple database schemas, each of them corresponding to a module within the `workflow_calcium_imaging.pipeline` file. + +from workflow_calcium_imaging.pipeline import lab, subject, session, scan, imaging + +# ## Workflow diagram +# +# Plot the workflow diagram. In relational databases, the entities (i.e. rows) in different tables are connected to each other. Visualization of this relationship helps one to write accurate queries. For the calcium imaging workflow, this connection is as follows: + +# + tags=[] +dj.Diagram(lab.Lab) + dj.Diagram(subject.Subject) + dj.Diagram(session.Session) + \ +dj.Diagram(scan) + dj.Diagram(imaging) +# - + +subject.Subject() + +scan.Scan() + +imaging.Fluorescence() + +# ## Fetch data from the database +# +# Fetch a fluorescence trace for a single mask and plot these values. + +imaging.Fluorescence.Trace() + +# Restrict the table with specific criteria. + +query_trace = imaging.Fluorescence.Trace & 'subject="subject3"' \ + & 'session_datetime="2022-09-01 19:16:44"' \ + & 'mask_id=120' +query_trace + +# Fetch a fluorescence trace from the database. + +trace = (query_trace).fetch('fluorescence') + +# Plot the fluorescence trace. + +# + +sampling_rate = (scan.ScanInfo & 'subject="subject3"' & 'session_datetime="2022-09-01 19:16:44"').fetch1('fps') + +pyplot.plot(np.r_[:trace.size] * 1/sampling_rate, trace, 'k') + +pyplot.title('Fluorescence trace for mask 120',labelsize=14) +pyplot.tick_params(labelsize=14) +pyplot.set_xlabel('Time (s)') +pyplot.set_ylabel('Activity (a.u.)') +# - + +# ## Run analysis +# +# The workflow has already been run for with a parameter set (paramset_idx=1). Let's re-run Suite2p with a different parameter set, changing the cell diameter to 10 microns. + +dj.Diagram(imaging.Processing)-2 + +imaging.ProcessingTask() + +imaging.ProcessingParamSet() + +params_suite2p = {'look_one_level_down': 0.0, + 'fast_disk': [], + 'delete_bin': False, + 'mesoscan': False, + 'h5py': [], + 'h5py_key': 'data', + 'save_path0': [], + 'subfolders': [], + 'nplanes': 1, + 'nchannels': 1, + 'functional_chan': 1, + 'tau': 1.0, + 'fs': 10.0, + 'force_sktiff': False, + 'preclassify': 0.0, + 'save_mat': False, + 'combined': True, + 'aspect': 1.0, + 'do_bidiphase': False, + 'bidiphase': 0.0, + 'do_registration': True, + 'keep_movie_raw': False, + 'nimg_init': 300, + 'batch_size': 500, + 'maxregshift': 0.1, + 'align_by_chan': 1, + 'reg_tif': False, + 'reg_tif_chan2': False, + 'subpixel': 10, + 'smooth_sigma': 1.15, + 'th_badframes': 1.0, + 'pad_fft': False, + 'nonrigid': True, + 'block_size': [128, 128], + 'snr_thresh': 1.2, + 'maxregshiftNR': 5.0, + '1Preg': False, + 'spatial_hp': 50.0, + 'pre_smooth': 2.0, + 'spatial_taper': 50.0, + 'roidetect': True, + 'sparse_mode': False, + 'diameter': 10, + 'spatial_scale': 0, + 'connected': True, + 'nbinned': 5000, + 'max_iterations': 20, + 'threshold_scaling': 1.0, + 'max_overlap': 0.75, + 'high_pass': 100.0, + 'inner_neuropil_radius': 2, + 'min_neuropil_pixels': 350, + 'allow_overlap': False, + 'chan2_thres': 0.65, + 'baseline': 'maximin', + 'win_baseline': 60.0, + 'sig_baseline': 10.0, + 'prctile_baseline': 8.0, + 'neucoeff': 0.7, + 'xrange': np.array([0, 0]), + 'yrange': np.array([0, 0])} + +imaging.ProcessingParamSet.insert_new_params(processing_method='suite2p', + paramset_idx=1, + params=params_suite2p, + paramset_desc='diameter=10') + +imaging.ProcessingParamSet() + +# + +os.makedirs('/home/inbox/0_1_0a2/subject3/210107_run00_orientation_8dir/suite2p_1', exist_ok=True) + +imaging.ProcessingTask.insert1(dict(subject='subject3', + session_datetime='2022-09-01 19:16:44', + scan_id=0, + paramset_idx=1, + processing_output_dir='subject3/210107_run00_orientation_8dir/suite2p_1')) +# - + +imaging.ProcessingTask() + +# Run Suite2p for the new parameter set and save the results to the respective tables. + +# + +populate_settings = dict(display_progress=True) + +imaging.Processing.populate(**populate_settings) + +key = (imaging.ProcessingTask & 'subject="subject3"' & 'session_datetime="2022-09-01 19:16:44"').fetch1('KEY') + +imaging.Curation().create1_from_processing_task(key) + +imaging.MotionCorrection.populate(**populate_settings) + +imaging.Segmentation.populate(**populate_settings) + +imaging.Fluorescence.populate(**populate_settings) + +imaging.Activity.populate(**populate_settings) +# - + +# ## Summary and next steps +# +# In this notebook we explored how to query and fetch data from the database, and re-run analysis with new parameters. Next, please explore more of the features of the DataJoint Elements in the other notebooks. Once you are ready to begin setting up your pipeline, fork this repository on GitHub and begin adapting it for your projects requirements. diff --git a/requirements.txt b/requirements.txt index 93457dd..a097493 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ element-event @ git+https://github.com/datajoint/element-event.git element-interface @ git+https://github.com/datajoint/element-interface.git ipykernel>=6.0.1 jupytext>=1.13.7 -sbxreader \ No newline at end of file +sbxreader +suite2p @ git+https://github.com/datajoint-company/suite2p.git \ No newline at end of file diff --git a/workflow_calcium_imaging/version.py b/workflow_calcium_imaging/version.py index ebe1990..0f8a416 100644 --- a/workflow_calcium_imaging/version.py +++ b/workflow_calcium_imaging/version.py @@ -3,4 +3,4 @@ Update the Docker image tag in `docker-compose-test.yaml` and `docker-compose-dev.yaml` to match """ -__version__ = "0.1.2" +__version__ = "0.1.3"