From dc2dc28684a1e27e3dfd0dddd676d38b3e554df1 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Tue, 20 Sep 2022 17:09:29 -0500 Subject: [PATCH 01/12] Update version and changelog --- CHANGELOG.md | 5 +++++ workflow_calcium_imaging/version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9452f4..f4413a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. +## [0.2.0] - 2022-09-20 + ++ Add - Notebook for Allen Institute workshop + ## [0.1.2] - 2022-09-19 + Update - Increment version to test CI/CD @@ -39,6 +43,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and + Add - Containerization for pytests + Comment - Phase previously designated 0.1.0 -> 0.0.0 +[0.2.0]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.2.0 [0.1.2]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.2 [0.1.1]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.1 [0.1.0]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.0 diff --git a/workflow_calcium_imaging/version.py b/workflow_calcium_imaging/version.py index ebe1990..23b6780 100644 --- a/workflow_calcium_imaging/version.py +++ b/workflow_calcium_imaging/version.py @@ -3,4 +3,4 @@ Update the Docker image tag in `docker-compose-test.yaml` and `docker-compose-dev.yaml` to match """ -__version__ = "0.1.2" +__version__ = "0.2.0" From c63facbd9aa5a56565ce5877f44d133e8a0dcde8 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Tue, 20 Sep 2022 20:03:20 -0500 Subject: [PATCH 02/12] Add workshop notebook --- notebooks/2022-allen-institute-workshop.ipynb | 403 ++++++++++++++++++ 1 file changed, 403 insertions(+) create mode 100644 notebooks/2022-allen-institute-workshop.ipynb diff --git a/notebooks/2022-allen-institute-workshop.ipynb b/notebooks/2022-allen-institute-workshop.ipynb new file mode 100644 index 0000000..98b9b72 --- /dev/null +++ b/notebooks/2022-allen-institute-workshop.ipynb @@ -0,0 +1,403 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "11e51a9c", + "metadata": {}, + "source": [ + "# Allen Institute Calcium Imaging Workshop\n", + "September 22, 2022\n", + "+ In this notebook, we will show some of the advantages of working with DataJoint workflows, including how to interact with a database in Python and how re-run an analysis.\n", + "\n", + "+ Other notebooks in this directory describe the process for running the analysis steps in more detail.\n", + "\n", + "+ This notebook is meant to be run on CodeBook (`https://codebook.datajoint.io`) which contains example data.\n", + "\n", + "+ First run the `01-configure` and `04-automate` notebooks to set up your environment and load example data into the database, respectively." + ] + }, + { + "cell_type": "markdown", + "id": "14a6ba1d", + "metadata": {}, + "source": [ + "## Configuration\n", + "\n", + "Import the relevant packages." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1e0295c-57bf-49d2-9a1e-931d0e53264b", + "metadata": {}, + "outputs": [], + "source": [ + "import datajoint as dj\n", + "import numpy as np\n", + "from matplotlib import pyplot" + ] + }, + { + "cell_type": "markdown", + "id": "e652916f", + "metadata": {}, + "source": [ + "Enter database credentials. A DataJoint workflow requires a connection to an existing relational database. The connection setup parameters are defined in the `dj.config` python dictionary." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fdc35bd8-2381-4769-9f43-33a7f3fd2332", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "dj.config['custom'] = {'database.prefix': '_allen_ophys_',\n", + " 'imaging_root_data_dir': '/home/inbox/0_1_0a2/'}" + ] + }, + { + "cell_type": "markdown", + "id": "7c545770", + "metadata": {}, + "source": [ + "Import the workflow. The current workflow is composed of multiple database schemas, each of them corresponding to a module within the `workflow_calcium_imaging.pipeline` file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ee08b24f", + "metadata": {}, + "outputs": [], + "source": [ + "from workflow_calcium_imaging.pipeline import lab, subject, session, scan, imaging" + ] + }, + { + "cell_type": "markdown", + "id": "2bd6d86b", + "metadata": {}, + "source": [ + "## Workflow diagram\n", + "\n", + "Plot the workflow diagram. In relational databases, the entities (i.e. rows) in different tables are connected to each other. Visualization of this relationship helps one to write accurate queries. For the calcium imaging workflow, this connection is as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8dd6d3ee", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "dj.Diagram(lab.Lab) + dj.Diagram(subject.Subject) + dj.Diagram(session.Session) + \\\n", + "dj.Diagram(scan) + dj.Diagram(imaging)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1caa7ced-93f6-4182-aa57-61bfbf961d38", + "metadata": {}, + "outputs": [], + "source": [ + "subject.Subject()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a110dca3-9149-40dd-8776-5b61476ccda1", + "metadata": {}, + "outputs": [], + "source": [ + "scan.Scan()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a4ea243", + "metadata": {}, + "outputs": [], + "source": [ + "imaging.Fluorescence()" + ] + }, + { + "cell_type": "markdown", + "id": "5ca53cb7", + "metadata": {}, + "source": [ + "## Fetch data from the database\n", + "\n", + "Fetch a fluorescence trace for a single mask and plot these values." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "082bcda0", + "metadata": {}, + "outputs": [], + "source": [ + "imaging.Fluorescence.Trace()" + ] + }, + { + "cell_type": "markdown", + "id": "0e3ebf0f", + "metadata": {}, + "source": [ + "Restrict the table with specific criteria." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "955af65d", + "metadata": {}, + "outputs": [], + "source": [ + "imaging.Fluorescence.Trace & 'subject=\"\"' \\\n", + " & 'session_datetime=\"2020-08-08\"' \\\n", + " & 'mask_id=120'" + ] + }, + { + "cell_type": "markdown", + "id": "509d5335", + "metadata": {}, + "source": [ + "Fetch a fluorescence trace from the database." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "96c1a6cd", + "metadata": {}, + "outputs": [], + "source": [ + "trace = (imaging.Fluorescence.Trace & 'subject=\"\"' \\\n", + " & 'session_datetime=\"2020-08-08\"' \\\n", + " & 'mask_id=120').fetch('fluorescence')" + ] + }, + { + "cell_type": "markdown", + "id": "a9bc7039", + "metadata": {}, + "source": [ + "Plot the fluorescence trace." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a9a2cd6a", + "metadata": {}, + "outputs": [], + "source": [ + "sampling_rate = (scan.ScanInfo & 'subject=\"\"' & 'session_datetime=\"2020-08-08\"').fetch1('fps')\n", + "\n", + "pyplot.plot(np.r_[:trace.size] * 1/sampling_rate, trace, 'k')\n", + "\n", + "pyplot.title('Fluorescence trace for mask 120',labelsize=14)\n", + "pyplot.tick_params(labelsize=14)\n", + "pyplot.set_xlabel('Time (s)')\n", + "pyplot.set_ylabel('Activity (a.u.)')" + ] + }, + { + "cell_type": "markdown", + "id": "68559c95-3c4e-4c6a-acbb-c06796a8399c", + "metadata": {}, + "source": [ + "## Run analysis\n", + "\n", + "The workflow has already been run for with a parameter set (paramset_idx=1). Let's re-run Suite2p with a different parameter set, changing the cell diameter to 10 microns." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b01dfad", + "metadata": {}, + "outputs": [], + "source": [ + "dj.Diagram(imaging.Processing)-2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ec4c0bb-c724-4c0a-add7-ee198dd632d0", + "metadata": {}, + "outputs": [], + "source": [ + "imaging.ProcessingTask()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7505c558", + "metadata": {}, + "outputs": [], + "source": [ + "imaging.ProcessingParamSet()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55196834", + "metadata": {}, + "outputs": [], + "source": [ + "params_suite2p = {'look_one_level_down': 0.0,\n", + " 'fast_disk': [],\n", + " 'delete_bin': False,\n", + " 'mesoscan': False,\n", + " 'h5py': [],\n", + " 'h5py_key': 'data',\n", + " 'save_path0': [],\n", + " 'subfolders': [],\n", + " 'nplanes': 1,\n", + " 'nchannels': 1,\n", + " 'functional_chan': 1,\n", + " 'tau': 1.0,\n", + " 'fs': 10.0,\n", + " 'force_sktiff': False,\n", + " 'preclassify': 0.0,\n", + " 'save_mat': False,\n", + " 'combined': True,\n", + " 'aspect': 1.0,\n", + " 'do_bidiphase': False,\n", + " 'bidiphase': 0.0,\n", + " 'do_registration': True,\n", + " 'keep_movie_raw': False,\n", + " 'nimg_init': 300,\n", + " 'batch_size': 500,\n", + " 'maxregshift': 0.1,\n", + " 'align_by_chan': 1,\n", + " 'reg_tif': False,\n", + " 'reg_tif_chan2': False,\n", + " 'subpixel': 10,\n", + " 'smooth_sigma': 1.15,\n", + " 'th_badframes': 1.0,\n", + " 'pad_fft': False,\n", + " 'nonrigid': True,\n", + " 'block_size': [128, 128],\n", + " 'snr_thresh': 1.2,\n", + " 'maxregshiftNR': 5.0,\n", + " '1Preg': False,\n", + " 'spatial_hp': 50.0,\n", + " 'pre_smooth': 2.0,\n", + " 'spatial_taper': 50.0,\n", + " 'roidetect': True,\n", + " 'sparse_mode': False,\n", + " 'diameter': 10,\n", + " 'spatial_scale': 0,\n", + " 'connected': True,\n", + " 'nbinned': 5000,\n", + " 'max_iterations': 20,\n", + " 'threshold_scaling': 1.0,\n", + " 'max_overlap': 0.75,\n", + " 'high_pass': 100.0,\n", + " 'inner_neuropil_radius': 2,\n", + " 'min_neuropil_pixels': 350,\n", + " 'allow_overlap': False,\n", + " 'chan2_thres': 0.65,\n", + " 'baseline': 'maximin',\n", + " 'win_baseline': 60.0,\n", + " 'sig_baseline': 10.0,\n", + " 'prctile_baseline': 8.0,\n", + " 'neucoeff': 0.7,\n", + " 'xrange': np.array([0, 0]),\n", + " 'yrange': np.array([0, 0])}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec0841aa", + "metadata": {}, + "outputs": [], + "source": [ + "imaging.ProcessingTask()" + ] + }, + { + "cell_type": "markdown", + "id": "637b9ec1", + "metadata": {}, + "source": [ + "Run Suite2p for the new parameter set and save the results to the respective tables." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5d6bc13", + "metadata": {}, + "outputs": [], + "source": [ + "populate_settings = dict(display_progress=True)\n", + "\n", + "imaging.Processing.populate(populate_settings)\n", + "\n", + "imaging.MotionCorrection.populate(populate_settings)\n", + "\n", + "imaging.Segmentation.populate(populate_settings)\n", + "\n", + "imaging.Fluorescence.populate(populate_settings)\n", + "\n", + "imaging.Activity.populate(populate_settings)" + ] + }, + { + "cell_type": "markdown", + "id": "4c8087a9", + "metadata": {}, + "source": [ + "## Summary and next steps\n", + "\n", + "In this notebook we explored how to query and fetch data from the database, and re-run analysis with new parameters. Next, please explore more of the features of the DataJoint Elements in the other notebooks. Once you are ready to begin setting up your pipeline, fork this repository on GitHub and begin adapting it for your projects requirements." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.7.9 ('workflow-calcium-imaging')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.9" + }, + "vscode": { + "interpreter": { + "hash": "2da0f701b958ef27dbb9ddfd0ee0eb0d75664c3cb2b4bc67fd0780a51e2b3863" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 8aa9f508852d623a4606730edb203ad441e5a339 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Tue, 20 Sep 2022 20:05:42 -0500 Subject: [PATCH 03/12] Update notebook --- notebooks/2022-allen-institute-workshop.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/notebooks/2022-allen-institute-workshop.ipynb b/notebooks/2022-allen-institute-workshop.ipynb index 98b9b72..f0269ef 100644 --- a/notebooks/2022-allen-institute-workshop.ipynb +++ b/notebooks/2022-allen-institute-workshop.ipynb @@ -352,15 +352,15 @@ "source": [ "populate_settings = dict(display_progress=True)\n", "\n", - "imaging.Processing.populate(populate_settings)\n", + "imaging.Processing.populate(**populate_settings)\n", "\n", - "imaging.MotionCorrection.populate(populate_settings)\n", + "imaging.MotionCorrection.populate(**populate_settings)\n", "\n", - "imaging.Segmentation.populate(populate_settings)\n", + "imaging.Segmentation.populate(**populate_settings)\n", "\n", - "imaging.Fluorescence.populate(populate_settings)\n", + "imaging.Fluorescence.populate(**populate_settings)\n", "\n", - "imaging.Activity.populate(populate_settings)" + "imaging.Activity.populate(**populate_settings)" ] }, { From 878592a872f0b1090f64cd53186c159e8b4f5c2f Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Tue, 20 Sep 2022 20:00:38 -0700 Subject: [PATCH 04/12] Update notebook --- notebooks/2022-allen-institute-workshop.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/2022-allen-institute-workshop.ipynb b/notebooks/2022-allen-institute-workshop.ipynb index f0269ef..9f7f9a8 100644 --- a/notebooks/2022-allen-institute-workshop.ipynb +++ b/notebooks/2022-allen-institute-workshop.ipynb @@ -7,7 +7,7 @@ "source": [ "# Allen Institute Calcium Imaging Workshop\n", "September 22, 2022\n", - "+ In this notebook, we will show some of the advantages of working with DataJoint workflows, including how to interact with a database in Python and how re-run an analysis.\n", + "+ In this notebook, we will show how to interact with a database in Python and how re-run an analysis.\n", "\n", "+ Other notebooks in this directory describe the process for running the analysis steps in more detail.\n", "\n", From fa6b5e2b973c2a682b5592780d721cf3fe0814bb Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Tue, 20 Sep 2022 20:06:28 -0700 Subject: [PATCH 05/12] Update notebook --- notebooks/2022-allen-institute-workshop.ipynb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/notebooks/2022-allen-institute-workshop.ipynb b/notebooks/2022-allen-institute-workshop.ipynb index 9f7f9a8..22deef3 100644 --- a/notebooks/2022-allen-institute-workshop.ipynb +++ b/notebooks/2022-allen-institute-workshop.ipynb @@ -375,6 +375,9 @@ } ], "metadata": { + "jupytext": { + "formats": "ipynb,py" + }, "kernelspec": { "display_name": "Python 3.7.9 ('workflow-calcium-imaging')", "language": "python", From d89d2ac00eb328767375b0a4d655179b2d81903c Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Tue, 20 Sep 2022 20:06:56 -0700 Subject: [PATCH 06/12] Add jupytext script --- .../2022-allen-institute-workshop.py | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 notebooks/py_scripts/2022-allen-institute-workshop.py diff --git a/notebooks/py_scripts/2022-allen-institute-workshop.py b/notebooks/py_scripts/2022-allen-institute-workshop.py new file mode 100644 index 0000000..ff71707 --- /dev/null +++ b/notebooks/py_scripts/2022-allen-institute-workshop.py @@ -0,0 +1,183 @@ +# --- +# jupyter: +# jupytext: +# formats: ipynb,py +# text_representation: +# extension: .py +# format_name: light +# format_version: '1.5' +# jupytext_version: 1.14.0 +# kernelspec: +# display_name: Python 3.7.9 ('workflow-calcium-imaging') +# language: python +# name: python3 +# --- + +# # Allen Institute Calcium Imaging Workshop +# September 22, 2022 +# + In this notebook, we will show how to interact with a database in Python and how re-run an analysis. +# +# + Other notebooks in this directory describe the process for running the analysis steps in more detail. +# +# + This notebook is meant to be run on CodeBook (`https://codebook.datajoint.io`) which contains example data. +# +# + First run the `01-configure` and `04-automate` notebooks to set up your environment and load example data into the database, respectively. + +# ## Configuration +# +# Import the relevant packages. + +import datajoint as dj +import numpy as np +from matplotlib import pyplot + +# Enter database credentials. A DataJoint workflow requires a connection to an existing relational database. The connection setup parameters are defined in the `dj.config` python dictionary. + +# + tags=[] +dj.config['custom'] = {'database.prefix': '_allen_ophys_', + 'imaging_root_data_dir': '/home/inbox/0_1_0a2/'} +# - + +# Import the workflow. The current workflow is composed of multiple database schemas, each of them corresponding to a module within the `workflow_calcium_imaging.pipeline` file. + +from workflow_calcium_imaging.pipeline import lab, subject, session, scan, imaging + +# ## Workflow diagram +# +# Plot the workflow diagram. In relational databases, the entities (i.e. rows) in different tables are connected to each other. Visualization of this relationship helps one to write accurate queries. For the calcium imaging workflow, this connection is as follows: + +# + tags=[] +dj.Diagram(lab.Lab) + dj.Diagram(subject.Subject) + dj.Diagram(session.Session) + \ +dj.Diagram(scan) + dj.Diagram(imaging) +# - + +subject.Subject() + +scan.Scan() + +imaging.Fluorescence() + +# ## Fetch data from the database +# +# Fetch a fluorescence trace for a single mask and plot these values. + +imaging.Fluorescence.Trace() + +# Restrict the table with specific criteria. + +imaging.Fluorescence.Trace & 'subject=""' \ + & 'session_datetime="2020-08-08"' \ + & 'mask_id=120' + +# Fetch a fluorescence trace from the database. + +trace = (imaging.Fluorescence.Trace & 'subject=""' \ + & 'session_datetime="2020-08-08"' \ + & 'mask_id=120').fetch('fluorescence') + +# Plot the fluorescence trace. + +# + +sampling_rate = (scan.ScanInfo & 'subject=""' & 'session_datetime="2020-08-08"').fetch1('fps') + +pyplot.plot(np.r_[:trace.size] * 1/sampling_rate, trace, 'k') + +pyplot.title('Fluorescence trace for mask 120',labelsize=14) +pyplot.tick_params(labelsize=14) +pyplot.set_xlabel('Time (s)') +pyplot.set_ylabel('Activity (a.u.)') +# - + +# ## Run analysis +# +# The workflow has already been run for with a parameter set (paramset_idx=1). Let's re-run Suite2p with a different parameter set, changing the cell diameter to 10 microns. + +dj.Diagram(imaging.Processing)-2 + +imaging.ProcessingTask() + +imaging.ProcessingParamSet() + +params_suite2p = {'look_one_level_down': 0.0, + 'fast_disk': [], + 'delete_bin': False, + 'mesoscan': False, + 'h5py': [], + 'h5py_key': 'data', + 'save_path0': [], + 'subfolders': [], + 'nplanes': 1, + 'nchannels': 1, + 'functional_chan': 1, + 'tau': 1.0, + 'fs': 10.0, + 'force_sktiff': False, + 'preclassify': 0.0, + 'save_mat': False, + 'combined': True, + 'aspect': 1.0, + 'do_bidiphase': False, + 'bidiphase': 0.0, + 'do_registration': True, + 'keep_movie_raw': False, + 'nimg_init': 300, + 'batch_size': 500, + 'maxregshift': 0.1, + 'align_by_chan': 1, + 'reg_tif': False, + 'reg_tif_chan2': False, + 'subpixel': 10, + 'smooth_sigma': 1.15, + 'th_badframes': 1.0, + 'pad_fft': False, + 'nonrigid': True, + 'block_size': [128, 128], + 'snr_thresh': 1.2, + 'maxregshiftNR': 5.0, + '1Preg': False, + 'spatial_hp': 50.0, + 'pre_smooth': 2.0, + 'spatial_taper': 50.0, + 'roidetect': True, + 'sparse_mode': False, + 'diameter': 10, + 'spatial_scale': 0, + 'connected': True, + 'nbinned': 5000, + 'max_iterations': 20, + 'threshold_scaling': 1.0, + 'max_overlap': 0.75, + 'high_pass': 100.0, + 'inner_neuropil_radius': 2, + 'min_neuropil_pixels': 350, + 'allow_overlap': False, + 'chan2_thres': 0.65, + 'baseline': 'maximin', + 'win_baseline': 60.0, + 'sig_baseline': 10.0, + 'prctile_baseline': 8.0, + 'neucoeff': 0.7, + 'xrange': np.array([0, 0]), + 'yrange': np.array([0, 0])} + +imaging.ProcessingTask() + +# Run Suite2p for the new parameter set and save the results to the respective tables. + +# + +populate_settings = dict(display_progress=True) + +imaging.Processing.populate(**populate_settings) + +imaging.MotionCorrection.populate(**populate_settings) + +imaging.Segmentation.populate(**populate_settings) + +imaging.Fluorescence.populate(**populate_settings) + +imaging.Activity.populate(**populate_settings) +# - + +# ## Summary and next steps +# +# In this notebook we explored how to query and fetch data from the database, and re-run analysis with new parameters. Next, please explore more of the features of the DataJoint Elements in the other notebooks. Once you are ready to begin setting up your pipeline, fork this repository on GitHub and begin adapting it for your projects requirements. From eb517d4c8fbfca5922593a9b33119f503d51ca34 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Tue, 20 Sep 2022 20:22:57 -0700 Subject: [PATCH 07/12] Update notebook --- notebooks/2022-allen-institute-workshop.ipynb | 52 ++++++++++++++++--- .../2022-allen-institute-workshop.py | 28 ++++++++-- 2 files changed, 69 insertions(+), 11 deletions(-) diff --git a/notebooks/2022-allen-institute-workshop.ipynb b/notebooks/2022-allen-institute-workshop.ipynb index 22deef3..7ddd738 100644 --- a/notebooks/2022-allen-institute-workshop.ipynb +++ b/notebooks/2022-allen-institute-workshop.ipynb @@ -35,7 +35,8 @@ "source": [ "import datajoint as dj\n", "import numpy as np\n", - "from matplotlib import pyplot" + "from matplotlib import pyplot\n", + "import os" ] }, { @@ -165,8 +166,8 @@ "metadata": {}, "outputs": [], "source": [ - "imaging.Fluorescence.Trace & 'subject=\"\"' \\\n", - " & 'session_datetime=\"2020-08-08\"' \\\n", + "imaging.Fluorescence.Trace & 'subject=\"subject3\"' \\\n", + " & 'session_datetime=\"2022-09-01 19:16:44\"' \\\n", " & 'mask_id=120'" ] }, @@ -185,8 +186,8 @@ "metadata": {}, "outputs": [], "source": [ - "trace = (imaging.Fluorescence.Trace & 'subject=\"\"' \\\n", - " & 'session_datetime=\"2020-08-08\"' \\\n", + "trace = (imaging.Fluorescence.Trace & 'subject=\"subject3\"' \\\n", + " & 'session_datetime=\"2022-09-01 19:16:44\"' \\\n", " & 'mask_id=120').fetch('fluorescence')" ] }, @@ -205,7 +206,7 @@ "metadata": {}, "outputs": [], "source": [ - "sampling_rate = (scan.ScanInfo & 'subject=\"\"' & 'session_datetime=\"2020-08-08\"').fetch1('fps')\n", + "sampling_rate = (scan.ScanInfo & 'subject=\"subject3\"' & 'session_datetime=\"2022-09-01 19:16:44\"').fetch1('fps')\n", "\n", "pyplot.plot(np.r_[:trace.size] * 1/sampling_rate, trace, 'k')\n", "\n", @@ -325,6 +326,45 @@ " 'yrange': np.array([0, 0])}" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "b42e5074", + "metadata": {}, + "outputs": [], + "source": [ + "imaging.ProcessingParamSet.insert_new_params(processing_method='suite2p', \n", + " paramset_idx=1, \n", + " params=params_suite2p,\n", + " paramset_desc='diameter=10')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41fe465c", + "metadata": {}, + "outputs": [], + "source": [ + "imaging.ProcessingParamSet()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5fb498b3", + "metadata": {}, + "outputs": [], + "source": [ + "os.makedirs('subject3/210107_run00_orientation_8dir/suite2p_1', exist_ok=True)\n", + "\n", + "imaging.ProcessingTask.insert1(dict(subject='subject3', \n", + " session_datetime='2022-09-01 19:16:44', \n", + " scan_id=0,\n", + " paramset_idx=1,\n", + " processing_output_dir='subject3/210107_run00_orientation_8dir/suite2p_1'))" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/notebooks/py_scripts/2022-allen-institute-workshop.py b/notebooks/py_scripts/2022-allen-institute-workshop.py index ff71707..bf496eb 100644 --- a/notebooks/py_scripts/2022-allen-institute-workshop.py +++ b/notebooks/py_scripts/2022-allen-institute-workshop.py @@ -30,6 +30,7 @@ import datajoint as dj import numpy as np from matplotlib import pyplot +import os # Enter database credentials. A DataJoint workflow requires a connection to an existing relational database. The connection setup parameters are defined in the `dj.config` python dictionary. @@ -65,20 +66,20 @@ # Restrict the table with specific criteria. -imaging.Fluorescence.Trace & 'subject=""' \ - & 'session_datetime="2020-08-08"' \ +imaging.Fluorescence.Trace & 'subject="subject3"' \ + & 'session_datetime="2022-09-01 19:16:44"' \ & 'mask_id=120' # Fetch a fluorescence trace from the database. -trace = (imaging.Fluorescence.Trace & 'subject=""' \ - & 'session_datetime="2020-08-08"' \ +trace = (imaging.Fluorescence.Trace & 'subject="subject3"' \ + & 'session_datetime="2022-09-01 19:16:44"' \ & 'mask_id=120').fetch('fluorescence') # Plot the fluorescence trace. # + -sampling_rate = (scan.ScanInfo & 'subject=""' & 'session_datetime="2020-08-08"').fetch1('fps') +sampling_rate = (scan.ScanInfo & 'subject="subject3"' & 'session_datetime="2022-09-01 19:16:44"').fetch1('fps') pyplot.plot(np.r_[:trace.size] * 1/sampling_rate, trace, 'k') @@ -160,6 +161,23 @@ 'xrange': np.array([0, 0]), 'yrange': np.array([0, 0])} +imaging.ProcessingParamSet.insert_new_params(processing_method='suite2p', + paramset_idx=1, + params=params_suite2p, + paramset_desc='diameter=10') + +imaging.ProcessingParamSet() + +# + +os.makedirs('subject3/210107_run00_orientation_8dir/suite2p_1', exist_ok=True) + +imaging.ProcessingTask.insert1(dict(subject='subject3', + session_datetime='2022-09-01 19:16:44', + scan_id=0, + paramset_idx=1, + processing_output_dir='subject3/210107_run00_orientation_8dir/suite2p_1')) +# - + imaging.ProcessingTask() # Run Suite2p for the new parameter set and save the results to the respective tables. From 5897b66be091ef20189470a85ca94efdee54a46e Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Tue, 20 Sep 2022 20:53:27 -0700 Subject: [PATCH 08/12] Update requirements --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 93457dd..a097493 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ element-event @ git+https://github.com/datajoint/element-event.git element-interface @ git+https://github.com/datajoint/element-interface.git ipykernel>=6.0.1 jupytext>=1.13.7 -sbxreader \ No newline at end of file +sbxreader +suite2p @ git+https://github.com/datajoint-company/suite2p.git \ No newline at end of file From c8ef7f2453c5d28c5b47da07f66a1498b93da22a Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Tue, 20 Sep 2022 20:54:55 -0700 Subject: [PATCH 09/12] Update version and changelog --- CHANGELOG.md | 4 ++-- workflow_calcium_imaging/version.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4413a7..04104e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. -## [0.2.0] - 2022-09-20 +## [0.1.3] - 2022-09-20 + Add - Notebook for Allen Institute workshop @@ -43,7 +43,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and + Add - Containerization for pytests + Comment - Phase previously designated 0.1.0 -> 0.0.0 -[0.2.0]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.2.0 +[0.1.3]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.3 [0.1.2]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.2 [0.1.1]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.1 [0.1.0]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.0 diff --git a/workflow_calcium_imaging/version.py b/workflow_calcium_imaging/version.py index 23b6780..0f8a416 100644 --- a/workflow_calcium_imaging/version.py +++ b/workflow_calcium_imaging/version.py @@ -3,4 +3,4 @@ Update the Docker image tag in `docker-compose-test.yaml` and `docker-compose-dev.yaml` to match """ -__version__ = "0.2.0" +__version__ = "0.1.3" From 399344a30cb44de4dee6b6e5466a1d39faa4337c Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Tue, 20 Sep 2022 20:57:58 -0700 Subject: [PATCH 10/12] Add absolute path --- notebooks/2022-allen-institute-workshop.ipynb | 2 +- notebooks/py_scripts/2022-allen-institute-workshop.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/2022-allen-institute-workshop.ipynb b/notebooks/2022-allen-institute-workshop.ipynb index 7ddd738..f22885c 100644 --- a/notebooks/2022-allen-institute-workshop.ipynb +++ b/notebooks/2022-allen-institute-workshop.ipynb @@ -356,7 +356,7 @@ "metadata": {}, "outputs": [], "source": [ - "os.makedirs('subject3/210107_run00_orientation_8dir/suite2p_1', exist_ok=True)\n", + "os.makedirs('/home/inbox/0_1_0a2/subject3/210107_run00_orientation_8dir/suite2p_1', exist_ok=True)\n", "\n", "imaging.ProcessingTask.insert1(dict(subject='subject3', \n", " session_datetime='2022-09-01 19:16:44', \n", diff --git a/notebooks/py_scripts/2022-allen-institute-workshop.py b/notebooks/py_scripts/2022-allen-institute-workshop.py index bf496eb..29777e3 100644 --- a/notebooks/py_scripts/2022-allen-institute-workshop.py +++ b/notebooks/py_scripts/2022-allen-institute-workshop.py @@ -169,7 +169,7 @@ imaging.ProcessingParamSet() # + -os.makedirs('subject3/210107_run00_orientation_8dir/suite2p_1', exist_ok=True) +os.makedirs('/home/inbox/0_1_0a2/subject3/210107_run00_orientation_8dir/suite2p_1', exist_ok=True) imaging.ProcessingTask.insert1(dict(subject='subject3', session_datetime='2022-09-01 19:16:44', From ac5ed4b6fea4f22423ea2fe6c5e3000b76a130f9 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Tue, 20 Sep 2022 21:07:18 -0700 Subject: [PATCH 11/12] Add insert to Curation table --- notebooks/2022-allen-institute-workshop.ipynb | 4 ++++ notebooks/py_scripts/2022-allen-institute-workshop.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/notebooks/2022-allen-institute-workshop.ipynb b/notebooks/2022-allen-institute-workshop.ipynb index f22885c..195bc27 100644 --- a/notebooks/2022-allen-institute-workshop.ipynb +++ b/notebooks/2022-allen-institute-workshop.ipynb @@ -394,6 +394,10 @@ "\n", "imaging.Processing.populate(**populate_settings)\n", "\n", + "key = (imaging.ProcessingTask & 'subject=\"subject3\"' & 'session_datetime=\"2022-09-01 19:16:44\"').fetch1('KEY')\n", + "\n", + "imaging.Curation().create1_from_processing_task(key)\n", + "\n", "imaging.MotionCorrection.populate(**populate_settings)\n", "\n", "imaging.Segmentation.populate(**populate_settings)\n", diff --git a/notebooks/py_scripts/2022-allen-institute-workshop.py b/notebooks/py_scripts/2022-allen-institute-workshop.py index 29777e3..e9f97a2 100644 --- a/notebooks/py_scripts/2022-allen-institute-workshop.py +++ b/notebooks/py_scripts/2022-allen-institute-workshop.py @@ -187,6 +187,10 @@ imaging.Processing.populate(**populate_settings) +key = (imaging.ProcessingTask & 'subject="subject3"' & 'session_datetime="2022-09-01 19:16:44"').fetch1('KEY') + +imaging.Curation().create1_from_processing_task(key) + imaging.MotionCorrection.populate(**populate_settings) imaging.Segmentation.populate(**populate_settings) From 11c6523917217933f1d16f65e3d301dd31999c93 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Tue, 20 Sep 2022 21:41:41 -0700 Subject: [PATCH 12/12] Update notebook --- notebooks/2022-allen-institute-workshop.ipynb | 11 +++++------ notebooks/py_scripts/2022-allen-institute-workshop.py | 11 +++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/notebooks/2022-allen-institute-workshop.ipynb b/notebooks/2022-allen-institute-workshop.ipynb index 195bc27..259b742 100644 --- a/notebooks/2022-allen-institute-workshop.ipynb +++ b/notebooks/2022-allen-institute-workshop.ipynb @@ -166,9 +166,10 @@ "metadata": {}, "outputs": [], "source": [ - "imaging.Fluorescence.Trace & 'subject=\"subject3\"' \\\n", - " & 'session_datetime=\"2022-09-01 19:16:44\"' \\\n", - " & 'mask_id=120'" + "query_trace = imaging.Fluorescence.Trace & 'subject=\"subject3\"' \\\n", + " & 'session_datetime=\"2022-09-01 19:16:44\"' \\\n", + " & 'mask_id=120'\n", + "query_trace" ] }, { @@ -186,9 +187,7 @@ "metadata": {}, "outputs": [], "source": [ - "trace = (imaging.Fluorescence.Trace & 'subject=\"subject3\"' \\\n", - " & 'session_datetime=\"2022-09-01 19:16:44\"' \\\n", - " & 'mask_id=120').fetch('fluorescence')" + "trace = (query_trace).fetch('fluorescence')" ] }, { diff --git a/notebooks/py_scripts/2022-allen-institute-workshop.py b/notebooks/py_scripts/2022-allen-institute-workshop.py index e9f97a2..4322c41 100644 --- a/notebooks/py_scripts/2022-allen-institute-workshop.py +++ b/notebooks/py_scripts/2022-allen-institute-workshop.py @@ -66,15 +66,14 @@ # Restrict the table with specific criteria. -imaging.Fluorescence.Trace & 'subject="subject3"' \ - & 'session_datetime="2022-09-01 19:16:44"' \ - & 'mask_id=120' +query_trace = imaging.Fluorescence.Trace & 'subject="subject3"' \ + & 'session_datetime="2022-09-01 19:16:44"' \ + & 'mask_id=120' +query_trace # Fetch a fluorescence trace from the database. -trace = (imaging.Fluorescence.Trace & 'subject="subject3"' \ - & 'session_datetime="2022-09-01 19:16:44"' \ - & 'mask_id=120').fetch('fluorescence') +trace = (query_trace).fetch('fluorescence') # Plot the fluorescence trace.