extract_Berg.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nwb_extraction import generated_formatted_features_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "berg_path = Path('/external/rprshnas01/netdata_kcni/stlab/Public/AIBS_patchseq_2020/human/ephys/000023')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "berg_nwbs = list(berg_path.glob('*.nwb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'hdmf-common' version 1.1.3 because version 1.5.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n",
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'core' version 2.2.5 because version 2.3.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start: 0.5, end: 1.4999799999999999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Could not compute input/access resistance ratio (sr: None, ir:: None)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Blowout is not available', 'Electrode 0 is not available', 'Seal is not available', 'Breakin sweep not found']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'hdmf-common' version 1.1.3 because version 1.5.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n",
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'core' version 2.2.5 because version 2.3.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start: 0.5, end: 1.4999799999999999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Could not compute input/access resistance ratio (sr: None, ir:: None)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Blowout is not available', 'Electrode 0 is not available', 'Seal is not available', 'Breakin sweep not found']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'hdmf-common' version 1.1.3 because version 1.5.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n",
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'core' version 2.2.5 because version 2.3.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n",
      "WARNING:root:sweep 19: ['Recording stopped before completing the experiment epoch']\n",
      "WARNING:root:sweep 20: ['Recording stopped before completing the experiment epoch']\n",
      "WARNING:root:sweep 21: ['Recording stopped before completing the experiment epoch']\n",
      "WARNING:root:sweep 22: ['Recording stopped before completing the experiment epoch']\n",
      "WARNING:root:sweep 23: ['Recording stopped before completing the experiment epoch']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start: 0.5, end: 1.4999799999999999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Could not compute input/access resistance ratio (sr: None, ir:: None)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Blowout is not available', 'Electrode 0 is not available', 'Seal is not available', 'Breakin sweep not found']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'hdmf-common' version 1.1.3 because version 1.5.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n",
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'core' version 2.2.5 because version 2.3.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n",
      "WARNING:root:sweep 7: ['stim epoch is missing', 'experiment epoch is missing']\n",
      "WARNING:root:sweep 17: ['stim epoch is missing', 'experiment epoch is missing']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start: 0.5, end: 1.4999799999999999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Could not compute input/access resistance ratio (sr: None, ir:: None)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Blowout is not available', 'Electrode 0 is not available', 'Seal is not available', 'Breakin sweep not found']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'hdmf-common' version 1.1.3 because version 1.5.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n",
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'core' version 2.2.5 because version 2.3.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start: 0.5, end: 1.4999799999999999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Could not compute input/access resistance ratio (sr: None, ir:: None)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Blowout is not available', 'Electrode 0 is not available', 'Seal is not available', 'Breakin sweep not found']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'hdmf-common' version 1.1.3 because version 1.5.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n",
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'core' version 2.2.5 because version 2.3.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start: 0.5, end: 1.4999799999999999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Could not compute input/access resistance ratio (sr: None, ir:: None)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Blowout is not available', 'Electrode 0 is not available', 'Seal is not available', 'Breakin sweep not found']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'hdmf-common' version 1.1.3 because version 1.5.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n",
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'core' version 2.2.5 because version 2.3.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n",
      "WARNING:root:sweep 31: ['Recording stopped before completing the experiment epoch']\n",
      "WARNING:root:sweep 32: ['Recording stopped before completing the experiment epoch']\n",
      "WARNING:root:sweep 33: ['Recording stopped before completing the experiment epoch']\n",
      "WARNING:root:sweep 35: ['Recording stopped before completing the experiment epoch']\n",
      "WARNING:root:sweep 36: ['Recording stopped before completing the experiment epoch']\n",
      "WARNING:root:sweep 37: ['Recording stopped before completing the experiment epoch']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start: 0.5, end: 1.4999799999999999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/ipfx/spike_features.py:87: RuntimeWarning: invalid value encountered in less\n",
      "  width_levels[width_levels < v[spike_indexes]] = thresh_to_peak_levels[width_levels < v[spike_indexes]]\n",
      "WARNING:root:Could not compute input/access resistance ratio (sr: None, ir:: None)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Blowout is not available', 'Electrode 0 is not available', 'Seal is not available', 'Breakin sweep not found']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'hdmf-common' version 1.1.3 because version 1.5.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n",
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'core' version 2.2.5 because version 2.3.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n",
      "WARNING:root:sweep 40: ['Recording stopped before completing the experiment epoch']\n",
      "WARNING:root:sweep 41: ['Recording stopped before completing the experiment epoch']\n",
      "WARNING:root:sweep 42: ['Recording stopped before completing the experiment epoch']\n",
      "WARNING:root:sweep 43: ['Recording stopped before completing the experiment epoch']\n",
      "WARNING:root:sweep 44: ['Recording stopped before completing the experiment epoch']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start: 0.5, end: 1.4999799999999999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Could not compute input/access resistance ratio (sr: None, ir:: None)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Blowout is not available', 'Electrode 0 is not available', 'Seal is not available', 'Breakin sweep not found']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'hdmf-common' version 1.1.3 because version 1.5.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n",
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'core' version 2.2.5 because version 2.3.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start: 0.5, end: 1.4999799999999999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Could not compute input/access resistance ratio (sr: None, ir:: None)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Blowout is not available', 'Electrode 0 is not available', 'Seal is not available', 'Breakin sweep not found']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'hdmf-common' version 1.1.3 because version 1.5.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n",
      "/nethome/kcni/dhoward/.anaconda3/envs/nwbephys/lib/python3.6/site-packages/hdmf/spec/namespace.py:533: UserWarning: Ignoring cached namespace 'core' version 2.2.5 because version 2.3.0 is already loaded.\n",
      "  % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start: 0.5, end: 1.4999799999999999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Could not compute input/access resistance ratio (sr: None, ir:: None)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Blowout is not available', 'Electrode 0 is not available', 'Seal is not available', 'Breakin sweep not found']\n",
      "CPU times: user 57.4 s, sys: 348 ms, total: 57.7 s\n",
      "Wall time: 1min\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "feats = []\n",
    "for nwb in berg_nwbs[:10]:\n",
    "    nwb_feats = generated_formatted_features_output(str(nwb))\n",
    "    feats.append(nwb_feats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_df = pd.DataFrame(feats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>v_baseline</th>\n",
       "      <th>rheobase_i</th>\n",
       "      <th>fi_fit_slope</th>\n",
       "      <th>sag</th>\n",
       "      <th>vm_for_sag</th>\n",
       "      <th>input_resistance</th>\n",
       "      <th>tau</th>\n",
       "      <th>hero_adapt</th>\n",
       "      <th>hero_avg_rate</th>\n",
       "      <th>...</th>\n",
       "      <th>rheo_trough_t</th>\n",
       "      <th>rheo_slow_trough_t</th>\n",
       "      <th>rheo_peak_v</th>\n",
       "      <th>qc_blowout_mv</th>\n",
       "      <th>qc_electrode_0_pa</th>\n",
       "      <th>qc_recording_date</th>\n",
       "      <th>qc_seal_gohm</th>\n",
       "      <th>qc_input_resistance_mohm</th>\n",
       "      <th>qc_initial_access_resistance_mohm</th>\n",
       "      <th>qc_input_access_resistance_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sub-770227957_ses-770611326_icephys.nwb</td>\n",
       "      <td>-79.085518</td>\n",
       "      <td>130.0</td>\n",
       "      <td>0.165003</td>\n",
       "      <td>0.007619</td>\n",
       "      <td>-87.643753</td>\n",
       "      <td>107.812494</td>\n",
       "      <td>0.018137</td>\n",
       "      <td>0.030283</td>\n",
       "      <td>9.00018</td>\n",
       "      <td>...</td>\n",
       "      <td>0.75540</td>\n",
       "      <td>0.75540</td>\n",
       "      <td>46.362499</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2018-10-31 03:37:36</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sub-611526465_ses-611616224_icephys.nwb</td>\n",
       "      <td>-73.851969</td>\n",
       "      <td>190.0</td>\n",
       "      <td>0.073941</td>\n",
       "      <td>0.017478</td>\n",
       "      <td>-84.031258</td>\n",
       "      <td>86.588606</td>\n",
       "      <td>0.021303</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.00002</td>\n",
       "      <td>...</td>\n",
       "      <td>0.81374</td>\n",
       "      <td>0.81374</td>\n",
       "      <td>51.843750</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2017-08-22 04:39:59</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sub-672637567_ses-672650708_icephys.nwb</td>\n",
       "      <td>-80.893845</td>\n",
       "      <td>200.0</td>\n",
       "      <td>0.077803</td>\n",
       "      <td>0.044364</td>\n",
       "      <td>-88.968750</td>\n",
       "      <td>64.843751</td>\n",
       "      <td>0.016678</td>\n",
       "      <td>0.062738</td>\n",
       "      <td>8.00016</td>\n",
       "      <td>...</td>\n",
       "      <td>0.71904</td>\n",
       "      <td>0.71904</td>\n",
       "      <td>46.531254</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2018-03-14 01:13:33</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sub-758233315_ses-758763787_icephys.nwb</td>\n",
       "      <td>-60.092258</td>\n",
       "      <td>150.0</td>\n",
       "      <td>0.029311</td>\n",
       "      <td>0.024248</td>\n",
       "      <td>-69.518753</td>\n",
       "      <td>89.218900</td>\n",
       "      <td>0.032980</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.00002</td>\n",
       "      <td>...</td>\n",
       "      <td>0.70618</td>\n",
       "      <td>NaN</td>\n",
       "      <td>63.162498</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2018-09-26 19:42:47</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sub-592433014_ses-592558145_icephys.nwb</td>\n",
       "      <td>-66.525501</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.148440</td>\n",
       "      <td>0.075644</td>\n",
       "      <td>-87.000008</td>\n",
       "      <td>163.269326</td>\n",
       "      <td>0.039818</td>\n",
       "      <td>0.321140</td>\n",
       "      <td>6.00012</td>\n",
       "      <td>...</td>\n",
       "      <td>0.67542</td>\n",
       "      <td>NaN</td>\n",
       "      <td>41.937500</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2017-05-31 19:45:09</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  filename  v_baseline  rheobase_i  \\\n",
       "0  sub-770227957_ses-770611326_icephys.nwb  -79.085518       130.0   \n",
       "1  sub-611526465_ses-611616224_icephys.nwb  -73.851969       190.0   \n",
       "2  sub-672637567_ses-672650708_icephys.nwb  -80.893845       200.0   \n",
       "3  sub-758233315_ses-758763787_icephys.nwb  -60.092258       150.0   \n",
       "4  sub-592433014_ses-592558145_icephys.nwb  -66.525501        50.0   \n",
       "\n",
       "   fi_fit_slope       sag  vm_for_sag  input_resistance       tau  hero_adapt  \\\n",
       "0      0.165003  0.007619  -87.643753        107.812494  0.018137    0.030283   \n",
       "1      0.073941  0.017478  -84.031258         86.588606  0.021303         NaN   \n",
       "2      0.077803  0.044364  -88.968750         64.843751  0.016678    0.062738   \n",
       "3      0.029311  0.024248  -69.518753         89.218900  0.032980         NaN   \n",
       "4      0.148440  0.075644  -87.000008        163.269326  0.039818    0.321140   \n",
       "\n",
       "   hero_avg_rate  ...  rheo_trough_t  rheo_slow_trough_t  rheo_peak_v  \\\n",
       "0        9.00018  ...        0.75540             0.75540    46.362499   \n",
       "1        1.00002  ...        0.81374             0.81374    51.843750   \n",
       "2        8.00016  ...        0.71904             0.71904    46.531254   \n",
       "3        1.00002  ...        0.70618                 NaN    63.162498   \n",
       "4        6.00012  ...        0.67542                 NaN    41.937500   \n",
       "\n",
       "   qc_blowout_mv  qc_electrode_0_pa    qc_recording_date  qc_seal_gohm  \\\n",
       "0           None               None  2018-10-31 03:37:36          None   \n",
       "1           None               None  2017-08-22 04:39:59          None   \n",
       "2           None               None  2018-03-14 01:13:33          None   \n",
       "3           None               None  2018-09-26 19:42:47          None   \n",
       "4           None               None  2017-05-31 19:45:09          None   \n",
       "\n",
       "   qc_input_resistance_mohm  qc_initial_access_resistance_mohm  \\\n",
       "0                      None                               None   \n",
       "1                      None                               None   \n",
       "2                      None                               None   \n",
       "3                      None                               None   \n",
       "4                      None                               None   \n",
       "\n",
       "   qc_input_access_resistance_ratio  \n",
       "0                              None  \n",
       "1                              None  \n",
       "2                              None  \n",
       "3                              None  \n",
       "4                              None  \n",
       "\n",
       "[5 rows x 35 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "manifest = pd.read_csv('/external/rprshnas01/netdata_kcni/stlab/Public/AIBS_patchseq_2020/human/2020-07-08_human_file_manifest/2020-07-08_human_file_manifest.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_id</th>\n",
       "      <th>file_name</th>\n",
       "      <th>file_size</th>\n",
       "      <th>sha256_checksum</th>\n",
       "      <th>md5_checksum</th>\n",
       "      <th>technique</th>\n",
       "      <th>file_type</th>\n",
       "      <th>archive</th>\n",
       "      <th>archive_uri</th>\n",
       "      <th>project</th>\n",
       "      <th>cell_specimen_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>541549258</td>\n",
       "      <td>541549258.marker</td>\n",
       "      <td>786</td>\n",
       "      <td>1cd5d6a0a4c4736b3a2fa1b92c25ba019bbfba56fe7d87...</td>\n",
       "      <td>8a20768e94628fb5b4a5dc6c503b165b</td>\n",
       "      <td>morphology</td>\n",
       "      <td>marker_swc</td>\n",
       "      <td>BIL</td>\n",
       "      <td>ftp://download.brainlib.org:8811/biccn/lein/ps...</td>\n",
       "      <td>hIVSCC-MET</td>\n",
       "      <td>541549258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>541549258</td>\n",
       "      <td>541549258_raw.swc</td>\n",
       "      <td>403725</td>\n",
       "      <td>e7e4d9a7ac3467413b7a87a8710c667abd9f683b7d1ec0...</td>\n",
       "      <td>b8d814c75951bad1200bb547c1c3d58b</td>\n",
       "      <td>morphology</td>\n",
       "      <td>raw_swc</td>\n",
       "      <td>BIL</td>\n",
       "      <td>ftp://download.brainlib.org:8811/biccn/lein/ps...</td>\n",
       "      <td>hIVSCC-MET</td>\n",
       "      <td>541549258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>541549258</td>\n",
       "      <td>541549258_transformed.swc</td>\n",
       "      <td>668106</td>\n",
       "      <td>340f091f397d34ee88e6a6b56c262e4061d75fca99f68b...</td>\n",
       "      <td>348772abbf79697b06d1ab54e2497f82</td>\n",
       "      <td>morphology</td>\n",
       "      <td>transformed_swc</td>\n",
       "      <td>BIL</td>\n",
       "      <td>ftp://download.brainlib.org:8811/biccn/lein/ps...</td>\n",
       "      <td>hIVSCC-MET</td>\n",
       "      <td>541549258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>541549227</td>\n",
       "      <td>sub-541516760_ses-541549227_icephys.nwb</td>\n",
       "      <td>29377521</td>\n",
       "      <td>c76358d07fb27a0c4d6f22e305ee5f36f3fea53d9e6e98...</td>\n",
       "      <td>598597e3bf47137afbc73d6eb4a20490</td>\n",
       "      <td>intracellular_electrophysiology</td>\n",
       "      <td>nwb</td>\n",
       "      <td>DANDI</td>\n",
       "      <td>https://girder.dandiarchive.org/api/v1/item/5f...</td>\n",
       "      <td>hIVSCC-MET</td>\n",
       "      <td>541549258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>541557114</td>\n",
       "      <td>541557114.marker</td>\n",
       "      <td>609</td>\n",
       "      <td>42e75ed5470b6b0860f60d92817b59273f4b21b08d8e5e...</td>\n",
       "      <td>a7af6d625b22300a8e5c328bdb401c0d</td>\n",
       "      <td>morphology</td>\n",
       "      <td>marker_swc</td>\n",
       "      <td>BIL</td>\n",
       "      <td>ftp://download.brainlib.org:8811/biccn/lein/ps...</td>\n",
       "      <td>hIVSCC-MET</td>\n",
       "      <td>541557114</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     file_id                                file_name  file_size  \\\n",
       "0  541549258                         541549258.marker        786   \n",
       "1  541549258                        541549258_raw.swc     403725   \n",
       "2  541549258                541549258_transformed.swc     668106   \n",
       "3  541549227  sub-541516760_ses-541549227_icephys.nwb   29377521   \n",
       "4  541557114                         541557114.marker        609   \n",
       "\n",
       "                                     sha256_checksum  \\\n",
       "0  1cd5d6a0a4c4736b3a2fa1b92c25ba019bbfba56fe7d87...   \n",
       "1  e7e4d9a7ac3467413b7a87a8710c667abd9f683b7d1ec0...   \n",
       "2  340f091f397d34ee88e6a6b56c262e4061d75fca99f68b...   \n",
       "3  c76358d07fb27a0c4d6f22e305ee5f36f3fea53d9e6e98...   \n",
       "4  42e75ed5470b6b0860f60d92817b59273f4b21b08d8e5e...   \n",
       "\n",
       "                       md5_checksum                        technique  \\\n",
       "0  8a20768e94628fb5b4a5dc6c503b165b                       morphology   \n",
       "1  b8d814c75951bad1200bb547c1c3d58b                       morphology   \n",
       "2  348772abbf79697b06d1ab54e2497f82                       morphology   \n",
       "3  598597e3bf47137afbc73d6eb4a20490  intracellular_electrophysiology   \n",
       "4  a7af6d625b22300a8e5c328bdb401c0d                       morphology   \n",
       "\n",
       "         file_type archive                                        archive_uri  \\\n",
       "0       marker_swc     BIL  ftp://download.brainlib.org:8811/biccn/lein/ps...   \n",
       "1          raw_swc     BIL  ftp://download.brainlib.org:8811/biccn/lein/ps...   \n",
       "2  transformed_swc     BIL  ftp://download.brainlib.org:8811/biccn/lein/ps...   \n",
       "3              nwb   DANDI  https://girder.dandiarchive.org/api/v1/item/5f...   \n",
       "4       marker_swc     BIL  ftp://download.brainlib.org:8811/biccn/lein/ps...   \n",
       "\n",
       "      project  cell_specimen_id  \n",
       "0  hIVSCC-MET         541549258  \n",
       "1  hIVSCC-MET         541549258  \n",
       "2  hIVSCC-MET         541549258  \n",
       "3  hIVSCC-MET         541549258  \n",
       "4  hIVSCC-MET         541557114  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "manifest.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_df = feature_df.merge(manifest.loc[:, ['file_id', 'file_name', 'cell_specimen_id']], left_on='filename', right_on='file_name', how='left').drop('file_name', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>v_baseline</th>\n",
       "      <th>rheobase_i</th>\n",
       "      <th>fi_fit_slope</th>\n",
       "      <th>sag</th>\n",
       "      <th>vm_for_sag</th>\n",
       "      <th>input_resistance</th>\n",
       "      <th>tau</th>\n",
       "      <th>hero_adapt</th>\n",
       "      <th>hero_avg_rate</th>\n",
       "      <th>...</th>\n",
       "      <th>rheo_peak_v</th>\n",
       "      <th>qc_blowout_mv</th>\n",
       "      <th>qc_electrode_0_pa</th>\n",
       "      <th>qc_recording_date</th>\n",
       "      <th>qc_seal_gohm</th>\n",
       "      <th>qc_input_resistance_mohm</th>\n",
       "      <th>qc_initial_access_resistance_mohm</th>\n",
       "      <th>qc_input_access_resistance_ratio</th>\n",
       "      <th>file_id</th>\n",
       "      <th>cell_specimen_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sub-770227957_ses-770611326_icephys.nwb</td>\n",
       "      <td>-79.085518</td>\n",
       "      <td>130.0</td>\n",
       "      <td>0.165003</td>\n",
       "      <td>0.007619</td>\n",
       "      <td>-87.643753</td>\n",
       "      <td>107.812494</td>\n",
       "      <td>0.018137</td>\n",
       "      <td>0.030283</td>\n",
       "      <td>9.00018</td>\n",
       "      <td>...</td>\n",
       "      <td>46.362499</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2018-10-31 03:37:36</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>770611326</td>\n",
       "      <td>770611354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sub-611526465_ses-611616224_icephys.nwb</td>\n",
       "      <td>-73.851969</td>\n",
       "      <td>190.0</td>\n",
       "      <td>0.073941</td>\n",
       "      <td>0.017478</td>\n",
       "      <td>-84.031258</td>\n",
       "      <td>86.588606</td>\n",
       "      <td>0.021303</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.00002</td>\n",
       "      <td>...</td>\n",
       "      <td>51.843750</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2017-08-22 04:39:59</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>611616224</td>\n",
       "      <td>611616275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sub-672637567_ses-672650708_icephys.nwb</td>\n",
       "      <td>-80.893845</td>\n",
       "      <td>200.0</td>\n",
       "      <td>0.077803</td>\n",
       "      <td>0.044364</td>\n",
       "      <td>-88.968750</td>\n",
       "      <td>64.843751</td>\n",
       "      <td>0.016678</td>\n",
       "      <td>0.062738</td>\n",
       "      <td>8.00016</td>\n",
       "      <td>...</td>\n",
       "      <td>46.531254</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2018-03-14 01:13:33</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>672650708</td>\n",
       "      <td>672650722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sub-758233315_ses-758763787_icephys.nwb</td>\n",
       "      <td>-60.092258</td>\n",
       "      <td>150.0</td>\n",
       "      <td>0.029311</td>\n",
       "      <td>0.024248</td>\n",
       "      <td>-69.518753</td>\n",
       "      <td>89.218900</td>\n",
       "      <td>0.032980</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.00002</td>\n",
       "      <td>...</td>\n",
       "      <td>63.162498</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2018-09-26 19:42:47</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>758763787</td>\n",
       "      <td>758763848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sub-592433014_ses-592558145_icephys.nwb</td>\n",
       "      <td>-66.525501</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.148440</td>\n",
       "      <td>0.075644</td>\n",
       "      <td>-87.000008</td>\n",
       "      <td>163.269326</td>\n",
       "      <td>0.039818</td>\n",
       "      <td>0.321140</td>\n",
       "      <td>6.00012</td>\n",
       "      <td>...</td>\n",
       "      <td>41.937500</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2017-05-31 19:45:09</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>592558145</td>\n",
       "      <td>592558182</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  filename  v_baseline  rheobase_i  \\\n",
       "0  sub-770227957_ses-770611326_icephys.nwb  -79.085518       130.0   \n",
       "1  sub-611526465_ses-611616224_icephys.nwb  -73.851969       190.0   \n",
       "2  sub-672637567_ses-672650708_icephys.nwb  -80.893845       200.0   \n",
       "3  sub-758233315_ses-758763787_icephys.nwb  -60.092258       150.0   \n",
       "4  sub-592433014_ses-592558145_icephys.nwb  -66.525501        50.0   \n",
       "\n",
       "   fi_fit_slope       sag  vm_for_sag  input_resistance       tau  hero_adapt  \\\n",
       "0      0.165003  0.007619  -87.643753        107.812494  0.018137    0.030283   \n",
       "1      0.073941  0.017478  -84.031258         86.588606  0.021303         NaN   \n",
       "2      0.077803  0.044364  -88.968750         64.843751  0.016678    0.062738   \n",
       "3      0.029311  0.024248  -69.518753         89.218900  0.032980         NaN   \n",
       "4      0.148440  0.075644  -87.000008        163.269326  0.039818    0.321140   \n",
       "\n",
       "   hero_avg_rate  ...  rheo_peak_v  qc_blowout_mv  qc_electrode_0_pa  \\\n",
       "0        9.00018  ...    46.362499           None               None   \n",
       "1        1.00002  ...    51.843750           None               None   \n",
       "2        8.00016  ...    46.531254           None               None   \n",
       "3        1.00002  ...    63.162498           None               None   \n",
       "4        6.00012  ...    41.937500           None               None   \n",
       "\n",
       "     qc_recording_date  qc_seal_gohm  qc_input_resistance_mohm  \\\n",
       "0  2018-10-31 03:37:36          None                      None   \n",
       "1  2017-08-22 04:39:59          None                      None   \n",
       "2  2018-03-14 01:13:33          None                      None   \n",
       "3  2018-09-26 19:42:47          None                      None   \n",
       "4  2017-05-31 19:45:09          None                      None   \n",
       "\n",
       "   qc_initial_access_resistance_mohm  qc_input_access_resistance_ratio  \\\n",
       "0                               None                              None   \n",
       "1                               None                              None   \n",
       "2                               None                              None   \n",
       "3                               None                              None   \n",
       "4                               None                              None   \n",
       "\n",
       "     file_id  cell_specimen_id  \n",
       "0  770611326         770611354  \n",
       "1  611616224         611616275  \n",
       "2  672650708         672650722  \n",
       "3  758763787         758763848  \n",
       "4  592558145         592558182  \n",
       "\n",
       "[5 rows x 37 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#feature_df.to_csv('./human_ephys_features_berg_w_QC.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nwbephys",
   "language": "python",
   "name": "nwbephys"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}