spherized_profiles/spherize-batch-effects.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adjust batch effects with a spherize transform\n",
    "\n",
    "Here, we load in all normalized profiles (level 4a) data across all plates and apply a spherize transform using the DMSO profiles as the background distribution.\n",
    "\n",
    "We've previously observed that sphering (aka whitening) the data successfully adjusts for technical artifacts induced by batch to batch variation and plate position effects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pathlib\n",
    "import subprocess\n",
    "import pandas as pd\n",
    "\n",
    "from pycytominer import normalize, feature_select\n",
    "from pycytominer.cyto_utils import output, infer_cp_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dir = pathlib.Path(\"../profiles/\")\n",
    "batches = [\"2016_04_01_a549_48hr_batch1\", \"2017_12_05_Batch2\"]\n",
    "\n",
    "suffixes = {\n",
    "    \"whole_plate\": \"_normalized.csv.gz\",\n",
    "    \"dmso\": \"_normalized_dmso.csv.gz\"\n",
    "}\n",
    "\n",
    "plates = {\n",
    "    batch: [x.name for x in pathlib.Path(f\"{input_dir}/{batch}\").iterdir() if \".DS_Store\" not in x.name]\n",
    "    for batch in batches\n",
    "}\n",
    "\n",
    "files = {\n",
    "    batch: {\n",
    "        suffix: [pathlib.Path(f\"{input_dir}/{batch}/{x}/{x}{suffixes[suffix]}\") for x in plates[batch]]\n",
    "        for suffix in suffixes\n",
    "    }\n",
    "    for batch in batches\n",
    "}\n",
    "\n",
    "feature_select_ops = [\n",
    "    \"variance_threshold\",\n",
    "    \"correlation_threshold\",\n",
    "    \"drop_na_columns\",\n",
    "    \"blocklist\",\n",
    "]\n",
    "\n",
    "na_cut = 0\n",
    "corr_threshold = 0.95\n",
    "output_dir = \"profiles\"\n",
    "\n",
    "full_blocklist_file = pathlib.Path(\"../utils/consensus_blocklist.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Now processing profiles/2016_04_01_a549_48hr_batch1_dmso_spherized_profiles_with_input_normalized_by_whole_plate.csv.gz...\n",
      "(52223, 1810)\n",
      "(52223, 830)\n",
      "Now processing profiles/2016_04_01_a549_48hr_batch1_dmso_spherized_profiles_with_input_normalized_by_dmso.csv.gz...\n",
      "(52223, 1810)\n",
      "(52223, 1055)\n",
      "Now processing profiles/2017_12_05_Batch2_dmso_spherized_profiles_with_input_normalized_by_whole_plate.csv.gz...\n",
      "(51447, 2229)\n",
      "(51447, 557)\n",
      "Now processing profiles/2017_12_05_Batch2_dmso_spherized_profiles_with_input_normalized_by_dmso.csv.gz...\n",
      "(51447, 2229)\n",
      "(51447, 763)\n"
     ]
    }
   ],
   "source": [
    "for batch in batches:\n",
    "    for suffix in suffixes:\n",
    "        output_file = pathlib.Path(\n",
    "            f\"{output_dir}/{batch}_dmso_spherized_profiles_with_input_normalized_by_{suffix}.csv.gz\"\n",
    "        )\n",
    "        print(f\"Now processing {output_file}...\")\n",
    "\n",
    "        profile_df = pd.concat([pd.read_csv(x) for x in files[batch][suffix]]).reset_index(drop=True)\n",
    "        print(profile_df.shape)\n",
    "        \n",
    "        # Step 1: Perform feature selection\n",
    "        if batch == \"2017_12_05_Batch2\":\n",
    "            profile_df = (\n",
    "                profile_df\n",
    "                .groupby([\"Metadata_cell_line\", \"Metadata_time_point\"])\n",
    "                .apply(\n",
    "                    lambda x: feature_select(\n",
    "                        profiles=x,\n",
    "                        operation=feature_select_ops,\n",
    "                        na_cutoff=na_cut,\n",
    "                        corr_threshold=corr_threshold,\n",
    "                        blocklist_file=full_blocklist_file\n",
    "                    )\n",
    "                )\n",
    "            )\n",
    "            \n",
    "            # Drop features that weren't selected in the grouped splits\n",
    "            profile_df = feature_select(\n",
    "                profiles=profile_df,\n",
    "                operation=\"drop_na_columns\",\n",
    "                na_cutoff=na_cut\n",
    "            )\n",
    "        else:\n",
    "            profile_df = feature_select(\n",
    "                profiles=profile_df,\n",
    "                operation=feature_select_ops,\n",
    "                na_cutoff=na_cut,\n",
    "                corr_threshold=corr_threshold,\n",
    "                blocklist_file=full_blocklist_file\n",
    "            )\n",
    "\n",
    "        # Step 2: Spherize transform\n",
    "        if batch == \"2017_12_05_Batch2\":\n",
    "            spherize_df = (\n",
    "                profile_df\n",
    "                .groupby([\"Metadata_cell_line\", \"Metadata_time_point\"])\n",
    "                .apply(\n",
    "                    lambda x: normalize(\n",
    "                        profiles=x,\n",
    "                        features=\"infer\",\n",
    "                        meta_features=\"infer\",\n",
    "                        samples=\"Metadata_broad_sample == 'DMSO'\",\n",
    "                        method=\"spherize\"\n",
    "                    )\n",
    "                )\n",
    "            )\n",
    "        else:\n",
    "            spherize_df = normalize(\n",
    "                profiles=profile_df,\n",
    "                features=\"infer\",\n",
    "                meta_features=\"infer\",\n",
    "                samples=\"Metadata_broad_sample == 'DMSO'\",\n",
    "                method=\"spherize\"\n",
    "            )\n",
    "\n",
    "        print(spherize_df.shape)\n",
    "        spherize_df.head()\n",
    "\n",
    "        # Step 3: Output profiles\n",
    "        output(\n",
    "            df=spherize_df,\n",
    "            output_filename=output_file\n",
    "        )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}