consensus/build-consensus-signatures.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Consensus signatures\n",
    "\n",
    "Here, we generate consensus signatures for two batches of the LINCS Drug Repurposing Hub Cell Painting subset.\n",
    "Consensus signatures are level 5 data.\n",
    "\n",
    "See the project [README.md](README.md) for more details.\n",
    "\n",
    "This notebook generates eight files; one per plate normalization and consensus normalization strategy, with and without feature selection.\n",
    "\n",
    "|Feature selection | Plate Normalization | Consensus Normalization | Consensus Suffix |\n",
    "|:---------------- | :------------------: | :------------------------: | -----------------: |\n",
    "| No  | DMSO | Median | `<BATCH>_consensus_median_dmso.csv.gz` |\n",
    "| No  | DMSO | MODZ | `<BATCH>_consensus_modz_dmso.csv.gz` |\n",
    "| No  | Whole Plate | Median | `<BATCH>_consensus_median.csv.gz` |\n",
    "| No  | Whole Plate | MODZ | `<BATCH>_consensus_modz.csv.gz` |\n",
    "| Yes | DMSO | Median | `<BATCH>_consensus_median_feature_select_dmso.csv.gz` |\n",
    "| Yes | DMSO | MODZ | `<BATCH>_consensus_modz_feature_select_dmso.csv.gz` |\n",
    "| Yes | Whole Plate | Median | `<BATCH>_consensus_median_feature_select.csv.gz` |\n",
    "| Yes | Whole Plate | MODZ | `<BATCH>_consensus_modz_feature_select.csv.gz` |\n",
    "\n",
    "We generate eight files per batch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 1;\n",
       "                var nbb_unformatted_code = \"%load_ext nb_black\";\n",
       "                var nbb_formatted_code = \"%load_ext nb_black\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%load_ext nb_black"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 2;\n",
       "                var nbb_unformatted_code = \"import os\\nimport pathlib\\nimport numpy as np\\nimport pandas as pd\\n\\nfrom pycytominer import aggregate, feature_select\\n\\nfrom pycytominer import consensus\\nfrom pycytominer.cyto_utils import infer_cp_features, output\";\n",
       "                var nbb_formatted_code = \"import os\\nimport pathlib\\nimport numpy as np\\nimport pandas as pd\\n\\nfrom pycytominer import aggregate, feature_select\\n\\nfrom pycytominer import consensus\\nfrom pycytominer.cyto_utils import infer_cp_features, output\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "import pathlib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from pycytominer import aggregate, feature_select\n",
    "\n",
    "from pycytominer import consensus\n",
    "from pycytominer.cyto_utils import infer_cp_features, output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 3;\n",
       "                var nbb_unformatted_code = \"# Set constants\\nbatches = [\\\"2016_04_01_a549_48hr_batch1\\\", \\\"2017_12_05_Batch2\\\"]\\noperations = [\\\"median\\\", \\\"modz\\\"]\\nprimary_dose_mapping = [0.04, 0.12, 0.37, 1.11, 3.33, 10, 20]\\n\\n# Aggregating columns\\nreplicate_cols = [\\n    \\\"Metadata_Plate_Map_Name\\\",\\n    \\\"Metadata_cell_id\\\",\\n    \\\"Metadata_broad_sample\\\",\\n    \\\"Metadata_pert_well\\\",\\n    \\\"Metadata_mmoles_per_liter\\\",\\n    \\\"Metadata_dose_recode\\\",\\n    \\\"Metadata_time_point\\\",\\n    \\\"Metadata_moa\\\",\\n    \\\"Metadata_target\\\",\\n]\\n\\n# feature selection operations\\nfeature_select_ops = [\\n    \\\"drop_na_columns\\\",\\n    \\\"variance_threshold\\\",\\n    \\\"correlation_threshold\\\",\\n    \\\"blocklist\\\",\\n]\\n\\n# Output option\\nfloat_format = \\\"%5g\\\"\\ncompression_options = {\\\"method\\\": \\\"gzip\\\", \\\"mtime\\\": 1}\\n\\n# Two different blocklist feature sets\\ncommit = \\\"838ac2eee8bee09b50a1ec8077201c01f7882c69\\\"\\ntraditional_blocklist_file = f\\\"https://raw.githubusercontent.com/cytomining/pycytominer/{commit}/pycytominer/data/blocklist_features.txt\\\"\\noutlier_blocklist_file = pathlib.Path(\\\"../utils/outlier_blocklist_features.txt\\\")\\nfull_blocklist_file = pathlib.Path(\\\"../utils/consensus_blocklist.txt\\\")\";\n",
       "                var nbb_formatted_code = \"# Set constants\\nbatches = [\\\"2016_04_01_a549_48hr_batch1\\\", \\\"2017_12_05_Batch2\\\"]\\noperations = [\\\"median\\\", \\\"modz\\\"]\\nprimary_dose_mapping = [0.04, 0.12, 0.37, 1.11, 3.33, 10, 20]\\n\\n# Aggregating columns\\nreplicate_cols = [\\n    \\\"Metadata_Plate_Map_Name\\\",\\n    \\\"Metadata_cell_id\\\",\\n    \\\"Metadata_broad_sample\\\",\\n    \\\"Metadata_pert_well\\\",\\n    \\\"Metadata_mmoles_per_liter\\\",\\n    \\\"Metadata_dose_recode\\\",\\n    \\\"Metadata_time_point\\\",\\n    \\\"Metadata_moa\\\",\\n    \\\"Metadata_target\\\",\\n]\\n\\n# feature selection operations\\nfeature_select_ops = [\\n    \\\"drop_na_columns\\\",\\n    \\\"variance_threshold\\\",\\n    \\\"correlation_threshold\\\",\\n    \\\"blocklist\\\",\\n]\\n\\n# Output option\\nfloat_format = \\\"%5g\\\"\\ncompression_options = {\\\"method\\\": \\\"gzip\\\", \\\"mtime\\\": 1}\\n\\n# Two different blocklist feature sets\\ncommit = \\\"838ac2eee8bee09b50a1ec8077201c01f7882c69\\\"\\ntraditional_blocklist_file = f\\\"https://raw.githubusercontent.com/cytomining/pycytominer/{commit}/pycytominer/data/blocklist_features.txt\\\"\\noutlier_blocklist_file = pathlib.Path(\\\"../utils/outlier_blocklist_features.txt\\\")\\nfull_blocklist_file = pathlib.Path(\\\"../utils/consensus_blocklist.txt\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Set constants\n",
    "batches = [\"2016_04_01_a549_48hr_batch1\", \"2017_12_05_Batch2\"]\n",
    "operations = [\"median\", \"modz\"]\n",
    "primary_dose_mapping = [0.04, 0.12, 0.37, 1.11, 3.33, 10, 20]\n",
    "\n",
    "# Aggregating columns\n",
    "replicate_cols = [\n",
    "    \"Metadata_Plate_Map_Name\",\n",
    "    \"Metadata_cell_id\",\n",
    "    \"Metadata_broad_sample\",\n",
    "    \"Metadata_pert_well\",\n",
    "    \"Metadata_mmoles_per_liter\",\n",
    "    \"Metadata_dose_recode\",\n",
    "    \"Metadata_time_point\",\n",
    "    \"Metadata_moa\",\n",
    "    \"Metadata_target\",\n",
    "]\n",
    "\n",
    "# feature selection operations\n",
    "feature_select_ops = [\n",
    "    \"drop_na_columns\",\n",
    "    \"variance_threshold\",\n",
    "    \"correlation_threshold\",\n",
    "    \"blocklist\",\n",
    "]\n",
    "\n",
    "# Output option\n",
    "float_format = \"%5g\"\n",
    "compression_options = {\"method\": \"gzip\", \"mtime\": 1}\n",
    "\n",
    "# Two different blocklist feature sets (traditional and outlier)\n",
    "commit = \"838ac2eee8bee09b50a1ec8077201c01f7882c69\"\n",
    "traditional_blocklist_file = f\"https://raw.githubusercontent.com/cytomining/pycytominer/{commit}/pycytominer/data/blocklist_features.txt\"\n",
    "outlier_blocklist_file = pathlib.Path(\"../utils/outlier_blocklist_features.txt\")\n",
    "\n",
    "full_blocklist_file = pathlib.Path(\"../utils/consensus_blocklist.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(87, 1)\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 4;\n",
       "                var nbb_unformatted_code = \"# Procees blocklist output\\nblocklist_df = pd.read_csv(traditional_blocklist_file)\\noutlier_blocklist_df = pd.read_csv(outlier_blocklist_file, names=[\\\"blocklist\\\"])\\n\\nfull_blocklist_df = (\\n    pd.concat([blocklist_df, outlier_blocklist_df], axis=\\\"rows\\\")\\n    .drop_duplicates()\\n    .reset_index(drop=True)\\n)\\n\\n# Note, we need to output a file to be compatible with pycytominer\\nprint(full_blocklist_df.shape)\\n\\nfull_blocklist_df.to_csv(full_blocklist_file, sep=\\\",\\\", index=False)\";\n",
       "                var nbb_formatted_code = \"# Procees blocklist output\\nblocklist_df = pd.read_csv(traditional_blocklist_file)\\noutlier_blocklist_df = pd.read_csv(outlier_blocklist_file, names=[\\\"blocklist\\\"])\\n\\nfull_blocklist_df = (\\n    pd.concat([blocklist_df, outlier_blocklist_df], axis=\\\"rows\\\")\\n    .drop_duplicates()\\n    .reset_index(drop=True)\\n)\\n\\n# Note, we need to output a file to be compatible with pycytominer\\nprint(full_blocklist_df.shape)\\n\\nfull_blocklist_df.to_csv(full_blocklist_file, sep=\\\",\\\", index=False)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Procees blocklist output\n",
    "blocklist_df = pd.read_csv(traditional_blocklist_file)\n",
    "outlier_blocklist_df = pd.read_csv(outlier_blocklist_file, names=[\"blocklist\"])\n",
    "\n",
    "full_blocklist_df = (\n",
    "    pd.concat([blocklist_df, outlier_blocklist_df], axis=\"rows\")\n",
    "    .drop_duplicates()\n",
    "    .reset_index(drop=True)\n",
    ")\n",
    "\n",
    "# Note, we need to output a file to be compatible with pycytominer\n",
    "print(full_blocklist_df.shape)\n",
    "\n",
    "full_blocklist_df.to_csv(full_blocklist_file, sep=\",\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "136\n",
      "134\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 5;\n",
       "                var nbb_unformatted_code = \"# Set file information\\nfile_bases = {\\n    \\\"whole_plate\\\": {\\n        \\\"input_file_suffix\\\": \\\"_normalized.csv.gz\\\",\\n        \\\"output_file_suffix\\\": \\\".csv.gz\\\",\\n    },\\n    \\\"dmso\\\": {\\n        \\\"input_file_suffix\\\": \\\"_normalized_dmso.csv.gz\\\",\\n        \\\"output_file_suffix\\\": \\\"_dmso.csv.gz\\\",\\n    },\\n}\\n\\nfile_path_info = {\\\"profile_dir\\\": {}, \\\"plate_dirs\\\": {}, \\\"plates\\\": {}}\\nfor batch in batches:\\n    file_path_info[\\\"profile_dir\\\"][batch] = pathlib.Path(\\\"..\\\", \\\"profiles\\\", batch)\\n    file_path_info[\\\"plate_dirs\\\"][batch] = [\\n        x\\n        for x in file_path_info[\\\"profile_dir\\\"][batch].iterdir()\\n        if x.name != \\\".DS_Store\\\"\\n    ]\\n    file_path_info[\\\"plates\\\"][batch] = [\\n        x.name for x in file_path_info[\\\"plate_dirs\\\"][batch]\\n    ]\\n    print(len(file_path_info[\\\"plates\\\"][batch]))\\n\\n# The output directory is also the batch name\\nfor batch in batches:\\n    pathlib.Path(batch).mkdir(exist_ok=True)\";\n",
       "                var nbb_formatted_code = \"# Set file information\\nfile_bases = {\\n    \\\"whole_plate\\\": {\\n        \\\"input_file_suffix\\\": \\\"_normalized.csv.gz\\\",\\n        \\\"output_file_suffix\\\": \\\".csv.gz\\\",\\n    },\\n    \\\"dmso\\\": {\\n        \\\"input_file_suffix\\\": \\\"_normalized_dmso.csv.gz\\\",\\n        \\\"output_file_suffix\\\": \\\"_dmso.csv.gz\\\",\\n    },\\n}\\n\\nfile_path_info = {\\\"profile_dir\\\": {}, \\\"plate_dirs\\\": {}, \\\"plates\\\": {}}\\nfor batch in batches:\\n    file_path_info[\\\"profile_dir\\\"][batch] = pathlib.Path(\\\"..\\\", \\\"profiles\\\", batch)\\n    file_path_info[\\\"plate_dirs\\\"][batch] = [\\n        x\\n        for x in file_path_info[\\\"profile_dir\\\"][batch].iterdir()\\n        if x.name != \\\".DS_Store\\\"\\n    ]\\n    file_path_info[\\\"plates\\\"][batch] = [\\n        x.name for x in file_path_info[\\\"plate_dirs\\\"][batch]\\n    ]\\n    print(len(file_path_info[\\\"plates\\\"][batch]))\\n\\n# The output directory is also the batch name\\nfor batch in batches:\\n    pathlib.Path(batch).mkdir(exist_ok=True)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Set file information\n",
    "file_bases = {\n",
    "    \"whole_plate\": {\n",
    "        \"input_file_suffix\": \"_normalized.csv.gz\",\n",
    "        \"output_file_suffix\": \".csv.gz\",\n",
    "    },\n",
    "    \"dmso\": {\n",
    "        \"input_file_suffix\": \"_normalized_dmso.csv.gz\",\n",
    "        \"output_file_suffix\": \"_dmso.csv.gz\",\n",
    "    },\n",
    "}\n",
    "\n",
    "file_path_info = {\"profile_dir\": {}, \"plate_dirs\": {}, \"plates\": {}}\n",
    "for batch in batches:\n",
    "    file_path_info[\"profile_dir\"][batch] = pathlib.Path(\"..\", \"profiles\", batch)\n",
    "    file_path_info[\"plate_dirs\"][batch] = [\n",
    "        x\n",
    "        for x in file_path_info[\"profile_dir\"][batch].iterdir()\n",
    "        if x.name != \".DS_Store\"\n",
    "    ]\n",
    "    file_path_info[\"plates\"][batch] = [\n",
    "        x.name for x in file_path_info[\"plate_dirs\"][batch]\n",
    "    ]\n",
    "    print(len(file_path_info[\"plates\"][batch]))\n",
    "\n",
    "# The output directory is also the batch name\n",
    "for batch in batches:\n",
    "    pathlib.Path(batch).mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and Process Data\n",
    "\n",
    "We load data per plate, concatenate, add make minor modifications.\n",
    "\n",
    "We perform this operation once per batch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(52223, 1811)\n",
      "(52223, 1811)\n",
      "(51447, 2229)\n",
      "(51447, 2229)\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 6;\n",
       "                var nbb_unformatted_code = \"# Load Data\\nall_profiles_dfs = {batch: {} for batch in batches}\\ncp_features = {batch: {} for batch in batches}\\n\\nfor batch in batches:\\n    for norm_strat, norm_file_base in file_bases.items():\\n        file_base = norm_file_base[\\\"input_file_suffix\\\"]\\n        all_profiles_df = []\\n        for plate_dir in file_path_info[\\\"plate_dirs\\\"][batch]:\\n            plate = plate_dir.name\\n            plate_file = plate_dir / f\\\"{plate}{file_base}\\\"\\n            plate_df = pd.read_csv(plate_file)\\n            all_profiles_df.append(plate_df)\\n\\n        # Concatenate profiles\\n        all_profiles_df = pd.concat(all_profiles_df, axis=\\\"rows\\\")\\n\\n        # Add time metadata for batch 1 data\\n        if batch == \\\"2016_04_01_a549_48hr_batch1\\\":\\n            all_profiles_df = all_profiles_df.assign(Metadata_time_point=\\\"48H\\\")\\n\\n        # Recode missing MOA and target values to be \\\"unknown\\\"\\n        all_profiles_df.Metadata_moa = all_profiles_df.Metadata_moa.fillna(\\\"unknown\\\")\\n        all_profiles_df.Metadata_target = all_profiles_df.Metadata_target.fillna(\\n            \\\"unknown\\\"\\n        )\\n\\n        # Store concatenated data frame\\n        all_profiles_dfs[batch][norm_strat] = all_profiles_df\\n\\n        # Determine every CellProfiler feature measured\\n        cp_features[batch][norm_strat] = infer_cp_features(\\n            all_profiles_dfs[batch][norm_strat]\\n        )\\n\\n        # Clean up\\n        print(all_profiles_df.shape)\\n        del all_profiles_df\";\n",
       "                var nbb_formatted_code = \"# Load Data\\nall_profiles_dfs = {batch: {} for batch in batches}\\ncp_features = {batch: {} for batch in batches}\\n\\nfor batch in batches:\\n    for norm_strat, norm_file_base in file_bases.items():\\n        file_base = norm_file_base[\\\"input_file_suffix\\\"]\\n        all_profiles_df = []\\n        for plate_dir in file_path_info[\\\"plate_dirs\\\"][batch]:\\n            plate = plate_dir.name\\n            plate_file = plate_dir / f\\\"{plate}{file_base}\\\"\\n            plate_df = pd.read_csv(plate_file)\\n            all_profiles_df.append(plate_df)\\n\\n        # Concatenate profiles\\n        all_profiles_df = pd.concat(all_profiles_df, axis=\\\"rows\\\")\\n\\n        # Add time metadata for batch 1 data\\n        if batch == \\\"2016_04_01_a549_48hr_batch1\\\":\\n            all_profiles_df = all_profiles_df.assign(Metadata_time_point=\\\"48H\\\")\\n\\n        # Recode missing MOA and target values to be \\\"unknown\\\"\\n        all_profiles_df.Metadata_moa = all_profiles_df.Metadata_moa.fillna(\\\"unknown\\\")\\n        all_profiles_df.Metadata_target = all_profiles_df.Metadata_target.fillna(\\n            \\\"unknown\\\"\\n        )\\n\\n        # Store concatenated data frame\\n        all_profiles_dfs[batch][norm_strat] = all_profiles_df\\n\\n        # Determine every CellProfiler feature measured\\n        cp_features[batch][norm_strat] = infer_cp_features(\\n            all_profiles_dfs[batch][norm_strat]\\n        )\\n\\n        # Clean up\\n        print(all_profiles_df.shape)\\n        del all_profiles_df\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Load Data\n",
    "all_profiles_dfs = {batch: {} for batch in batches}\n",
    "cp_features = {batch: {} for batch in batches}\n",
    "\n",
    "for batch in batches:\n",
    "    for norm_strat, norm_file_base in file_bases.items():\n",
    "        file_base = norm_file_base[\"input_file_suffix\"]\n",
    "        all_profiles_df = []\n",
    "        for plate_dir in file_path_info[\"plate_dirs\"][batch]:\n",
    "            plate = plate_dir.name\n",
    "            plate_file = plate_dir / f\"{plate}{file_base}\"\n",
    "            plate_df = pd.read_csv(plate_file)\n",
    "            all_profiles_df.append(plate_df)\n",
    "\n",
    "        # Concatenate profiles\n",
    "        all_profiles_df = pd.concat(all_profiles_df, axis=\"rows\")\n",
    "\n",
    "        # Add time metadata for batch 1 data\n",
    "        if batch == \"2016_04_01_a549_48hr_batch1\":\n",
    "            all_profiles_df = all_profiles_df.assign(Metadata_time_point=\"48H\")\n",
    "\n",
    "        # Recode missing MOA and target values to be \"unknown\"\n",
    "        all_profiles_df.Metadata_moa = all_profiles_df.Metadata_moa.fillna(\"unknown\")\n",
    "        all_profiles_df.Metadata_target = all_profiles_df.Metadata_target.fillna(\n",
    "            \"unknown\"\n",
    "        )\n",
    "\n",
    "        # Store concatenated data frame\n",
    "        all_profiles_dfs[batch][norm_strat] = all_profiles_df\n",
    "\n",
    "        # Determine every CellProfiler feature measured\n",
    "        cp_features[batch][norm_strat] = infer_cp_features(\n",
    "            all_profiles_dfs[batch][norm_strat]\n",
    "        )\n",
    "\n",
    "        # Clean up\n",
    "        print(all_profiles_df.shape)\n",
    "        del all_profiles_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create consensus profiles\n",
    "\n",
    "Create profiles with and without feature selection.\n",
    "\n",
    "We generate two different consensus profiles for each of the normalization strategies, with and without feature selection.\n",
    "This generates eight different files _per batch_."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Now processing batch: 2016_04_01_a549_48hr_batch1\n",
      "  Now calculating median consensus for whole_plate normalization\n",
      "  There are 10752 median consensus profiles for whole_plate normalization\n",
      "  Now feature selecting on median consensus for whole_plate normalization\n",
      "  There are 501 features in median consensus profiles for whole_plate normalization\n",
      "  Now calculating modz consensus for whole_plate normalization\n",
      "  There are 10752 modz consensus profiles for whole_plate normalization\n",
      "  Now feature selecting on modz consensus for whole_plate normalization\n",
      "  There are 441 features in modz consensus profiles for whole_plate normalization\n",
      "  Now calculating median consensus for dmso normalization\n",
      "  There are 10752 median consensus profiles for dmso normalization\n",
      "  Now feature selecting on median consensus for dmso normalization\n",
      "  There are 561 features in median consensus profiles for dmso normalization\n",
      "  Now calculating modz consensus for dmso normalization\n",
      "  There are 10752 modz consensus profiles for dmso normalization\n",
      "  Now feature selecting on modz consensus for dmso normalization\n",
      "  There are 496 features in modz consensus profiles for dmso normalization\n",
      "\n",
      "\n",
      "Now processing batch: 2017_12_05_Batch2\n",
      "  Now calculating median consensus for whole_plate normalization\n",
      "  There are 10368 median consensus profiles for whole_plate normalization\n",
      "  Now feature selecting on median consensus for whole_plate normalization\n",
      "  There are 829 features in median consensus profiles for whole_plate normalization\n",
      "  Now calculating modz consensus for whole_plate normalization\n",
      "  There are 10368 modz consensus profiles for whole_plate normalization\n",
      "  Now feature selecting on modz consensus for whole_plate normalization\n",
      "  There are 757 features in modz consensus profiles for whole_plate normalization\n",
      "  Now calculating median consensus for dmso normalization\n",
      "  There are 10368 median consensus profiles for dmso normalization\n",
      "  Now feature selecting on median consensus for dmso normalization\n",
      "  There are 903 features in median consensus profiles for dmso normalization\n",
      "  Now calculating modz consensus for dmso normalization\n",
      "  There are 10368 modz consensus profiles for dmso normalization\n",
      "  Now feature selecting on modz consensus for dmso normalization\n",
      "  There are 811 features in modz consensus profiles for dmso normalization\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 7;\n",
       "                var nbb_unformatted_code = \"all_consensus_dfs = {batch: {} for batch in batches}\\nfor batch in batches:\\n    print(f\\\"Now processing batch: {batch}\\\")\\n    for norm_strat in file_bases:\\n        all_profiles_df = all_profiles_dfs[batch][norm_strat]\\n        cp_norm_features = cp_features[batch][norm_strat]\\n\\n        consensus_profiles = {}\\n        for operation in operations:\\n            print(\\n                f\\\"  Now calculating {operation} consensus for {norm_strat} normalization\\\"\\n            )\\n\\n            consensus_profiles[operation] = {}\\n\\n            consensus_profiles[operation][\\\"no_feat_select\\\"] = consensus(\\n                profiles=all_profiles_df,\\n                replicate_columns=replicate_cols,\\n                operation=operation,\\n                features=cp_norm_features,\\n            )\\n\\n            # How many DMSO profiles per well?\\n            print(\\n                f\\\"  There are {consensus_profiles[operation]['no_feat_select'].shape[0]} {operation} consensus profiles for {norm_strat} normalization\\\"\\n            )\\n\\n            # Perform feature selection\\n            print(\\n                f\\\"  Now feature selecting on {operation} consensus for {norm_strat} normalization\\\"\\n            )\\n            consensus_profiles[operation][\\\"feat_select\\\"] = feature_select(\\n                profiles=consensus_profiles[operation][\\\"no_feat_select\\\"],\\n                features=\\\"infer\\\",\\n                operation=feature_select_ops,\\n                blocklist_file=full_blocklist_file\\n            )\\n\\n            # How many features in feature selected profile?\\n            print(\\n                f\\\"  There are {consensus_profiles[operation]['feat_select'].shape[1]} features in {operation} consensus profiles for {norm_strat} normalization\\\"\\n            )\\n\\n        all_consensus_dfs[batch][norm_strat] = consensus_profiles\\n    print(\\\"\\\\n\\\")\";\n",
       "                var nbb_formatted_code = \"all_consensus_dfs = {batch: {} for batch in batches}\\nfor batch in batches:\\n    print(f\\\"Now processing batch: {batch}\\\")\\n    for norm_strat in file_bases:\\n        all_profiles_df = all_profiles_dfs[batch][norm_strat]\\n        cp_norm_features = cp_features[batch][norm_strat]\\n\\n        consensus_profiles = {}\\n        for operation in operations:\\n            print(\\n                f\\\"  Now calculating {operation} consensus for {norm_strat} normalization\\\"\\n            )\\n\\n            consensus_profiles[operation] = {}\\n\\n            consensus_profiles[operation][\\\"no_feat_select\\\"] = consensus(\\n                profiles=all_profiles_df,\\n                replicate_columns=replicate_cols,\\n                operation=operation,\\n                features=cp_norm_features,\\n            )\\n\\n            # How many DMSO profiles per well?\\n            print(\\n                f\\\"  There are {consensus_profiles[operation]['no_feat_select'].shape[0]} {operation} consensus profiles for {norm_strat} normalization\\\"\\n            )\\n\\n            # Perform feature selection\\n            print(\\n                f\\\"  Now feature selecting on {operation} consensus for {norm_strat} normalization\\\"\\n            )\\n            consensus_profiles[operation][\\\"feat_select\\\"] = feature_select(\\n                profiles=consensus_profiles[operation][\\\"no_feat_select\\\"],\\n                features=\\\"infer\\\",\\n                operation=feature_select_ops,\\n                blocklist_file=full_blocklist_file,\\n            )\\n\\n            # How many features in feature selected profile?\\n            print(\\n                f\\\"  There are {consensus_profiles[operation]['feat_select'].shape[1]} features in {operation} consensus profiles for {norm_strat} normalization\\\"\\n            )\\n\\n        all_consensus_dfs[batch][norm_strat] = consensus_profiles\\n    print(\\\"\\\\n\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "all_consensus_dfs = {batch: {} for batch in batches}\n",
    "for batch in batches:\n",
    "    print(f\"Now processing batch: {batch}\")\n",
    "    for norm_strat in file_bases:\n",
    "        all_profiles_df = all_profiles_dfs[batch][norm_strat]\n",
    "        cp_norm_features = cp_features[batch][norm_strat]\n",
    "\n",
    "        consensus_profiles = {}\n",
    "        for operation in operations:\n",
    "            print(\n",
    "                f\"  Now calculating {operation} consensus for {norm_strat} normalization\"\n",
    "            )\n",
    "\n",
    "            consensus_profiles[operation] = {}\n",
    "\n",
    "            consensus_profiles[operation][\"no_feat_select\"] = consensus(\n",
    "                profiles=all_profiles_df,\n",
    "                replicate_columns=replicate_cols,\n",
    "                operation=operation,\n",
    "                features=cp_norm_features,\n",
    "            )\n",
    "\n",
    "            # How many DMSO profiles per well?\n",
    "            print(\n",
    "                f\"  There are {consensus_profiles[operation]['no_feat_select'].shape[0]} {operation} consensus profiles for {norm_strat} normalization\"\n",
    "            )\n",
    "\n",
    "            # Perform feature selection\n",
    "            print(\n",
    "                f\"  Now feature selecting on {operation} consensus for {norm_strat} normalization\"\n",
    "            )\n",
    "            consensus_profiles[operation][\"feat_select\"] = feature_select(\n",
    "                profiles=consensus_profiles[operation][\"no_feat_select\"],\n",
    "                features=\"infer\",\n",
    "                operation=feature_select_ops,\n",
    "                blocklist_file=full_blocklist_file,\n",
    "            )\n",
    "\n",
    "            # How many features in feature selected profile?\n",
    "            print(\n",
    "                f\"  There are {consensus_profiles[operation]['feat_select'].shape[1]} features in {operation} consensus profiles for {norm_strat} normalization\"\n",
    "            )\n",
    "\n",
    "        all_consensus_dfs[batch][norm_strat] = consensus_profiles\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merge and output consensus signatures\n",
    "\n",
    "Output with and without feature selection."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Now processing batch: 2016_04_01_a549_48hr_batch1\n",
      "  Now Writing: Feature selection: No; Consensus Operation: median; Norm Strategy: whole_plate\n",
      "  File: 2016_04_01_a549_48hr_batch1/2016_04_01_a549_48hr_batch1_consensus_median.csv.gz\n",
      "(10752, 1790)\n",
      "  Now Writing: Feature selection: Yes; Consensus Operation: median; Norm Strategy: whole_plate\n",
      "  File: 2016_04_01_a549_48hr_batch1/2016_04_01_a549_48hr_batch1_consensus_median_feature_select.csv.gz\n",
      "(10752, 501)\n",
      "  Now Writing: Feature selection: No; Consensus Operation: modz; Norm Strategy: whole_plate\n",
      "  File: 2016_04_01_a549_48hr_batch1/2016_04_01_a549_48hr_batch1_consensus_modz.csv.gz\n",
      "(10752, 1790)\n",
      "  Now Writing: Feature selection: Yes; Consensus Operation: modz; Norm Strategy: whole_plate\n",
      "  File: 2016_04_01_a549_48hr_batch1/2016_04_01_a549_48hr_batch1_consensus_modz_feature_select.csv.gz\n",
      "(10752, 441)\n",
      "  Now Writing: Feature selection: No; Consensus Operation: median; Norm Strategy: dmso\n",
      "  File: 2016_04_01_a549_48hr_batch1/2016_04_01_a549_48hr_batch1_consensus_median_dmso.csv.gz\n",
      "(10752, 1790)\n",
      "  Now Writing: Feature selection: Yes; Consensus Operation: median; Norm Strategy: dmso\n",
      "  File: 2016_04_01_a549_48hr_batch1/2016_04_01_a549_48hr_batch1_consensus_median_feature_select_dmso.csv.gz\n",
      "(10752, 561)\n",
      "  Now Writing: Feature selection: No; Consensus Operation: modz; Norm Strategy: dmso\n",
      "  File: 2016_04_01_a549_48hr_batch1/2016_04_01_a549_48hr_batch1_consensus_modz_dmso.csv.gz\n",
      "(10752, 1790)\n",
      "  Now Writing: Feature selection: Yes; Consensus Operation: modz; Norm Strategy: dmso\n",
      "  File: 2016_04_01_a549_48hr_batch1/2016_04_01_a549_48hr_batch1_consensus_modz_feature_select_dmso.csv.gz\n",
      "(10752, 496)\n",
      "\n",
      "\n",
      "Now processing batch: 2017_12_05_Batch2\n",
      "  Now Writing: Feature selection: No; Consensus Operation: median; Norm Strategy: whole_plate\n",
      "  File: 2017_12_05_Batch2/2017_12_05_Batch2_consensus_median.csv.gz\n",
      "(10368, 2207)\n",
      "  Now Writing: Feature selection: Yes; Consensus Operation: median; Norm Strategy: whole_plate\n",
      "  File: 2017_12_05_Batch2/2017_12_05_Batch2_consensus_median_feature_select.csv.gz\n",
      "(10368, 829)\n",
      "  Now Writing: Feature selection: No; Consensus Operation: modz; Norm Strategy: whole_plate\n",
      "  File: 2017_12_05_Batch2/2017_12_05_Batch2_consensus_modz.csv.gz\n",
      "(10368, 2207)\n",
      "  Now Writing: Feature selection: Yes; Consensus Operation: modz; Norm Strategy: whole_plate\n",
      "  File: 2017_12_05_Batch2/2017_12_05_Batch2_consensus_modz_feature_select.csv.gz\n",
      "(10368, 757)\n",
      "  Now Writing: Feature selection: No; Consensus Operation: median; Norm Strategy: dmso\n",
      "  File: 2017_12_05_Batch2/2017_12_05_Batch2_consensus_median_dmso.csv.gz\n",
      "(10368, 2207)\n",
      "  Now Writing: Feature selection: Yes; Consensus Operation: median; Norm Strategy: dmso\n",
      "  File: 2017_12_05_Batch2/2017_12_05_Batch2_consensus_median_feature_select_dmso.csv.gz\n",
      "(10368, 903)\n",
      "  Now Writing: Feature selection: No; Consensus Operation: modz; Norm Strategy: dmso\n",
      "  File: 2017_12_05_Batch2/2017_12_05_Batch2_consensus_modz_dmso.csv.gz\n",
      "(10368, 2207)\n",
      "  Now Writing: Feature selection: Yes; Consensus Operation: modz; Norm Strategy: dmso\n",
      "  File: 2017_12_05_Batch2/2017_12_05_Batch2_consensus_modz_feature_select_dmso.csv.gz\n",
      "(10368, 811)\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 8;\n",
       "                var nbb_unformatted_code = \"for batch in batches:\\n    print(f\\\"Now processing batch: {batch}\\\")\\n    for norm_strat in file_bases:\\n        file_suffix = file_bases[norm_strat][\\\"output_file_suffix\\\"]\\n        for operation in operations:\\n\\n            # No feature selection\\n            consensus_file = f\\\"{batch}_consensus_{operation}{file_suffix}\\\"\\n            consensus_file = pathlib.Path(batch, consensus_file)\\n\\n            consensus_df = all_consensus_dfs[batch][norm_strat][operation][\\n                \\\"no_feat_select\\\"\\n            ]\\n\\n            print(\\n                f\\\"  Now Writing: Feature selection: No; Consensus Operation: {operation}; Norm Strategy: {norm_strat}\\\"\\n            )\\n            print(f\\\"  File: {consensus_file}\\\")\\n            print(consensus_df.shape)\\n\\n            output(\\n                df=consensus_df,\\n                output_filename=consensus_file,\\n                sep=\\\",\\\",\\n                float_format=float_format,\\n                compression_options=compression_options,\\n            )\\n\\n            # With feature selection\\n            consensus_feat_df = all_consensus_dfs[batch][norm_strat][operation][\\n                \\\"feat_select\\\"\\n            ]\\n\\n            consensus_feat_file = (\\n                f\\\"{batch}_consensus_{operation}_feature_select{file_suffix}\\\"\\n            )\\n            consensus_feat_file = pathlib.Path(batch, consensus_feat_file)\\n\\n            print(\\n                f\\\"  Now Writing: Feature selection: Yes; Consensus Operation: {operation}; Norm Strategy: {norm_strat}\\\"\\n            )\\n            print(f\\\"  File: {consensus_feat_file}\\\")\\n            print(consensus_feat_df.shape)\\n\\n            output(\\n                df=consensus_feat_df,\\n                output_filename=consensus_feat_file,\\n                sep=\\\",\\\",\\n                float_format=float_format,\\n                compression_options=compression_options,\\n            )\\n    print(\\\"\\\\n\\\")\";\n",
       "                var nbb_formatted_code = \"for batch in batches:\\n    print(f\\\"Now processing batch: {batch}\\\")\\n    for norm_strat in file_bases:\\n        file_suffix = file_bases[norm_strat][\\\"output_file_suffix\\\"]\\n        for operation in operations:\\n\\n            # No feature selection\\n            consensus_file = f\\\"{batch}_consensus_{operation}{file_suffix}\\\"\\n            consensus_file = pathlib.Path(batch, consensus_file)\\n\\n            consensus_df = all_consensus_dfs[batch][norm_strat][operation][\\n                \\\"no_feat_select\\\"\\n            ]\\n\\n            print(\\n                f\\\"  Now Writing: Feature selection: No; Consensus Operation: {operation}; Norm Strategy: {norm_strat}\\\"\\n            )\\n            print(f\\\"  File: {consensus_file}\\\")\\n            print(consensus_df.shape)\\n\\n            output(\\n                df=consensus_df,\\n                output_filename=consensus_file,\\n                sep=\\\",\\\",\\n                float_format=float_format,\\n                compression_options=compression_options,\\n            )\\n\\n            # With feature selection\\n            consensus_feat_df = all_consensus_dfs[batch][norm_strat][operation][\\n                \\\"feat_select\\\"\\n            ]\\n\\n            consensus_feat_file = (\\n                f\\\"{batch}_consensus_{operation}_feature_select{file_suffix}\\\"\\n            )\\n            consensus_feat_file = pathlib.Path(batch, consensus_feat_file)\\n\\n            print(\\n                f\\\"  Now Writing: Feature selection: Yes; Consensus Operation: {operation}; Norm Strategy: {norm_strat}\\\"\\n            )\\n            print(f\\\"  File: {consensus_feat_file}\\\")\\n            print(consensus_feat_df.shape)\\n\\n            output(\\n                df=consensus_feat_df,\\n                output_filename=consensus_feat_file,\\n                sep=\\\",\\\",\\n                float_format=float_format,\\n                compression_options=compression_options,\\n            )\\n    print(\\\"\\\\n\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for batch in batches:\n",
    "    print(f\"Now processing batch: {batch}\")\n",
    "    for norm_strat in file_bases:\n",
    "        file_suffix = file_bases[norm_strat][\"output_file_suffix\"]\n",
    "        for operation in operations:\n",
    "\n",
    "            # No feature selection\n",
    "            consensus_file = f\"{batch}_consensus_{operation}{file_suffix}\"\n",
    "            consensus_file = pathlib.Path(batch, consensus_file)\n",
    "\n",
    "            consensus_df = all_consensus_dfs[batch][norm_strat][operation][\n",
    "                \"no_feat_select\"\n",
    "            ]\n",
    "\n",
    "            print(\n",
    "                f\"  Now Writing: Feature selection: No; Consensus Operation: {operation}; Norm Strategy: {norm_strat}\"\n",
    "            )\n",
    "            print(f\"  File: {consensus_file}\")\n",
    "            print(consensus_df.shape)\n",
    "\n",
    "            output(\n",
    "                df=consensus_df,\n",
    "                output_filename=consensus_file,\n",
    "                sep=\",\",\n",
    "                float_format=float_format,\n",
    "                compression_options=compression_options,\n",
    "            )\n",
    "\n",
    "            # With feature selection\n",
    "            consensus_feat_df = all_consensus_dfs[batch][norm_strat][operation][\n",
    "                \"feat_select\"\n",
    "            ]\n",
    "\n",
    "            consensus_feat_file = (\n",
    "                f\"{batch}_consensus_{operation}_feature_select{file_suffix}\"\n",
    "            )\n",
    "            consensus_feat_file = pathlib.Path(batch, consensus_feat_file)\n",
    "\n",
    "            print(\n",
    "                f\"  Now Writing: Feature selection: Yes; Consensus Operation: {operation}; Norm Strategy: {norm_strat}\"\n",
    "            )\n",
    "            print(f\"  File: {consensus_feat_file}\")\n",
    "            print(consensus_feat_df.shape)\n",
    "\n",
    "            output(\n",
    "                df=consensus_feat_df,\n",
    "                output_filename=consensus_feat_file,\n",
    "                sep=\",\",\n",
    "                float_format=float_format,\n",
    "                compression_options=compression_options,\n",
    "            )\n",
    "    print(\"\\n\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}