et_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extracting eye-tracking data outputs from SR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import modules\n",
    "\n",
    "import os\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "import numpy as np\n",
    "import glob\n",
    "import shutil\n",
    "#import mne"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to format ID\n",
    "def format_id(id_str):\n",
    "    parts = id_str.split('_')\n",
    "    #if len(parts) != 2:\n",
    "        #raise ValueError(\"Invalid ID format. Should be 'XXX_XX' or 'XXXX_XX' or 'XXX_X' or 'XXXX_XX'.\")\n",
    "    \n",
    "    numeric_part = parts[0]\n",
    "    letter_part = parts[1]\n",
    "    \n",
    "    #if not numeric_part.isdigit():\n",
    "     #   print(\"Numeric part before '_' should consist of digits only.\")\n",
    "    \n",
    "    \n",
    "    # Ensure numeric part is 4 digits long by padding with zeros if necessary\n",
    "    padded_numeric_part = numeric_part.zfill(4)\n",
    "    \n",
    "    formatted_id = f\"{padded_numeric_part}_{letter_part}\"\n",
    "    \n",
    "    return formatted_id\n",
    "\n",
    "# Fuction to fix experiment builder ID's so that they match the ET ID's\n",
    "def eb_id_transform(file):\n",
    "    file = file.upper()\n",
    "    file = file.replace('Q', \"\")\n",
    "\n",
    "    if \"_\" not in file:\n",
    "        # Add \"_\" right before the first letter from the end \n",
    "        # Find the index of the first alphanumeric character\n",
    "        for i, char in enumerate(file):\n",
    "            if char.isalpha():\n",
    "                break      \n",
    "        # Insert \"_\" before the first alphanumeric character found\n",
    "        file = file[:i] + '_' + file[i:]\n",
    "        \n",
    "    # add 0's to the end of the file name to make it 4 digits\n",
    "    file = format_id(file)\n",
    "    return file\n",
    "\n",
    "# Function to check if file name contains task information\n",
    "def has_task_info(file_name, task_info):\n",
    "    return task_info in file_name\n",
    "\n",
    "## Function to check if file name contains task information\n",
    "def process_participants(task_folder, eeg_q1k_subjects_df, missing_eeg, et_subjects, transformed_et):\n",
    "    \"\"\"\n",
    "    Process participant folders to track missing EEG data.\n",
    "    \n",
    "    :param task_folder: Path to the current task folder.\n",
    "    :param eeg_q1k_subjects_df: DataFrame containing EEG subjects data.\n",
    "    :param missing_eeg: List to store participants with missing EEG data.\n",
    "    :param et_subjects: List to store participant names.\n",
    "    :param transformed_et: List to store transformed participant IDs.\n",
    "    \"\"\"    \n",
    "    for participant in os.listdir(task_folder):\n",
    "        participant_folder = os.path.join(task_folder, participant)\n",
    "        \n",
    "        et_subjects.append(participant)\n",
    "        transformed_id = eb_id_transform(participant)\n",
    "        transformed_et.append(transformed_id)\n",
    "        \n",
    "        if transformed_id in eeg_q1k_subjects_df.et_ID.values:\n",
    "            # Retrieve new participant ID if available\n",
    "            new_participant = eeg_q1k_subjects_df.loc[eeg_q1k_subjects_df['et_ID'] == transformed_id, 'q1k_ID'].values\n",
    "            if new_participant.size > 0:\n",
    "                new_id = new_participant[0]\n",
    "        else:\n",
    "            missing_eeg.append(participant)\n",
    "    return new_id\n",
    "\n",
    "# Function to copy and paste participant files\n",
    "def process_participant_files(task, participant, participant_folder, transformed_id, new_participant, final_output_dir):\n",
    "    # Process .edf and .txt files in one pass\n",
    "    for file_name in os.listdir(participant_folder):\n",
    "        if file_name.endswith('.edf'):\n",
    "            #new_file_name = f\"{new_participant}_{task}.{file_name.split('.')[-1]}\"\n",
    "            new_file_name = f\"{new_participant}_{task}.asc\"\n",
    "            source_path = os.path.join(participant_folder, file_name)\n",
    "            destination_path = os.path.join(final_output_dir, new_participant, new_file_name)\n",
    "            #shutil.copy(source_path, destination_path)\n",
    "            print('new_file_name: ' + new_file_name)\n",
    "            print('source_path: ' + source_path)\n",
    "            print('destination_path: ' + destination_path)\n",
    "            command = \"edf2asc -y -input \" + source_path + \" \" + destination_path\n",
    "            print(command)\n",
    "            !{command}\n",
    "\n",
    "        #elif file_name.endswith('.txt') and has_task_info(file_name, task):\n",
    "        #    new_file_name = f\"{new_participant}_{task}.{file_name.split('.')[-1]}\"\n",
    "        #    source_path = os.path.join(participant_folder, file_name)\n",
    "        #    destination_path = os.path.join(final_output_dir, new_participant, new_file_name)\n",
    "        #    shutil.copy(source_path, destination_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create reference file to map Q1K IDs to eye tracking data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "eeg_q1k_subjects= []\n",
    "truncated_eeg_q1k_subjects = []\n",
    "family_id_subjects = []\n",
    "site = \"HSJ\" #'MHC' or 'HSJ'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a list of tasks  \n",
    "#tasks = ['GO', 'NSP', 'AS', 'PLR','VS','FSP','REST', 'SSAEP', 'SSVEP','TMMN','TO']\n",
    "task_id_in = 'VEP'\n",
    "task_id_out = 'VEP'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../../sourcedata/et/Q1K_HSJ_100114_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_10043_F1',\n",
       " '../../sourcedata/et/Q1K_HSJ_10064_M1',\n",
       " '../../sourcedata/et/archive',\n",
       " '../../sourcedata/et/Q1K_HSJ_100114_S2',\n",
       " '../../sourcedata/et/Q1K_HSJ_100162_S1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100108_F1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100129_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_100150_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_1525-1024_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_100147_F2',\n",
       " '../../sourcedata/et/Q1K_HSJ_10086_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100123_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_100162_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100100_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_1525-1024_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100105_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_1525-1026_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_100152_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_100123_F1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100114_S1',\n",
       " '../../sourcedata/et/Q1K_HSJ_10083_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_10064_S1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100131_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_100134_F1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100111_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_10086_S1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100119_F1',\n",
       " '../../sourcedata/et/Q1K_HSJ_10083_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_100111_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_100129_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100159_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_1525-1026_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_1525-1001_F1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100162_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_100100_S1',\n",
       " '../../sourcedata/et/Q1K_HSJ_10064_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_100131_S1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100105_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_100111_F1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100147_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_1525_1037_F1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100147_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100157_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_1525-1001_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_10050_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100134_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_100150_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100152_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100114_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_10093_O1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100105_F1',\n",
       " '../../sourcedata/et/Q1K_HSJ_10050_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_100146_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_100134_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_10053_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_10083_F1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100123_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_1525_1037_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_1525-1001_S1',\n",
       " '../../sourcedata/et/Q1K_HSJ_1525-1006_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_1525-1001_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_100100_F1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100157_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100154_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_10086_F1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100119_S1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100131_M1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100114_F1',\n",
       " '../../sourcedata/et/Q1K_HSJ_100134_S1',\n",
       " '../../sourcedata/et/Q1K_HSJ_10043_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_100126_P',\n",
       " '../../sourcedata/et/Q1K_HSJ_10062_P']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#glob.glob(f\"../../../../../../Sharing/CHUSJ-Q1K-PILOT/experimental/{site}/sourcedata/eeg/*\")\n",
    "glob.glob(f\"../../sourcedata/et/*\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q1K_HSJ_100152_P\n",
      "152_P\n",
      "Q1K_HSJ_10083_M1\n",
      "83_M1\n",
      "Q1K_HSJ_1525_1052_S1\n",
      "1052_S1\n",
      "Q1K_HSJ_100123_F1\n",
      "123_F1\n",
      "Q1K_HSJ_100114_S1\n",
      "114_S1\n",
      "Q1K_HSJ_10064_S1\n",
      "64_S1\n",
      "Q1K_HSJ_1525-1006_P\n",
      "1006_P\n",
      "Q1K_HSJ_100134_F1\n",
      "134_F1\n",
      "Q1K_HSJ_1525-1033_S1\n",
      "1033_S1\n",
      "Q1K_HSJ_100131_P\n",
      "131_P\n",
      "Q1K_HSJ_1025-1042_P\n",
      "1042_P\n",
      "Q1K_HSJ_1025-1061_F1\n",
      "1061_F1\n",
      "Q1K_HSJ_100162_M1\n",
      "162_M1\n",
      "Q1K_HSJ_100100_P\n",
      "100_P\n",
      "Q1K_HSJ_100105_M1\n",
      "105_M1\n",
      "Q1K_HSJ_100162_P\n",
      "162_P\n",
      "Q1K_HSJ_1525_1009_M1\n",
      "1009_M1\n",
      "Q1K_HSJ_100100_S1\n",
      "100_S1\n",
      "Q1K_HSJ_1525-1080_F1\n",
      "1080_F1\n",
      "Q1K_HSJ_10086_S1\n",
      "86_S1\n",
      "Q1K_HSJ_1525_1052_P\n",
      "1052_P\n",
      "Q1K_HSJ_1525-1026_P\n",
      "1026_P\n",
      "Q1K_HSJ_100111_M1\n",
      "111_M1\n",
      "Q1K_HSJ_1025-1061_M\n",
      "1061_M\n",
      "Q1K_HSJ_100119_F1\n",
      "119_F1\n",
      "Q1K_HSJ_100111_P\n",
      "111_P\n",
      "Q1K_HSJ_100159_M1\n",
      "159_M1\n",
      "Q1K_HSJ_1525-1045_P\n",
      "1045_P\n",
      "Q1K_HSJ_100129_M1\n",
      "129_M1\n",
      "Q1K_HSJ_100114_S2\n",
      "114_S2\n",
      "Q1K_HSJ_100128_P\n",
      "128_P\n",
      "Q1K_HSJ_1525-1045_S1\n",
      "1045_S1\n",
      "Q1K_HSJ_100162_S1\n",
      "162_S1\n",
      "Q1K_HSJ_1525-1024_P\n",
      "1024_P\n",
      "Q1K_HSJ_1525_1018_P\n",
      "1018_P\n",
      "Q1K_HSJ_1025-1061_S3\n",
      "1061_S3\n",
      "Q1K_HSJ_1525-1033_M1\n",
      "1033_M1\n",
      "__Q1K_HSJ_10083_M1\n",
      "83_M1\n",
      "Q1K_HSJ_1525-1024_M1\n",
      "1024_M1\n",
      "Q1K_HSJ_10043_F1\n",
      "43_F1\n",
      "Q1K_HSJ_1525-1057_P\n",
      "1057_P\n",
      "Q1K_HSJ_100114_M1\n",
      "114_M1\n",
      "Q1K_HSJ_1525_1052_M1\n",
      "1052_M1\n",
      "Q1K_HSJ_10064_M1\n",
      "64_M1\n",
      "Q1K_HSJ_1025-1042_M1\n",
      "1042_M1\n",
      "Q1K_HSJ_1525-1026_M1\n",
      "1026_M1\n",
      "Q1K_HSJ_1525-1080_P\n",
      "1080_P\n",
      "Q1K_HSJ_10064_P\n",
      "64_P\n",
      "Q1K_HSJ_1525-1001_F1\n",
      "1001_F1\n",
      "Q1K_HSJ_10086_M1\n",
      "86_M1\n",
      "Q1K_HSJ_100147_F2\n",
      "147_F2\n",
      "Q1K_HSJ_100123_P\n",
      "123_P\n",
      "Q1K_HSJ_1525_1009_S1\n",
      "1009_S1\n",
      "Q1K_HSJ_1525_1009_P\n",
      "1009_P\n",
      "Q1K_HSJ_100108_F1\n",
      "108_F1\n",
      "Q1K_HSJ_1525-1057_M1\n",
      "1057_M1\n",
      "Q1K_HSJ_100104_C1\n",
      "104_C1\n",
      "Q1K_HSJ_100129_P\n",
      "129_P\n",
      "Q1K_HSJ_1525-1067_P\n",
      "1067_P\n",
      "Q1K_HSJ_10083_P\n",
      "83_P\n",
      "Q1K_HSJ_100150_P\n",
      "150_P\n",
      "Q1K_HSJ_100154_P\n",
      "154_P\n",
      "Q1K_HSJ_1525_1037_F1\n",
      "1037_F1\n",
      "Q1K_HSJ_100119_S1\n",
      "119_S1\n",
      "Q1K_HSJ_10086_F1\n",
      "86_F1\n",
      "Q1K_HSJ_1525-1001_M1\n",
      "1001_M1\n",
      "Q1K_HSJ_100131_M1\n",
      "131_M1\n",
      "Q1K_HSJ_1525-1021_P\n",
      "1021_P\n",
      "Q1K_HSJ_1525-1057_F1\n",
      "1057_F1\n",
      "Q1K_HSJ_100100_F1\n",
      "100_F1\n",
      "Q1K_HSJ_1525-1028_S1\n",
      "1028_S1\n",
      "Q1K_HSJ_100157_M1\n",
      "157_M1\n",
      "Q1K_HSJ_1525-1073_P\n",
      "1073_P\n",
      "Q1K_HSJ_1525-1067_M1\n",
      "1067_M1\n",
      "Q1K_HSJ_100126_P\n",
      "126_P\n",
      "Q1K_HSJ_100114_F1\n",
      "114_F1\n",
      "Q1K_HSJ_1525_1052_F1\n",
      "1052_F1\n",
      "Q1K_HSJ_1525-1001_P\n",
      "1001_P\n",
      "Q1K_HSJ_1525-1073_M1\n",
      "1073_M1\n",
      "Q1K_HSJ_1525_1037_P\n",
      "1037_P\n",
      "Q1K_HSJ_100134_S1\n",
      "134_S1\n",
      "Q1K_HSJ_10050_P\n",
      "50_P\n",
      "Q1K_HSJ_1525-1080_M1\n",
      "1080_M1\n",
      "Q1K_HSJ_100157_P\n",
      "157_P\n",
      "Q1K_HSJ_1525-1067_S1\n",
      "1067_S1\n",
      "Q1K_HSJ_100147_M1\n",
      "147_M1\n",
      "Q1K_HSJ_100150_M1\n",
      "150_M1\n",
      "Q1K_HSJ_10050_M1\n",
      "50_M1\n",
      "Q1K_HSJ_100134_P\n",
      "134_P\n",
      "Q1K_HSJ_1525-1001_S1\n",
      "1001_S1\n",
      "Q1K_HSJ_100131_S1\n",
      "131_S1\n",
      "Q1K_HSJ_100105_P\n",
      "105_P\n",
      "Q1K_HSJ_100111_F1\n",
      "111_F1\n",
      "Q1K_HSJ_1525-1006_M1\n",
      "1006_M1\n",
      "Q1K_HSJ_100119_M1\n",
      "119_M1\n",
      "Q1K_HSJ_1525-1028_P\n",
      "1028_P\n",
      "Q1K_HSJ_1525-1021_F1\n",
      "1021_F1\n",
      "Q1K_HSJ_1525-1028_S2\n",
      "1028_S2\n",
      "Q1K_HSJ_100147_P\n",
      "147_P\n",
      "Q1K_HSJ_100146_P\n",
      "146_P\n",
      "Q1K_HSJ_10043_P\n",
      "43_P\n",
      "Q1K_HSJ_100134_M1\n",
      "134_M1\n",
      "Q1K_HSJ_10062_P\n",
      "62_P\n",
      "Q1K_HSJ_100104_P\n",
      "104_P\n",
      "Q1K_HSJ_1525-1012_P\n",
      "1012_P\n",
      "Q1K_HSJ_10053_M1\n",
      "53_M1\n",
      "Q1K_HSJ_100123_M1\n",
      "123_M1\n",
      "Q1K_HSJ_1525-1033_P\n",
      "1033_P\n",
      "Q1K_HSJ_10083_F1\n",
      "83_F1\n",
      "Q1K_HSJ_1525_1042_F1\n",
      "1042_F1\n",
      "Q1K_HSJ_100152_M1\n",
      "152_M1\n",
      "Q1K_HSJ_100105_F1\n",
      "105_F1\n",
      "Q1K_HSJ_1525-1045_F1\n",
      "1045_F1\n",
      "Q1K_HSJ_100114_P\n",
      "114_P\n",
      "Q1K_HSJ_10093_O1\n",
      "93_O1\n",
      "Q1K_HSJ_1525-1061_P\n",
      "1061_P\n",
      "_Q1K_HSJ_10083_M1\n",
      "83_M1\n",
      "Q1K_HSJ_10053_P\n",
      "53_P\n"
     ]
    }
   ],
   "source": [
    "# Create a list of all files in the EEG folders\n",
    "\n",
    "#for site in sites: \n",
    "    #for file in glob.glob(f\"../../../../../../Sharing/CHUSJ-Q1K-PILOT/experimental/{site}/sourcedata/eeg/*\"):\n",
    "for file in glob.glob(f\"../../../../../Sharing/CHUSJ-Q1K-PILOT/experimental/HSJ/sourcedata/eeg/*\"):    \n",
    "    subject_id = file.split('/')[-1]\n",
    "    # Skip sessions that have already been processed\n",
    "    print(subject_id)\n",
    "    eeg_q1k_subjects.append(subject_id)\n",
    "    if \"1025\" in subject_id:\n",
    "        truncated_id=subject_id.split('1025')[1][1:]\n",
    "    #    print(subject_id)\n",
    "    elif \"1525\" in subject_id:\n",
    "        truncated_id=subject_id.split('1525')[1][1:]\n",
    "    elif \"HSJ\" in subject_id:\n",
    "        truncated_id=subject_id.split('Q1K_HSJ_100')[1]\n",
    "    elif \"MHC\" in subject_id:\n",
    "        truncated_id=subject_id.split('Q1K_MHC_200')[1]\n",
    "\n",
    "    truncated_eeg_q1k_subjects.append(truncated_id)\n",
    "    print(truncated_id)\n",
    "    length = len(subject_id)\n",
    "    family_id = truncated_id.split('_')[0]\n",
    "    family_id_subjects.append(family_id)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "eeg_q1k_subjects_df = pd.DataFrame({'q1k_ID': eeg_q1k_subjects, 'et_ID': truncated_eeg_q1k_subjects,\n",
    "                                    'family_ID': family_id_subjects})\n",
    "# Add 0s to the et_ID ID to make it 4 digits\n",
    "eeg_q1k_subjects_df['et_ID'] = eeg_q1k_subjects_df['et_ID'].apply(lambda x: format_id(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>q1k_ID</th>\n",
       "      <th>et_ID</th>\n",
       "      <th>family_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1K_HSJ_100152_P</td>\n",
       "      <td>0152_P</td>\n",
       "      <td>152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1K_HSJ_10083_M1</td>\n",
       "      <td>0083_M1</td>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1K_HSJ_1525_1052_S1</td>\n",
       "      <td>1052_S1</td>\n",
       "      <td>1052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1K_HSJ_100123_F1</td>\n",
       "      <td>0123_F1</td>\n",
       "      <td>123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1K_HSJ_100114_S1</td>\n",
       "      <td>0114_S1</td>\n",
       "      <td>114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>Q1K_HSJ_100114_P</td>\n",
       "      <td>0114_P</td>\n",
       "      <td>114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>Q1K_HSJ_10093_O1</td>\n",
       "      <td>0093_O1</td>\n",
       "      <td>93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>Q1K_HSJ_1525-1061_P</td>\n",
       "      <td>1061_P</td>\n",
       "      <td>1061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>_Q1K_HSJ_10083_M1</td>\n",
       "      <td>0083_M1</td>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>Q1K_HSJ_10053_P</td>\n",
       "      <td>0053_P</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>118 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   q1k_ID    et_ID family_ID\n",
       "0        Q1K_HSJ_100152_P   0152_P       152\n",
       "1        Q1K_HSJ_10083_M1  0083_M1        83\n",
       "2    Q1K_HSJ_1525_1052_S1  1052_S1      1052\n",
       "3       Q1K_HSJ_100123_F1  0123_F1       123\n",
       "4       Q1K_HSJ_100114_S1  0114_S1       114\n",
       "..                    ...      ...       ...\n",
       "113      Q1K_HSJ_100114_P   0114_P       114\n",
       "114      Q1K_HSJ_10093_O1  0093_O1        93\n",
       "115   Q1K_HSJ_1525-1061_P   1061_P      1061\n",
       "116     _Q1K_HSJ_10083_M1  0083_M1        83\n",
       "117       Q1K_HSJ_10053_P   0053_P        53\n",
       "\n",
       "[118 rows x 3 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eeg_q1k_subjects_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are a total of 116 unique participants\n"
     ]
    }
   ],
   "source": [
    "print(\"There are a total of\" , len(eeg_q1k_subjects_df.et_ID.unique()), \"unique participants\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HSJ\n",
      "VEP\n"
     ]
    }
   ],
   "source": [
    "# Define root directory for all eye-tracking tasks \n",
    "root_dir = '../../../../../Sharing/CHUSJ-Q1K-PILOT/experimental'\n",
    "\n",
    "# Define the output directory for the processed data\n",
    "output_dir = f'../../sourcedata/et/'\n",
    "\n",
    "if not os.path.exists(output_dir):\n",
    "    os.makedirs(output_dir)\n",
    "\n",
    "# Track participants with missing EEG data\n",
    "missing_eeg = []\n",
    "et_subjects=[]\n",
    "transformed_et=[]\n",
    "test_missing=[]\n",
    "versions = [\"version_warnings_on\", \"version_warnings_off\", \"version_warnings_off/version_warnings_off\"]\n",
    "\n",
    "# Iterate thourgh each site\n",
    "#for site in sites: \n",
    "print(site)\n",
    "print(task_id_in)\n",
    "if site == \"HSJ\":\n",
    "    for version in versions: \n",
    "        et_dir= os.path.join(root_dir, f\"{site}/sourcedata/et/Q1K_EB_2.2.299/\", f\"{version}/\")\n",
    "        print(et_dir)\n",
    "        # Iterate through each task\n",
    "        #for task in tasks:\n",
    "        task_folders = glob.glob(os.path.join(et_dir, f\"*_{task_id_in}_*/results/\"))\n",
    "        for task_folder in task_folders:\n",
    "            print(task_folder)\n",
    "            # Iterate over participant folders in current task folder \n",
    "            participants = os.listdir(task_folder)\n",
    "            for participant in participants:\n",
    "                print(participant)\n",
    "                participant_folder = os.path.join(task_folder, participant)\n",
    "                # print(participant_folder)\n",
    "                et_subjects.append(participant)\n",
    "\n",
    "                # Process .edf file \n",
    "                \n",
    "                # Create new partcipant name based on Q1K ID\n",
    "                transformed_id= eb_id_transform(participant) # Transform the ID to match the EEG ID\n",
    "                transformed_et.append(transformed_id) # Append the transformed ID to the list\n",
    "                if transformed_id in eeg_q1k_subjects_df.et_ID.values: # Check if the transformed ID is in the EEG ID list\n",
    "                    new_participant = eeg_q1k_subjects_df.loc[eeg_q1k_subjects_df['et_ID'] == transformed_id].q1k_ID.values[0]\n",
    "                    print(new_participant)\n",
    "                    final_output_dir = output_dir\n",
    "                    if not os.path.exists(os.path.join(final_output_dir, new_participant)):\n",
    "                        os.makedirs(os.path.join(final_output_dir, new_participant))\n",
    "                else:\n",
    "                    missing_eeg.append(participant)\n",
    "                    final_output_dir = os.path.join(output_dir + \"/archive/missing_eeg/\")\n",
    "                    #new_participant = participant\n",
    "                    continue\n",
    "\n",
    "                if not os.path.exists(os.path.join(final_output_dir, new_participant)):\n",
    "                     os.makedirs(os.path.join(final_output_dir, new_participant))\n",
    "\n",
    "            # Process participant files (EDF and TXT)\n",
    "                process_participant_files(task_id_out, participant, participant_folder, \n",
    "                                            transformed_id, new_participant, final_output_dir)\n",
    "                et_subjects.append(new_participant)\n",
    "        \n",
    "elif site == \"MHC\":\n",
    "        et_dir= os.path.join(root_dir, f\"{site}/sourcedata/et/Q1K_EB_2.2.299/\")\n",
    "        print(et_dir)\n",
    "        # Iterate through each task\n",
    "#        for task in task_id_in:\n",
    "        print(task_id_in)\n",
    "        task_folders = glob.glob(os.path.join(et_dir, f\"*_{task_id_in}_*/results/\"))\n",
    "        print(task_folders)\n",
    "        for task_folder in task_folders:\n",
    "            print(task_folder)\n",
    "            # Iterate over participant folders in current task folder\n",
    "            participants = os.listdir(task_folder)\n",
    "\n",
    "            for participant in participants:\n",
    "                participant_folder = os.path.join(task_folder, participant)\n",
    "                print(participant_folder)\n",
    "                et_subjects.append(participant)\n",
    "                # Process .edf file \n",
    "                # Create new partcipant name based on Q1K ID\n",
    "                transformed_id= eb_id_transform(participant)\n",
    "                transformed_et.append(transformed_id)\n",
    "                if transformed_id in eeg_q1k_subjects_df.et_ID.values:\n",
    "                    new_participant = eeg_q1k_subjects_df.loc[eeg_q1k_subjects_df['et_ID'] == transformed_id].q1k_ID.values[0]\n",
    "                    final_output_dir = output_dir\n",
    "                    #print(new_participant)\n",
    "                    if not os.path.exists(os.path.join(final_output_dir, new_participant)):\n",
    "                            os.makedirs(os.path.join(final_output_dir, new_participant))\n",
    "                else:\n",
    "                    missing_eeg.append(participant)\n",
    "                    final_output_dir = os.path.join(output_dir + \"/archive/missing_eeg/\")\n",
    "                    new_participant = participant\n",
    "\n",
    "                    continue\n",
    "                print(new_participant)\n",
    "\n",
    "                process_participant_files(task_id_in, participant, participant_folder, \n",
    "                                            transformed_id, new_participant, final_output_dir)\n",
    "                et_subjects.append(new_participant)\n",
    "                # print(new_participant)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Save the list of participants with eyetracking data but missing EEG data inside teh archive folder\n",
    "missing_eeg_df = pd.DataFrame({'participants': list(set(missing_eeg))})\n",
    "missing_eeg_df.to_csv(os.path.join(output_dir, 'archive/missing_eeg/q1k_missing_eeg.csv'), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save look up table\n",
    "eeg_q1k_subjects_df.to_csv(os.path.join(output_dir, 'archive/missing_eeg/et_eeg_lookup_table.csv'), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}