preprocess_mimic_iv.ipynb

import pandas as pd
from tqdm import tqdm
import pickle
import numpy as np

mimic_data_dir = '/home/anand/UHIF/UniHPF-master/UniHPF-master/mimic4/'

# Get all ICU stays.
icu = pd.read_csv(mimic_data_dir+'icustays.csv', usecols=['subject_id', 'hadm_id', 'stay_id', 'intime', 'outtime'])
icu = icu.loc[icu.intime.notna()]
icu = icu.loc[icu.outtime.notna()]

# Filter out pediatric patients.
pat = pd.read_csv(mimic_data_dir+'patients.csv', usecols=['subject_id', 'dod', 'gender','anchor_year','anchor_age'])
icu = icu.merge(pat, on='subject_id', how='left')
#adm_eth_icu = pd.read_csv(mimic_data_dir+'admissions.csv', usecols=['subject_id', 'ethnicity'])
#icu = icu.merge(adm_eth_icu, on='subject_id', how='left')
icu['intime'] = pd.to_datetime(icu.intime)

icu['yob']= pat['anchor_year'] - pat['anchor_age']
#icu['dob'] = pd.to_datetime(icu.dob)
icu['age'] = icu.intime.map(lambda x:x.year) - icu['yob']
icu = icu.loc[icu.age>=18] #53k icustays


# Extract chartevents for icu stays.
ch = []
for chunk in tqdm(pd.read_csv(mimic_data_dir+'chartevents.csv', chunksize=10000000,
                usecols = ['hadm_id', 'stay_id', 'itemid', 'charttime', 'value', 'valuenum', 'valueuom'])):
    chunk = chunk.loc[chunk.stay_id.isin(icu.stay_id)]
    #chunk = chunk.loc[chunk['ERROR']!=1]
    chunk = chunk.loc[chunk.charttime.notna()]
    #chunk.drop(columns=['ERROR'], inplace=True)
    ch.append(chunk)
del chunk
print ('Done')
ch = pd.concat(ch)
print ('Done')
ch = ch.loc[~(ch.value.isna() & ch.valuenum.isna())]
ch['TABLE'] = 'chart'
print ('Done')

# Extract labevents for admissions.
la = pd.read_csv(mimic_data_dir+'labevents.csv',usecols = ['hadm_id', 'itemid', 'charttime', 'value', 'valuenum', 'valueuom'])
la = la.loc[la.hadm_id.isin(icu.hadm_id)]
la.hadm_id = la.hadm_id.astype(int)
la = la.loc[la.charttime.notna()]
la = la.loc[~(la.value.isna() & la.valuenum.isna())]
la['stay_id'] = np.nan
la['TABLE'] = 'lab'

# Extract bp events. Remove outliers. Make sure median values of CareVue and MetaVision items are close.
dbp = [8368, 220051, 225310, 8555, 8441, 220180, 8502, 8440, 8503, 8504, 8507, 8506, 224643, 227242]
sbp = [51, 220050, 225309, 6701, 455, 220179, 3313, 3315, 442, 3317, 3323, 3321, 224167, 227243]
mbp = [52, 220052, 225312, 224, 6702, 224322, 456, 220181, 3312, 3314, 3316, 3322, 3320, 443]
ch_bp = ch.loc[ch.itemid.isin(dbp+sbp+mbp)]
ch_bp = ch_bp.loc[(ch_bp.valuenum>=0)&(ch_bp.valuenum<=375)]
ch_bp.loc[ch_bp.itemid.isin(dbp), 'name'] = 'DBP'
ch_bp.loc[ch_bp.itemid.isin(sbp), 'name'] = 'SBP'
ch_bp.loc[ch_bp.itemid.isin(mbp), 'name'] = 'MBP'
ch_bp['valueuom'] = 'mmHg'
ch_bp['value'] = None
events = ch_bp.copy()
del ch_bp

# Extract GCS events. Checked for outliers.
gcs_eye = [184, 220739]
gcs_motor = [454, 223901]
gcs_verbal = [723, 223900]
ch_gcs = ch.loc[ch.itemid.isin(gcs_eye+gcs_motor+gcs_verbal)]
ch_gcs.loc[ch_gcs.itemid.isin(gcs_eye), 'name'] = 'GCS_eye'
ch_gcs.loc[ch_gcs.itemid.isin(gcs_motor), 'name'] = 'GCS_motor'
ch_gcs.loc[ch_gcs.itemid.isin(gcs_verbal), 'name'] = 'GCS_verbal'
ch_gcs['valueuom'] = None
ch_gcs['value'] = None
events = pd.concat([events, ch_gcs])
del ch_gcs

# Extract heart_rate events. Remove outliers.
hr = [211, 220045]
ch_hr = ch.loc[ch.itemid.isin(hr)]
ch_hr = ch_hr.loc[(ch_hr.valuenum>=0)&(ch_hr.valuenum<=390)]
ch_hr['name'] = 'HR'
ch_hr['valueuom'] = 'bpm'
ch_hr['value'] = None
events = pd.concat([events, ch_hr])
del ch_hr

# Extract respiratory_rate events. Remove outliers. Checked unit consistency.
rr = [618, 220210, 3603, 224689, 614, 651, 224422, 615, 224690, 619, 224688, 227860, 227918]
ch_rr = ch.loc[ch.itemid.isin(rr)]
ch_rr = ch_rr.loc[(ch_rr.valuenum>=0)&(ch_rr.valuenum<=330)]
ch_rr['name'] = 'RR'
ch_rr['valueuom'] = 'brpm'
ch_rr['value'] = None
events = pd.concat([events, ch_rr])
del ch_rr

# Extract temperature events. Convert F to C. Remove outliers.
temp_c = [3655, 677, 676, 223762]
temp_f = [223761, 678, 679, 3654]
ch_temp_c = ch.loc[ch.itemid.isin(temp_c)]
ch_temp_f = ch.loc[ch.itemid.isin(temp_f)]
ch_temp_f.valuenum = (ch_temp_f.valuenum-32)*5/9
ch_temp = pd.concat([ch_temp_c, ch_temp_f])
del ch_temp_c
del ch_temp_f
ch_temp = ch_temp.loc[(ch_temp.valuenum>=14.2)&(ch_temp.valuenum<=47)]
ch_temp['name'] = 'Temperature'
ch_temp['valueuom'] = 'C'
ch_temp['value'] = None
events = pd.concat([events, ch_temp])
del ch_temp

# Extract weight events. Convert lb to kg. Remove outliers.
we_kg = [224639, 226512, 226846, 763]
we_lb = [226531]
ch_we_kg = ch.loc[ch.itemid.isin(we_kg)]
ch_we_lb = ch.loc[ch.itemid.isin(we_lb)]
ch_we_lb.valuenum = ch_we_lb.valuenum * 0.453592
ch_we = pd.concat([ch_we_kg, ch_we_lb])
del ch_we_kg
del ch_we_lb
ch_we = ch_we.loc[(ch_we.valuenum>=0)&(ch_we.valuenum<=300)]
ch_we['name'] = 'Weight'
ch_we['valueuom'] = 'kg'
ch_we['value'] = None
events = pd.concat([events, ch_we])
del ch_we

# Extract height events. Convert in to cm. 
he_in = [1394, 226707]
he_cm = [226730]
ch_he_in = ch.loc[ch.itemid.isin(he_in)]
ch_he_cm = ch.loc[ch.itemid.isin(he_cm)]
ch_he_in.valuenum = ch_he_in.valuenum * 2.54
ch_he = pd.concat([ch_he_in, ch_he_cm])
del ch_he_in
del ch_he_cm
ch_he = ch_he.loc[(ch_he.valuenum>=0)&(ch_he.valuenum<=275)]
ch_he['name'] = 'Height'
ch_he['valueuom'] = 'cm'
ch_he['value'] = None
events = pd.concat([events, ch_he])
del ch_he

# Extract fio2 events. Convert % to fraction. Remove outliers.
fio2 = [3420, 223835, 3422, 189, 727, 190]
ch_fio2 = ch.loc[ch.itemid.isin(fio2)]
idx = ch_fio2.valuenum>1.0
ch_fio2.loc[idx, 'valuenum'] = ch_fio2.loc[idx, 'valuenum'] / 100
ch_fio2 = ch_fio2.loc[(ch_fio2.valuenum>=0.2)&(ch_fio2.valuenum<=1)]
ch_fio2['name'] = 'FiO2'
ch_fio2['valueuom'] = None
ch_fio2['value'] = None
events = pd.concat([events, ch_fio2])
del ch_fio2

# Extract capillary refill rate events. Convert to binary.
cr = [3348, 115, 8377, 224308, 223951]
ch_cr = ch.loc[ch.itemid.isin(cr)]
ch_cr = ch_cr.loc[~(ch_cr.value=='Other/Remarks')]
idx = (ch_cr.value=='Normal <3 Seconds')|(ch_cr.value=='Normal <3 secs')
ch_cr.loc[idx, 'valuenum'] = 0
idx = (ch_cr.value=='Abnormal >3 Seconds')|(ch_cr.value=='Abnormal >3 secs')
ch_cr.loc[idx, 'valuenum'] = 1
ch_cr['valueuom'] = None
ch_cr['name'] = 'CRR'
events = pd.concat([events, ch_cr])
del ch_cr

# Extract glucose events. Remove outliers.
gl_bl = [225664, 1529, 811, 807, 3745, 50809]
gl_wb = [226537]
gl_se = [220621, 50931]

ev_blgl = pd.concat((ch.loc[ch.itemid.isin(gl_bl)], la.loc[la.itemid.isin(gl_bl)]))
ev_blgl = ev_blgl.loc[(ev_blgl.valuenum>=0)&(ev_blgl.valuenum<=2200)]
ev_blgl['name'] = 'Glucose (Blood)'
ev_wbgl = pd.concat((ch.loc[ch.itemid.isin(gl_wb)], la.loc[la.itemid.isin(gl_wb)]))
ev_wbgl = ev_wbgl.loc[(ev_wbgl.valuenum>=0)&(ev_wbgl.valuenum<=2200)]
ev_wbgl['name'] = 'Glucose (Whole Blood)'
ev_segl = pd.concat((ch.loc[ch.itemid.isin(gl_se)], la.loc[la.itemid.isin(gl_se)]))
ev_segl = ev_segl.loc[(ev_segl.valuenum>=0)&(ev_segl.valuenum<=2200)]
ev_segl['name'] = 'Glucose (Serum)'

ev_gl = pd.concat((ev_blgl, ev_wbgl, ev_segl))
del ev_blgl, ev_wbgl, ev_segl
ev_gl['valueuom'] = 'mg/dL'
ev_gl['value'] = None
events = pd.concat([events, ev_gl])
del ev_gl

# Extract bilirubin events. Remove outliers.
br_to = [50885]
br_di = [50883]
br_in = [50884]
ev_br = pd.concat((ch.loc[ch.itemid.isin(br_to+br_di+br_in)], la.loc[la.itemid.isin(br_to+br_di+br_in)]))
ev_br = ev_br.loc[(ev_br.valuenum>=0)&(ev_br.valuenum<=66)]
ev_br.loc[ev_br.itemid.isin(br_to), 'name'] = 'Bilirubin (Total)'
ev_br.loc[ev_br.itemid.isin(br_di), 'name'] = 'Bilirubin (Direct)'
ev_br.loc[ev_br.itemid.isin(br_in), 'name'] = 'Bilirubin (Indirect)'
ev_br['valueuom'] = 'mg/dL'
ev_br['value'] = None
events = pd.concat([events, ev_br])
del ev_br

# Extract intubated events.
itb = [50812]
la_itb = la.loc[la.itemid.isin(itb)]
idx = (la_itb.value=='INTUBATED')
la_itb.loc[idx, 'valuenum'] = 1
idx = (la_itb.value=='NOT INTUBATED')
la_itb.loc[idx, 'valuenum'] = 0
la_itb['valueuom'] = None
la_itb['name'] = 'Intubated'
events = pd.concat([events, la_itb])
del la_itb

# Extract multiple events. Remove outliers.
o2sat = [834, 50817, 8498, 220227, 646, 220277]
sod = [50983, 50824]
pot = [50971, 50822]
mg = [50960]
po4 = [50970]
ca_total = [50893]
ca_free = [50808]
wbc = [51301, 51300]
hct = [50810, 51221]
hgb = [51222, 50811]
cl = [50902, 50806]
bic = [50882, 50803]
alt = [50861]
alp = [50863]
ast = [50878]
alb = [50862]
lac = [50813]
ld = [50954]
usg = [51498]
ph_ur = [51491, 51094, 220734, 1495, 1880, 1352, 6754, 7262]
ph_bl = [50820]
po2 = [50821]
pco2 = [50818]
tco2 = [50804]
be = [50802]
monos = [51254]
baso = [51146]
eos = [51200]
neuts = [51256]
lym_per = [51244, 51245]
lym_abs = [51133]
pt = [51274]
ptt = [51275]
inr = [51237]
agap = [50868]
bun = [51006]
cr_bl = [50912]
cr_ur = [51082]
mch = [51248]
mchc = [51249]
mcv = [51250]
rdw = [51277]
plt = [51265]
rbc = [51279]

features = {'O2 Saturation': [o2sat, [0,100], '%'],
            'Sodium': [sod, [0,250], 'mEq/L'], 
            'Potassium': [pot, [0,15], 'mEq/L'], 
            'Magnesium': [mg, [0,22], 'mg/dL'], 
            'Phosphate': [po4, [0,22], 'mg/dL'],
            'Calcium Total': [ca_total, [0,40], 'mg/dL'],
            'Calcium Free': [ca_free, [0,10], 'mmol/L'],
            'WBC': [wbc, [0,1100], 'K/uL'], 
            'Hct': [hct, [0,100], '%'], 
            'Hgb': [hgb, [0,30], 'g/dL'], 
            'Chloride': [cl, [0,200], 'mEq/L'],
            'Bicarbonate': [bic, [0,66], 'mEq/L'],
            'ALT': [alt, [0,11000], 'IU/L'],
            'ALP': [alp, [0,4000], 'IU/L'],
            'AST': [ast, [0,22000], 'IU/L'],
            'Albumin': [alb, [0,10], 'g/dL'],
            'Lactate': [lac, [0,33], 'mmol/L'],
            'LDH': [ld, [0,35000], 'IU/L'],
            'SG Urine': [usg, [0,2], ''],
            'pH Urine': [ph_ur, [0,14], ''],
            'pH Blood': [ph_bl, [0,14], ''],
            'PO2': [po2, [0,770], 'mmHg'],
            'PCO2': [pco2, [0,220], 'mmHg'],
            'Total CO2': [tco2, [0,65], 'mEq/L'],
            'Base Excess': [be, [-31, 28], 'mEq/L'],
            'Monocytes': [monos, [0,100], '%'],
            'Basophils': [baso, [0,100], '%'],
            'Eoisinophils': [eos, [0,100], '%'],
            'Neutrophils': [neuts, [0,100], '%'],
            'Lymphocytes': [lym_per, [0,100], '%'],
            'Lymphocytes (Absolute)': [lym_abs, [0,25000], '#/uL'],
            'PT': [pt, [0,150], 'sec'],
            'PTT': [ptt, [0,150], 'sec'],
            'INR': [inr, [0,150], ''],
            'Anion Gap': [agap, [0,55], 'mg/dL'],
            'BUN': [bun, [0,275], 'mEq/L'],
            'Creatinine Blood': [cr_bl, [0,66], 'mg/dL'],
            'Creatinine Urine': [cr_ur, [0,650], 'mg/dL'],
            'MCH': [mch, [0,50], 'pg'],
            'MCHC': [mchc, [0,50], '%'],
            'MCV': [mcv, [0,150], 'fL'],
            'RDW': [rdw, [0,37], '%'],
            'Platelet Count': [plt, [0,2200], 'K/uL'],
            'RBC': [rbc, [0,14], 'm/uL']
            }

for k, v in features.items():
    print (k)
    ev_k = pd.concat((ch.loc[ch.itemid.isin(v[0])], la.loc[la.itemid.isin(v[0])]))
    ev_k = ev_k.loc[(ev_k.valuenum>=v[1][0])&(ev_k.valuenum<=v[1][1])]
    ev_k['name'] = k
    ev_k['valueuom'] = v[2]
    ev_k['value'] = None
    assert (ev_k.valuenum.isna().sum()==0)
    events = pd.concat([events, ev_k])
del ev_k

# Free some memory.
del ch, la

# Extract outputevents.
oe = pd.read_csv(mimic_data_dir+'outputevents.csv', usecols = ['stay_id', 'itemid', 'charttime', 'value', 'valueuom'])
oe = oe.loc[oe.value.notna()]
oe['valuenum'] = oe.value
oe.value = None
oe = oe.loc[oe.stay_id.isin(icu.stay_id)]
oe.stay_id = oe.stay_id.astype(int)
oe['TABLE'] = 'output'

# Extract information about output items from d_items.csv.
items = pd.read_csv(mimic_data_dir+'d_items.csv', usecols=['itemid', 'label', 'abbreviation', 'unitname', 'param_type'])
items.loc[items.label.isna(), 'label'] = ''
items.label = items.label.str.lower()
oeitems = oe[['itemid']].drop_duplicates()
oeitems = oeitems.merge(items, on='itemid', how='left')

# Extract multiple events. Replace outliers with median.
uf = [40286]
keys = ['urine', 'foley', 'void', 'nephrostomy', 'condom', 'drainage bag']
cond = pd.concat([oeitems.label.str.contains(k) for k in keys], axis=1).any(axis='columns')
ur = list(oeitems.loc[cond].itemid)
keys = ['stool', 'fecal', 'colostomy', 'ileostomy', 'rectal']
cond = pd.concat([oeitems.label.str.contains(k) for k in keys], axis=1).any(axis='columns')
st = list(oeitems.loc[cond].itemid)
ct = list(oeitems.loc[oeitems.label.str.contains('chest tube')].itemid) + [226593, 226590, 226591, 226595, 226592]
gs = [40059, 40052, 226576, 226575, 226573, 40051, 226630]
ebl = [40064, 226626, 40491, 226629]
em = [40067, 226571, 40490, 41015, 40427]
jp = list(oeitems.loc[oeitems.label.str.contains('jackson')].itemid)
res = [227510, 227511, 42837, 43892, 44909, 44959]
pre = [40060, 226633]

features = {'Ultrafiltrate': [uf, [0,7000],'mL'],
            'Urine': [ur, [0,2500], 'mL'],
            'Stool': [st, [0,4000], 'mL'],
            'Chest Tube': [ct, [0,2500], 'mL'],
            'Gastric': [gs, [0,4000], 'mL'],
            'EBL': [ebl, [0,10000], 'mL'],
#             'Pre-admission': [pre, [0,13000], 'mL'], # Repeated by mistake.
            'Emesis': [em, [0,2000], 'mL'],
            'Jackson-Pratt': [jp, [0,2000], 'ml'],
            'Residual': [res, [0, 1050], 'mL'],
            'Pre-admission Output': [pre, [0, 13000], 'ml']
            }

for k, v in features.items():
    print (k)
    ev_k = oe.loc[oe.itemid.isin(v[0])]
    ind = (ev_k.valuenum>=v[1][0])&(ev_k.valuenum<=v[1][1])
    med = ev_k.valuenum.loc[ind].median()
    ev_k.loc[~ind, 'valuenum'] = med
    ev_k['name'] = k
    ev_k['valueuom'] = v[2]
    events = pd.concat([events, ev_k])
del ev_k

# Extract CV and MV inputevents.
ie = pd.read_csv(mimic_data_dir+'inputevents.csv',
    usecols = ['stay_id', 'itemid', 'starttime', 'endtime',
               'amount', 'amountuom'])
ie = ie.loc[ie.amount.notna()]
ie = ie.loc[ie.stay_id.isin(icu.stay_id)]

# Split MV intervals hourly.
ie.starttime = pd.to_datetime(ie.starttime)
ie.endtime = pd.to_datetime(ie.endtime)
ie['TD'] = ie.endtime - ie.starttime
new_ie = ie.loc[ie.TD<=pd.Timedelta(1,'h')].drop(columns=['starttime', 'TD'])
ie = ie.loc[ie.TD>pd.Timedelta(1,'h')]
new_rows = []
for _,row in tqdm(ie.iterrows()):
    icuid, iid, amo, uom, stm, td = row.stay_id, row.itemid, row.amount, row.amountuom, row.starttime, row.TD
    td = td.total_seconds()/60
    num_hours = td // 60
    hour_amount = 60*amo/td
    for i in range(1,int(num_hours)+1):
        new_rows.append([icuid, iid, stm+pd.Timedelta(i,'h'), hour_amount, uom])
    rem_mins = td % 60
    if rem_mins>0:
        new_rows.append([icuid, iid, row['endtime'], rem_mins*amo/td, uom])
new_rows = pd.DataFrame(new_rows, columns=['stay_id', 'itemid', 'endtime', 'amount', 'amountuom'])
new_ie = pd.concat((new_ie, new_rows))
ie = new_ie.copy()
del new_ie
ie['TABLE'] = 'inputevents' 
ie.rename(columns={'endtime':'charttime'}, inplace=True)
ie.rename(columns={'amount':'valuenum', 'amountuom':'valueuom'}, inplace=True)
events.charttime = pd.to_datetime(events.charttime)

# Convert mcg->mg, L->ml.
ind = (ie.valueuom=='mcg')
ie.loc[ind, 'valuenum'] = ie.loc[ind, 'valuenum']*0.001
ie.loc[ind, 'valueuom'] = 'mg'
ind = (ie.valueuom=='L')
ie.loc[ind, 'valuenum'] = ie.loc[ind, 'valuenum']*1000
ie.loc[ind, 'valueuom'] = 'ml'

# Extract Vasopressin events. Remove outliers.
vaso = [30051, 222315]
ev_vaso = ie.loc[ie.itemid.isin(vaso)]
ind1 = (ev_vaso.valuenum==0)
ind2 = ev_vaso.valueuom.isin(['U','units'])
ind3 = (ev_vaso.valuenum>=0)&(ev_vaso.valuenum<=400)
ind = ((ind2&ind3)|ind1)
med = ev_vaso.valuenum.loc[ind].median()
ev_vaso.loc[~ind, 'valuenum'] = med
ev_vaso['valueuom'] = 'units'
ev_vaso['name'] = 'Vasopressin'
events = pd.concat([events, ev_vaso])
del ev_vaso

# Extract Vancomycin events. Convert dose,g to mg. Remove outliers.
vanc = [225798]
ev_vanc = ie.loc[ie.itemid.isin(vanc)]
ind = ev_vanc.valueuom.isin(['mg'])
ev_vanc.loc[ind, 'valuenum'] = ev_vanc.loc[ind, 'valuenum']*0.001 
ev_vanc['valueuom'] = 'g'
ind = (ev_vanc.valuenum>=0)&(ev_vanc.valuenum<=8)
med = ev_vanc.valuenum.loc[ind].median()
ev_vanc.loc[~ind, 'valuenum'] = med
ev_vanc['name'] = 'Vacomycin'
events = pd.concat([events, ev_vanc])
del ev_vanc

# Extract Calcium Gluconate events. Convert units. Remove outliers.
cagl = [30023, 221456, 227525, 42504, 43070, 45699, 46591, 44346, 46291]
ev_cagl = ie.loc[ie.itemid.isin(cagl)]
ind = ev_cagl.valueuom.isin(['mg'])
ev_cagl.loc[ind, 'valuenum'] = ev_cagl.loc[ind, 'valuenum']*0.001 
ind1 = (ev_cagl.valuenum==0)
ind2 = ev_cagl.valueuom.isin(['mg', 'gm', 'grams'])
ind3 = (ev_cagl.valuenum>=0)&(ev_cagl.valuenum<=200)
ind = (ind2&ind3)|ind1
med = ev_cagl.valuenum.loc[ind].median()
ev_cagl.loc[~ind, 'valuenum'] = med
ev_cagl['valueuom'] = 'g'
ev_cagl['name'] = 'Calcium Gluconate'
events = pd.concat([events, ev_cagl])
del ev_cagl

# Extract Furosemide events. Remove outliers.
furo = [30123, 221794, 228340]
ev_furo = ie.loc[ie.itemid.isin(furo)]
ind1 = (ev_furo.valuenum==0)
ind2 = (ev_furo.valueuom=='mg')
ind3 = (ev_furo.valuenum>=0)&(ev_furo.valuenum<=250)
ind = ind1|(ind2&ind3)
med = ev_furo.valuenum.loc[ind].median()
ev_furo.loc[~ind, 'valuenum'] = med
ev_furo['valueuom'] = 'mg'
ev_furo['name'] = 'Furosemide'
events = pd.concat([events, ev_furo])
del ev_furo

# Extract Famotidine events. Remove outliers.
famo = [225907]
ev_famo = ie.loc[ie.itemid.isin(famo)]
ind1 = (ev_famo.valuenum==0)
ind2 = (ev_famo.valueuom=='dose')
ind3 = (ev_famo.valuenum>=0)&(ev_famo.valuenum<=1)
ind = ind1|(ind2&ind3)
med = ev_famo.valuenum.loc[ind].median()
ev_famo.loc[~ind, 'valuenum'] = med
ev_famo['valueuom'] = 'dose'
ev_famo['name'] = 'Famotidine'
events = pd.concat([events, ev_famo])
del ev_famo

# Extract Piperacillin events. Convert units. Remove outliers.
pipe = [225893, 225892]
ev_pipe = ie.loc[ie.itemid.isin(pipe)]
ind1 = (ev_pipe.valuenum==0)
ind2 = (ev_pipe.valueuom=='dose')
ind3 = (ev_pipe.valuenum>=0)&(ev_pipe.valuenum<=1)
ind = ind1|(ind2&ind3)
med = ev_pipe.valuenum.loc[ind].median()
ev_pipe.loc[~ind, 'valuenum'] = med
ev_pipe['valueuom'] = 'dose'
ev_pipe['name'] = 'Piperacillin'
events = pd.concat([events, ev_pipe])
del ev_pipe

# Extract Cefazolin events. Convert units. Remove outliers.
cefa = [225850]
ev_cefa = ie.loc[ie.itemid.isin(cefa)]
ind1 = (ev_cefa.valuenum==0)
ind2 = (ev_cefa.valueuom=='dose')
ind3 = (ev_cefa.valuenum>=0)&(ev_cefa.valuenum<=2)
ind = ind1|(ind2&ind3)
med = ev_cefa.valuenum.loc[ind].median()
ev_cefa.loc[~ind, 'valuenum'] = med 
ev_cefa['valueuom'] = 'dose'
ev_cefa['name'] = 'Cefazolin'
events = pd.concat([events, ev_cefa])
del ev_cefa

# Extract Fiber events. Remove outliers.
fibe = [225936, 30166, 30073, 227695, 30088, 225928, 226051, 226050, 226048, 45381, 45597, 227699, 227696, 44218, 45406, 44675, 226049, 44202, 45370, 227698, 226027, 42106, 43994, 45865, 44318, 42091, 44699, 44010, 43134, 44045, 43088, 42641, 45691, 45515, 45777, 42663, 42027, 44425, 45657, 45775, 44631, 44106, 42116, 44061, 44887, 42090, 42831, 45541, 45497, 46789, 44765, 42050]
ev_fibe = ie.loc[ie.itemid.isin(fibe)]
ind1 = (ev_fibe.valuenum==0)
ind2 = (ev_fibe.valueuom=='ml')
ind3 = (ev_fibe.valuenum>=0)&(ev_fibe.valuenum<=1600)
ind = ind1|(ind2&ind3)
med = ev_fibe.valuenum.loc[ind].median()
ev_fibe.loc[~ind, 'valuenum'] = med 
ev_fibe['name'] = 'Fiber'
ev_fibe['valueuom'] = 'ml'
events = pd.concat([events, ev_fibe])
del ev_fibe

# Extract Pantoprazole events. Remove outliers.
pant = [225910, 40549, 41101, 41583, 44008, 40700, 40550]
ev_pant = ie.loc[ie.itemid.isin(pant)]
ind = (ev_pant.valuenum>0)
ev_pant.loc[ind, 'valuenum'] = 1
ind = (ev_pant.valuenum>=0)
med = ev_pant.valuenum.loc[ind].median()
ev_pant.loc[~ind, 'valuenum'] = med
ev_pant['name'] = 'Pantoprazole'
ev_pant['valueuom'] = 'dose'
events = pd.concat([events, ev_pant])
del ev_pant

# Extract Magnesium Sulphate events. Remove outliers.
masu = [222011, 30027, 227524]
ev_masu = ie.loc[ie.itemid.isin(masu)]
ind = (ev_masu.valueuom=='mg')
ev_masu.loc[ind, 'valuenum'] = ev_masu.loc[ind, 'valuenum']*0.001
ind1 = (ev_masu.valuenum==0)
ind2 = ev_masu.valueuom.isin(['gm', 'grams', 'mg'])
ind3 = (ev_masu.valuenum>=0)&(ev_masu.valuenum<=125)
ind = ind1|(ind2&ind3)
med = ev_masu.valuenum.loc[ind].median()
ev_masu.loc[~ind, 'valuenum'] = med 
ev_masu['valueuom'] = 'g'
ev_masu['name'] = 'Magnesium Sulphate'
events = pd.concat([events, ev_masu])
del ev_masu

# Extract Potassium Chloride events. Remove outliers.
poch = [30026, 225166, 227536]
ev_poch = ie.loc[ie.itemid.isin(poch)]
ind1 = (ev_poch.valuenum==0)
ind2 = ev_poch.valueuom.isin(['mEq', 'mEq.'])
ind3 = (ev_poch.valuenum>=0)&(ev_poch.valuenum<=501)
ind = ind1|(ind2&ind3)
med = ev_poch.valuenum.loc[ind].median()
ev_poch.loc[~ind, 'valuenum'] = med 
ev_poch['valueuom'] = 'mEq'
ev_poch['name'] = 'KCl'
events = pd.concat([events, ev_poch])
del ev_poch

# Extract multiple events. Remove outliers.
mida = [30124, 221668]
prop = [30131, 222168]
albu25 = [220862, 30009]
albu5 = [220864, 30008]
ffpl = [30005, 220970]
lora = [30141, 221385]
mosu = [30126, 225154]
game = [30144, 225799]
lari = [30021, 225828]
milr = [30125, 221986]
crys = [30101, 226364, 30108, 226375]
hepa = [30025, 225975, 225152]
prbc = [30001, 225168, 30104, 226368, 227070]
poin = [30056, 226452, 30109, 226377]
neos = [30128, 221749, 30127]
pigg = [226089, 30063]
nigl = [30121, 222056, 30049]
nipr = [30050, 222051]
meto = [225974]
nore = [30120, 221906, 30047]
coll = [30102, 226365, 30107, 226376]
hyzi = [221828]
gtfl = [226453, 30059]
hymo = [30163, 221833]
fent = [225942, 30118, 221744, 30149]
inre = [30045, 223258, 30100]
inhu = [223262]
ingl = [223260]
innp = [223259]
nana = [30140]
d5wa = [30013, 220949]
doth = [30015, 225823, 30060, 225825, 220950, 30016, 30061, 225827, 225941, 30160, 220952, 30159, 30014, 30017, 228142, 228140, 45360, 228141, 41550]
nosa = [225158, 30018]
hans = [30020, 225159]
stwa = [225944, 30065]
frwa = [30058, 225797, 41430, 40872, 41915, 43936, 41619, 42429, 44492, 46169, 42554]
solu = [225943]
dopa = [30043, 221662]
epin = [30119, 221289, 30044]
amio = [30112, 221347, 228339, 45402]
tpnu = [30032, 225916, 225917, 30096]
msbo = [227523]
pcbo = [227522]
prad = [30054, 226361]

features = {'Midazolam': [mida, [0, 500], 'mg'],
            'Propofol': [prop, [0, 12000], 'mg'],
            'Albumin 25%': [albu25, [0, 750], 'ml'],
            'Albumin 5%': [albu5, [0, 1300], 'ml'],
            'Fresh Frozen Plasma': [ffpl, [0, 33000], 'ml'],
            'Lorazepam': [lora, [0, 300], 'mg'],
            'Morphine Sulfate': [mosu, [0, 4000], 'mg'],
            'Gastric Meds': [game, [0, 7000], 'ml'],
            'Lactated Ringers': [lari, [0, 17000], 'ml'],
            'Milrinone': [milr, [0, 50], 'ml'],
            'OR/PACU Crystalloid': [crys, [0, 22000], 'ml'],
            'Packed RBC': [prbc, [0, 17250], 'ml'],
            'PO intake': [poin, [0, 11000], 'ml'],
            'Neosynephrine': [neos, [0, 1200], 'mg'],
            'Piggyback': [pigg, [0, 1000], 'ml'],
            'Nitroglycerine': [nigl, [0, 350], 'mg'],
            'Nitroprusside': [nipr, [0, 430], 'mg'],
            'Metoprolol': [meto, [0, 151], 'mg'],
            'Norepinephrine': [nore, [0, 80], 'mg'],
            'Colloid': [coll, [0, 20000], 'ml'],
            'Hydralazine': [hyzi, [0, 80], 'mg'],
            'GT Flush': [gtfl, [0, 2100], 'ml'],
            'Hydromorphone': [hymo, [0, 125], 'mg'],
            'Fentanyl': [fent, [0, 20], 'mg'],
            'Insulin Regular': [inre, [0, 1500], 'units'],
            'Insulin Humalog': [inhu, [0, 340], 'units'],
            'Insulin largine': [ingl, [0, 150], 'units'],
            'Insulin NPH': [innp, [0, 100], 'units'],
            'Unknown': [nana, [0, 1100], 'ml'],
            'D5W': [d5wa, [0,11000], 'ml'],
            'Dextrose Other': [doth, [0,4000], 'ml'],
            'Normal Saline': [nosa, [0, 11000], 'ml'],
            'Half Normal Saline': [hans, [0, 2000], 'ml'],
            'Sterile Water': [stwa, [0, 10000], 'ml'],
            'Free Water': [frwa, [0, 2500], 'ml'],
            'Solution': [solu, [0, 1500], 'ml'],
            'Dopamine': [dopa, [0, 1300], 'mg'],
            'Epinephrine': [epin, [0, 100], 'mg'],
            'Amiodarone': [amio, [0, 1200], 'mg'],
            'TPN': [tpnu, [0, 1600], 'ml'],
            'Magnesium Sulfate (Bolus)': [msbo, [0, 250], 'ml'],
            'KCl (Bolus)': [pcbo, [0, 500], 'ml'],
            'Pre-admission Intake': [prad, [0, 30000], 'ml']
            }

for k, v in features.items():
    print (k)
    ev_k = ie.loc[ie.itemid.isin(v[0])]
    ind = (ev_k.valuenum>=v[1][0])&(ev_k.valuenum<=v[1][1])
    med = ev_k.valuenum.loc[ind].median()
    ev_k.loc[~ind, 'valuenum'] = med
    ev_k['name'] = k
    ev_k['valueuom'] = v[2]
    events = pd.concat([events, ev_k])
del ev_k

# Extract heparin events. (Missed earlier.)
ev_k = ie.loc[ie.itemid.isin(hepa)]
ind1 = ev_k.valueuom.isin(['U', 'units'])
ind2 = (ev_k.valuenum>=0)&(ev_k.valuenum<=25300)
ind = (ind1&ind2)
med = ev_k.valuenum.loc[ind].median()
ev_k.loc[~ind, 'valuenum'] = med
ev_k['name'] = 'Heparin'
ev_k['valueuom'] = 'units'
events = pd.concat([events, ev_k])
del ev_k

# Extract weight events from MV inputevents.
ie_mv = pd.read_csv(mimic_data_dir+'inputevents.csv', usecols = ['stay_id', 'starttime', 'patientweight'])
ie_mv = ie_mv.drop_duplicates()
ie_mv = ie_mv.loc[ie_mv.stay_id.isin(icu.stay_id)]
ie_mv.rename(columns={'starttime':'charttime', 'patientweight':'valuenum'}, inplace=True)
ie_mv = ie_mv.loc[(ie_mv.valuenum>=0)&(ie_mv.valuenum<=300)]
ie_mv['valueuom'] = 'kg'
ie_mv['name'] = 'Weight'
events = pd.concat([events, ie_mv])
del ie_mv

# Save data.
events.to_csv('mimic_iv_events.csv', index=False)
icu.to_csv('mimic_iv_icu.csv', index=False)

import pandas as pd
from tqdm import tqdm
import pickle
import numpy as np

mimic_data_dir = '/home/anand/UHIF/UniHPF-master/UniHPF-master/mimic4/'

# Read extracted time series data.
events = pd.read_csv('mimic_iv_events.csv', usecols=['hadm_id', 'stay_id', 'charttime', 'valuenum', 'TABLE', 'name'])
icu = pd.read_csv('mimic_iv_icu.csv')
# Convert times to type datetime.
events.charttime = pd.to_datetime(events.charttime)
icu.intime = pd.to_datetime(icu.intime)
icu.outtime = pd.to_datetime(icu.outtime)

# Assign stay_id to rows without it. Remove rows that can't be assigned one.
icu['icustay_times'] = icu.apply(lambda x:[x.stay_id, x.intime, x.outtime], axis=1)
adm_icu_times = icu.groupby('hadm_id').agg({'icustay_times':list}).reset_index()
icu.drop(columns=['icustay_times'], inplace=True)
events = events.merge(adm_icu_times, on=['hadm_id'], how='left')
idx = events.stay_id.isna()
tqdm.pandas()
def f(x):
    chart_time = x.charttime
    for icu_times in x.icustay_times:
        if icu_times[1]<=chart_time<=icu_times[2]:
            return icu_times[0]
events.loc[idx, 'stay_id'] = (events.loc[idx]).progress_apply(f, axis=1)
events.drop(columns=['icustay_times'], inplace=True)
events = events.loc[events.stay_id.notna()]
events.drop(columns=['hadm_id'], inplace=True)

# Filter icu table.
icu = icu.loc[icu.stay_id.isin(events.stay_id)]

# Get rel_charttime in minutes.
events = events.merge(icu[['stay_id', 'intime']], on='stay_id', how='left')
events['rel_charttime'] = events.charttime-events.intime
events.drop(columns=['intime', 'charttime'], inplace=True)
events.rel_charttime = events.rel_charttime.dt.total_seconds()//60

# Save current icu table.
icu_full = icu.copy()

# Get icustays which lasted for atleast 24 hours.
icu = icu.loc[(icu.outtime-icu.intime)>=pd.Timedelta(24,'h')]

# Get icustays with patient alive for atleast 24 hours.
adm = pd.read_csv(mimic_data_dir+'admissions.csv', usecols=['hadm_id', 'deathtime'])
icu = icu.merge(adm, on='hadm_id', how='left')
icu.deathtime = pd.to_datetime(icu.deathtime)
icu = icu.loc[((icu.deathtime-icu.intime)>=pd.Timedelta(24,'h'))|icu.deathtime.isna()]

# Get icustays with aleast one event in first 24h.
icu = icu.loc[icu.stay_id.isin(events.loc[events.rel_charttime<24*60].stay_id)]

# Get sup and unsup icustays.
all_icustays = np.array(icu_full.stay_id)
sup_icustays = np.array(icu.stay_id)
unsup_icustays = np.setdiff1d(all_icustays, sup_icustays)
all_icustays = np.concatenate((sup_icustays, unsup_icustays), axis=-1)

print("all_icustays",all_icustays)

# Get ts_ind.
def inv_list(x, start=0):
    d = {}
    for i in range(len(x)):
        d[x[i]] = i
    return d
icustay_to_ind = inv_list(all_icustays)
events['ts_ind'] = events.stay_id.map(icustay_to_ind)

# Rename some columns.
events.rename(columns={'rel_charttime':'minute', 'name':'variable', 'valuenum':'value'}, inplace=True)


# Add gender and age.
icu_full['ts_ind'] = icu_full.stay_id.map(icustay_to_ind)
data_age = icu_full[['ts_ind', 'age']]
data_age['variable'] = 'age'
data_age.rename(columns={'age':'value'}, inplace=True)
data_gen = icu_full[['ts_ind', 'gender']]
data_gen.loc[data_gen.gender=='M', 'gender'] = 0
data_gen.loc[data_gen.gender=='F', 'gender'] = 1
data_gen['variable'] = 'gender'
data_gen.rename(columns={'gender':'value'}, inplace=True)
data = pd.concat((data_age, data_gen), ignore_index=True)
data['minute'] = 0
events = pd.concat((data, events), ignore_index=True)


# Drop duplicate events.
events.drop_duplicates(inplace=True)

icu_full.to_csv('mimic_iv_icu_full.csv', index=False)

#events.to_csv('mimic_iii_events_full.csv', index=False)

# Add mortality label.
adm = pd.read_csv(mimic_data_dir+'admissions.csv', usecols=['hadm_id', 'hospital_expire_flag'])
oc = icu_full[['ts_ind', 'hadm_id', 'subject_id']].merge(adm, on='hadm_id', how='left')
oc = oc.rename(columns={'hospital_expire_flag': 'in_hospital_mortality'})

# # Get code to phenotype index mapping.
# phen_def = yaml.load(open('hcup_ccs_2015_definitions.yaml'), Loader=yaml.FullLoader)
# keep = ["Acute and unspecified renal failure", "Acute cerebrovascular disease", 'Acute myocardial infarction', 
#         'Cardiac dysrhythmias', 'Chronic kidney disease', 'Chronic obstructive pulmonary disease and bronchiectasis', 
#         'Complications of surgical procedures or medical care', 'Conduction disorders', 'Congestive heart failure; nonhypertensive', 
#         'Coronary atherosclerosis and other heart disease', 'Diabetes mellitus with complications', 
#         'Diabetes mellitus without complication', 'Disorders of lipid metabolism', 'Essential hypertension',
#         'Fluid and electrolyte disorders', 'Gastrointestinal hemorrhage', 'Hypertension with complications and secondary hypertension',
#         'Other liver diseases', 'Other lower respiratory disease', 'Other upper respiratory disease',
#         'Pleurisy; pneumothorax; pulmonary collapse', 'Pneumonia (except that caused by tuberculosis or sexually transmitted disease)', 
#         'Respiratory failure; insufficiency; arrest (adult)', 'Septicemia (except in labor)', 'Shock']
# phen_def = {k:phen_def[k]['codes'] for k in keep}
# phen_ind = inv_list(sorted(keep))
# phen_def = {phen_ind[k]:v for (k,v) in phen_def.items()}
# code_to_phen = {}
# for (k,v) in phen_def.items():
#     for c in v:
#         code_to_phen[c] = k

# # Add phen label.
# dx = pd.read_csv(mimic_data_dir+'DIAGNOSES_ICD.csv', usecols=['hadm_id', 'ICD9_CODE']).drop_duplicates()
# dx = dx.loc[dx.hadm_id.isin(oc.hadm_id)]
# dx = dx.loc[dx.ICD9_CODE.isin(code_to_phen)]
# dx['phenotypes'] = dx.ICD9_CODE.map(code_to_phen)
# dx = dx.groupby('hadm_id').agg({'phenotypes':set})
# oc = oc.merge(dx, on='hadm_id')
# P = len(keep)
# def multi_hot(x):
#     mh = np.zeros(P)
#     for i in x:
#         mh[i] = 1
#     return mh
# oc['phenotypes'] = oc['phenotypes'].apply(multi_hot)
icu.to_csv('icu_to_see_sup_subj.csv',index=False)
# Get train-valid-test split for sup task.
all_sup_subjects = icu.subject_id.unique()
np.random.seed(2023)
np.random.shuffle(all_sup_subjects)
S = len(all_sup_subjects)
print("all sup sub",S)
bp1, bp2 = int(0.64*S), int(0.8*S)
train_sub = all_sup_subjects[:bp1]
valid_sub = all_sup_subjects[bp1:bp2]
test_sub = all_sup_subjects[bp2:]
icu['ts_ind'] = icu.stay_id.map(icustay_to_ind)
icu.to_csv('icu_final_train_val_test_ind.csv',index=False)
train_ind = np.array(icu.loc[icu.subject_id.isin(train_sub)].ts_ind)
valid_ind = np.array(icu.loc[icu.subject_id.isin(valid_sub)].ts_ind)
test_ind = np.array(icu.loc[icu.subject_id.isin(test_sub)].ts_ind)

# Filter columns.
events = events[['ts_ind', 'minute', 'variable', 'value', 'TABLE']]

# Convert minute to hour.
events['hour'] = events['minute']/60
events.drop(columns=['minute'], inplace=True)

# Aggregate data.
events['value'] = events['value'].astype(float)
events.loc[events['TABLE'].isna(), 'TABLE'] = 'N/A'
events = events.groupby(['ts_ind', 'hour', 'variable']).agg({'value':'mean', 'TABLE':'unique'}).reset_index()
def f(x):
    if len(x)==0:
        return x[0]
    else:
        return ','.join(x)
events['TABLE'] = events['TABLE'].apply(f)

# Save data.
pickle.dump([events, oc, train_ind, valid_ind, test_ind], open('mimic_iv_preprocessed.pkl','wb'))

# Normalize data and save. 
ts, oc, train_ind, valid_ind, test_ind = pickle.load(open('mimic_iv_preprocessed.pkl','rb'))
means_stds = ts.groupby('variable').agg({'value':['mean', 'std']})
means_stds.columns = [col[1] for col in means_stds.columns]
means_stds.loc[means_stds['std']==0, 'std'] = 1
ts = ts.merge(means_stds.reset_index(), on='variable', how='left')
ii = ~ts.variable.isin(['age', 'gender'])
ts.loc[ii, 'value'] = (ts.loc[ii, 'value']-ts.loc[ii, 'mean'])/ts.loc[ii, 'std']
ts.to_csv('ts.csv', index=False)
oc.to_csv('oc.csv', index=False)
print(len(train_ind), len(valid_ind), len(test_ind))
print(train_ind, valid_ind, test_ind)


#print(ts.head())
#print(oc.head())
#print(train_ind, valid_ind, test_ind)
pickle.dump([ts, oc, train_ind, valid_ind, test_ind], open('mimic_iv_preprocessed.pkl','wb'))


#adm = pd.read_csv(mimic_data_dir+'admissions.csv', usecols=['hadm_id', 'hospital_expire_flag','ethnicity'])
#oc_ethn = icu_full[['ts_ind', 'hadm_id', 'subject_id']].merge(adm, on='hadm_id', how='left')
#oc_ethn = oc_ethn.rename(columns={'hospital_expire_flag': 'in_hospital_mortality'})
#oc_ethn.to_csv('mimic_iii_ethnicity.csv', index=False)