autolabel.py

# -*- coding: utf-8 -*-
"""AutoLabel.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1CWvRi_X9iGeMGIVDjDvPc8TNxo1EBB4e
"""

from __future__ import absolute_import, division, print_function, unicode_literals
import tensorflow as tf
import numpy as np
import pandas as pd
import scipy as sp
import sklearn
import math
from sklearn.preprocessing import StandardScaler
import os
import glob
import random
import matplotlib.pyplot as plt
from google.colab import drive
drive.mount('/content/drive')
from keras.utils import to_categorical
from tensorflow.keras.layers import Dense, Flatten, Conv2D, Conv1D, MaxPool1D, Dropout, CuDNNLSTM, TimeDistributed, Input, concatenate
from tensorflow.keras import Model
from tqdm import tqdm, tqdm_notebook

def normalise_data(training, testing, n):
    
    length_training = len(training)
    length_testing = len(testing)
    
    training = np.expand_dims(np.ndarray.flatten(training, order='C'), axis=1)
    testing = np.expand_dims(np.ndarray.flatten(testing, order='C'), axis=1)
    
    # Normalise the data for pca
    scaler = StandardScaler()
    scaler.fit(training)
    normalised_training = np.reshape(scaler.transform(training), (length_training, n, 3), order='C')
    normalised_testing = np.reshape(scaler.transform(testing), (length_testing, n, 3), order='C')
   
    print('Data normalised')
  
    return normalised_training, normalised_testing

def n_all_markers(base_dir, train_subjects, test_subjects, model, n): # This function creates a model using all marker input data available

    training_x = []
    testing_x = []
    train_features = []
    test_features = []
    train_features_list = []
    test_features_list = []
    training_y = []
    testing_y = []
    train_labels_list = []
    test_labels_list = []
    n_training_y = []
    n_testing_y = []
    n_train_labels_list = []
    n_test_labels_list = []

    # Dictionary
    marker_labels = ['FLE', 'FME', 'THIGH', 'FAM', 'TAM', 'KNEE', 'FCC', 'FM2', 'TF', 'FMT', 'ASIS', 'PSIS',
              'oFLE', 'oFME', 'oTHIGH', 'oFAM', 'oTAM', 'oKNEE', 'oFCC', 'oFM2', 'oTF', 'oFMT', 'oASIS', 'oPSIS']
    d = {}

    for i,j in enumerate(marker_labels):
        d[j] = i
    
    print('Loading training data...')
    
    os.chdir(base_dir)

    # Find all matrices created by MATLAB programme, first do right leg stance for training data subjects
    for subject in tqdm_notebook(train_subjects):
        os.chdir(base_dir + subject)
        for name in glob.glob('r_?'):
            FLE_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFLE(x)', 'RFLE(y)', 'RFLE(z)'])
            FME_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFME(x)', 'RFME(y)', 'RFME(z)'])
            T1_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RT1(x)', 'RT1(y)', 'RT1(z)'])
            T2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RT2(x)', 'RT2(y)', 'RT2(z)'])
            T3_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RT3(x)', 'RT3(y)', 'RT3(z)'])
            FAM_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFAM(x)', 'RFAM(y)', 'RFAM(z)'])
            TAM_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RTAM(x)', 'RTAM(y)', 'RTAM(z)'])
            C1_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RC1(x)', 'RC1(y)', 'RC1(z)'])
            C2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RC2(x)', 'RC2(y)', 'RC2(z)'])
            C3_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RC3(x)', 'RC3(y)', 'RC3(z)'])
            FCC_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFCC(x)', 'RFCC(y)', 'RFCC(z)'])
            FM2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFM2(x)', 'RFM2(y)', 'RFM2(z)'])
            TF_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RTF(x)', 'RTF(y)', 'RTF(z)'])
            FMT_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFMT(x)', 'RFMT(y)', 'RFMT(z)'])
            ASIS_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RASIS(x)', 'RASIS(y)', 'RASIS(z)'])
            PSIS_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RPSIS(x)', 'RPSIS(y)', 'RPSIS(z)'])                                                                                 
            oFLE_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFLE(x)', 'LFLE(y)', 'LFLE(z)'])
            oFME_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFME(x)', 'LFME(y)', 'LFME(z)'])
            oT1_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LT1(x)', 'LT1(y)', 'LT1(z)'])
            oT2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LT2(x)', 'LT2(y)', 'LT2(z)'])
            oT3_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LT3(x)', 'LT3(y)', 'LT3(z)'])
            oFAM_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFAM(x)', 'LFAM(y)', 'LFAM(z)'])
            oTAM_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LTAM(x)', 'LTAM(y)', 'LTAM(z)'])
            oC1_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LC1(x)', 'LC1(y)', 'LC1(z)'])
            oC2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LC2(x)', 'LC2(y)', 'LC2(z)'])
            oC3_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LC3(x)', 'LC3(y)', 'LC3(z)'])
            oFCC_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFCC(x)', 'LFCC(y)', 'LFCC(z)'])
            oFM2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFM2(x)', 'LFM2(y)', 'LFM2(z)'])
            oTF_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LTF(x)', 'LTF(y)', 'LTF(z)'])
            oFMT_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFMT(x)', 'LFMT(y)', 'LFMT(z)'])
            oASIS_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LASIS(x)', 'LASIS(y)', 'LASIS(z)'])
            oPSIS_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LPSIS(x)', 'LPSIS(y)', 'LPSIS(z)'])                                                                                  
          
            for i in range(n, len(FLE_in), n):            
                # Create features for model to use
                matrix_FLE = FLE_in[i-n:i]
                matrix_FME = FME_in[i-n:i]
                matrix_T1 = T1_in[i-n:i]
                matrix_T2 = T2_in[i-n:i]                                                                              
                matrix_T3 = T3_in[i-n:i]
                matrix_FAM = FAM_in[i-n:i]
                matrix_TAM = TAM_in[i-n:i]
                matrix_C1 = C1_in[i-n:i]
                matrix_C2 = C2_in[i-n:i]
                matrix_C3 = C3_in[i-n:i]     
                matrix_FCC = FCC_in[i-n:i]                                                                              
                matrix_FM2 = FM2_in[i-n:i]
                matrix_TF = TF_in[i-n:i]                                                                              
                matrix_FMT = FMT_in[i-n:i]    
                matrix_ASIS = ASIS_in[i-n:i]                                                                              
                matrix_PSIS = PSIS_in[i-n:i]
                matrix_oFLE = oFLE_in[i-n:i]
                matrix_oFME = oFME_in[i-n:i]
                matrix_oT1 = oT1_in[i-n:i]
                matrix_oT2 = oT2_in[i-n:i]                                                                              
                matrix_oT3 = oT3_in[i-n:i]
                matrix_oFAM = oFAM_in[i-n:i]
                matrix_oTAM = oTAM_in[i-n:i]
                matrix_oC1 = oC1_in[i-n:i]
                matrix_oC2 = oC2_in[i-n:i]
                matrix_oC3 = oC3_in[i-n:i]     
                matrix_oFCC = oFCC_in[i-n:i]                                                                              
                matrix_oFM2 = oFM2_in[i-n:i]
                matrix_oTF = oTF_in[i-n:i]                                                                              
                matrix_oFMT = oFMT_in[i-n:i]    
                matrix_oASIS = oASIS_in[i-n:i]                                                                              
                matrix_oPSIS = oPSIS_in[i-n:i]   

                train_features = np.stack([matrix_FLE, matrix_FME, matrix_T1, matrix_T2, matrix_T3, matrix_FAM, matrix_TAM, matrix_C1,
                                         matrix_C2, matrix_C3, matrix_FCC, matrix_FM2, matrix_TF, matrix_FMT, matrix_ASIS, matrix_PSIS,
                                         matrix_oFLE, matrix_oFME, matrix_oT1, matrix_oT2, matrix_oT3, matrix_oFAM, matrix_oTAM, matrix_oC1,
                                         matrix_oC2, matrix_oC3, matrix_oFCC, matrix_oFM2, matrix_oTF, matrix_oFMT, matrix_oASIS, matrix_oPSIS])

                train_features_list.append(train_features)


                train_labels = np.expand_dims(np.stack([d['FLE'], d['FME'], d['THIGH'], d['THIGH'], d['THIGH'], d['FAM'], d['TAM'], d['KNEE'],
                                                        d['KNEE'], d['KNEE'], d['FCC'], d['FM2'], d['TF'], d['FMT'], d['ASIS'], d['PSIS'],
                                                        d['oFLE'], d['oFME'], d['oTHIGH'], d['oTHIGH'], d['oTHIGH'], d['oFAM'], d['oTAM'], d['oKNEE'],
                                                        d['oKNEE'], d['oKNEE'], d['oFCC'], d['oFM2'], d['oTF'], d['oFMT'], d['oASIS'], d['oPSIS']]), axis=1)

                train_labels_list.append(train_labels)
        
        # Now repeat for left leg stance
        for name in glob.glob('l_?'):            
            FLE_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFLE(x)', 'LFLE(y)', 'LFLE(z)'])
            FME_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFME(x)', 'LFME(y)', 'LFME(z)'])
            T1_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LT1(x)', 'LT1(y)', 'LT1(z)'])
            T2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LT2(x)', 'LT2(y)', 'LT2(z)'])
            T3_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LT3(x)', 'LT3(y)', 'LT3(z)'])
            FAM_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFAM(x)', 'LFAM(y)', 'LFAM(z)'])
            TAM_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LTAM(x)', 'LTAM(y)', 'LTAM(z)'])
            C1_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LC1(x)', 'LC1(y)', 'LC1(z)'])
            C2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LC2(x)', 'LC2(y)', 'LC2(z)'])
            C3_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LC3(x)', 'LC3(y)', 'LC3(z)'])
            FCC_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFCC(x)', 'LFCC(y)', 'LFCC(z)'])
            FM2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFM2(x)', 'LFM2(y)', 'LFM2(z)'])
            TF_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LTF(x)', 'LTF(y)', 'LTF(z)'])
            FMT_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFMT(x)', 'LFMT(y)', 'LFMT(z)'])
            ASIS_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LASIS(x)', 'LASIS(y)', 'LASIS(z)'])
            PSIS_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LPSIS(x)', 'LPSIS(y)', 'LPSIS(z)'])                                                                                 
            oFLE_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFLE(x)', 'RFLE(y)', 'RFLE(z)'])
            oFME_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFME(x)', 'RFME(y)', 'RFME(z)'])
            oT1_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RT1(x)', 'RT1(y)', 'RT1(z)'])
            oT2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RT2(x)', 'RT2(y)', 'RT2(z)'])
            oT3_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RT3(x)', 'RT3(y)', 'RT3(z)'])
            oFAM_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFAM(x)', 'RFAM(y)', 'RFAM(z)'])
            oTAM_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RTAM(x)', 'RTAM(y)', 'RTAM(z)'])
            oC1_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RC1(x)', 'RC1(y)', 'RC1(z)'])
            oC2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RC2(x)', 'RC2(y)', 'RC2(z)'])
            oC3_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RC3(x)', 'RC3(y)', 'RC3(z)'])
            oFCC_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFCC(x)', 'RFCC(y)', 'RFCC(z)'])
            oFM2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFM2(x)', 'RFM2(y)', 'RFM2(z)'])
            oTF_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RTF(x)', 'RTF(y)', 'RTF(z)'])
            oFMT_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFMT(x)', 'RFMT(y)', 'RFMT(z)'])
            oASIS_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RASIS(x)', 'RASIS(y)', 'RASIS(z)'])
            oPSIS_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RPSIS(x)', 'RPSIS(y)', 'RPSIS(z)'])
                                                                                              
            for i in range(n, len(FLE_in), n):
                # Create features for model to use
                matrix_FLE = FLE_in[i-n:i]
                matrix_FME = FME_in[i-n:i]
                matrix_T1 = T1_in[i-n:i]
                matrix_T2 = T2_in[i-n:i]                                                                              
                matrix_T3 = T3_in[i-n:i]
                matrix_FAM = FAM_in[i-n:i]
                matrix_TAM = TAM_in[i-n:i]
                matrix_C1 = C1_in[i-n:i]
                matrix_C2 = C2_in[i-n:i]
                matrix_C3 = C3_in[i-n:i]     
                matrix_FCC = FCC_in[i-n:i]                                                                              
                matrix_FM2 = FM2_in[i-n:i]
                matrix_TF = TF_in[i-n:i]                                                                              
                matrix_FMT = FMT_in[i-n:i]    
                matrix_ASIS = ASIS_in[i-n:i]                                                                              
                matrix_PSIS = PSIS_in[i-n:i]
                matrix_oFLE = oFLE_in[i-n:i]
                matrix_oFME = oFME_in[i-n:i]
                matrix_oT1 = oT1_in[i-n:i]
                matrix_oT2 = oT2_in[i-n:i]                                                                              
                matrix_oT3 = oT3_in[i-n:i]
                matrix_oFAM = oFAM_in[i-n:i]
                matrix_oTAM = oTAM_in[i-n:i]
                matrix_oC1 = oC1_in[i-n:i]
                matrix_oC2 = oC2_in[i-n:i]
                matrix_oC3 = oC3_in[i-n:i]     
                matrix_oFCC = oFCC_in[i-n:i]                                                                              
                matrix_oFM2 = oFM2_in[i-n:i]
                matrix_oTF = oTF_in[i-n:i]                                                                              
                matrix_oFMT = oFMT_in[i-n:i]    
                matrix_oASIS = oASIS_in[i-n:i]                                                                              
                matrix_oPSIS = oPSIS_in[i-n:i]                                                                              

                train_features = np.stack([matrix_FLE, matrix_FME, matrix_T1, matrix_T2, matrix_T3, matrix_FAM, matrix_TAM, matrix_C1,
                                         matrix_C2, matrix_C3, matrix_FCC, matrix_FM2, matrix_TF, matrix_FMT, matrix_ASIS, matrix_PSIS,
                                         matrix_oFLE, matrix_oFME, matrix_oT1, matrix_oT2, matrix_oT3, matrix_oFAM, matrix_oTAM, matrix_oC1,
                                         matrix_oC2, matrix_oC3, matrix_oFCC, matrix_oFM2, matrix_oTF, matrix_oFMT, matrix_oASIS, matrix_oPSIS])

                train_features_list.append(train_features)
                
                
                train_labels = np.expand_dims(np.stack([d['FLE'], d['FME'], d['THIGH'], d['THIGH'], d['THIGH'], d['FAM'], d['TAM'], d['KNEE'],
                                                        d['KNEE'], d['KNEE'], d['FCC'], d['FM2'], d['TF'], d['FMT'], d['ASIS'], d['PSIS'],
                                                        d['oFLE'], d['oFME'], d['oTHIGH'], d['oTHIGH'], d['oTHIGH'], d['oFAM'], d['oTAM'], d['oKNEE'],
                                                        d['oKNEE'], d['oKNEE'], d['oFCC'], d['oFM2'], d['oTF'], d['oFMT'], d['oASIS'], d['oPSIS']]), axis=1)
                                
                train_labels_list.append(train_labels)
                

        os.chdir("..")
        
    # Create stack of these lists, containing all features, at all times, from all trials, in one matrix. This will serve as data to train the model.
    training_x = np.vstack(train_features_list)
    training_y = np.vstack(train_labels_list)

    print('Loading testing data...')
    
    # Now repeat as above for remaining subjects to use as testing data
    for subject in tqdm_notebook(test_subjects):
        os.chdir(base_dir + subject)
        for name in glob.glob('r_?'):
            FLE_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFLE(x)', 'RFLE(y)', 'RFLE(z)'])
            FME_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFME(x)', 'RFME(y)', 'RFME(z)'])
            T1_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RT1(x)', 'RT1(y)', 'RT1(z)'])
            T2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RT2(x)', 'RT2(y)', 'RT2(z)'])
            T3_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RT3(x)', 'RT3(y)', 'RT3(z)'])
            FAM_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFAM(x)', 'RFAM(y)', 'RFAM(z)'])
            TAM_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RTAM(x)', 'RTAM(y)', 'RTAM(z)'])
            C1_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RC1(x)', 'RC1(y)', 'RC1(z)'])
            C2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RC2(x)', 'RC2(y)', 'RC2(z)'])
            C3_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RC3(x)', 'RC3(y)', 'RC3(z)'])
            FCC_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFCC(x)', 'RFCC(y)', 'RFCC(z)'])
            FM2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFM2(x)', 'RFM2(y)', 'RFM2(z)'])
            TF_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RTF(x)', 'RTF(y)', 'RTF(z)'])
            FMT_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFMT(x)', 'RFMT(y)', 'RFMT(z)'])
            ASIS_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RASIS(x)', 'RASIS(y)', 'RASIS(z)'])
            PSIS_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RPSIS(x)', 'RPSIS(y)', 'RPSIS(z)'])                                                                                 
            oFLE_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFLE(x)', 'LFLE(y)', 'LFLE(z)'])
            oFME_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFME(x)', 'LFME(y)', 'LFME(z)'])
            oT1_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LT1(x)', 'LT1(y)', 'LT1(z)'])
            oT2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LT2(x)', 'LT2(y)', 'LT2(z)'])
            oT3_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LT3(x)', 'LT3(y)', 'LT3(z)'])
            oFAM_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFAM(x)', 'LFAM(y)', 'LFAM(z)'])
            oTAM_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LTAM(x)', 'LTAM(y)', 'LTAM(z)'])
            oC1_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LC1(x)', 'LC1(y)', 'LC1(z)'])
            oC2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LC2(x)', 'LC2(y)', 'LC2(z)'])
            oC3_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LC3(x)', 'LC3(y)', 'LC3(z)'])
            oFCC_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFCC(x)', 'LFCC(y)', 'LFCC(z)'])
            oFM2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFM2(x)', 'LFM2(y)', 'LFM2(z)'])
            oTF_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LTF(x)', 'LTF(y)', 'LTF(z)'])
            oFMT_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFMT(x)', 'LFMT(y)', 'LFMT(z)'])
            oASIS_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LASIS(x)', 'LASIS(y)', 'LASIS(z)'])
            oPSIS_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LPSIS(x)', 'LPSIS(y)', 'LPSIS(z)'])

            for i in range(n, len(FLE_in), n):
                # Create features for model to use
                matrix_FLE = FLE_in[i-n:i]
                matrix_FME = FME_in[i-n:i]
                matrix_T1 = T1_in[i-n:i]
                matrix_T2 = T2_in[i-n:i]                                                                              
                matrix_T3 = T3_in[i-n:i]
                matrix_FAM = FAM_in[i-n:i]
                matrix_TAM = TAM_in[i-n:i]
                matrix_C1 = C1_in[i-n:i]
                matrix_C2 = C2_in[i-n:i]
                matrix_C3 = C3_in[i-n:i]     
                matrix_FCC = FCC_in[i-n:i]                                                                              
                matrix_FM2 = FM2_in[i-n:i]
                matrix_TF = TF_in[i-n:i]                                                                              
                matrix_FMT = FMT_in[i-n:i]    
                matrix_ASIS = ASIS_in[i-n:i]                                                                              
                matrix_PSIS = PSIS_in[i-n:i]
                matrix_oFLE = oFLE_in[i-n:i]
                matrix_oFME = oFME_in[i-n:i]
                matrix_oT1 = oT1_in[i-n:i]
                matrix_oT2 = oT2_in[i-n:i]                                                                              
                matrix_oT3 = oT3_in[i-n:i]
                matrix_oFAM = oFAM_in[i-n:i]
                matrix_oTAM = oTAM_in[i-n:i]
                matrix_oC1 = oC1_in[i-n:i]
                matrix_oC2 = oC2_in[i-n:i]
                matrix_oC3 = oC3_in[i-n:i]     
                matrix_oFCC = oFCC_in[i-n:i]                                                                              
                matrix_oFM2 = oFM2_in[i-n:i]
                matrix_oTF = oTF_in[i-n:i]                                                                              
                matrix_oFMT = oFMT_in[i-n:i]    
                matrix_oASIS = oASIS_in[i-n:i]                                                                              
                matrix_oPSIS = oPSIS_in[i-n:i]   

                test_features = np.stack([matrix_FLE, matrix_FME, matrix_T1, matrix_T2, matrix_T3, matrix_FAM, matrix_TAM, matrix_C1,
                                         matrix_C2, matrix_C3, matrix_FCC, matrix_FM2, matrix_TF, matrix_FMT, matrix_ASIS, matrix_PSIS,
                                         matrix_oFLE, matrix_oFME, matrix_oT1, matrix_oT2, matrix_oT3, matrix_oFAM, matrix_oTAM, matrix_oC1,
                                         matrix_oC2, matrix_oC3, matrix_oFCC, matrix_oFM2, matrix_oTF, matrix_oFMT, matrix_oASIS, matrix_oPSIS])

                test_features_list.append(test_features)
                
                
                test_labels = np.expand_dims(np.stack([d['FLE'], d['FME'], d['THIGH'], d['THIGH'], d['THIGH'], d['FAM'], d['TAM'], d['KNEE'],
                                                        d['KNEE'], d['KNEE'], d['FCC'], d['FM2'], d['TF'], d['FMT'], d['ASIS'], d['PSIS'],
                                                        d['oFLE'], d['oFME'], d['oTHIGH'], d['oTHIGH'], d['oTHIGH'], d['oFAM'], d['oTAM'], d['oKNEE'],
                                                        d['oKNEE'], d['oKNEE'], d['oFCC'], d['oFM2'], d['oTF'], d['oFMT'], d['oASIS'], d['oPSIS']]), axis=1)
                                
                test_labels_list.append(test_labels)


        # Now repeat for left leg stance
        for name in glob.glob('l_?'):
            FLE_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFLE(x)', 'LFLE(y)', 'LFLE(z)'])
            FME_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFME(x)', 'LFME(y)', 'LFME(z)'])
            T1_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LT1(x)', 'LT1(y)', 'LT1(z)'])
            T2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LT2(x)', 'LT2(y)', 'LT2(z)'])
            T3_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LT3(x)', 'LT3(y)', 'LT3(z)'])
            FAM_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFAM(x)', 'LFAM(y)', 'LFAM(z)'])
            TAM_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LTAM(x)', 'LTAM(y)', 'LTAM(z)'])
            C1_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LC1(x)', 'LC1(y)', 'LC1(z)'])
            C2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LC2(x)', 'LC2(y)', 'LC2(z)'])
            C3_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LC3(x)', 'LC3(y)', 'LC3(z)'])
            FCC_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFCC(x)', 'LFCC(y)', 'LFCC(z)'])
            FM2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFM2(x)', 'LFM2(y)', 'LFM2(z)'])
            TF_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LTF(x)', 'LTF(y)', 'LTF(z)'])
            FMT_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LFMT(x)', 'LFMT(y)', 'LFMT(z)'])
            ASIS_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LASIS(x)', 'LASIS(y)', 'LASIS(z)'])
            PSIS_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['LPSIS(x)', 'LPSIS(y)', 'LPSIS(z)'])                                                                                 
            oFLE_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFLE(x)', 'RFLE(y)', 'RFLE(z)'])
            oFME_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFME(x)', 'RFME(y)', 'RFME(z)'])
            oT1_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RT1(x)', 'RT1(y)', 'RT1(z)'])
            oT2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RT2(x)', 'RT2(y)', 'RT2(z)'])
            oT3_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RT3(x)', 'RT3(y)', 'RT3(z)'])
            oFAM_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFAM(x)', 'RFAM(y)', 'RFAM(z)'])
            oTAM_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RTAM(x)', 'RTAM(y)', 'RTAM(z)'])
            oC1_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RC1(x)', 'RC1(y)', 'RC1(z)'])
            oC2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RC2(x)', 'RC2(y)', 'RC2(z)'])
            oC3_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RC3(x)', 'RC3(y)', 'RC3(z)'])
            oFCC_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFCC(x)', 'RFCC(y)', 'RFCC(z)'])
            oFM2_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFM2(x)', 'RFM2(y)', 'RFM2(z)'])
            oTF_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RTF(x)', 'RTF(y)', 'RTF(z)'])
            oFMT_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RFMT(x)', 'RFMT(y)', 'RFMT(z)'])
            oASIS_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RASIS(x)', 'RASIS(y)', 'RASIS(z)'])
            oPSIS_in = pd.read_csv(name, sep = '\t', index_col=False, skiprows = 0, usecols = ['RPSIS(x)', 'RPSIS(y)', 'RPSIS(z)'])

            for i in range(n, len(FLE_in), n):
                # Create features for model to use
                matrix_FLE = FLE_in[i-n:i]
                matrix_FME = FME_in[i-n:i]
                matrix_T1 = T1_in[i-n:i]
                matrix_T2 = T2_in[i-n:i]                                                                              
                matrix_T3 = T3_in[i-n:i]
                matrix_FAM = FAM_in[i-n:i]
                matrix_TAM = TAM_in[i-n:i]
                matrix_C1 = C1_in[i-n:i]
                matrix_C2 = C2_in[i-n:i]
                matrix_C3 = C3_in[i-n:i]     
                matrix_FCC = FCC_in[i-n:i]                                                                              
                matrix_FM2 = FM2_in[i-n:i]
                matrix_TF = TF_in[i-n:i]                                                                              
                matrix_FMT = FMT_in[i-n:i]    
                matrix_ASIS = ASIS_in[i-n:i]                                                                              
                matrix_PSIS = PSIS_in[i-n:i]
                matrix_oFLE = oFLE_in[i-n:i]
                matrix_oFME = oFME_in[i-n:i]
                matrix_oT1 = oT1_in[i-n:i]
                matrix_oT2 = oT2_in[i-n:i]                                                                              
                matrix_oT3 = oT3_in[i-n:i]
                matrix_oFAM = oFAM_in[i-n:i]
                matrix_oTAM = oTAM_in[i-n:i]
                matrix_oC1 = oC1_in[i-n:i]
                matrix_oC2 = oC2_in[i-n:i]
                matrix_oC3 = oC3_in[i-n:i]     
                matrix_oFCC = oFCC_in[i-n:i]                                                                              
                matrix_oFM2 = oFM2_in[i-n:i]
                matrix_oTF = oTF_in[i-n:i]                                                                              
                matrix_oFMT = oFMT_in[i-n:i]    
                matrix_oASIS = oASIS_in[i-n:i]                                                                              
                matrix_oPSIS = oPSIS_in[i-n:i]                                                                              

                test_features = np.stack([matrix_FLE, matrix_FME, matrix_T1, matrix_T2, matrix_T3, matrix_FAM, matrix_TAM, matrix_C1,
                                         matrix_C2, matrix_C3, matrix_FCC, matrix_FM2, matrix_TF, matrix_FMT, matrix_ASIS, matrix_PSIS,
                                         matrix_oFLE, matrix_oFME, matrix_oT1, matrix_oT2, matrix_oT3, matrix_oFAM, matrix_oTAM, matrix_oC1,
                                         matrix_oC2, matrix_oC3, matrix_oFCC, matrix_oFM2, matrix_oTF, matrix_oFMT, matrix_oASIS, matrix_oPSIS])

                test_features_list.append(test_features)
                
                
                test_labels = np.expand_dims(np.stack([d['FLE'], d['FME'], d['THIGH'], d['THIGH'], d['THIGH'], d['FAM'], d['TAM'], d['KNEE'],
                                                        d['KNEE'], d['KNEE'], d['FCC'], d['FM2'], d['TF'], d['FMT'], d['ASIS'], d['PSIS'],
                                                        d['oFLE'], d['oFME'], d['oTHIGH'], d['oTHIGH'], d['oTHIGH'], d['oFAM'], d['oTAM'], d['oKNEE'],
                                                        d['oKNEE'], d['oKNEE'], d['oFCC'], d['oFM2'], d['oTF'], d['oFMT'], d['oASIS'], d['oPSIS']]), axis=1)
                                
                test_labels_list.append(test_labels)
                

        os.chdir("..")

        
    # Create stack of these lists, containing all features, at all times, from all trials, in one matrix. This will serve as data to test the model.
    testing_x = np.vstack(test_features_list)
    testing_y = np.vstack(test_labels_list)

    
    print('Data loaded')
    
    (n_training_x, n_testing_x) = normalise_data(training_x, testing_x, n)
     
    
    (predictions, simple_predictions, ensemble_predictions, labels) = ensemble_nn(n_training_x, training_y, n_testing_x, testing_y, n)
    
    
    (errors, accuracy) = analysis(predictions, simple_predictions, ensemble_predictions, labels, marker_labels)
    
    print('Errors:\t' + str(errors))
    print('Ensemble Accuracy:\t' + str(accuracy))
    
    
    
    
    
    return accuracy

def mean_max_min(features, n):

    # features size is (32,n,3)
    
    calculated_features = np.zeros((3,n,3))
    
    # Find mean, max, min; create new matrix with these values
    calculated_features[0,:,:] = np.mean(features, axis=0)
    calculated_features[1,:,:] = np.max(features, axis=0)
    calculated_features[2,:,:] = np.min(features, axis=0)
                
    return calculated_features

def mean_max_min_3d(features, n):

    # features size is (32,n,3)
    
    calculated_features = np.zeros((len(features),n,3,3))
    
    # Find mean, max, min; create new matrix with these values
    calculated_features[:,:,:,0] = np.mean(features, axis=0)
    calculated_features[:,:,:,1] = np.max(features, axis=0)
    calculated_features[:,:,:,2] = np.min(features, axis=0)
                
    return calculated_features

def corrupt(features, n): # Statically corrupt data for generator
    
    features_corrupt = features.copy()
    
    #features shape is (trials,32,n,3)
    
    # Randomly corrupt 10% of data
    for z in range(len(features_corrupt[0])):
        for i in range(32):
            for j in range(n):
                for k in range(3):
                    r = random.randint(0,n)
                    if r < n//10: 
                        features_corrupt[z,i,j,k] = 0

    return features_corrupt

def generate_batch_matrices(features, labels, n, batch_size): # Generate data
  
  # Create empty arrays to contain batch of features and labels
  batch_features_3d = np.zeros((batch_size, 3, n, 3))
  batch_features_2d = np.zeros((batch_size, n, 3))
  batch_labels = np.zeros((batch_size, 24))

  while True:    
      for i in range(batch_size):
            
            # Choose random index in features
            index1 = random.randint(0,len(features)-1)
            index2 = random.randint(0,31)
            
            batch_features_3d[i,:,:,:] = mean_max_min(features[index1,:,:,:], n)
            batch_features_2d[i,:,:] = features[index1,index2,:,:]
            batch_labels[i,:] = labels[index1,index2,:]

      yield [batch_features_3d, batch_features_2d], batch_labels

def ensemble_nn(training_x, training_y, testing_x, testing_y, n):
  
    # Reshape data to necessary dimensions
    training_x = np.reshape(training_x, (-1,32,n,3))
    testing_x = np.reshape(testing_x, (-1,32,n,3))
    training_y = to_categorical(np.reshape(training_y, (-1,32,1)))
    testing_y = to_categorical(np.reshape(testing_y, (-1,32,1)))
    
    training_x = corrupt(training_x, n)
    testing_x = corrupt(testing_x, n)
    
    training_x_3d = np.zeros((len(training_x),32,n,3,3))
    testing_x_3d = np.zeros((len(testing_x),32,n,3,3))
    for i in range(len(training_x)):
        training_x_3d[i,:,:,:,:] = mean_max_min_3d(training_x[i,:,:,:], n)
    for i in range(len(testing_x)):
        testing_x_3d[i,:,:,:,:] = mean_max_min_3d(testing_x[i,:,:,:], n)
    
    training_x = np.reshape(training_x, (-1,n,3))
    testing_x = np.reshape(testing_x, (-1,n,3))
    training_x_3d = np.reshape(training_x_3d, (-1,n,3,3))
    testing_x_3d = np.reshape(testing_x_3d, (-1,n,3,3))
    
    training_y = np.reshape(training_y, (-1,24))
    testing_y = np.reshape(testing_y, (-1,24))

    
    # Set up parallel model layers
    input_3D = tf.keras.Input(shape=(n,3,3))
    conv1 = tf.keras.layers.Conv2D(filters=128, kernel_size=16, strides=1, padding='same', data_format='channels_last', activation='relu')(input_3D)
    conv2 = tf.keras.layers.Conv2D(filters=64, kernel_size=16, strides=1, padding='same', data_format='channels_last', activation='relu')(conv1)
    conv3 = tf.keras.layers.Conv2D(filters=32, kernel_size=16, strides=1, padding='same', data_format='channels_last', activation='relu')(conv2)
    conv4 = tf.keras.layers.Conv2D(filters=16, kernel_size=16, strides=1, padding='same', data_format='channels_last', activation='relu')(conv3)
    conv5 = tf.keras.layers.Conv2D(filters=1, kernel_size=16, strides=1, padding='same', data_format='channels_last', activation='relu')(conv4)
    flatten1 = tf.keras.layers.Flatten()(conv5)
    dense1 = tf.keras.layers.Dense(128, activation='relu')(flatten1)
    dense2 = tf.keras.layers.Dense(128, activation='relu')(dense1)
    
    input_2D = tf.keras.Input(shape=(n,3))
    conv6 = tf.keras.layers.Conv1D(filters=1024, kernel_size=16, strides=1, padding='same', activation='relu')(input_2D)
    conv7 = tf.keras.layers.Conv1D(filters=1024, kernel_size=16, strides=1, padding='same', activation='relu')(conv6)
    conv8 = tf.keras.layers.Conv1D(filters=1024, kernel_size=16, strides=1, padding='same', activation='relu')(conv7)
    flatten2 = tf.keras.layers.Flatten()(conv8)
    dense3 = tf.keras.layers.Dense(128, activation='relu')(flatten2)
    dense4 = tf.keras.layers.Dense(128, activation='relu')(dense3)
    
    merged_vector = tf.keras.layers.concatenate([dense2, dense4])
    dense5 = tf.keras.layers.Dense(128, activation='relu')(merged_vector)
    dropout1 = tf.keras.layers.Dropout(0.25)(dense5)
    dense6 = tf.keras.layers.Dense(24, activation='softmax')(dropout1)
    
    model = Model(inputs=[input_3D, input_2D], outputs=dense6)

    
    # Set up simple model layers
    simple_input_3D = tf.keras.Input(shape=(n,3,3))
    simple_input_2D = tf.keras.Input(shape=(n,3))
    simple_conv1 = tf.keras.layers.Conv1D(filters=512, kernel_size=16, strides=1, padding='same', activation='relu')(simple_input_2D)
    simple_conv2 = tf.keras.layers.Conv1D(filters=512, kernel_size=16, strides=1, padding='same', activation='relu')(simple_conv1)
    simple_conv3 = tf.keras.layers.Conv1D(filters=512, kernel_size=16, strides=1, padding='same', activation='relu')(simple_conv2)
    simple_flatten = tf.keras.layers.Flatten()(simple_conv3)
    simple_dense1 = tf.keras.layers.Dense(128, activation='relu')(simple_flatten)
    simple_dense2 = tf.keras.layers.Dense(24, activation='softmax')(simple_dense1)
    
    simple_model = Model(inputs=[simple_input_3D, simple_input_2D], outputs=simple_dense2)

    
    # Decaying learning rate for optimizer
    lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
        0.001,
        decay_steps = 1000,
        decay_rate = 0.95,
        staircase = True)
    
    opt_adam = tf.keras.optimizers.Adam(lr_schedule)
    
    
    # Compile models
    model.compile(optimizer=opt_adam,
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])
    
    simple_model.compile(optimizer=opt_adam,
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    
#     # Fit models using generators
#     BS = 128  
         
#     train_gen = generate_batch_matrices(training_x, training_y, n, batch_size=BS)
#     valid_gen = generate_batch_matrices(testing_x, testing_y, n, batch_size=BS)
    
#     print('Fitting parallel model')
#     EPOCHS = 5#100
#     model.fit_generator(train_gen,
#                         steps_per_epoch=training_x.shape[0] // BS,
#                         validation_data=valid_gen,
#                         validation_steps=testing_x.shape[0] // BS,
#                         epochs=EPOCHS, verbose=1, use_multiprocessing=True, shuffle=True)
    
#     print('Fitting simple model')
#     EPOCHS = 5#150 
#     simple_model.fit_generator(train_gen,
#                         steps_per_epoch=training_x.shape[0] // BS,
#                         validation_data=valid_gen,
#                         validation_steps=testing_x.shape[0] // BS,
#                         epochs=EPOCHS, verbose=1, use_multiprocessing=True, shuffle=True)

    BS = 6000
    EPOCHS = 100    
    model.fit([training_x_3d, training_x], training_y, batch_size=BS, epochs=EPOCHS, 
              validation_data=[[testing_x_3d, testing_x], testing_y], verbose=1, shuffle=False)
    
    EPOCHS = 150    
    simple_model.fit([training_x_3d, training_x], training_y, batch_size=BS, epochs=EPOCHS, 
              validation_data=[[testing_x_3d, testing_x], testing_y], verbose=1, shuffle=False)
    
    
#     # Find predictions and results
#     print('Analysing results...')
#     test, labels = valid_gen.__next__()

#     raw_predictions = model.predict(test)
#     simple_raw_predictions = simple_model.predict(test)
#     ensemble_raw_predictions = np.mean(np.stack((raw_predictions, raw_predictions, simple_raw_predictions, simple_raw_predictions, simple_raw_predictions), axis=-1), axis=-1)
      
#     predictions = np.argmax(raw_predictions, axis=1)
#     simple_predictions = np.argmax(simple_raw_predictions, axis=1)
#     ensemble_predictions = np.argmax(ensemble_raw_predictions, axis=1)
    
#     labels = np.argmax(labels, axis=1)

#     results = model.evaluate_generator(valid_gen, steps=testing_x.shape[0] // BS, use_multiprocessing=True)
#     accuracy_score = (results[1])
#     simple_results = model.evaluate_generator(valid_gen, steps=testing_x.shape[0] // BS, use_multiprocessing=True)
#     simple_accuracy_score = (simple_results[1])

    raw_predictions = model.predict([testing_x_3d, testing_x])
    simple_raw_predictions = simple_model.predict([testing_x_3d, testing_x])
    ensemble_raw_predictions = np.mean(np.stack((raw_predictions, raw_predictions, simple_raw_predictions, simple_raw_predictions, simple_raw_predictions), axis=-1), axis=-1)
    
    predictions = np.argmax(raw_predictions, axis=1)
    simple_predictions = np.argmax(simple_raw_predictions, axis=1)
    ensemble_predictions = np.argmax(ensemble_raw_predictions, axis=1)

    labels = np.argmax(testing_y, axis=1)
    
    return predictions, simple_predictions, ensemble_predictions, labels

def analysis(predictions, simple_predictions, ensemble_predictions, labels, marker_labels):

    # Plot graphs comparing predictions vs. testing results
    fig1 = plt.figure(figsize=(15,5))
    plt.plot(predictions[:60], 'b')
    plt.plot(simple_predictions[:60], 'y')
    plt.plot(ensemble_predictions[:60], 'g')
    plt.plot(labels[:60], 'k')
    plt.yticks(np.arange(24), marker_labels)
    plt.ylabel('Marker')
    plt.title('Predictions')
    plt.show()
    
    # Determine which markers are mislabelled
    incorrect = np.zeros(24)
    errors = 0
    for i in range(len(ensemble_predictions)):
        if ensemble_predictions[i] != labels[i]:
            incorrect[labels[i]] += 1
            errors += 1
            print(str(marker_labels[labels[i]]) +' incorrectly labelled as ' + str(marker_labels[ensemble_predictions[i]]))
            if predictions[i] == labels[i]:
                print('correctly labelled by parallel model')
            elif simple_predictions[i] == labels[i]:
                print('correctly labelled by simple model')
            else:
                print('correctly labelled by neither model')
    
    
    fig2 = plt.figure(figsize=(15,5))
    plt.bar(np.arange(24), incorrect, align='center', alpha=0.5)
    plt.xticks(np.arange(24), marker_labels)
    plt.xlabel('Marker')
    plt.ylabel('Errors')
    plt.title('Number of errors for each marker')
    plt.show()
    
    ensemble_accuracy = 1 - errors/len(labels)
    
    return errors, ensemble_accuracy

def cross_validate(base_dir, folds, model, markers, n): # This function uses cross-validation to determine the true performance of the model using data from selected markers as inputs
    
    print('Cross-validating')
    
    start = 0
    total = len(os.listdir(base_dir))
    end = subs_per_fold = total // folds

    accuracy_scores = []


    with tqdm_notebook(total=folds) as pbar:
      
        print('---------------------------------------')
      
        for k in range(folds):

            test_subjects = os.listdir(base_dir)[start:end]
            train_subjects = [i for i in os.listdir(base_dir) if i not in test_subjects]

            # Trains and tests for each fold, returning scores, using desired markers
            (accuracy_score) = n_all_markers(base_dir, train_subjects, test_subjects, model, n)

            accuracy_scores.append(accuracy_score)

            start += subs_per_fold
            end += subs_per_fold
            
            pbar.update(1)
              

    # Find mean scores, finds cross-validated scores
    cv_accuracy = np.mean(accuracy_scores)
    
    return cv_accuracy

def settings(base_dir, folds, model, markers, n): # Take in settings and apply desired model
    
    # Print settings information
    print('Settings')
    print('Model:\t\t' + model)
    print('Base directory:\t' + str(base_dir))
    print('Folds:\t\t' + str(folds))
    print('Markers:\t' + markers)
    print('---------------------------------------')
    
    (cv_accuracy) = cross_validate(base_dir, folds, model, markers, n)
    
    # Print cross-validated scores
    print('---------------------------------------')
    print('CV Accuracy:\t' + str(cv_accuracy))

# Machine learning model to use
model = 'neural network'

# Location of processed matrices containing kinematics and forceplate data
base_directory = '/content/drive/My Drive/Colab Notebooks/Processed Force and Kinematics Data/'
  
# Number of lines to use for calculating features
n = 50

# Number of folds to use in k-fold cross-validation
num_folds = 5

# Which markers to use; 'all' or 'selected'
markers = 'all'


settings(base_directory, num_folds, model, markers, n)