pie_predict.py

"""
The code implementation of the paper:

A. Rasouli, I. Kotseruba, T. Kunic, and J. Tsotsos, "PIE: A Large-Scale Dataset and Models for Pedestrian Intention Estimation and
Trajectory Prediction", ICCV 2019.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""
import os
import time
import pickle
import numpy as np

from keras.layers import Input, RepeatVector, Dense, Permute
from keras.layers import Concatenate, Multiply, Dropout
from keras.layers.recurrent import LSTM
from keras.models import Model, load_model
from keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint
from keras.optimizers import RMSprop
from keras import regularizers

class PIEPredict(object):
    """
    An encoder decoder model for pedestrian trajectory prediction

    Attributes:
       _num_hidden_units: Number of LSTM hidden units
       _regularizer_value: The value of L2 regularizer for training
       _regularizer: Training regularizer set as L2
       _activation: LSTM actications
       _embed_size: The size embedding unit applied to the representation produced by encoder
       _embed_dropout: the dropout of embedding unit

    Model attributes: The following attributes will be set during training depending on the training data
       _observe_length: Observation duration in frames (number of time steps of the encoder)
       _predict_length: Prediciton duration in frames (number of time steps of the decoder)
       _encoder_feature_size: The number of data points entering the encoder, e.g. for 'bounding boxes' the size is 4 (x1 y1 x2 y2)
       _decoder_feature_size: The number of data points entering the decoder, e.g. for 'speed' the size is 1
       _prediction_size: The size of the decoder output after dense layer, e.g. for 'bounding boxes' the size is 4 (x1 y1 x2 y2)

    Methods:
        get_tracks: Generates trajectory tracks by sampling from pedestrian sequences
        get_data_helper: Create training data by combining sequences with different modalities
        get_data: Generates training and testing data
        log_configs: Writes model and training configurations to a file
        train: Trains the model
        test: Tests the model
        pie_encdec: Generates the network model
        create_lstm_model: A helper function for creating an LSTM unit
        attention_temporal: Temporal attention custom layer
        attention_element: Elementwise attention custom layer
    """
    def __init__(self,
                 num_hidden_units=256,
                 regularizer_val=0.0001,
                 activation='softsign',
                 embed_size=64,
                 embed_dropout=0):

        # Network parameters
        self._num_hidden_units = num_hidden_units
        self._regularizer_value = regularizer_val
        self._regularizer = regularizers.l2(regularizer_val)

        self._activation = activation
        self._embed_size = embed_size
        self._embed_dropout = embed_dropout

        # model parameters
        self._observe_length = 15
        self._predict_length = 15

        self._encoder_feature_size = 4
        self._decoder_feature_size = 4

        self._prediction_size = 4

    def get_tracks(self, dataset, data_types, observe_length, predict_length, overlap, normalize):
        """
        Generates tracks by sampling from pedestrian sequences
        :param dataset: The raw data passed to the method
        :param data_types: Specification of types of data for encoder and decoder. Data types depend on datasets. e.g.
        JAAD has 'bbox', 'ceneter' and PIE in addition has 'obd_speed', 'heading_angle', etc.
        :param observe_length: The length of the observation (i.e. time steps of the encoder)
        :param predict_length: The length of the prediction (i.e. time steps of the decoder)
        :param overlap: How much the sampled tracks should overlap. A value between [0,1) should be selected
        :param normalize: Whether to normalize center/bounding box coordinates, i.e. convert to velocities. NOTE: when
        the tracks are normalized, observation length becomes 1 step shorter, i.e. first step is removed.
        :return: A dictinary containing sampled tracks for each data modality
        """
        #  Calculates the overlap in terms of number of frames
        seq_length = observe_length + predict_length
        overlap_stride = observe_length if overlap == 0 else \
            int((1 - overlap) * observe_length)
        overlap_stride = 1 if overlap_stride < 1 else overlap_stride

        #  Check the validity of keys selected by user as data type
        d = {}
        for dt in data_types:
            try:
                d[dt] = dataset[dt]
            except KeyError:
                raise ('Wrong data type is selected %s' % dt)

        d['image'] = dataset['image']
        d['pid'] = dataset['pid']

        #  Sample tracks from sequneces
        for k in d.keys():
            tracks = []
            for track in d[k]:
                tracks.extend([track[i:i + seq_length] for i in
                               range(0, len(track) - seq_length + 1, overlap_stride)])
            d[k] = tracks

        #  Normalize tracks by subtracting bbox/center at first time step from the rest
        if normalize:
            if 'bbox' in data_types:
                for i in range(len(d['bbox'])):
                    d['bbox'][i] = np.subtract(d['bbox'][i][1:], d['bbox'][i][0]).tolist()
            if 'center' in data_types:
                for i in range(len(d['center'])):
                    d['center'][i] = np.subtract(d['center'][i][1:], d['center'][i][0]).tolist()
            #  Adjusting the length of other data types
            for k in d.keys():
                if k != 'bbox' and k != 'center':
                    for i in range(len(d[k])):
                        d[k][i] = d[k][i][1:]

        return d

    def get_data_helper(self, data, data_type):
        """
        A helper function for data generation that combines different data types into a single representation
        :param data: A dictionary of different data types
        :param data_type: The data types defined for encoder and decoder input/output
        :return: A unified data representation as a list
        """
        if not data_type:
            return []
        d = []
        for dt in data_type:
            if dt == 'image':
                continue
            d.append(np.array(data[dt]))

        #  Concatenate different data points into a single representation
        if len(d) > 1:
            return np.concatenate(d, axis=2)
        else:
            return d[0]

    def get_data(self, data, **model_opts):
        """
        Main data generation function for training/testing
        :param data: The raw data
        :param model_opts: Control parameters for data generation characteristics (see below for default values)
        :return: A dictionary containing training and testing data
        """
        opts = {
            'normalize_bbox': True,
            'track_overlap': 0.5,
            'observe_length': 15,
            'predict_length': 45,
            'enc_input_type': ['bbox'],
            'dec_input_type': [],
            'prediction_type': ['bbox']
        }

        for key, value in model_opts.items():
            assert key in opts.keys(), 'wrong data parameter %s' % key
            opts[key] = value

        observe_length = opts['observe_length']
        data_types = set(opts['enc_input_type'] + opts['dec_input_type'] + opts['prediction_type'])
        data_tracks = self.get_tracks(data, data_types, observe_length,
                                      opts['predict_length'], opts['track_overlap'],
                                      opts['normalize_bbox'])

        if opts['normalize_bbox']:
            observe_length -= 1

        obs_slices = {}
        pred_slices = {}

        #  Generate observation/prediction sequences from the tracks
        for k in data_tracks.keys():
            obs_slices[k] = []
            pred_slices[k] = []
            obs_slices[k].extend([d[0:observe_length] for d in data_tracks[k]])
            pred_slices[k].extend([d[observe_length:] for d in data_tracks[k]])

        # Generate observation data input to encoder
        enc_input = self.get_data_helper(obs_slices, opts['enc_input_type'])

        # Generate data for prediction decoder
        dec_input = self.get_data_helper(pred_slices, opts['dec_input_type'])
        pred_target = self.get_data_helper(pred_slices, opts['prediction_type'])

        if not len(dec_input) > 0:
            dec_input = np.zeros(shape=pred_target.shape)

        return {'obs_image': obs_slices['image'],
                'obs_pid': obs_slices['pid'],
                'pred_image': pred_slices['image'],
                'pred_pid': pred_slices['pid'],
                'enc_input': enc_input,
                'dec_input': dec_input,
                'pred_target': pred_target,
                'model_opts': opts}

    def get_path(self,
                 file_name='',
                 save_folder='models',
                 dataset='pie',
                 model_type='trajectory',
                 save_root_folder='data/'):
        """
        A path generator method for saving model and config data. It create directories if needed.
        :param file_name: The actual save file name , e.g. 'model.h5'
        :param save_folder: The name of folder containing the saved files
        :param dataset: The name of the dataset used
        :param save_root_folder: The root folder
        :return: The full path for the model name and the path to the final folder
        """
        save_path = os.path.join(save_root_folder, dataset, model_type, save_folder)
        if not os.path.exists(save_path):
            os.makedirs(save_path)
        return os.path.join(save_path, file_name), save_path

    def log_configs(self, config_path, batch_size, epochs,
                    lr, loss, learning_scheduler, opts):
        """
        Logs the parameters of the model and training
        :param config_path: The path to save the file
        :param batch_size: Batch size of training
        :param epochs: Number of epochs for training
        :param lr: Learning rate of training
        :param loss: Type of loss function
        :param learning_scheduler: Whether learning scheduler was used
        :param opts: Model options (see get_data)
        """
        # Save config and training param files
        with open(config_path, 'wt') as fid:
            fid.write("####### Model options #######\n")
            for k in opts:
                fid.write("%s: %s\n" % (k, str(opts[k])))

            fid.write("\n####### Network config #######\n")
            fid.write("%s: %s\n" % ('hidden_units', str(self._num_hidden_units)))
            fid.write("%s: %s\n" % ('reg_value ', str(self._regularizer_value)))
            fid.write("%s: %s\n" % ('activation', str(self._activation)))
            fid.write("%s: %s\n" % ('embed_size', str(self._embed_size)))
            fid.write("%s: %s\n" % ('embed_dropout', str(self._embed_dropout)))

            fid.write("%s: %s\n" % ('observe_length', str(self._observe_length)))
            fid.write("%s: %s\n" % ('predict_length ', str(self._predict_length)))
            fid.write("%s: %s\n" % ('encoder_feature_size', str(self._encoder_feature_size)))
            fid.write("%s: %s\n" % ('decoder_feature_size', str(self._decoder_feature_size)))
            fid.write("%s: %s\n" % ('prediction_size', str(self._prediction_size)))

            fid.write("\n####### Training config #######\n")
            fid.write("%s: %s\n" % ('batch_size', str(batch_size)))
            fid.write("%s: %s\n" % ('epochs', str(epochs)))
            fid.write("%s: %s\n" % ('lr', str(lr)))
            fid.write("%s: %s\n" % ('loss', str(loss)))
            fid.write("%s: %s\n" % ('learning_scheduler', str(learning_scheduler)))

        print('Wrote configs to {}'.format(config_path))

    def train(self, data_train, data_val,
              batch_size=64,
              epochs=60,
              lr=0.001,
              loss='mse',
              learning_scheduler=True,
              **model_opts):
        """
        Training method for the model
        :param data_train: Training data
        :param data_val: Validation data
        :param batch_size: Batch size of training
        :param epochs: Number of epochs for training
        :param lr: Learning rate of training
        :param loss: Type of loss function
        :param learning_scheduler: Whether learning scheduler was used
        :param model_opts: Data generation parameters (see get_data)
        :return: The path to where the final model is saved
        """
        optimizer = RMSprop(lr=lr)

        train_data = self.get_data(data_train, **model_opts)
        val_data = self.get_data(data_val, **model_opts)

        print("Number of samples:\n Train: %d \n Val: %d \n"
              % (train_data['enc_input'].shape[0], val_data['enc_input'].shape[0]))

        self._observe_length = train_data['enc_input'].shape[1]
        self._predict_length = train_data['pred_target'].shape[1]

        self._encoder_feature_size = train_data['enc_input'].shape[2]
        self._decoder_feature_size = train_data['dec_input'].shape[2]

        # Set the output sizes
        self._prediction_size = train_data['pred_target'].shape[2]

        # Set path names for saving configs and model
        model_folder_name = time.strftime("%d%b%Y-%Hh%Mm%Ss")

        if 'bbox' in model_opts['prediction_type']:
            model_type = 'trajectory'
        else:
            model_type = 'speed'
        print(model_type)

        model_path, _ = self.get_path(save_folder=model_folder_name,
                                      model_type=model_type,
                                      file_name='model.h5')

        # Save data parameters
        opts_path, _ = self.get_path(save_folder=model_folder_name,
                                     model_type=model_type,
                                     file_name='model_opts.pkl')


        with open(opts_path, 'wb') as fid:
            pickle.dump(train_data['model_opts'], fid,
                        pickle.HIGHEST_PROTOCOL)

        # save training and model parameters
        config_path, _ = self.get_path(save_folder=model_folder_name,
                                       model_type=model_type,
                                       file_name='configs.txt')
        self.log_configs(config_path, batch_size, epochs,
                         lr, loss, learning_scheduler,
                         train_data['model_opts'])

        pie_model = self.pie_encdec()

        # Generate training data
        train_data = ([train_data['enc_input'],
                       train_data['dec_input']],
                      train_data['pred_target'])

        val_data = ([val_data['enc_input'],
                     val_data['dec_input']],
                    val_data['pred_target'])

        pie_model.compile(loss=loss, optimizer=optimizer)

        print("##############################################")
        print(" Training for predicting sequences of size %d" % self._predict_length)
        print("##############################################")

        checkpoint = ModelCheckpoint(filepath=model_path,
                                     save_best_only=True,
                                     save_weights_only=False,
                                     monitor='val_loss')
        call_backs = [checkpoint]

        #  Setting up learning schedulers
        if learning_scheduler:
            early_stop = EarlyStopping(monitor='val_loss',
                                       min_delta=1.0, patience=10,
                                       verbose=1)
            plateau_sch = ReduceLROnPlateau(monitor='val_loss',
                                            factor=0.2, patience=5,
                                            min_lr=1e-07, verbose=1)
            call_backs.extend([early_stop, plateau_sch])

        history = pie_model.fit(x=train_data[0], y=train_data[1],
                                batch_size=batch_size, epochs=epochs,
                                validation_data=val_data, verbose=1,
                                callbacks=call_backs)

        print('Train model is saved to {}'.format(model_path))

        history_path, saved_files_path = self.get_path(save_folder=model_folder_name,
                                                       model_type=model_type,
                                                       file_name='history.pkl')

        with open(history_path, 'wb') as fid:
            pickle.dump(history.history, fid, pickle.HIGHEST_PROTOCOL)

        return saved_files_path

    def test(self, data_test, model_path=''):
        """
        Testing method for the model
        :param data_test: Testing data
        :param model_path: The path to where the model to be tested is saved
        :return: Mean squared error (MSE) of the prediction
        """
        test_model = load_model(os.path.join(model_path, 'model.h5'))
        test_model.summary()

        with open(os.path.join(model_path, 'model_opts.pkl'), 'rb') as fid:
            try:
                model_opts = pickle.load(fid)
            except:
                model_opts = pickle.load(fid, encoding='bytes')

        test_data = self.get_data(data_test, **model_opts)
        test_obs_data = [test_data['enc_input'], test_data['dec_input']]
        test_target_data = test_data['pred_target']

        test_results = test_model.predict(test_obs_data, batch_size=2048, verbose=1)

        perf = {}
        #  Performance on bounding boxes
        performance = np.square(test_target_data - test_results)
        perf['mse'] = performance.mean(axis=None)
        perf['mse_last'] = performance[:, -1, :].mean(axis=None)

        # print("MSE  %f" % perf['mse'])
        # print("mse-15: %.2f\nmse-30: %.2f\nmse-45: %.2f"
        #       % (perf['mse-15'], perf['mse-30'], perf['mse']))
        # print("MSE last %f" % perf['mse_last'])

        if model_opts['prediction_type'][0] == 'bbox':
            #  Performance on centers (displacement)
            model_opts['normalize_bbox'] = False
            test_data = self.get_data(data_test, **model_opts)
            test_obs_data_org = [test_data['enc_input'], test_data['dec_input']]
            test_target_data_org = test_data['pred_target']

            results_org = test_results + np.expand_dims(test_obs_data_org[0][:, 0, 0:4], axis=1)

            #  Performance measures for centers
            res_centers = np.zeros(shape=(test_results.shape[0], test_results.shape[1], 2))
            centers = np.zeros(shape=(test_results.shape[0], test_results.shape[1], 2))
            for b in range(test_results.shape[0]):
                for s in range(test_results.shape[1]):
                    centers[b, s, 0] = (test_target_data_org[b, s, 2] + test_target_data_org[b, s, 0]) / 2
                    centers[b, s, 1] = (test_target_data_org[b, s, 3] + test_target_data_org[b, s, 1]) / 2
                    res_centers[b, s, 0] = (results_org[b, s, 2] + results_org[b, s, 0]) / 2
                    res_centers[b, s, 1] = (results_org[b, s, 3] + results_org[b, s, 1]) / 2

            c_performance = np.square(centers - res_centers)
            perf['center_mse'] = c_performance.mean(axis=None)
            perf['center_mse_last'] = c_performance[:, -1, :].mean(axis=None)

            # # print("Center MSE  %f" % perf['center_mse'])
            # print("c-mse-15: %.2f\nc-mse-30: %.2f\nc-mse-45: %.2f"
            #       % (perf['c-mse-15'], perf['c-mse-30'], perf['center_mse']))
            # print("Center MSE last %f" % perf['center_mse_last'])

        save_results_path = os.path.join(model_path,
                                         '{:.2f}.pkl'.format(perf['mse']))
        save_performance_path = os.path.join(model_path,
                                         '{:.2f}.txt'.format(perf['mse']))

        with open(save_performance_path, 'wt') as fid:
            for k in sorted(perf.keys()):
                fid.write("%s: %s\n" % (k, str(perf[k])))

        if not os.path.exists(save_results_path):
            try:
                results = {'img_seqs': data_test['pred_image'],
                           'results': test_results,
                           'gt': test_target_data,
                           'performance': perf}
            except:
                results = {'img_seqs': [],
                           'results': test_results,
                           'gt': test_target_data,
                           'performance': perf}

            with open(save_results_path, 'wb') as fid:
                pickle.dump(results, fid, pickle.HIGHEST_PROTOCOL)

        return perf

    def test_final(self, data_test, traj_model_path='', intent_model_path='', speed_model_path=''):

        intent_path = os.path.join(intent_model_path, 'ped_intents.pkl')
        with open(intent_path, 'rb') as fid:
            try:
                intent = pickle.load(fid)
            except:
                intent = pickle.load(fid, encoding='bytes')

        model_opts = {'normalize_bbox': True,
                       'track_overlap': 0.5,  # 0.8 for jaad, 0.5 pie
                       'observe_length': 15,
                       'predict_length': 45,
                       'enc_input_type': ['bbox'],
                       'dec_input_type': [],
                       'prediction_type': ['bbox']}

        box_data = self.get_data(data_test, **model_opts)

        intent_dic = {}
        for pid, img, r in zip(intent['ped_id'], intent['images'], intent['results']):
            img_name = img[0].split('/')[-1].split('.')[0]
            p_id = pid[0][0]
            if p_id in intent_dic.keys():
                intent_dic[p_id][img_name] = r
            else:
                intent_dic[p_id] = {img_name: r}

        int_data = np.zeros(shape=(box_data['pred_target'].shape[0], box_data['pred_target'].shape[1], 1))
        obs_pids = box_data['obs_pid']
        obs_images = box_data['obs_image']

        intent_list = []
        last_ped = ''
        for i in range(len(obs_pids)):
            pid = obs_pids[i][0][0]
            if pid != last_ped:
                intent_list = []
                last_ped = pid

            # check if the current id exists in intention results
            if pid in intent_dic:
                img_name = obs_images[i][0].split('/')[-1].split('.')[0]
                # if id exists we check if there is a value for the given sequence
                if img_name in intent_dic[pid]:
                    intent_result = intent_dic[pid][img_name]
                    intent_list.append(intent_result)
                    int_data[i] = np.array([intent_result] * box_data['pred_target'].shape[1])
                else:
                    # if there is no value we check whether previously not value was observed
                    if intent_list == []:
                        int_data[i] = np.array([[0.5]] * box_data['pred_target'].shape[1])
                    else:
                        # if there is no value we check and list is not empty, we use average results
                        int_avg = np.mean(np.array(intent_list))
                        int_data[i] = np.array([[int_avg]] * box_data['pred_target'].shape[1])
            else:
                # if the id does not exist just populate with 0.5
                int_data[i] = np.array([[0.5]] * box_data['pred_target'].shape[1])

        #speed_path = '/home/aras/PycharmProjects/release_code/test/speed_models/pie_speed'
        speed_model = load_model(os.path.join(speed_model_path, 'model.h5'))

        #bis_path = '/home/aras/PycharmProjects/release_code/test/box_intent_speed'
        box_intent_speed_model = load_model(os.path.join(traj_model_path, 'model.h5'))

        ################## run speed model ####################
        model_opts['enc_input_type'] = ['obd_speed']
        model_opts['prediction_type'] = ['obd_speed']
        speed_data = self.get_data(data_test, **model_opts)

        _speed_data = [speed_data['enc_input'], speed_data['dec_input']]
        speed_results = speed_model.predict(_speed_data,
                                            batch_size=2056,
                                            verbose=1)

        # speed intent
        int_speed = np.concatenate([int_data, speed_results], axis=2)
        test_results = box_intent_speed_model.predict([box_data['enc_input'], int_speed],
                                                      batch_size=2056, verbose=1)

        # Performance measures for bounding boxes
        perf = {}
        performance = np.square(test_results - box_data['pred_target'])
        perf['mse-15'] = performance[:, 0:15, :].mean(axis=None)
        perf['mse-30'] = performance[:, 0:30, :].mean(axis=None)  # 15:30
        perf['mse-45'] = performance.mean(axis=None)
        perf['mse-last'] = performance[:, -1, :].mean(axis=None)

        # print("mse-15: %.2f\nmse-30: %.2f\nmse-45: %.2f"
        #       % (perf['mse-15'], perf['mse-30'], perf['mse-45']))
        # print("mse-last %.2f\n" % (perf['mse-last']))

        #  Performance on centers (displacement)
        model_opts['normalize_bbox'] = False
        model_opts['enc_input_type'] = ['bbox']
        model_opts['prediction_type'] = ['bbox']
        test_data = self.get_data(data_test, **model_opts)
        test_obs_data_org = [test_data['enc_input'], test_data['dec_input']]
        test_target_data_org = test_data['pred_target']

        results_org = test_results + np.expand_dims(test_obs_data_org[0][:, 0, 0:4], axis=1)

        #  Performance measures for centers
        res_centers = np.zeros(shape=(test_results.shape[0], test_results.shape[1], 2))
        centers = np.zeros(shape=(test_results.shape[0], test_results.shape[1], 2))
        for b in range(test_results.shape[0]):
            for s in range(test_results.shape[1]):
                centers[b, s, 0] = (test_target_data_org[b, s, 2] + test_target_data_org[b, s, 0]) / 2
                centers[b, s, 1] = (test_target_data_org[b, s, 3] + test_target_data_org[b, s, 1]) / 2
                res_centers[b, s, 0] = (results_org[b, s, 2] + results_org[b, s, 0]) / 2
                res_centers[b, s, 1] = (results_org[b, s, 3] + results_org[b, s, 1]) / 2

        c_performance = np.square(centers - res_centers)
        perf['c-mse-15'] = c_performance[:, 0:15, :].mean(axis=None)
        perf['c-mse-30'] = c_performance[:, 0:30, :].mean(axis=None)  # 0:30
        perf['c-mse-45'] = c_performance.mean(axis=None)
        perf['c-mse-last'] = c_performance[:, -1, :].mean(axis=None)

        # print("c-mse-15: %.2f\nc-mse-30: %.2f\nc-mse-45: %.2f" \
        #       % (perf['c-mse-15'], perf['c-mse-30'], perf['c-mse-45']))
        # print("c-mse-last: %.2f\n" % (perf['c-mse-last']))

        save_results_path = os.path.join(traj_model_path,
                                         '{:.2f}.pkl'.format(perf['mse-45']))
        save_performance_path = os.path.join(traj_model_path,
                                             '{:.2f}.txt'.format(perf['mse-45']))

        with open(save_performance_path, 'wt') as fid:
            for k in sorted(perf.keys()):
                fid.write("%s: %s\n" % (k, str(perf[k])))

        if not os.path.exists(save_results_path):
            try:
                results = {'img_seqs': box_data['pred_image'],
                           'results': test_results,
                           'gt': box_data['pred_target'],
                           'performance': perf}
            except:
                results = {'img_seqs': [],
                           'results': test_results,
                           'gt': box_data['pred_target'],
                           'performance': perf}
            with open(save_results_path, 'wb') as fid:
                pickle.dump(results, fid, pickle.HIGHEST_PROTOCOL)
        return perf  # performance

    def pie_encdec(self):
        """
        Generates the encoder decoder method
        :return: An instance of the network model
        """

        # Generate input data. the shapes is (sequence_lenght,length of flattened features)
        _encoder_input = Input(shape=(self._observe_length, self._encoder_feature_size),
                               name='encoder_input')

        # Temporal attention module
        _attention_net = self.attention_temporal(_encoder_input, self._observe_length)

        # Generate Encoder LSTM Unit
        encoder_model = self.create_lstm_model(name='encoder_network')
        _encoder_outputs_states = encoder_model(_attention_net)
        _encoder_states = _encoder_outputs_states[1:]

        # Generate Decoder LSTM unit
        decoder_model = self.create_lstm_model(name='decoder_network', r_state=False)
        _hidden_input = RepeatVector(self._predict_length)(_encoder_states[0])
        _decoder_input = Input(shape=(self._predict_length, self._decoder_feature_size),
                               name='pred_decoder_input')

        # Embedding unit on the output of Encoder
        _embedded_hidden_input = Dense(self._embed_size, activation='relu')(_hidden_input)
        _embedded_hidden_input = Dropout(self._embed_dropout,
                                         name='dropout_dec_input')(_embedded_hidden_input)

        decoder_concat_inputs = Concatenate(axis=2)([_embedded_hidden_input, _decoder_input])

        # Self attention unit
        att_input_dim = self._embed_size + self._decoder_feature_size
        decoder_concat_inputs = self.attention_element(decoder_concat_inputs, att_input_dim)

        # Initialize the decoder with encoder states
        decoder_output = decoder_model(decoder_concat_inputs,
                                       initial_state=_encoder_states)
        decoder_output = Dense(self._prediction_size,
                               activation='linear',
                               name='decoder_dense')(decoder_output)

        net_model = Model(inputs=[_encoder_input, _decoder_input],
                          outputs=decoder_output)
        net_model.summary()

        return net_model

    def create_lstm_model(self, name='lstm', r_state=True, r_sequence=True):
        """
        A Helper function that generates an instance of LSTM
        :param name: Name of the layer
        :param r_state: Whether to return states
        :param r_sequence: Whether to return sequences
        :return: An LSTM instance
        """

        return LSTM(units=self._num_hidden_units,
                    return_state=r_state,
                    return_sequences=r_sequence,
                    stateful=False,
                    kernel_regularizer=self._regularizer,
                    recurrent_regularizer=self._regularizer,
                    bias_regularizer=self._regularizer,
                    activity_regularizer=None,
                    activation=self._activation,
                    name=name)

    # Custom layers
    def attention_temporal(self, input_data, sequence_length):
        """
        A temporal attention layer
        :param input_data: Network input
        :param sequence_length: Length of the input sequence
        :return: The output of attention layer
        """
        a = Permute((2, 1))(input_data)
        a = Dense(sequence_length, activation='sigmoid')(a)
        a_probs = Permute((2, 1))(a)
        output_attention_mul = Multiply()([input_data, a_probs])
        return output_attention_mul

    def attention_element(self, input_data, input_dim):
        """
        A self-attention unit
        :param input_data: Network input
        :param input_dim: The feature dimension of the input
        :return: The output of the attention network
        """
        input_data_probs = Dense(input_dim, activation='sigmoid')(input_data)  # sigmoid
        output_attention_mul = Multiply()([input_data, input_data_probs])  # name='att_mul'
        return output_attention_mul