Action Detection-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d009b44",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%pip install mediapipe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d26e663a",
   "metadata": {},
   "outputs": [],
   "source": [
    "!python3 --version\n",
    "!python3 -m pip --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bff95b8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install --upgrade pip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f8fa28e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8bef0eca",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1845804a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c694187",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "16cf443f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os \n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "import cv2\n",
    "import mediapipe as mp"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "798e1512",
   "metadata": {},
   "source": [
    "# KeyPoints using MP holistic\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6fef6750",
   "metadata": {},
   "outputs": [],
   "source": [
    "mp_holistic = mp.solutions.holistic  # holistic model\n",
    "mp_drawing = mp.solutions.drawing_utils # drawing utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64b09f6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def mediapipe_detection(img,model) : \n",
    "    img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n",
    "    img.flags.writeable = 0\n",
    "    results = model.process(img)\n",
    "    img.flags.writeable = 1\n",
    "    img = cv2.cvtColor(img,cv2.COLOR_RGB2BGR)\n",
    "    return img, results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7db0ec3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_landmarks(img,results):\n",
    "    mp_drawing.draw_landmarks(img,results.face_landmarks,mp_holistic.FACEMESH_TESSELATION)\n",
    "    mp_drawing.draw_landmarks(img,results.pose_landmarks,mp_holistic.POSE_CONNECTIONS)\n",
    "    mp_drawing.draw_landmarks(img,results.left_hand_landmarks,mp_holistic.HAND_CONNECTIONS)\n",
    "    mp_drawing.draw_landmarks(img,results.right_hand_landmarks,mp_holistic.HAND_CONNECTIONS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cdaea80",
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_styled_landmarks(img,results):\n",
    "    # Draw face connections \n",
    "    mp_drawing.draw_landmarks(img,results.face_landmarks,mp_holistic.FACEMESH_TESSELATION,\n",
    "                              # dots color\n",
    "                             mp_drawing.DrawingSpec(color=(80,110,10),thickness=1,circle_radius=1),\n",
    "                              # connection lines color\n",
    "                             mp_drawing.DrawingSpec(color=(80,256,121),thickness=1,circle_radius=1)\n",
    "                              \n",
    "                             )\n",
    "    # Draw Body land marks \n",
    "    mp_drawing.draw_landmarks(img,results.pose_landmarks,mp_holistic.POSE_CONNECTIONS,\n",
    "                             mp_drawing.DrawingSpec(color=(80,22,10),thickness=2,circle_radius=4),\n",
    "                             mp_drawing.DrawingSpec(color=(80,44,121),thickness=2,circle_radius=2)\n",
    "                              \n",
    "                             )\n",
    "    # Draw left hand landmarks\n",
    "    mp_drawing.draw_landmarks(img,results.left_hand_landmarks,mp_holistic.HAND_CONNECTIONS,\n",
    "                             mp_drawing.DrawingSpec(color=(121,22,76),thickness=2,circle_radius=4),\n",
    "                             mp_drawing.DrawingSpec(color=(121,44,250),thickness=2,circle_radius=2)\n",
    "                              )\n",
    "    # Draw right hand landmarks\n",
    "    mp_drawing.draw_landmarks(img,results.right_hand_landmarks,mp_holistic.HAND_CONNECTIONS,\n",
    "                             mp_drawing.DrawingSpec(color=(245,117,66),thickness=2,circle_radius=4),\n",
    "                             mp_drawing.DrawingSpec(color=(245,66,230),thickness=2,circle_radius=2)\n",
    "                              )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33020de8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# cap = cv2.VideoCapture(0)\n",
    "# with mp_holistic.Holistic(min_detection_confidence=.5,min_tracking_confidence=.5) as holistic:\n",
    "#     while cap.isOpened():\n",
    "#         ret,frame = cap.read()\n",
    "        \n",
    "#         # make detecitons \n",
    "#         img, results = mediapipe_detection(frame,holistic)\n",
    "#         print(results)\n",
    "        \n",
    "#         # drawing land marks\n",
    "#         draw_styled_landmarks(img,results)\n",
    "         \n",
    "#         cv2.imshow('Video',img)\n",
    "#         if cv2.waitKey(1) & 0xFF ==ord('q'):\n",
    "#             break\n",
    "#     cap.release()\n",
    "#     cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f2a5de0",
   "metadata": {},
   "source": [
    "# Extract Keypoint Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cdbe64bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_keypoints(results):\n",
    "    pose = np.array([[res.x,res.y,res.z,res.visibility] for res in results.pose_landmarks.landmark]).flatten() if results.pose_landmarks else np.zeros(33*4)\n",
    "    face = np.array([[res.x,res.y,res.z] for res in results.face_landmarks.landmark]).flatten() if results.face_landmarks else np.zeros(468*3)\n",
    "    lh = np.array([[res.x,res.y,res.z,] for res in results.left_hand_landmarks.landmark]).flatten() if results.left_hand_landmarks else np.zeros(21*3)  \n",
    "    rh = np.array([[res.x,res.y,res.z,] for res in results.right_hand_landmarks.landmark]).flatten() if results.right_hand_landmarks else np.zeros(21*3)  \n",
    "    return np.concatenate([pose,face,lh,rh])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1c85ac9",
   "metadata": {},
   "source": [
    "# Setup Folders for collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c11e1595",
   "metadata": {},
   "outputs": [],
   "source": [
    "#  Path fo exportinh data , numpy arrays\n",
    "DATA_PATH = os.path.join('MP_DATA')\n",
    "\n",
    "#  actions that we try to detect\n",
    "actions  = np.array(['HELLO',\"THANKS\",\"VICTORY\"])\n",
    "\n",
    "# 30 videos worth of data\n",
    "no_sequences= 30\n",
    "\n",
    "#  videos are going to be 30 frames in length\n",
    "sequence_lenght = 30 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "576875f5",
   "metadata": {},
   "outputs": [],
   "source": [
    " for action in actions:\n",
    "        for sequence in range(no_sequences):\n",
    "            try: \n",
    "                os.makedirs(os.path.join(DATA_PATH,action, str(sequence)))\n",
    "            except:\n",
    "                pass"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3158568",
   "metadata": {},
   "source": [
    "# Collect keypoint values for training and testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53863ce2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# cap = cv2.VideoCapture(0)\n",
    "# with mp_holistic.Holistic(min_detection_confidence=.5,min_tracking_confidence=.5) as holistic:\n",
    "# #     Loop through actions\n",
    "#     for action in actions:\n",
    "# #         loop through sequences aka videos\n",
    "#         for sequence in range(no_sequences):\n",
    "# #         Loop through video lrnght aka sequence length\n",
    "#             for frame_num in range(sequence_lenght):\n",
    "            \n",
    "#                 ret,frame = cap.read()\n",
    "\n",
    "#                 # make detecitons \n",
    "#                 img, results = mediapipe_detection(frame,holistic)\n",
    "#                 print(results)\n",
    "\n",
    "#                 # drawing land marks\n",
    "#                 draw_styled_landmarks(img,results)\n",
    "                \n",
    "# #                  wait logic \n",
    "#                 if frame_num==0:\n",
    "#                     cv2.putText(img,'STARTING COLLECTING',(120,200),\n",
    "#                                 cv2.FONT_HERSHEY_COMPLEX,1,(0,255,0),1,cv2.LINE_AA)\n",
    "#                     cv2.putText(img,'Collecting Frames for {} Video Number {}'.format(action,sequence),(15,12),\n",
    "#                                 cv2.FONT_HERSHEY_COMPLEX,0.5,(0,0,255),1,cv2.LINE_AA)\n",
    "#                     cv2.waitKey(2000)\n",
    "#                 else:\n",
    "#                     cv2.putText(img,'Collecting Frames for {} Video Number {}'.format(action,sequence),(15,12),\n",
    "#                                 cv2.FONT_HERSHEY_COMPLEX,0.5,(0,0,255),1,cv2.LINE_AA)                    \n",
    "# #                 New export keypoints \n",
    "#                 keypoints = extract_keypoints(results)\n",
    "#                 npy_path = os.path.join(DATA_PATH,action,str(sequence),str(frame_num))\n",
    "#                 np.save(npy_path,keypoints)\n",
    "            \n",
    "            \n",
    "#                 cv2.imshow('Video',img)\n",
    "#                 if cv2.waitKey(1) & 0xFF ==ord('q'):\n",
    "#                     break\n",
    "#     cap.release()\n",
    "#     cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "649d3ec7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# cap.release()\n",
    "# cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e511b4c5",
   "metadata": {},
   "source": [
    "# Preprocess Data and Create Labels and features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e4665f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a6bf50b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!pip install tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "433d5681",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1ae82c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "label_map = {label:num for num ,label in enumerate(actions)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9d0f5a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "label_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06c3c9ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "sequencs, labels = [],[]\n",
    "for action in actions:\n",
    "    for sequence in range(no_sequences):\n",
    "        window=[]\n",
    "        for frame_num in range(sequence_lenght):\n",
    "            res= np.load(os.path.join(DATA_PATH,action,str(sequence),\"{}.npy\".format(frame_num)))\n",
    "            window.append(res)\n",
    "        sequencs.append(window)\n",
    "        labels.append(label_map[action])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2933b7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "sequencs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f0ebd56",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array(sequencs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8169c569",
   "metadata": {},
   "outputs": [],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9158043",
   "metadata": {},
   "outputs": [],
   "source": [
    "y = to_categorical(labels).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc59aa51",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b578a6a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=.05)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7778049",
   "metadata": {},
   "source": [
    "# Build and train LSTM Neural Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e5bcb65",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import LSTM,Dense\n",
    "from tensorflow.keras.callbacks import TensorBoard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb2c60a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir = os.path.join('Logs')\n",
    "tb_callback = TensorBoard(log_dir=log_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94a5a1c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(LSTM(64,return_sequences=True,activation = \"relu\",input_shape=(30,1662)))\n",
    "model.add(LSTM(128,return_sequences=True,activation = \"relu\"))\n",
    "model.add(LSTM(64,return_sequences=False,activation = \"relu\"))\n",
    "model.add(Dense(64,activation='relu'))\n",
    "model.add(Dense(32,activation='relu'))\n",
    "model.add(Dense(actions.shape[0],activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96971437",
   "metadata": {},
   "outputs": [],
   "source": [
    "res = [.7,.2,.1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae72c607",
   "metadata": {},
   "outputs": [],
   "source": [
    "actions[np.argmax(res)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "696d4d78",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['categorical_accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b165f4be",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit(X_train,y_train,epochs=2000,callbacks=[tb_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a13ae7c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6dbb282",
   "metadata": {},
   "outputs": [],
   "source": [
    "res = np.argmax(model.predict(X_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50558fc8",
   "metadata": {},
   "source": [
    "res[0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91bff01e",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5706f10d",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat_labels = [np.argmax(i) for i in y_hat]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "108e896d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a317dc1f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4e5ce62",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d734e26",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae290e1d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4958a0ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "actions[np.argmax([y_test[1]])]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94d80bd6",
   "metadata": {},
   "source": [
    "# Save Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e829dc30",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('Actions.keras')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3cc7508a",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights('Actions.keras')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c14209ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('Banglore_home_prices_model.pickle','wb') as f:\n",
    "    pickle.dump(lr_clf,f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7fc3470e",
   "metadata": {},
   "source": [
    "# Evaluation using confusion Matrix and accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "277ec71b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import multilabel_confusion_matrix,accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb5131f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat = model.predict(X_train)\n",
    "ytrue = np.argmax(y_train,axis=1).tolist()\n",
    "yhat = np.argmax(yhat,axis=1).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43cbe191",
   "metadata": {},
   "outputs": [],
   "source": [
    "cm = multilabel_confusion_matrix(ytrue,yhat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "154aad2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bffbaa2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sn\n",
    "plt.figure(figsize=(10,7))\n",
    "sn.heatmap(cm[0],annot=True,fmt='d')\n",
    "plt.title(\"HELLO\")\n",
    "plt.xlabel('Predicted')\n",
    "plt.ylabel('True')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3886f7d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sn\n",
    "plt.figure(figsize=(10,7))\n",
    "sn.heatmap(cm[1],annot=True,fmt='d')\n",
    "plt.title(\"THANK YOU\")\n",
    "plt.xlabel('Predicted')\n",
    "plt.ylabel('True')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "033c58bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sn\n",
    "plt.figure(figsize=(10,7))\n",
    "sn.heatmap(cm[2],annot=True,fmt='d')\n",
    "plt.title(\"VICTORY\")\n",
    "plt.xlabel('Predicted')\n",
    "plt.ylabel('True')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6db8aaa7",
   "metadata": {},
   "source": [
    "# test in real time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f08f4426",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#  New detection variables\n",
    "sequence = []\n",
    "sentence = []\n",
    "threshold=.4\n",
    "cap = cv2.VideoCapture(0)\n",
    "with mp_holistic.Holistic(min_detection_confidence=.5,min_tracking_confidence=.5) as holistic:\n",
    "    while cap.isOpened():\n",
    "        ret,frame = cap.read()\n",
    "        \n",
    "        # make detecitons \n",
    "        img, results = mediapipe_detection(frame,holistic)\n",
    "#         print(results)\n",
    "        \n",
    "#         ########################### prediction logic\n",
    "        keypoints = extract_keypoints(results)\n",
    "        sequence.insert(0,keypoints)\n",
    "        sequence  = sequence[:30]\n",
    "        \n",
    "        if len(sequence)==30:\n",
    "            res = model.predict(np.expand_dims(sequence,axis=0))[0]\n",
    "            print(actions[np.argmax(res)])\n",
    "\n",
    "            \n",
    "#          ###############   Viz logic ##################\n",
    "        if res[np.argmax(res)] > threshold:\n",
    "            if len(sentence) >0 :\n",
    "                if actions[np.argmax(res)] != sentence[-1]:\n",
    "                    sentence.append(actions[np.argmax(res)])\n",
    "                else:\n",
    "                    sentence.append(actions[np.argmax(res)])\n",
    "        if len(sentence)>5:\n",
    "            sentence  =sentence[-5:]\n",
    "        \n",
    "        cv2.rectangle(img,(0,0),(640,40),(245,117,16),-1)\n",
    "        cv2.putText(img,' '.join(sentence),(3,30),\n",
    "                   cv2.FONT_HERSHEY_COMPLEX,1,(255,255,255),2,cv2.LINE_AA)\n",
    "\n",
    "\n",
    "        # #################################### drawing land marks\n",
    "        draw_styled_landmarks(img,results)\n",
    "         \n",
    "        cv2.imshow('Video',img)\n",
    "        if cv2.waitKey(1) & 0xFF ==ord('q'):\n",
    "            break\n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1b019cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.expand_dims(X_test[0],axis=0).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfd378fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "colors = [(245,117,16), (117,245,16), (16,117,245)]\n",
    "def prob_viz(res, actions, input_frame, colors):\n",
    "    output_frame = input_frame.copy()\n",
    "    for num, prob in enumerate(res):\n",
    "        cv2.rectangle(output_frame, (0,60+num*40), (int(prob*100), 90+num*40), colors[num], -1)\n",
    "        cv2.putText(output_frame, actions[num], (0, 85+num*40), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,255), 2, cv2.LINE_AA)\n",
    "        \n",
    "    return output_frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ee41e45",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(18,18))\n",
    "plt.imshow(prob_viz(res,actions,img,colors))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a1e9ba0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. New detection variables\n",
    "sequence = []\n",
    "sentence = []\n",
    "predictions = []\n",
    "threshold = 0.5\n",
    "\n",
    "cap = cv2.VideoCapture(0)\n",
    "# Set mediapipe model \n",
    "with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:\n",
    "    while cap.isOpened():\n",
    "\n",
    "        # Read feed\n",
    "        ret, frame = cap.read()\n",
    "\n",
    "        # Make detections\n",
    "        img, results = mediapipe_detection(frame, holistic)\n",
    "        print(results)\n",
    "        \n",
    "        # Draw landmarks\n",
    "#         draw_styled_landmarks(image, results)\n",
    "        \n",
    "        # 2. Prediction logic\n",
    "        keypoints = extract_keypoints(results)\n",
    "        sequence.append(keypoints)\n",
    "        sequence = sequence[-30:]\n",
    "        \n",
    "        if len(sequence) == 30:\n",
    "            res = model.predict(np.expand_dims(sequence, axis=0))[0]\n",
    "            print(actions[np.argmax(res)])\n",
    "            predictions.append(np.argmax(res))\n",
    "            \n",
    "            \n",
    "        #3. Viz logic\n",
    "            if np.unique(predictions[-10:])[0]==np.argmax(res): \n",
    "                if res[np.argmax(res)] > threshold: \n",
    "                    \n",
    "                    if len(sentence) > 0: \n",
    "                        if actions[np.argmax(res)] != sentence[-1]:\n",
    "                            sentence.append(actions[np.argmax(res)])\n",
    "                    else:\n",
    "                        sentence.append(actions[np.argmax(res)])\n",
    "\n",
    "            if len(sentence) > 5: \n",
    "                sentence = sentence[-5:]\n",
    "\n",
    "            # #### Viz probabilities\n",
    "            img = prob_viz(res, actions, img, colors)\n",
    "            \n",
    "        cv2.rectangle(img, (0,0), (640, 40), (245, 117, 16), -1)\n",
    "        cv2.putText(img, ' '.join(sentence), (3,30), \n",
    "                       cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA)\n",
    "        \n",
    "        # Show to screen\n",
    "        cv2.imshow('OpenCV Feed', img)\n",
    "\n",
    "        # Break gracefully\n",
    "        if cv2.waitKey(10) & 0xFF == ord('q'):\n",
    "            break\n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "659ddfe6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a0fb1dc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}