-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathcameraBackground.py
146 lines (118 loc) · 6 KB
/
cameraBackground.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
from mediapipe.framework.formats import landmark_pb2
from mediapipe import solutions
from mediapipe.tasks import python
from mediapipe.tasks.python import vision
import cv2
import json
import mediapipe as mp
import numpy as np
import os
# Add this near the top of your script, after your imports
cv2.namedWindow('Face Mesh and Hands', cv2.WINDOW_NORMAL)
from dataclasses import dataclass
from typing import Tuple, Dict
@dataclass
class DrawingSpec:
color: Tuple[int, int, int]
thickness: int
FACEMESH_LEFT_IRIS = mp.solutions.face_mesh.FACEMESH_LEFT_IRIS
FACEMESH_RIGHT_IRIS = mp.solutions.face_mesh.FACEMESH_RIGHT_IRIS
def iris_connection_style():
face_mesh_iris_connections_style = {}
left_spec = DrawingSpec(color=(0, 255, 0), thickness=2)
right_spec = DrawingSpec(color=(255, 0, 0), thickness=2)
for connection in FACEMESH_LEFT_IRIS:
face_mesh_iris_connections_style[connection] = left_spec
for connection in FACEMESH_RIGHT_IRIS:
face_mesh_iris_connections_style[connection] = right_spec
return face_mesh_iris_connections_style
def draw_landmarks_on_image(image, face_detection_result, hand_detection_result):
if face_detection_result.face_landmarks:
for face_landmarks in face_detection_result.face_landmarks:
face_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
face_landmarks_proto.landmark.extend([
landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in face_landmarks
])
solutions.drawing_utils.draw_landmarks(
image=image,
landmark_list=face_landmarks_proto,
connections=mp.solutions.face_mesh.FACEMESH_TESSELATION,
landmark_drawing_spec=None,
connection_drawing_spec=mp.solutions.drawing_styles.get_default_face_mesh_tesselation_style())
solutions.drawing_utils.draw_landmarks(
image=image,
landmark_list=face_landmarks_proto,
connections=mp.solutions.face_mesh.FACEMESH_CONTOURS,
landmark_drawing_spec=None,
connection_drawing_spec=mp.solutions.drawing_styles.get_default_face_mesh_contours_style())
solutions.drawing_utils.draw_landmarks(
image=image,
landmark_list=face_landmarks_proto,
connections=mp.solutions.face_mesh.FACEMESH_IRISES,
landmark_drawing_spec=None,
connection_drawing_spec=iris_connection_style())
# draw hand landmarks
if hand_detection_result.hand_landmarks:
for hand_landmarks in hand_detection_result.hand_landmarks:
hand_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
hand_landmarks_proto.landmark.extend([
landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in hand_landmarks
])
solutions.drawing_utils.draw_landmarks(
image,
hand_landmarks_proto,
mp.solutions.hands.HAND_CONNECTIONS,
mp.solutions.drawing_styles.get_default_hand_landmarks_style(),
mp.solutions.drawing_styles.get_default_hand_connections_style())
return image
# create facelandmarker and handlandmarker with gpu support
face_model_path = os.path.join('models', 'face_landmarker.task')
hand_model_path = os.path.join('models', 'hand_landmarker.task')
base_options = python.BaseOptions(model_asset_path=face_model_path, delegate=python.BaseOptions.Delegate.CPU)
face_options = vision.FaceLandmarkerOptions(base_options=base_options,
output_face_blendshapes=True,
output_facial_transformation_matrixes=True,
num_faces=1)
face_detector = vision.FaceLandmarker.create_from_options(face_options)
base_options = python.BaseOptions(model_asset_path=hand_model_path, delegate=python.BaseOptions.Delegate.CPU)
hand_options = vision.HandLandmarkerOptions(base_options=base_options,
num_hands=2)
hand_detector = vision.HandLandmarker.create_from_options(hand_options)
# initialize webcam with lower resolution
cap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 800)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 600)
cap.set(cv2.CAP_PROP_FPS, 60)
# target display size
display_width, display_height = 800, 600
from collections import deque
jaw_open_values = deque(maxlen=40)
cv2.resizeWindow('Face Mesh and Hands', 1024, 768) # or whatever size you want
while True:
ret, frame = cap.read()
frame = cv2.flip(frame, 1)
if not ret:
break
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb_frame)
face_result = face_detector.detect(mp_image)
hand_result = hand_detector.detect(mp_image)
frame_data = {"face_landmarks": [], "hand_landmarks": []}
if face_result.face_landmarks:
for blendshapes in face_result.face_blendshapes:
jaw_open = next((b.score for b in blendshapes if b.category_name == 'jawOpen'), None)
if jaw_open is not None:
jaw_open_values.append(jaw_open)
frame_data["face_landmarks"] = [[{"x": p.x, "y": p.y, "z": p.z} for p in face] for face in face_result.face_landmarks]
if hand_result.hand_landmarks:
frame_data["hand_landmarks"] = [[{"x": p.x, "y": p.y, "z": p.z} for p in hand] for hand in hand_result.hand_landmarks]
with open('landmark_data.json', 'w') as f:
json.dump(frame_data, f)
annotated_image = draw_landmarks_on_image(rgb_frame, face_result, hand_result)
annotated_image = cv2.cvtColor(annotated_image, cv2.COLOR_RGB2BGR)
display_image = cv2.resize(annotated_image, (display_width, display_height), interpolation=cv2.INTER_LINEAR)
cv2.imshow('Face Mesh and Hands', display_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()