[Python] How to Count Visible Landmarks with MediaPipe Face Mesh and ZED2 Camera in Python?

I’m working on a project that uses a ZED2 camera with the MediaPipe Face Mesh model to detect and visualize facial landmarks. The goal is to count the number of visible landmarks dynamically, especially when the face is partially turned or covered. However, I’m facing an issue where the number of landmarks always stays at 0.

What’s happening: The code is consistently returning 0 landmarks even when the face is totally visible.

What I expect: When the face is turned, partially covered, or in different orientations, the number of visible landmarks should dynamically change.

Here's the code:

import pyzed.sl as sl
import cv2
import mediapipe as mp

# ZED Initialization
zed = sl.Camera()
init_params = sl.InitParameters()
init_params.camera_resolution = sl.RESOLUTION.HD720
init_params.depth_mode = sl.DEPTH_MODE.ULTRA
init_params.coordinate_units = sl.UNIT.METER

if zed.open(init_params) != sl.ERROR_CODE.SUCCESS:
print("ZED could not be opened.")
exit(1)

runtime_parameters = sl.RuntimeParameters()
image_zed = sl.Mat()

# Mediapipe FaceMesh Setup
mp_face_mesh = mp.solutions.face_mesh
face_mesh = mp_face_mesh.FaceMesh(static_image_mode=False, max_num_faces=1, refine_landmarks=True)

def count_visible_landmarks(landmarks):
visible_count = 0
# Iterate through all landmarks and count the ones with visibility > 0.5
for landmark in landmarks:
if landmark.visibility > 0.5: # Threshold for visibility
visible_count += 1
return visible_count

def update():
if zed.grab(runtime_parameters) == sl.ERROR_CODE.SUCCESS:
# Retrieve the current frame from the ZED camera
zed.retrieve_image(image_zed, sl.VIEW.LEFT)
img = image_zed.get_data()

# Convert the image from BGRA to RGB (for use with Mediapipe)
img_rgb = cv2.cvtColor(img, cv2.COLOR_BGRA2RGB)

# Process the image using Mediapipe's face mesh
results = face_mesh.process(img_rgb)

if results.multi_face_landmarks:
# For each face detected, calculate the number of visible landmarks
for face_landmarks in results.multi_face_landmarks:
visible_landmarks_count = count_visible_landmarks(face_landmarks.landmark)
print(f"Visible landmarks: {visible_landmarks_count}")
else:
print("No face detected.")

# Continuously call the update function until the program is stopped
while True:
update()
if cv2.waitKey(1) & 0xFF == ord('q'):
break

# Close the camera and windows after the program ends
zed.close()
cv2.destroyAllWindows()

Continue reading...