[Python] MediaPipe hand detection works with direct camera access but fails with camera...

I have two different implementations of a hand gesture detection system using MediaPipe, where one works perfectly and the other doesn't.

Working Implementation:

from flask import Flask, render_template, Response
import cv2
import mediapipe as mp
import numpy as np # Added numpy import
import time

app = Flask(__name__)

class FingerCounter:
def __init__(self):
self.mp_hands = mp.solutions.hands
self.hands = self.mp_hands.Hands(
max_num_hands=2,
min_detection_confidence=0.5,
min_tracking_confidence=0.5
)
self.mp_draw = mp.solutions.drawing_utils

def count_fingers(self, hand_landmarks, handedness):
tip_ids = [4, 8, 12, 16, 20]
fingers = []

# Thumb - different logic for left and right hands
if handedness.classification[0].label == 'Right':
if hand_landmarks.landmark[tip_ids[0]].x < hand_landmarks.landmark[tip_ids[0] - 1].x:
fingers.append(1)
else:
fingers.append(0)
else: # Left hand
if hand_landmarks.landmark[tip_ids[0]].x > hand_landmarks.landmark[tip_ids[0] - 1].x:
fingers.append(1)
else:
fingers.append(0)

# Other fingers - same for both hands
for id in range(1, 5):
if hand_landmarks.landmark[tip_ids[id]].y < hand_landmarks.landmark[tip_ids[id] - 2].y:
fingers.append(1)
else:
fingers.append(0)

return fingers.count(1)

def process_frame(self, frame):
image = cv2.flip(frame, 1)
image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
results = self.hands.process(image_rgb)

if results.multi_hand_landmarks:
for idx, (hand_landmarks, handedness) in enumerate(zip(results.multi_hand_landmarks, results.multi_handedness)):
self.mp_draw.draw_landmarks(
image,
hand_landmarks,
self.mp_hands.HAND_CONNECTIONS,
self.mp_draw.DrawingSpec(color=(0, 255, 0), thickness=2),
self.mp_draw.DrawingSpec(color=(255, 0, 0), thickness=2)
)

# Count fingers with handedness information
finger_count = self.count_fingers(hand_landmarks, handedness)

# Position text based on hand
text_x = 50 if handedness.classification[0].label == 'Left' else image.shape[1] - 250
cv2.putText(
image,
f'{handedness.classification[0].label}: {finger_count}',
(text_x, 50),
cv2.FONT_HERSHEY_SIMPLEX,
1,
(255, 255, 255),
3
)
cv2.putText(
image,
f'{handedness.classification[0].label}: {finger_count}',
(text_x, 50),
cv2.FONT_HERSHEY_SIMPLEX,
1,
(0, 0, 255),
2
)

# Update total count
cv2.putText(
image,
f'Total Fingers: {sum(1 for hand in results.multi_hand_landmarks for _ in range(self.count_fingers(hand, results.multi_handedness[results.multi_hand_landmarks.index(hand)])))}',
(image.shape[1]//2 - 100, 100),
cv2.FONT_HERSHEY_SIMPLEX,
1,
(255, 255, 255),
3
)
cv2.putText(
image,
f'Total Fingers: {sum(1 for hand in results.multi_hand_landmarks for _ in range(self.count_fingers(hand, results.multi_handedness[results.multi_hand_landmarks.index(hand)])))}',
(image.shape[1]//2 - 100, 100),
cv2.FONT_HERSHEY_SIMPLEX,
1,
(0, 0, 255),
2
)

return image

counter = FingerCounter()

def get_camera():
# Try different camera indices
for index in range(10):
cap = cv2.VideoCapture(index)
if cap.isOpened():
return cap
return None

camera = get_camera()
if camera is None:
print("No camera found!")

def generate_frames():
while True:
if camera is None:
# Create a black frame with error message if no camera
frame = np.zeros((480, 640, 3), dtype=np.uint8)
cv2.putText(frame, "No camera found!", (200, 240),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
ret, buffer = cv2.imencode('.jpg', frame)
frame = buffer.tobytes()
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')
time.sleep(1)
continue

success, frame = camera.read()
if not success:
break
else:
processed_frame = counter.process_frame(frame)
ret, buffer = cv2.imencode('.jpg', processed_frame)
frame = buffer.tobytes()
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')

@app.route('/')
def index():
return render_template('index.html')

@app.route('/video_feed')
def video_feed():
return Response(generate_frames(),
mimetype='multipart/x-mixed-replace; boundary=frame')

if __name__ == '__main__':
app.run(host='127.0.0.1', port=8080)

1. index.html:

<!-- templates/index.html -->
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Finger Counter Web App</title>
<style>
body {
font-family: Arial, sans-serif;
display: flex;
flex-direction: column;
align-items: center;
background-color: #f0f0f0;
margin: 0;
padding: 20px;
}

.container {
max-width: 800px;
width: 100%;
text-align: center;
}

.video-container {
margin: 20px auto;
width: 640px;
height: 480px;
background-color: #000;
border-radius: 10px;
overflow: hidden;
}

#video_feed {
width: 100%;
height: 100%;
object-fit: cover;
}

h1 {
color: #2C3E50;
margin-bottom: 20px;
}

.status {
margin-top: 20px;
font-size: 18px;
color: #333;
padding: 10px;
background-color: white;
border-radius: 5px;
box-shadow: 0 2px 5px rgba(0,0,0,0.1);
}
</style>
</head>
<body>
<div class="container">
<h1>Hand Gesture Finger Counter</h1>
<div class="video-container">
<img id="video_feed" src="{{ url_for('video_feed') }}" alt="Video feed">
</div>
<div class="status">
Show your hand to the camera to count fingers
</div>
</div>
</body>
</html>

Issue Description:

• The first implementation (shown above) works perfectly both locally and in production
• It accurately detects hands and counts fingers for both left and right hands
• The second implementation (i have added a github repo link for the second implementation), with camera permission button:
• Successfully starts the camera when clicked
• Shows video feed correctly
• But MediaPipe hand detection fails to initialize/work

Second Implementation Github Repo

Continue reading...