[Python] getting coordinate info from mediapipe hand landmark and writing json file

I wrote a code for obtaining time series data(in json format) of two hands' skeletone coordinates from a sign language video.

Basicly The json file's format should be

There are 272 frames and as you can know, each frames should have one Left hand coordinates and Right hand ones.

I haven't checked every single frames but it seems that Right hands' coordinates are ok. But some of Left hands' coordinates are missing.

And another problem is the information of left hand should come first, but it seems that information of right habds seem to come first on some frames.

I guess there would be problem just with the video or the code can't detect the skeletone.

What should I do?

The below is the code

import json
import cv2
import mediapipe as mp

mp_drawing = mp.solutions.drawing_utils
mp_hands = mp.solutions.hands
hands = mp_hands.Hands()

HAND_LANDMARKS = [
"wrist", "thumb_cmc", "thumb_mcp", "thumb_ip", "thumb_tip",
"index_finger_mcp", "index_finger_pip", "index_finger_dip", "index_finger_tip",
"middle_finger_mcp", "middle_finger_pip", "middle_finger_dip", "middle_finger_tip",
"ring_finger_mcp", "ring_finger_pip", "ring_finger_dip", "ring_finger_tip",
"pinky_mcp", "pinky_pip", "pinky_dip", "pinky_tip"
]

def output_video(input_video):
frame_width = int(input_video.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(input_video.get(cv2.CAP_PROP_FRAME_HEIGHT))
fps = int(input_video.get(cv2.CAP_PROP_FPS))
return cv2.VideoWriter(
'output/result.mov',
cv2.VideoWriter_fourcc(*'avc1'),
fps,
(frame_width, frame_height)
)

def extract_landmarks(hand_landmarks):
""" Extracting coordinates of hand landmark and returning them to list """
landmarks = []
for (idx, landmark) in enumerate(hand_landmarks.landmark):
landmarks.append({
'x': landmark.x,
'y': landmark.y,
'body_part': HAND_LANDMARKS[idx]
})
return landmarks

def write_json(output_json_path: str, result_json):
""" saving as json format """
with open(output_json_path, 'w') as json_file:
json.dump(result_json, json_file, indent=4)

def process():
""" extracting hand landmark from video and saving to json """
frame_index = 0
result_json = [] # List for saving data from all frames

capture = cv2.VideoCapture('[video path]')
output = output_video(capture)

while capture.isOpened():
ret, frame = capture.read()
if not ret:
break

# frame to RGB and detect Hand pose
results = hands.process(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
frame_data = {'frame': frame_index, 'hands': []} # Current frame data

# if hand being detected
if results.multi_hand_landmarks and results.multi_handedness:
for hand_landmarks, handedness in zip(results.multi_hand_landmarks, results.multi_handedness):
hand_label = handedness.classification[0].label # "Left" or "Right"

# drawing hand landmark
mp_drawing.draw_landmarks(
frame, hand_landmarks, mp_hands.HAND_CONNECTIONS
)

# Saving the detected hand info
frame_data['hands'].append({
'hand': hand_label,
'landmarks': extract_landmarks(hand_landmarks)
})

result_json.append(frame_data) # Adding current frame data
frame_index += 1
output.write(frame)

write_json('[json file path]', result_json)
capture.release()
output.release()

process()


And these are the frames with problem { "frame": 75, "hands": [] },

(frame missing information of hand landmark)

{
"frame": 76,
"hands": [
{
"hand": "Right",
"landmarks": [
{
"x": 0.5663378834724426,
"y": 0.6888471245765686,
"body_part": "wrist"
},


... (Information of Left hand should come first but right one came first) (Or missing left hands' or right hands' information) (Or 2 left or right hands' information on one frame)

What should I do to solve this problem?

Continue reading...