[Python] Segments too short for OpenSmile feature processing

I am currently working on creating a time series of feature vectors from video input with consistent capture at 30 fps, ignoring the fps of the input video. I have decided to use PyAV to split the video input into audio and video streams, and grab frames each 1/30th second and the audio segment at that time. The frames are then fed into MediaPipe Face Landmarker and the audio segment into Opensmile LLD feature extractor and appended to the time-series together. I am having trouble as of now because OpenSMILE wont extract features because it thinks my segments are too small. How would I go about solving this? This is the main file:

from mediapipe.tasks.python import vision
import mediapipe as mp
import numpy as np
import cv2
import numpy as np
import opensmile
import av

base = mp.tasks.BaseOptions(model_asset_path="face_landmarker.task")
options = vision.FaceLandmarkerOptions(base_options=base,
num_faces=1)
detector = vision.FaceLandmarker.create_from_options(options)

smile = opensmile.Smile(
feature_set=opensmile.FeatureSet.eGeMAPSv02,
feature_level=opensmile.FeatureLevel.LowLevelDescriptors
)

def get_features(image_rgb, audio_signal, sr):
print("Starting feature extraction")
# Grab feature array from facial landmarks using MediaPipe
image = mp.Image(
image_format=mp.ImageFormat.SRGB,
data=image_rgb
)

result = detector.detect(image)

face = result.face_landmarks[0]
face_features = np.array([[lm.x, lm.y, lm.z] for lm in face]).flatten()

print(audio_signal)
audio_features = smile.process_signal(audio_signal, sr).to_numpy().flatten()

print(audio_features)
# Concatenate
final_array = np.concatenate((face_features, audio_features))
print("Saved array!")

return final_array

def mead_feature_list(video_path):
target_fps = 30
window = 1.0 / target_fps

container = av.open(video_path)
audio_stream = container.streams.audio[0]
video_stream = container.streams.video[0]

audio_buffer = []
for packet in container.demux(audio_stream):
for frame in packet.decode():
t = frame.pts * audio_stream.time_base
audio_buffer.append((t, frame.to_ndarray()))

container.seek(0)
video_stream = container.streams.video[0]

time_series = []
next_t = 0.0
audio_index = 0

for frame in container.decode(video_stream):
t = frame.pts * video_stream.time_base
if t < next_t:
continue

image = frame.to_ndarray(format="rgb24")

audio_segment = []
while audio_index < len(audio_buffer):
audio_t, samples = audio_buffer[audio_index]
if audio_t < t + window:
audio_segment.append(samples)
audio_index += 1
else:
break

expected_samples = int(window * audio_stream.rate)

if len(audio_segment) == 0:
audio_data = np.zeros(expected_samples, dtype="float32")
else:
audio_data = np.concatenate(audio_segment, axis=0).astype("float32") / 32768.0

if len(audio_data) < expected_samples:
pad = expected_samples - len(audio_data)
audio_data = np.pad(audio_data, (0, pad), mode='constant')

if audio_data.ndim == 2 and audio_data.shape[1] > 1:
audio_data = audio_data.mean(axis=1)

print(audio_segment)
time_series.append(get_features(image, audio_data, audio_stream.rate))
next_t += window


return time_series

print(mead_feature_list("test_video/video_test_first.mp4"))


And here is the log, no errors but Opensmile fills array audio_features with NaNs as you can see:


(mead_env) WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
C:\Users\user\Documents\code\ml\mead\micro-expression-analysis-device-mead>python mead_feature_extract_main.py
W0000 00:00:1763949432.629800 11316 face landmarker_graph.cc:174] Sets FaceBlendshapesGraph acceleration to xnnpack by default. INFO: Created TensorFlow Lite XNNPACK delegate for CPU.
We000 00:00:1763949432.709222 1308 inference_feedback_manager.cc:114] Feedback manager requires a model with a single signature inference. Disabling support for feedback tensors.
We000 00:00:1763949432.746479 1388 inference_feedback_manager.cc:114] Feedback manager requires a model with a single signature inference. Disabling support for feedback tensors.
[]
Starting feature extraction
[0. 0. 0. ... . . .]
C:\Users\user\miniconda3\envs\mead_env\Lib\site-packages\opensmile\core\smile.py: 297: UserWarning: Segment too short, filling with
NaN.
warnings.warn(UserWarning("Segment too short, filling with NaN."))
[nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan
nan nan nan nan nan nan nan]
Saved array!
[]
Starting feature extraction
[0. 0. 0. ... 0. 0. 0.]
C:\Users\user\miniconda3\envs\mead_env\Lib\site-packages\opensmile\core\smile.py: 297: UserWarning: Segment too short, filling with
NaN.
warnings.warn(UserWarning("Segment too short, filling with NaN."))
[nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan
nan nan nan nan nan nan nan]
Saved array!
[ ]

Continue reading...