Сегменты слишком короткие для обработки функций OpenSmile ⇐ Python

[Anonymous]

В настоящее время я работаю над созданием временного ряда векторов признаков на основе видеовхода с последовательным захватом со скоростью 30 кадров в секунду, игнорируя частоту кадров входного видео. Я решил использовать PyAV для разделения видеовхода на аудио- и видеопотоки и захвата кадров каждую 1/30 секунды и аудиосегмента в это время. Затем кадры передаются в MediaPipe Face Landmarker, а аудиосегмент — в экстрактор функций Opensmile LLD и вместе добавляются к временному ряду. На данный момент у меня возникли проблемы, потому что OpenSMILE не извлекает функции, поскольку считает, что мои сегменты слишком малы. Как мне решить эту проблему? Это основной файл:
from mediapipe.tasks.python import vision
import mediapipe as mp
import numpy as np
import cv2
import numpy as np
import opensmile
import av

base = mp.tasks.BaseOptions(model_asset_path="face_landmarker.task")
options = vision.FaceLandmarkerOptions(base_options=base,
num_faces=1)
detector = vision.FaceLandmarker.create_from_options(options)

smile = opensmile.Smile(
feature_set=opensmile.FeatureSet.eGeMAPSv02,
feature_level=opensmile.FeatureLevel.LowLevelDescriptors
)

def get_features(image_rgb, audio_signal, sr):
print("Starting feature extraction")
# Grab feature array from facial landmarks using MediaPipe
image = mp.Image(
image_format=mp.ImageFormat.SRGB,
data=image_rgb
)

result = detector.detect(image)

face = result.face_landmarks[0]
face_features = np.array([[lm.x, lm.y, lm.z] for lm in face]).flatten()

print(audio_signal)
audio_features = smile.process_signal(audio_signal, sr).to_numpy().flatten()

print(audio_features)
# Concatenate
final_array = np.concatenate((face_features, audio_features))
print("Saved array!")

return final_array

def mead_feature_list(video_path):
target_fps = 30
window = 1.0 / target_fps

container = av.open(video_path)
audio_stream = container.streams.audio[0]
video_stream = container.streams.video[0]

audio_buffer = []
for packet in container.demux(audio_stream):
for frame in packet.decode():
t = frame.pts * audio_stream.time_base
audio_buffer.append((t, frame.to_ndarray()))

container.seek(0)
video_stream = container.streams.video[0]

time_series = []
next_t = 0.0
audio_index = 0

for frame in container.decode(video_stream):
t = frame.pts * video_stream.time_base
if t < next_t:
continue

image = frame.to_ndarray(format="rgb24")

audio_segment = []
while audio_index < len(audio_buffer):
audio_t, samples = audio_buffer[audio_index]
if audio_t < t + window:
audio_segment.append(samples)
audio_index += 1
else:
break

expected_samples = int(window * audio_stream.rate)

if len(audio_segment) == 0:
audio_data = np.zeros(expected_samples, dtype="float32")
else:
audio_data = np.concatenate(audio_segment, axis=0).astype("float32") / 32768.0

if len(audio_data) < expected_samples:
pad = expected_samples - len(audio_data)
audio_data = np.pad(audio_data, (0, pad), mode='constant')

if audio_data.ndim == 2 and audio_data.shape[1] > 1:
audio_data = audio_data.mean(axis=1)

print(audio_segment)
time_series.append(get_features(image, audio_data, audio_stream.rate))
next_t += window

return time_series

print(mead_feature_list("test_video/video_test_first.mp4"))

И вот журнал, ошибок нет, но Opensmile заполняет массив audio_features значениями NaN, как вы можете видеть:

(mead_env) WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
C:\Users\user\Documents\code\ml\mead\micro-expression-analysis-device-mead>python mead_feature_extract_main.py
W0000 00:00:1763949432.629800 11316 face landmarker_graph.cc:174] Sets FaceBlendshapesGraph acceleration to xnnpack by default. INFO: Created TensorFlow Lite XNNPACK delegate for CPU.
We000 00:00:1763949432.709222 1308 inference_feedback_manager.cc:114] Feedback manager requires a model with a single signature inference. Disabling support for feedback tensors.
We000 00:00:1763949432.746479 1388 inference_feedback_manager.cc:114] Feedback manager requires a model with a single signature inference. Disabling support for feedback tensors.
[]
Starting feature extraction
[0. 0. 0. ... . . .]
C:\Users\user\miniconda3\envs\mead_env\Lib\site-packages\opensmile\core\smile.py: 297: UserWarning: Segment too short, filling with
NaN.
warnings.warn(UserWarning("Segment too short, filling with NaN."))
[nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan
nan nan nan nan nan nan nan]
Saved array!
[]
Starting feature extraction
[0. 0. 0. ... 0. 0. 0.]
C:\Users\user\miniconda3\envs\mead_env\Lib\site-packages\opensmile\core\smile.py: 297: UserWarning: Segment too short, filling with
NaN.
warnings.warn(UserWarning("Segment too short, filling with NaN."))
[nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan
nan nan nan nan nan nan nan]
Saved array!
[ ]


Подробнее здесь: https://stackoverflow.com/questions/798 ... processing