[Python] Python aiortc: How to record audio and video come from client in the same file?

I have an aiortc app, which html5 client send data (microphone,camera) into server.

In server side i sucessfully played this two streams seperatly.

But when i try to record using aiortc MediaRecorder helper class, only the voice is recording, and the video is dropped (mp4 format).

I think this is due sync issue.

The audio_frame and the video_frame (each pair of them) have different time_base. (I don't know if this is strange). But it also has different time.

I can share code, but couldn't help at all right now.

Edit: I also tried to synchronize them client side with no luck

// Synchronize tracks
async function synchronizeTracks(localStream) {
const videoTrack = localStream.getVideoTracks()[0];
const audioTrack = localStream.getAudioTracks()[0];

const syncedStream = new MediaStream();

// Add tracks to the synchronized stream
syncedStream.addTrack(videoTrack);
syncedStream.addTrack(audioTrack);

// Video and audio processors
const videoProcessor = new MediaStreamTrackProcessor({ track: videoTrack });
const audioProcessor = new MediaStreamTrackProcessor({ track: audioTrack });

const videoReader = videoProcessor.readable.getReader();
const audioReader = audioProcessor.readable.getReader();

const videoWriter = new MediaStreamTrackGenerator({ kind: "video" }).writable.getWriter();
const audioWriter = new MediaStreamTrackGenerator({ kind: "audio" }).writable.getWriter();

const syncThreshold = 5; // Maximum allowable time difference in milliseconds
let baseTimestamp = null;

async function processTracks() {
try {
while (true) {
const [videoResult, audioResult] = await Promise.all([
videoReader.read(),
audioReader.read(),
]);

if (videoResult.done || audioResult.done) break;

const videoFrame = videoResult.value;
const audioFrame = audioResult.value;

// Initialize base timestamp if needed
if (baseTimestamp === null) {
baseTimestamp = Math.min(videoFrame.timestamp, audioFrame.timestamp);
}

const videoRelativeTimestamp = videoFrame.timestamp - baseTimestamp;
const audioRelativeTimestamp = audioFrame.timestamp - baseTimestamp;

const timeDifference = videoRelativeTimestamp - audioRelativeTimestamp;

if (Math.abs(timeDifference) <= syncThreshold) {
// Frames are in sync
await videoWriter.write(videoFrame);
await audioWriter.write(audioFrame);
} else if (timeDifference > 0) {
// Video is ahead, wait for audio to catch up
await audioWriter.write(audioFrame);
// Reuse video frame on the next loop
videoReader.releaseLock();
} else {
// Audio is ahead, wait for video to catch up
await videoWriter.write(videoFrame);
// Reuse audio frame on the next loop
audioReader.releaseLock();
}

// Release frames
videoFrame.close();
audioFrame.close();
}
} catch (error) {
console.error("Error in track synchronization:", error);
} finally {
videoReader.releaseLock();
audioReader.releaseLock();
videoWriter.close();
audioWriter.close();
}
}

processTracks();

return syncedStream;
}

Continue reading...