[Python] Why is my LiveKit-based multimodal agent speaking twice in the room, even though...

I'm using LiveKit with a multimodal agent (OpenAI RealtimeModel) to conduct interactive interviews in a virtual room. The goal is to:

• Let the agent converse in real-time using audio (modalities=["audio"])
• Automatically disconnect the agent after a specified time
• Send a 1-minute warning message before disconnecting

✅ What's working:
Agent joins the room and responds to speech
Transcript is accurate (user + agent messages are logged correctly)
Interview ends as scheduled

❌ The problem:
When I join the room via the LiveKit Playground, I hear the agent's voice twice — it sounds like double-play of the response and both response are different
But the transcript only shows one instance of each message.

async def end_interview_at_time(ctx: JobContext, session, end_time: datetime, warning_time: datetime, my_shutdown_hook):
"""Task that handles interview timing and ending"""
warning_sent = False

while True:
current_time = datetime.now()

# Check if we need to send warning
if not warning_sent and current_time >= warning_time:
logger.info("Warning time reached, informing LLM to wrap up")
wrap_up_instruction = "You have 1 minute remaining. Please wrap up the by asking any final questions conclude."
await send_message(ctx, wrap_up_instruction)
warning_sent = True
logger.info("Wrap-up instruction sent to LLM")

# Check if interview should end
if current_time >= end_time:
logger.info("duration completed, shutting down")
# Add shutdown hook before ending the session
ctx.add_shutdown_callback(my_shutdown_hook)
# Clean up the room
print(f"Cleaning up the room {ctx.room.name} \n\n")
await room_manager.cleanup_room(ctx.room.name)
# Shutdown the agent
ctx.shutdown(reason="Interview duration completed")
break

await asyncio.sleep(10)

async def entrypoint(ctx: JobContext):
metadata = json.loads(ctx.job.metadata)
logger.info(f"\n\n job metadata type: {type(metadata)}")
logger.info(f"\n\n job metadata: {metadata}")

async def my_shutdown_hook():
print(json.dumps(transcript, indent=2))

# Store conversation in database

try:

await ctx.connect(auto_subscribe=AutoSubscribe.SUBSCRIBE_ALL)
await ctx.wait_for_participant()

# Set up timer
start_time = datetime.now()
end_time = start_time + timedelta(minutes=metadata.get("duration_minutes") + 0.25)
warning_time = end_time - timedelta(minutes=1)

model = openai.realtime.RealtimeModel(
instructions=prompts.get_system_prompt(),
voice="shimmer",
temperature=0.8,
modalities=["audio", "text"]
)
agent = MultimodalAgent(
model=model,
)
agent.start(ctx.room)

session = model.sessions[0]

# Start with welcome message
welcome_message = VisaInterviewPrompts.get_welcome_message(metadata.get("visa_type"))
# await send_voice_message(session, welcome_message)
await send_message(ctx, welcome_message)


@agent.on("user_speech_committed")
def on_user_speech_committed(msg: llm.ChatMessage):
# print(msg)
# if (msg.content is not None and isinstance(msg.content, list)):
# msg = "\n".join("[image]" if isinstance(x, llm.ChatImage) else x for x in msg)
transcript.append({
"timestamp": get_formatted_timestamp(interview_start_time),
"role": "applicant",
"content": msg
})
handle_query(msg)

@agent.on("agent_speech_committed")
def on_agent_speech_committed(msg: llm.ChatMessage):
# print("INTERVIEWER: ", msg.content)
transcript.append({
"timestamp": get_formatted_timestamp(interview_start_time),
"role": "interviewer",
"content": msg
})

def handle_query(msg: llm.ChatMessage):
session.conversation.item.create(
llm.ChatMessage(
role="user",
content=msg
)
)
session.response.create()

# Start the interview timer task
timer_task = asyncio.create_task(end_interview_at_time(ctx, session, end_time, warning_time, my_shutdown_hook))

# Wait for the timer task to complete (which will happen when the interview ends)
await timer_task

except Exception as e:
logger.error(f"Error in main interview loop: {e}")
try:
ctx.add_shutdown_callback(my_shutdown_hook)
# Clean up the room
await room_manager.cleanup_room(ctx.room.name)
# Shutdown the agent
ctx.shutdown(reason="Error Encountered")
except:
pass

if __name__ == "__main__":
cli.run_app(WorkerOptions(entrypoint_fnc=entrypoint, agent_name="visa-interview-agent"))



# token creation and dispatching the agent
token = (
AccessToken(
self.api_key,
self.api_secret
)
.with_identity(participant_name)
.with_grants(VideoGrants(room_join=True, room=room_name))
.with_room_config(
RoomConfiguration(
agents=[
RoomAgentDispatch(
agent_name=self.agent_name,
metadata=json.dumps(token_metadata),
)
]
)
)
.to_jwt()
)

Continue reading...