From e34365dea314f9cd0f8d8c59cf84001d0d93df19 Mon Sep 17 00:00:00 2001 From: Adrian Czerwiec Date: Fri, 6 Feb 2026 10:52:04 +0100 Subject: [PATCH 1/3] preserve game state --- .../backend/src/agent/elevenlabs/api.ts | 98 ------------------- .../src/agent/elevenlabs/audioInterface.ts | 45 --------- .../backend/src/agent/elevenlabs/session.ts | 48 --------- .../backend/src/controllers/notifications.ts | 14 +++ .../packages/backend/src/router.ts | 3 +- .../packages/web/src/assets/elevenlabs.svg | 14 --- .../packages/web/src/hooks/useAgentEvents.ts | 87 ++++++++-------- 7 files changed, 63 insertions(+), 246 deletions(-) delete mode 100644 deep-sea-stories/packages/backend/src/agent/elevenlabs/api.ts delete mode 100644 deep-sea-stories/packages/backend/src/agent/elevenlabs/audioInterface.ts delete mode 100644 deep-sea-stories/packages/backend/src/agent/elevenlabs/session.ts delete mode 100644 deep-sea-stories/packages/web/src/assets/elevenlabs.svg diff --git a/deep-sea-stories/packages/backend/src/agent/elevenlabs/api.ts b/deep-sea-stories/packages/backend/src/agent/elevenlabs/api.ts deleted file mode 100644 index f27a1ba..0000000 --- a/deep-sea-stories/packages/backend/src/agent/elevenlabs/api.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { ElevenLabsClient } from '@elevenlabs/elevenlabs-js'; -import { - ClientTools, - Conversation, -} from '@elevenlabs/elevenlabs-js/api/resources/conversationalAi/conversation/index.js'; -import type { Story } from '../../types.js'; -import { - getFirstMessageForStory, - getInstructionsForStory, - getToolDescriptionForStory, -} from '../../utils.js'; -import type { AgentConfig, VoiceAgentApi } from '../api.js'; -import type { VoiceAgentSession } from '../session.js'; -import { ForwardingAudioInterface } from './audioInterface.js'; -import { ElevenLabsSession } from './session.js'; - -export class ElevenLabsApi implements VoiceAgentApi { - private elevenLabs: ElevenLabsClient; - - constructor(apiKey: string) { - this.elevenLabs = new ElevenLabsClient({ apiKey }); - } - - async createAgentSession(config: AgentConfig): Promise { - const story = config.story; - const instructions = getInstructionsForStory(story); - const firstMessage = getFirstMessageForStory(story); - const endGameToolId = await this.ensureGameEndingTool(story); - - console.log( - `Creating ElevenLabs agent for story "${story.title}" (ID: ${story.id})`, - ); - - const prompt = { prompt: instructions, toolIds: [endGameToolId] }; - - const params = { - conversationConfig: { - conversation: { - maxDurationSeconds: config.gameTimeLimitSeconds, - }, - agent: { - firstMessage, - language: 'en', - prompt, - }, - }, - }; - - const { agentId } = - await this.elevenLabs.conversationalAi.agents.create(params); - - return await this.createElevenLabsSession(config, agentId); - } - - private async createElevenLabsSession(config: AgentConfig, agentId: string) { - const clientTools = new ClientTools(); - clientTools.register('endGame', (_) => config.onEndGame()); - - const audioInterface = new ForwardingAudioInterface(); - - const conversation = new Conversation({ - agentId, - requiresAuth: false, - audioInterface, - clientTools, - callbackAgentResponse: (response) => config.onTranscription(response), - }); - - return new ElevenLabsSession(audioInterface, conversation); - } - - private async ensureGameEndingTool(story: Story): Promise { - const toolName = 'endGame'; - - const toolDescription = getToolDescriptionForStory(story); - - const { tools: allTools } = - await this.elevenLabs.conversationalAi.tools.list(); - const existingTool = allTools.find( - ({ toolConfig }) => - toolConfig.type === 'client' && - toolConfig.name === toolName && - toolConfig.description === toolDescription, - ); - - if (existingTool) return existingTool.id; - - const createdTool = await this.elevenLabs.conversationalAi.tools.create({ - toolConfig: { - type: 'client', - name: toolName, - description: toolDescription, - }, - }); - - return createdTool.id; - } -} diff --git a/deep-sea-stories/packages/backend/src/agent/elevenlabs/audioInterface.ts b/deep-sea-stories/packages/backend/src/agent/elevenlabs/audioInterface.ts deleted file mode 100644 index 004b0f3..0000000 --- a/deep-sea-stories/packages/backend/src/agent/elevenlabs/audioInterface.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { AudioInterface } from '@elevenlabs/elevenlabs-js/api/resources/conversationalAi/conversation/index.js'; - -export class ForwardingAudioInterface extends AudioInterface { - private inputCallback: ((audio: Buffer) => void) | null = null; - private onAgentAudio: ((audio: Buffer) => void) | null = null; - private onInterrupt: (() => void) | null = null; - - setInterruptCallback(onInterrupt: () => void) { - this.onInterrupt = onInterrupt; - } - - setAgentAudioCallback(onAgentAudio: (audio: Buffer) => void) { - this.onAgentAudio = onAgentAudio; - } - - sendAudio(audio: Buffer): void { - if (!this.inputCallback) return; - - this.inputCallback(audio); - } - - start(inputCallback: (audio: Buffer) => void): void { - this.inputCallback = inputCallback; - } - - stop(): void { - this.inputCallback = null; - } - - output(buffer: Buffer): void { - if (buffer.length <= 0) { - console.warn('[Audio Interface] Received empty audio buffer from agent'); - return; - } - - if (!this.onAgentAudio) console.error('Agent callback missing!'); - - this.onAgentAudio?.(buffer); - } - - interrupt(): void { - console.warn('Agent interrupted!'); - this.onInterrupt?.(); - } -} diff --git a/deep-sea-stories/packages/backend/src/agent/elevenlabs/session.ts b/deep-sea-stories/packages/backend/src/agent/elevenlabs/session.ts deleted file mode 100644 index 7483acd..0000000 --- a/deep-sea-stories/packages/backend/src/agent/elevenlabs/session.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { Conversation } from '@elevenlabs/elevenlabs-js/api/resources/conversationalAi/conversation/index.js'; -import type { VoiceAgentSession } from '../session.js'; -import type { ForwardingAudioInterface } from './audioInterface.js'; - -export class ElevenLabsSession implements VoiceAgentSession { - private session: Conversation; - private audioInterface: ForwardingAudioInterface; - - constructor(audioInterface: ForwardingAudioInterface, session: Conversation) { - this.audioInterface = audioInterface; - this.session = session; - } - - sendAudio(audio: Buffer) { - this.audioInterface.sendAudio(this.boostAudioVolume(audio, 7.0)); - } - - registerInterruptionCallback(onInterrupt: () => void) { - this.audioInterface.setInterruptCallback(onInterrupt); - } - - registerAgentAudioCallback(onAgentAudio: (audio: Buffer) => void) { - this.audioInterface.setAgentAudioCallback(onAgentAudio); - } - - async announceTimeExpired() { - console.log('ElevenLabs session time expired (handled by platform)'); - } - - async open() { - await this.session.startSession(); - } - - async close(_wait: boolean) { - this.session.endSession(); - } - - private boostAudioVolume(audioBuffer: Buffer, gain = 2.0): Buffer { - for (let offset = 0; offset < audioBuffer.length - 1; offset += 2) { - const sample = audioBuffer.readInt16LE(offset); - const amplified = Math.round(sample * gain); - const clamped = Math.max(-32768, Math.min(32767, amplified)); - audioBuffer.writeInt16LE(clamped, offset); - } - - return audioBuffer; - } -} diff --git a/deep-sea-stories/packages/backend/src/controllers/notifications.ts b/deep-sea-stories/packages/backend/src/controllers/notifications.ts index 69b3992..f8c0ea0 100644 --- a/deep-sea-stories/packages/backend/src/controllers/notifications.ts +++ b/deep-sea-stories/packages/backend/src/controllers/notifications.ts @@ -4,6 +4,20 @@ import { tracked } from '@trpc/server'; import { z } from 'zod'; import { publicProcedure } from '../trpc.js'; +export const getEvents = publicProcedure + .input(z.object({ roomId: z.string() })) + .query(({ ctx, input }) => { + const history = ctx.notifierService.getEventHistory(input.roomId); + const lastEventId = + history.length > 0 + ? history[history.length - 1].id.toString() + : null; + return { + events: history.map(({ event }) => event), + lastEventId, + }; + }); + export const Notifications = publicProcedure .input( z.object({ diff --git a/deep-sea-stories/packages/backend/src/router.ts b/deep-sea-stories/packages/backend/src/router.ts index f82ae63..208cf0f 100644 --- a/deep-sea-stories/packages/backend/src/router.ts +++ b/deep-sea-stories/packages/backend/src/router.ts @@ -1,4 +1,4 @@ -import { Notifications } from './controllers/notifications.js'; +import { Notifications, getEvents } from './controllers/notifications.js'; import { createPeer } from './controllers/peers.js'; import { createRoom } from './controllers/rooms.js'; import { @@ -18,6 +18,7 @@ export const appRouter = router({ stopGame, getStories, Notifications, + getEvents, muteVoiceAgent, }); diff --git a/deep-sea-stories/packages/web/src/assets/elevenlabs.svg b/deep-sea-stories/packages/web/src/assets/elevenlabs.svg deleted file mode 100644 index 1880710..0000000 --- a/deep-sea-stories/packages/web/src/assets/elevenlabs.svg +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - diff --git a/deep-sea-stories/packages/web/src/hooks/useAgentEvents.ts b/deep-sea-stories/packages/web/src/hooks/useAgentEvents.ts index ee2f799..b9e646a 100644 --- a/deep-sea-stories/packages/web/src/hooks/useAgentEvents.ts +++ b/deep-sea-stories/packages/web/src/hooks/useAgentEvents.ts @@ -1,43 +1,50 @@ -import type { AgentEvent } from '@deep-sea-stories/common'; -import { useEffect, useState } from 'react'; -import { useTRPCClient } from '@/contexts/trpc'; +import type { AgentEvent } from "@deep-sea-stories/common"; +import { useEffect, useState } from "react"; +import { useTRPCClient } from "@/contexts/trpc"; export const useAgentEvents = (roomId?: string) => { - const [events, setEvents] = useState([]); - const trpcClient = useTRPCClient(); - - useEffect(() => { - if (!roomId) return; - - setEvents([]); - - const subscription = trpcClient.Notifications.subscribe( - { roomId, lastEventId: undefined }, - { - onStarted: () => { - console.log( - '[useAgentEvents] Subscription started successfully for', - roomId, - ); - }, - onData: (data: unknown) => { - const event = - data && typeof data === 'object' && 'data' in data - ? (data as { data: AgentEvent }).data - : (data as AgentEvent); - setEvents((prev) => [...prev, event]); - }, - onError: (error: unknown) => { - console.error('[useAgentEvents] Subscription error:', error); - }, - }, - ); - - return () => { - console.log('[useAgentEvents] Unsubscribing from', roomId); - subscription.unsubscribe(); - }; - }, [trpcClient, roomId]); - - return events; + const [events, setEvents] = useState([]); + const trpcClient = useTRPCClient(); + + useEffect(() => { + if (!roomId) return; + + setEvents([]); + + let subscription: + | ReturnType + | undefined; + let active = true; + + trpcClient.getEvents + .query({ roomId }) + .then(({ events: pastEvents, lastEventId }) => { + if (!active) return; + + setEvents(pastEvents); + + subscription = trpcClient.Notifications.subscribe( + { roomId, lastEventId }, + { + onData: (data: unknown) => { + const event = + data && typeof data === "object" && "data" in data + ? (data as { data: AgentEvent }).data + : (data as AgentEvent); + setEvents((prev) => [...prev, event]); + }, + onError: (error: unknown) => { + console.error("[useAgentEvents] Subscription error:", error); + }, + }, + ); + }); + + return () => { + active = false; + subscription?.unsubscribe(); + }; + }, [trpcClient, roomId]); + + return events; }; From 91c52eab1d3a8420c0123d11cd5be74506b8935d Mon Sep 17 00:00:00 2001 From: Adrian Czerwiec Date: Fri, 6 Feb 2026 11:09:58 +0100 Subject: [PATCH 2/3] display toast on error --- deep-sea-stories/packages/web/src/hooks/useAgentEvents.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deep-sea-stories/packages/web/src/hooks/useAgentEvents.ts b/deep-sea-stories/packages/web/src/hooks/useAgentEvents.ts index b9e646a..7fc9922 100644 --- a/deep-sea-stories/packages/web/src/hooks/useAgentEvents.ts +++ b/deep-sea-stories/packages/web/src/hooks/useAgentEvents.ts @@ -1,6 +1,8 @@ import type { AgentEvent } from "@deep-sea-stories/common"; import { useEffect, useState } from "react"; import { useTRPCClient } from "@/contexts/trpc"; +import { toast } from "@/components/ui/sonner"; +import { X } from "lucide-react"; export const useAgentEvents = (roomId?: string) => { const [events, setEvents] = useState([]); @@ -38,7 +40,8 @@ export const useAgentEvents = (roomId?: string) => { }, }, ); - }); + }) + .catch(() => toast("Failed to fetch events", X)); return () => { active = false; From 2c33a77c40ca1ab43e6cca8a8d10f177ef8cf1f2 Mon Sep 17 00:00:00 2001 From: Adrian Czerwiec Date: Fri, 6 Feb 2026 11:10:06 +0100 Subject: [PATCH 3/3] format --- .../backend/src/controllers/notifications.ts | 4 +- .../packages/web/src/hooks/useAgentEvents.ts | 100 +++++++++--------- 2 files changed, 51 insertions(+), 53 deletions(-) diff --git a/deep-sea-stories/packages/backend/src/controllers/notifications.ts b/deep-sea-stories/packages/backend/src/controllers/notifications.ts index f8c0ea0..3db6ba6 100644 --- a/deep-sea-stories/packages/backend/src/controllers/notifications.ts +++ b/deep-sea-stories/packages/backend/src/controllers/notifications.ts @@ -9,9 +9,7 @@ export const getEvents = publicProcedure .query(({ ctx, input }) => { const history = ctx.notifierService.getEventHistory(input.roomId); const lastEventId = - history.length > 0 - ? history[history.length - 1].id.toString() - : null; + history.length > 0 ? history[history.length - 1].id.toString() : null; return { events: history.map(({ event }) => event), lastEventId, diff --git a/deep-sea-stories/packages/web/src/hooks/useAgentEvents.ts b/deep-sea-stories/packages/web/src/hooks/useAgentEvents.ts index 7fc9922..3954572 100644 --- a/deep-sea-stories/packages/web/src/hooks/useAgentEvents.ts +++ b/deep-sea-stories/packages/web/src/hooks/useAgentEvents.ts @@ -1,53 +1,53 @@ -import type { AgentEvent } from "@deep-sea-stories/common"; -import { useEffect, useState } from "react"; -import { useTRPCClient } from "@/contexts/trpc"; -import { toast } from "@/components/ui/sonner"; -import { X } from "lucide-react"; +import type { AgentEvent } from '@deep-sea-stories/common'; +import { useEffect, useState } from 'react'; +import { useTRPCClient } from '@/contexts/trpc'; +import { toast } from '@/components/ui/sonner'; +import { X } from 'lucide-react'; export const useAgentEvents = (roomId?: string) => { - const [events, setEvents] = useState([]); - const trpcClient = useTRPCClient(); - - useEffect(() => { - if (!roomId) return; - - setEvents([]); - - let subscription: - | ReturnType - | undefined; - let active = true; - - trpcClient.getEvents - .query({ roomId }) - .then(({ events: pastEvents, lastEventId }) => { - if (!active) return; - - setEvents(pastEvents); - - subscription = trpcClient.Notifications.subscribe( - { roomId, lastEventId }, - { - onData: (data: unknown) => { - const event = - data && typeof data === "object" && "data" in data - ? (data as { data: AgentEvent }).data - : (data as AgentEvent); - setEvents((prev) => [...prev, event]); - }, - onError: (error: unknown) => { - console.error("[useAgentEvents] Subscription error:", error); - }, - }, - ); - }) - .catch(() => toast("Failed to fetch events", X)); - - return () => { - active = false; - subscription?.unsubscribe(); - }; - }, [trpcClient, roomId]); - - return events; + const [events, setEvents] = useState([]); + const trpcClient = useTRPCClient(); + + useEffect(() => { + if (!roomId) return; + + setEvents([]); + + let subscription: + | ReturnType + | undefined; + let active = true; + + trpcClient.getEvents + .query({ roomId }) + .then(({ events: pastEvents, lastEventId }) => { + if (!active) return; + + setEvents(pastEvents); + + subscription = trpcClient.Notifications.subscribe( + { roomId, lastEventId }, + { + onData: (data: unknown) => { + const event = + data && typeof data === 'object' && 'data' in data + ? (data as { data: AgentEvent }).data + : (data as AgentEvent); + setEvents((prev) => [...prev, event]); + }, + onError: (error: unknown) => { + console.error('[useAgentEvents] Subscription error:', error); + }, + }, + ); + }) + .catch(() => toast('Failed to fetch events', X)); + + return () => { + active = false; + subscription?.unsubscribe(); + }; + }, [trpcClient, roomId]); + + return events; };