diff --git a/deep-sea-stories/packages/backend/src/agent/elevenlabs/api.ts b/deep-sea-stories/packages/backend/src/agent/elevenlabs/api.ts deleted file mode 100644 index f27a1ba..0000000 --- a/deep-sea-stories/packages/backend/src/agent/elevenlabs/api.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { ElevenLabsClient } from '@elevenlabs/elevenlabs-js'; -import { - ClientTools, - Conversation, -} from '@elevenlabs/elevenlabs-js/api/resources/conversationalAi/conversation/index.js'; -import type { Story } from '../../types.js'; -import { - getFirstMessageForStory, - getInstructionsForStory, - getToolDescriptionForStory, -} from '../../utils.js'; -import type { AgentConfig, VoiceAgentApi } from '../api.js'; -import type { VoiceAgentSession } from '../session.js'; -import { ForwardingAudioInterface } from './audioInterface.js'; -import { ElevenLabsSession } from './session.js'; - -export class ElevenLabsApi implements VoiceAgentApi { - private elevenLabs: ElevenLabsClient; - - constructor(apiKey: string) { - this.elevenLabs = new ElevenLabsClient({ apiKey }); - } - - async createAgentSession(config: AgentConfig): Promise { - const story = config.story; - const instructions = getInstructionsForStory(story); - const firstMessage = getFirstMessageForStory(story); - const endGameToolId = await this.ensureGameEndingTool(story); - - console.log( - `Creating ElevenLabs agent for story "${story.title}" (ID: ${story.id})`, - ); - - const prompt = { prompt: instructions, toolIds: [endGameToolId] }; - - const params = { - conversationConfig: { - conversation: { - maxDurationSeconds: config.gameTimeLimitSeconds, - }, - agent: { - firstMessage, - language: 'en', - prompt, - }, - }, - }; - - const { agentId } = - await this.elevenLabs.conversationalAi.agents.create(params); - - return await this.createElevenLabsSession(config, agentId); - } - - private async createElevenLabsSession(config: AgentConfig, agentId: string) { - const clientTools = new ClientTools(); - clientTools.register('endGame', (_) => config.onEndGame()); - - const audioInterface = new ForwardingAudioInterface(); - - const conversation = new Conversation({ - agentId, - requiresAuth: false, - audioInterface, - clientTools, - callbackAgentResponse: (response) => config.onTranscription(response), - }); - - return new ElevenLabsSession(audioInterface, conversation); - } - - private async ensureGameEndingTool(story: Story): Promise { - const toolName = 'endGame'; - - const toolDescription = getToolDescriptionForStory(story); - - const { tools: allTools } = - await this.elevenLabs.conversationalAi.tools.list(); - const existingTool = allTools.find( - ({ toolConfig }) => - toolConfig.type === 'client' && - toolConfig.name === toolName && - toolConfig.description === toolDescription, - ); - - if (existingTool) return existingTool.id; - - const createdTool = await this.elevenLabs.conversationalAi.tools.create({ - toolConfig: { - type: 'client', - name: toolName, - description: toolDescription, - }, - }); - - return createdTool.id; - } -} diff --git a/deep-sea-stories/packages/backend/src/agent/elevenlabs/audioInterface.ts b/deep-sea-stories/packages/backend/src/agent/elevenlabs/audioInterface.ts deleted file mode 100644 index 004b0f3..0000000 --- a/deep-sea-stories/packages/backend/src/agent/elevenlabs/audioInterface.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { AudioInterface } from '@elevenlabs/elevenlabs-js/api/resources/conversationalAi/conversation/index.js'; - -export class ForwardingAudioInterface extends AudioInterface { - private inputCallback: ((audio: Buffer) => void) | null = null; - private onAgentAudio: ((audio: Buffer) => void) | null = null; - private onInterrupt: (() => void) | null = null; - - setInterruptCallback(onInterrupt: () => void) { - this.onInterrupt = onInterrupt; - } - - setAgentAudioCallback(onAgentAudio: (audio: Buffer) => void) { - this.onAgentAudio = onAgentAudio; - } - - sendAudio(audio: Buffer): void { - if (!this.inputCallback) return; - - this.inputCallback(audio); - } - - start(inputCallback: (audio: Buffer) => void): void { - this.inputCallback = inputCallback; - } - - stop(): void { - this.inputCallback = null; - } - - output(buffer: Buffer): void { - if (buffer.length <= 0) { - console.warn('[Audio Interface] Received empty audio buffer from agent'); - return; - } - - if (!this.onAgentAudio) console.error('Agent callback missing!'); - - this.onAgentAudio?.(buffer); - } - - interrupt(): void { - console.warn('Agent interrupted!'); - this.onInterrupt?.(); - } -} diff --git a/deep-sea-stories/packages/backend/src/agent/elevenlabs/session.ts b/deep-sea-stories/packages/backend/src/agent/elevenlabs/session.ts deleted file mode 100644 index 7483acd..0000000 --- a/deep-sea-stories/packages/backend/src/agent/elevenlabs/session.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { Conversation } from '@elevenlabs/elevenlabs-js/api/resources/conversationalAi/conversation/index.js'; -import type { VoiceAgentSession } from '../session.js'; -import type { ForwardingAudioInterface } from './audioInterface.js'; - -export class ElevenLabsSession implements VoiceAgentSession { - private session: Conversation; - private audioInterface: ForwardingAudioInterface; - - constructor(audioInterface: ForwardingAudioInterface, session: Conversation) { - this.audioInterface = audioInterface; - this.session = session; - } - - sendAudio(audio: Buffer) { - this.audioInterface.sendAudio(this.boostAudioVolume(audio, 7.0)); - } - - registerInterruptionCallback(onInterrupt: () => void) { - this.audioInterface.setInterruptCallback(onInterrupt); - } - - registerAgentAudioCallback(onAgentAudio: (audio: Buffer) => void) { - this.audioInterface.setAgentAudioCallback(onAgentAudio); - } - - async announceTimeExpired() { - console.log('ElevenLabs session time expired (handled by platform)'); - } - - async open() { - await this.session.startSession(); - } - - async close(_wait: boolean) { - this.session.endSession(); - } - - private boostAudioVolume(audioBuffer: Buffer, gain = 2.0): Buffer { - for (let offset = 0; offset < audioBuffer.length - 1; offset += 2) { - const sample = audioBuffer.readInt16LE(offset); - const amplified = Math.round(sample * gain); - const clamped = Math.max(-32768, Math.min(32767, amplified)); - audioBuffer.writeInt16LE(clamped, offset); - } - - return audioBuffer; - } -} diff --git a/deep-sea-stories/packages/backend/src/controllers/notifications.ts b/deep-sea-stories/packages/backend/src/controllers/notifications.ts index 69b3992..3db6ba6 100644 --- a/deep-sea-stories/packages/backend/src/controllers/notifications.ts +++ b/deep-sea-stories/packages/backend/src/controllers/notifications.ts @@ -4,6 +4,18 @@ import { tracked } from '@trpc/server'; import { z } from 'zod'; import { publicProcedure } from '../trpc.js'; +export const getEvents = publicProcedure + .input(z.object({ roomId: z.string() })) + .query(({ ctx, input }) => { + const history = ctx.notifierService.getEventHistory(input.roomId); + const lastEventId = + history.length > 0 ? history[history.length - 1].id.toString() : null; + return { + events: history.map(({ event }) => event), + lastEventId, + }; + }); + export const Notifications = publicProcedure .input( z.object({ diff --git a/deep-sea-stories/packages/backend/src/router.ts b/deep-sea-stories/packages/backend/src/router.ts index f82ae63..208cf0f 100644 --- a/deep-sea-stories/packages/backend/src/router.ts +++ b/deep-sea-stories/packages/backend/src/router.ts @@ -1,4 +1,4 @@ -import { Notifications } from './controllers/notifications.js'; +import { Notifications, getEvents } from './controllers/notifications.js'; import { createPeer } from './controllers/peers.js'; import { createRoom } from './controllers/rooms.js'; import { @@ -18,6 +18,7 @@ export const appRouter = router({ stopGame, getStories, Notifications, + getEvents, muteVoiceAgent, }); diff --git a/deep-sea-stories/packages/web/src/assets/elevenlabs.svg b/deep-sea-stories/packages/web/src/assets/elevenlabs.svg deleted file mode 100644 index 1880710..0000000 --- a/deep-sea-stories/packages/web/src/assets/elevenlabs.svg +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - diff --git a/deep-sea-stories/packages/web/src/hooks/useAgentEvents.ts b/deep-sea-stories/packages/web/src/hooks/useAgentEvents.ts index ee2f799..3954572 100644 --- a/deep-sea-stories/packages/web/src/hooks/useAgentEvents.ts +++ b/deep-sea-stories/packages/web/src/hooks/useAgentEvents.ts @@ -1,6 +1,8 @@ import type { AgentEvent } from '@deep-sea-stories/common'; import { useEffect, useState } from 'react'; import { useTRPCClient } from '@/contexts/trpc'; +import { toast } from '@/components/ui/sonner'; +import { X } from 'lucide-react'; export const useAgentEvents = (roomId?: string) => { const [events, setEvents] = useState([]); @@ -11,31 +13,39 @@ export const useAgentEvents = (roomId?: string) => { setEvents([]); - const subscription = trpcClient.Notifications.subscribe( - { roomId, lastEventId: undefined }, - { - onStarted: () => { - console.log( - '[useAgentEvents] Subscription started successfully for', - roomId, - ); - }, - onData: (data: unknown) => { - const event = - data && typeof data === 'object' && 'data' in data - ? (data as { data: AgentEvent }).data - : (data as AgentEvent); - setEvents((prev) => [...prev, event]); - }, - onError: (error: unknown) => { - console.error('[useAgentEvents] Subscription error:', error); - }, - }, - ); + let subscription: + | ReturnType + | undefined; + let active = true; + + trpcClient.getEvents + .query({ roomId }) + .then(({ events: pastEvents, lastEventId }) => { + if (!active) return; + + setEvents(pastEvents); + + subscription = trpcClient.Notifications.subscribe( + { roomId, lastEventId }, + { + onData: (data: unknown) => { + const event = + data && typeof data === 'object' && 'data' in data + ? (data as { data: AgentEvent }).data + : (data as AgentEvent); + setEvents((prev) => [...prev, event]); + }, + onError: (error: unknown) => { + console.error('[useAgentEvents] Subscription error:', error); + }, + }, + ); + }) + .catch(() => toast('Failed to fetch events', X)); return () => { - console.log('[useAgentEvents] Unsubscribing from', roomId); - subscription.unsubscribe(); + active = false; + subscription?.unsubscribe(); }; }, [trpcClient, roomId]);