-
Notifications
You must be signed in to change notification settings - Fork 3.1k
Add sounds for completed turns and user input requests #3892
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
1cec70e
8387bda
f38723f
9626d08
ac5b546
61cef12
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| import type { EnvironmentThreadShell } from "@t3tools/client-runtime/state/shell"; | ||
| import { TurnId } from "@t3tools/contracts"; | ||
| import { describe, expect, it } from "vite-plus/test"; | ||
| import { | ||
| captureThreadSoundState, | ||
| captureThreadSoundStateWhileSettingsHydrating, | ||
| deriveInteractionSoundCues, | ||
| } from "./interactionSounds"; | ||
|
|
||
| function makeThread(overrides: Partial<EnvironmentThreadShell> = {}): EnvironmentThreadShell { | ||
| return { | ||
| environmentId: "environment-1", | ||
| id: "thread-1", | ||
| projectId: "project-1", | ||
| title: "Thread", | ||
| modelSelection: null, | ||
| runtimeMode: "full-access", | ||
| interactionMode: "default", | ||
| branch: null, | ||
| worktreePath: null, | ||
| latestTurn: null, | ||
| createdAt: "2026-07-11T12:00:00.000Z", | ||
| updatedAt: "2026-07-11T12:00:00.000Z", | ||
| archivedAt: null, | ||
| session: null, | ||
| latestUserMessageAt: null, | ||
| hasPendingApprovals: false, | ||
| hasPendingUserInput: false, | ||
| hasActionableProposedPlan: false, | ||
| ...overrides, | ||
| } as EnvironmentThreadShell; | ||
| } | ||
|
|
||
| describe("interaction sounds", () => { | ||
| it("plays success when a turn becomes completed", () => { | ||
| const running = makeThread({ | ||
| latestTurn: { | ||
| turnId: TurnId.make("turn-1"), | ||
| state: "running", | ||
| requestedAt: "2026-07-11T12:00:00.000Z", | ||
| startedAt: "2026-07-11T12:00:01.000Z", | ||
| completedAt: null, | ||
| assistantMessageId: null, | ||
| }, | ||
| }); | ||
| const completed = makeThread({ | ||
| latestTurn: { | ||
| ...running.latestTurn!, | ||
| state: "completed", | ||
| completedAt: "2026-07-11T12:00:05.000Z", | ||
| }, | ||
| }); | ||
|
|
||
| expect(deriveInteractionSoundCues(captureThreadSoundState([running]), [completed])).toEqual([ | ||
| "success", | ||
| ]); | ||
| }); | ||
|
|
||
| it("plays bloom when a thread starts requesting user input", () => { | ||
| const thread = makeThread(); | ||
|
|
||
| expect( | ||
| deriveInteractionSoundCues(captureThreadSoundState([thread]), [ | ||
| makeThread({ hasPendingUserInput: true }), | ||
| ]), | ||
| ).toEqual(["bloom"]); | ||
| }); | ||
|
|
||
| it("plays bloom when a thread starts requesting approval", () => { | ||
| const thread = makeThread(); | ||
|
|
||
| expect( | ||
| deriveInteractionSoundCues(captureThreadSoundState([thread]), [ | ||
| makeThread({ hasPendingApprovals: true }), | ||
| ]), | ||
| ).toEqual(["bloom"]); | ||
| }); | ||
|
|
||
| it("does not replay cues for unchanged state", () => { | ||
| const thread = makeThread({ | ||
| hasPendingUserInput: true, | ||
| hasPendingApprovals: true, | ||
| latestTurn: { | ||
| turnId: TurnId.make("turn-1"), | ||
| state: "completed", | ||
| requestedAt: "2026-07-11T12:00:00.000Z", | ||
| startedAt: "2026-07-11T12:00:01.000Z", | ||
| completedAt: "2026-07-11T12:00:05.000Z", | ||
| assistantMessageId: null, | ||
| }, | ||
| }); | ||
|
|
||
| expect(deriveInteractionSoundCues(captureThreadSoundState([thread]), [thread])).toEqual([]); | ||
| }); | ||
|
|
||
| it("does not play cues while existing threads are first hydrated", () => { | ||
| const thread = makeThread({ | ||
| hasPendingUserInput: true, | ||
| latestTurn: { | ||
| turnId: TurnId.make("turn-1"), | ||
| state: "completed", | ||
| requestedAt: "2026-07-11T12:00:00.000Z", | ||
| startedAt: "2026-07-11T12:00:01.000Z", | ||
| completedAt: "2026-07-11T12:00:05.000Z", | ||
| assistantMessageId: null, | ||
| }, | ||
| }); | ||
|
|
||
| expect(deriveInteractionSoundCues(new Map(), [thread])).toEqual([]); | ||
| }); | ||
|
|
||
| it("preserves pre-hydration thread state so cues can play after settings hydrate", () => { | ||
| const running = makeThread({ | ||
| latestTurn: { | ||
| turnId: TurnId.make("turn-1"), | ||
| state: "running", | ||
| requestedAt: "2026-07-11T12:00:00.000Z", | ||
| startedAt: "2026-07-11T12:00:01.000Z", | ||
| completedAt: null, | ||
| assistantMessageId: null, | ||
| }, | ||
| }); | ||
| const completed = makeThread({ | ||
| latestTurn: { | ||
| ...running.latestTurn!, | ||
| state: "completed", | ||
| completedAt: "2026-07-11T12:00:05.000Z", | ||
| }, | ||
| }); | ||
|
|
||
| const seeded = captureThreadSoundStateWhileSettingsHydrating(null, [running]); | ||
| const frozen = captureThreadSoundStateWhileSettingsHydrating(seeded, [completed]); | ||
|
|
||
| expect(deriveInteractionSoundCues(frozen, [completed])).toEqual(["success"]); | ||
| }); | ||
|
|
||
| it("admits newly seen threads while settings are hydrating", () => { | ||
| const seeded = captureThreadSoundStateWhileSettingsHydrating(null, []); | ||
| const withThread = captureThreadSoundStateWhileSettingsHydrating(seeded, [ | ||
| makeThread({ hasPendingUserInput: true }), | ||
| ]); | ||
|
|
||
| expect( | ||
| deriveInteractionSoundCues(withThread, [makeThread({ hasPendingUserInput: true })]), | ||
| ).toEqual([]); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| import type { EnvironmentThreadShell } from "@t3tools/client-runtime/state/shell"; | ||
|
|
||
| export type InteractionSoundCue = "bloom" | "success"; | ||
|
|
||
| interface ThreadSoundState { | ||
| readonly completedTurn: string | null; | ||
| readonly hasPendingUserAction: boolean; | ||
| } | ||
|
|
||
| export type ThreadSoundStateByKey = ReadonlyMap<string, ThreadSoundState>; | ||
|
|
||
| function threadKey(thread: EnvironmentThreadShell): string { | ||
| return `${thread.environmentId}:${thread.id}`; | ||
| } | ||
|
|
||
| function completedTurn(thread: EnvironmentThreadShell): string | null { | ||
| const latestTurn = thread.latestTurn; | ||
| if (latestTurn?.state !== "completed" || latestTurn.completedAt === null) { | ||
| return null; | ||
| } | ||
| return `${latestTurn.turnId}:${latestTurn.completedAt}`; | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When a provider Useful? React with 👍 / 👎. |
||
| } | ||
|
|
||
| export function captureThreadSoundState( | ||
| threads: ReadonlyArray<EnvironmentThreadShell>, | ||
| ): ThreadSoundStateByKey { | ||
| return new Map( | ||
| threads.map((thread) => [ | ||
| threadKey(thread), | ||
| { | ||
| completedTurn: completedTurn(thread), | ||
| hasPendingUserAction: thread.hasPendingUserInput || thread.hasPendingApprovals, | ||
| }, | ||
| ]), | ||
| ); | ||
| } | ||
|
|
||
| /** | ||
| * While client settings are still hydrating, keep a sound baseline without | ||
| * advancing known thread state. Newly seen threads are admitted so later | ||
| * transitions can still produce cues once hydration completes. | ||
| */ | ||
| export function captureThreadSoundStateWhileSettingsHydrating( | ||
| previous: ThreadSoundStateByKey | null, | ||
| threads: ReadonlyArray<EnvironmentThreadShell>, | ||
| ): ThreadSoundStateByKey { | ||
| const next = captureThreadSoundState(threads); | ||
| if (previous === null) { | ||
| return next; | ||
| } | ||
|
|
||
| const merged = new Map(previous); | ||
| for (const [key, state] of next) { | ||
| if (!merged.has(key)) { | ||
| merged.set(key, state); | ||
| } | ||
| } | ||
| return merged; | ||
|
cursor[bot] marked this conversation as resolved.
|
||
| } | ||
|
|
||
| export function deriveInteractionSoundCues( | ||
| previous: ThreadSoundStateByKey, | ||
| threads: ReadonlyArray<EnvironmentThreadShell>, | ||
| ): InteractionSoundCue[] { | ||
| const cues: InteractionSoundCue[] = []; | ||
|
|
||
| for (const thread of threads) { | ||
| const prior = previous.get(threadKey(thread)); | ||
| const nextCompletedTurn = completedTurn(thread); | ||
|
|
||
| if (prior && nextCompletedTurn !== null && prior.completedTurn !== nextCompletedTurn) { | ||
| cues.push("success"); | ||
| } | ||
| const hasPendingUserAction = thread.hasPendingUserInput || thread.hasPendingApprovals; | ||
| if (prior && hasPendingUserAction && !prior.hasPendingUserAction) { | ||
| cues.push("bloom"); | ||
| } | ||
| } | ||
|
|
||
| return cues; | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,6 +9,7 @@ import { | |
| useNavigate, | ||
| } from "@tanstack/react-router"; | ||
| import { useEffect, useEffectEvent, useRef, useState } from "react"; | ||
| import { play } from "cuelume"; | ||
|
|
||
| import { APP_BASE_NAME, APP_DISPLAY_NAME, APP_STAGE_LABEL } from "../branding"; | ||
| import { resolveServerBackedAppDisplayName } from "../branding.logic"; | ||
|
|
@@ -27,7 +28,7 @@ import { | |
| toastManager, | ||
| } from "../components/ui/toast"; | ||
| import { resolveAndPersistPreferredEditor } from "../editorPreferences"; | ||
| import { useClientSettings } from "../hooks/useSettings"; | ||
| import { useClientSettings, useClientSettingsHydrated } from "../hooks/useSettings"; | ||
| import { | ||
| deriveLogicalProjectKeyFromSettings, | ||
| derivePhysicalProjectKeyFromPath, | ||
|
|
@@ -47,7 +48,18 @@ import { | |
| primaryServerConfigEventAtom, | ||
| primaryServerWelcomeAtom, | ||
| } from "../state/server"; | ||
| import { readProject, setActiveEnvironmentId, useActiveEnvironmentId } from "../state/entities"; | ||
| import { | ||
| readProject, | ||
| setActiveEnvironmentId, | ||
| useActiveEnvironmentId, | ||
| useThreadShells, | ||
| } from "../state/entities"; | ||
| import { | ||
| captureThreadSoundState, | ||
| captureThreadSoundStateWhileSettingsHydrating, | ||
| deriveInteractionSoundCues, | ||
| type ThreadSoundStateByKey, | ||
| } from "../interactionSounds"; | ||
| import { | ||
| createKeybindingsUpdateToastController, | ||
| type KeybindingsUpdateToastController, | ||
|
|
@@ -134,13 +146,41 @@ function RootRouteView() { | |
| <SlowRpcRequestToastCoordinator /> | ||
| <HostedStaticEnvironmentBootstrap /> | ||
| {primaryEnvironmentAuthenticated ? <EventRouter /> : null} | ||
| <InteractionSoundCoordinator /> | ||
| {primaryEnvironmentAuthenticated ? <ProviderUpdateLaunchNotification /> : null} | ||
| {appShell} | ||
| </AnchoredToastProvider> | ||
| </ToastProvider> | ||
| ); | ||
| } | ||
|
|
||
| function InteractionSoundCoordinator() { | ||
| const threads = useThreadShells(); | ||
| const completionSoundEnabled = useClientSettings((settings) => settings.enableCompletionSounds); | ||
| const settingsHydrated = useClientSettingsHydrated(); | ||
| const previousStateRef = useRef<ThreadSoundStateByKey | null>(null); | ||
|
|
||
| useEffect(() => { | ||
| if (!settingsHydrated) { | ||
| previousStateRef.current = captureThreadSoundStateWhileSettingsHydrating( | ||
| previousStateRef.current, | ||
| threads, | ||
| ); | ||
|
Comment on lines
+164
to
+168
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
On warm starts the shell layer first exposes the cached snapshot ( Useful? React with 👍 / 👎. |
||
| return; | ||
| } | ||
|
|
||
| const previous = previousStateRef.current; | ||
| if (completionSoundEnabled && previous !== null) { | ||
| for (const cue of deriveInteractionSoundCues(previous, threads)) { | ||
| play(cue); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
In browsers that require Useful? React with 👍 / 👎. |
||
| } | ||
| } | ||
| previousStateRef.current = captureThreadSoundState(threads); | ||
| }, [completionSoundEnabled, settingsHydrated, threads]); | ||
|
cursor[bot] marked this conversation as resolved.
|
||
|
|
||
| return null; | ||
| } | ||
|
|
||
| function DocumentTitleSync() { | ||
| const primaryServerVersion = | ||
| useAtomValue(primaryServerConfigAtom)?.environment.serverVersion ?? null; | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Interrupted finished turns skip sound
Medium Severity
completedTurnonly treatslatestTurn.state === "completed"as finished. Shell snapshots often end turns asinterruptedwith a non-nullcompletedAt(session teardown races and interrupt handling), whichresolveThreadAwarenessPhasealready counts as completed. Those finishes never emit thesuccesscue.Reviewed by Cursor Bugbot for commit 9626d08. Configure here.