mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-10-06 07:29:42 +02:00
feat(voice): dictate through the server's Claude Code login, no API key
The mic button previously needed a Deepgram API key, or fell back to the browser's Web Speech engine. It can now transcribe through the same speech-to-text service Claude Code's own /voice mode uses, so anyone signed in to Claude Code on the server gets dictation with no third-party account. Claude Code's voice mode cannot be driven directly: it opens the HOST's microphone (sox/arecord), and the CLI runs in a headless tmux pane while the human is in a browser somewhere else. So capture stays in the browser and only the transcription backend is borrowed. Audio goes browser -> Codeman -> Anthropic. The OAuth token never reaches the page: the browser sends PCM16 (16 kHz mono, produced by an AudioWorklet since MediaRecorder cannot emit raw PCM) and receives text. - GET /api/voice/status reports readiness and never the token - GET /ws/voice/stream relays one dictation, with the same Host/Origin upgrade guard as the terminal socket, plus caps on concurrency, stream length and frame size - credentials are read-only: Codeman never refreshes them, since a refresh rotates the refresh token and could sign the user out of their own CLI - claudeVoiceEnabled (synced, default OFF) gates the whole server side - voiceSettings.provider picks auto/claude/deepgram/webspeech; auto prefers Claude, then a configured Deepgram key, then the browser Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -24,4 +24,5 @@ export { registerSearchRoutes } from './search-routes.js';
|
||||
export { registerMeRoutes } from './me-routes.js';
|
||||
export { registerAdminRoutes } from './admin-routes.js';
|
||||
export { registerWsRoutes } from './ws-routes.js';
|
||||
export { registerVoiceRoutes } from './voice-routes.js';
|
||||
export { registerWebviewRoutes, tryWebviewRefererFallback } from './webview-routes.js';
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
/**
|
||||
* @fileoverview Claude voice dictation routes.
|
||||
*
|
||||
* - `GET /api/voice/status` — can this server transcribe? (settings gate + credential state)
|
||||
* - `GET /ws/voice/stream` — one dictation: PCM16 audio up, transcripts down
|
||||
*
|
||||
* Design and the upstream protocol: `docs/claude-voice-plan.md`. The relay itself
|
||||
* lives in `../voice-stream.ts`; this file is the auth, gating and lifetime shell
|
||||
* around it.
|
||||
*
|
||||
* ⚠️ `/api/voice/status` reports STATE, never the token: `{ available, reason,
|
||||
* subscriptionType?, expiresAt? }`. The Claude OAuth access token stays inside the
|
||||
* server process — the browser sends audio and receives text, nothing else.
|
||||
*
|
||||
* ⚠️ The WebSocket carries the same upgrade guard as `/ws/sessions/:id/terminal`
|
||||
* (allowed Host + same-site Origin, on top of the global auth hook that already ran
|
||||
* on the handshake). Without it a cross-site page could open a dictation stream on
|
||||
* the user's credentials and bill their subscription.
|
||||
*
|
||||
* ⚠️ The feature is OFF unless `claudeVoiceEnabled` is set: turning it on spends the
|
||||
* server owner's Claude subscription on transcription for anyone who can reach the
|
||||
* UI, which is a decision for the operator rather than a default.
|
||||
*/
|
||||
|
||||
import { createRequire } from 'module';
|
||||
import { FastifyInstance } from 'fastify';
|
||||
import type { WebSocket } from 'ws';
|
||||
import { ApiErrorCode, createErrorResponse } from '../../types.js';
|
||||
import { isAllowedRequestHost, isAllowedRequestOrigin, type HostPolicy } from '../network-auth-policy.js';
|
||||
import { readClaudeOAuthCredentials } from '../../claude-credentials.js';
|
||||
import { VoiceStreamRelay } from '../voice-stream.js';
|
||||
import { MAX_AUDIO_FRAME_BYTES, MAX_CONCURRENT_STREAMS } from '../../config/voice.js';
|
||||
import type { ConfigPort } from '../ports/index.js';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const { version: APP_VERSION } = require('../../../package.json') as { version: string };
|
||||
|
||||
/** Why voice is unavailable, in a form the frontend can branch on. */
|
||||
export type VoiceUnavailableReason = 'disabled' | 'no-credentials' | 'expired' | 'malformed';
|
||||
|
||||
export interface VoiceStatus {
|
||||
available: boolean;
|
||||
reason?: VoiceUnavailableReason;
|
||||
/** Display-only ('max', 'pro'); present when the credential store reported one. */
|
||||
subscriptionType?: string;
|
||||
expiresAt?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the server's dictation readiness. Split out and exported so the status
|
||||
* endpoint and the WebSocket upgrade cannot drift apart: the socket must never
|
||||
* accept a stream the status endpoint calls unavailable.
|
||||
*/
|
||||
export async function resolveVoiceStatus(enabled: boolean): Promise<VoiceStatus> {
|
||||
if (!enabled) return { available: false, reason: 'disabled' };
|
||||
const creds = await readClaudeOAuthCredentials();
|
||||
switch (creds.status) {
|
||||
case 'ok':
|
||||
return { available: true, subscriptionType: creds.subscriptionType, expiresAt: creds.expiresAt };
|
||||
case 'expired':
|
||||
return { available: false, reason: 'expired', expiresAt: creds.expiresAt };
|
||||
case 'malformed':
|
||||
return { available: false, reason: 'malformed' };
|
||||
default:
|
||||
return { available: false, reason: 'no-credentials' };
|
||||
}
|
||||
}
|
||||
|
||||
/** Live relays, server-wide. Dictation is human-paced, so the cap is small. */
|
||||
let activeStreams = 0;
|
||||
|
||||
/** Test seam: the cap is process-wide state, so suites must be able to reset it. */
|
||||
export function _resetVoiceStreamCountForTesting(): void {
|
||||
activeStreams = 0;
|
||||
}
|
||||
|
||||
/** Split a comma-separated keyterms query value into terms. */
|
||||
function parseKeyterms(raw: unknown): string[] {
|
||||
if (typeof raw !== 'string' || !raw) return [];
|
||||
return raw
|
||||
.split(',')
|
||||
.map((t) => t.trim())
|
||||
.filter(Boolean)
|
||||
.slice(0, 100);
|
||||
}
|
||||
|
||||
export function registerVoiceRoutes(app: FastifyInstance, ctx: ConfigPort, getHostPolicy: () => HostPolicy): void {
|
||||
app.get('/api/voice/status', async (_req, reply) => {
|
||||
try {
|
||||
return { success: true, data: await resolveVoiceStatus(await ctx.getClaudeVoiceEnabled()) };
|
||||
} catch {
|
||||
reply.code(500);
|
||||
return createErrorResponse(ApiErrorCode.INTERNAL_ERROR, 'Failed to read voice status');
|
||||
}
|
||||
});
|
||||
|
||||
app.get<{ Querystring: { language?: string; keyterms?: string } }>(
|
||||
'/ws/voice/stream',
|
||||
{ websocket: true },
|
||||
async (socket: WebSocket, req) => {
|
||||
// Cross-site upgrade guard first: this socket spends the operator's Claude
|
||||
// subscription, so it must be reachable only from Codeman's own origin.
|
||||
const policy = getHostPolicy();
|
||||
if (!isAllowedRequestHost(req.headers.host, policy) || !isAllowedRequestOrigin(req.headers.origin, policy)) {
|
||||
socket.close(4003, 'Forbidden');
|
||||
return;
|
||||
}
|
||||
|
||||
const status = await resolveVoiceStatus(await ctx.getClaudeVoiceEnabled());
|
||||
if (!status.available) {
|
||||
socket.close(4004, status.reason ?? 'unavailable');
|
||||
return;
|
||||
}
|
||||
// Re-read rather than trusting resolveVoiceStatus's discarded token: the
|
||||
// status helper deliberately never returns it.
|
||||
const creds = await readClaudeOAuthCredentials();
|
||||
if (creds.status !== 'ok' || !creds.accessToken) {
|
||||
socket.close(4004, 'no-credentials');
|
||||
return;
|
||||
}
|
||||
|
||||
if (activeStreams >= MAX_CONCURRENT_STREAMS) {
|
||||
socket.close(4008, 'Too many voice streams');
|
||||
return;
|
||||
}
|
||||
activeStreams++;
|
||||
|
||||
let released = false;
|
||||
const release = () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
activeStreams--;
|
||||
};
|
||||
|
||||
const send = (payload: Record<string, unknown>) => {
|
||||
if (socket.readyState !== 1) return;
|
||||
try {
|
||||
socket.send(JSON.stringify(payload));
|
||||
} catch {
|
||||
/* client vanished mid-write */
|
||||
}
|
||||
};
|
||||
|
||||
const relay = new VoiceStreamRelay({
|
||||
accessToken: creds.accessToken,
|
||||
appVersion: APP_VERSION,
|
||||
language: req.query.language,
|
||||
keyterms: parseKeyterms(req.query.keyterms),
|
||||
onReady: () => send({ t: 'ready' }),
|
||||
onTranscript: (text, final) => send({ t: 'transcript', text, final }),
|
||||
onError: (message) => send({ t: 'error', message }),
|
||||
onClose: () => {
|
||||
release();
|
||||
send({ t: 'closed' });
|
||||
if (socket.readyState === 1) {
|
||||
try {
|
||||
socket.close(1000, 'Voice stream ended');
|
||||
} catch {
|
||||
/* already closing */
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
// Handlers are attached synchronously before any further await
|
||||
// (@fastify/websocket drops messages that arrive before they exist).
|
||||
socket.on('message', (raw: Buffer, isBinary: boolean) => {
|
||||
if (isBinary) {
|
||||
if (raw.length === 0 || raw.length > MAX_AUDIO_FRAME_BYTES) return;
|
||||
relay.sendAudio(raw);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const msg = JSON.parse(String(raw)) as { t?: string };
|
||||
if (msg.t === 'finalize') relay.finalize();
|
||||
else if (msg.t === 'stop') relay.close();
|
||||
} catch {
|
||||
/* non-JSON control frame — ignore */
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('close', () => {
|
||||
relay.close();
|
||||
release();
|
||||
});
|
||||
socket.on('error', () => {
|
||||
relay.close();
|
||||
release();
|
||||
});
|
||||
|
||||
relay.connect();
|
||||
}
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user