mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-10-03 05:59:43 +02:00
feat(custom-model): Custom Model Endpoint Profiles (local or cloud, all harnesses)
Point any Codeman-supported harness (Claude, opencode, Codex, Gemini, Pi, Grok, DeepSeek, OMP) at a custom OpenAI-compatible endpoint instead of its native cloud backend, for a given session. Covers local hardware (llama.cpp, Ollama, vLLM, DGX Spark, Strix Halo) and cloud (Azure AI Foundry, OpenRouter). Off by default (customModelEndpointsEnabled, synced, default OFF). - Registry: capabilities.customModelInjection per CLI entry (env / configContentEnv / configDir / unsupported kinds) - Pure injection builder (custom-model-injection.ts) turning an endpoint + model id into the real env vars / config content per CLI - Endpoint store + CRUD routes (custom-model-hosts.ts, custom-model-routes.ts), discovery via GET /v1/models, SSRF-guarded - Session integration: Session.setCustomModel()/restartCli() (POST /api/sessions/:id/custom-model), reusing the existing respawn-pane -k primitive to restart the CLI process with new env - Multi-user hardening: every new redirect-capable env var added to its CLI's privilegedEnvKeys, closing a pre-existing gap where several were already reachable via the generic envOverrides field's prefix allowlist - Standalone scripts/test-local-llm-harnesses.mjs: spawns real CLI binaries against a real endpoint outside the web UI, independent of tmux/sessions - Mock-server contract tests (test/fixtures/mock-openai-server.ts) replaying every CLI's injected values through a real HTTP shape Real end-to-end validation against a live llama-swap server (inside a codeman/agent:llm-test Docker image with all 9 CLI binaries) found and fixed three real bugs before they shipped: - Codex's config.toml schema was wrong ([model].default table instead of a top-level model string + [model_providers.custom]); fixing it then surfaced a genuine, documented protocol incompatibility (Codex only speaks the Responses API since Feb 2026, which llama.cpp/llama-swap don't implement) - Claude Code's async session-title-generation call validates ANTHROPIC_DEFAULT_HAIKU_MODEL against its own internal model list and hangs the whole -p invocation on an unrecognized name; documented for chunk 6, worked around in the standalone script only (--bare is NOT safe for a real interactive session, which needs hooks) - The discovery route's authStyle: 'both' option (send both Authorization and api-key headers) reliably hung a real server; removed the option entirely rather than just changing the default Status: draft. Chunk 6 (frontend toolbar/settings UI) not yet built — see PR.md and deployment_plan.md for the full chunk breakdown and confidence table. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017HqNWfmtBU2KN29SvSVWB3
This commit is contained in:
Devvyn
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
a017e9a8e0
commit
41416566aa
@@ -0,0 +1,128 @@
|
||||
/**
|
||||
* @fileoverview Custom Model Endpoint Profiles CRUD + discovery
|
||||
* (deployment_plan.md). Endpoints are machine-level infra, like remote/docker
|
||||
* hosts, so writes are admin-only in multi-user mode
|
||||
* (`case-routes.ts`'s `/api/remote-hosts` is the pattern this mirrors).
|
||||
*
|
||||
* Discovery (`POST /:id/discover-models`) fetches `${baseUrl}/v1/models`.
|
||||
* `isBlockedWebviewUrl()` is the same synchronous hostname/link-local/cloud-
|
||||
* metadata check `webview-egress-policy.ts` uses for saved dashboard URLs —
|
||||
* reused here as a save-time and discover-time guard. It does NOT re-check
|
||||
* the DNS-RESOLVED address the way `webviewFetch()`'s undici lookup hook
|
||||
* does; wiring that dispatcher-level guard here is a followup, not done in
|
||||
* this pass, since this route is already admin-only in multi-user mode.
|
||||
*/
|
||||
|
||||
import type { FastifyInstance, FastifyRequest } from 'fastify';
|
||||
import { ApiErrorCode, createErrorResponse, type ApiResponse } from '../../types.js';
|
||||
import { isAdmin, parseBody } from '../route-helpers.js';
|
||||
import { isMultiUserMode } from '../../config/multiuser.js';
|
||||
import { getDataDir } from '../../config/instance.js';
|
||||
import { isBlockedWebviewUrl } from '../webview-egress-policy.js';
|
||||
import { CustomModelHostSchema } from '../schemas.js';
|
||||
import { readCustomModelHosts, writeCustomModelHosts, type CustomModelHost } from '../../custom-model-hosts.js';
|
||||
|
||||
const CODEMAN_CONFIG_DIR = getDataDir();
|
||||
const DISCOVER_TIMEOUT_MS = 8000;
|
||||
|
||||
function adminOnly(req: FastifyRequest, reply: { code: (n: number) => unknown }): ApiResponse<never> | null {
|
||||
if (!isMultiUserMode() || isAdmin(req)) return null;
|
||||
reply.code(403);
|
||||
return createErrorResponse(ApiErrorCode.FORBIDDEN, 'Admin only in multi-user mode');
|
||||
}
|
||||
|
||||
async function discoverModels(host: Pick<CustomModelHost, 'baseUrl' | 'apiKey' | 'authStyle'>): Promise<string[]> {
|
||||
const headers: Record<string, string> = {};
|
||||
const apiKey = host.apiKey?.trim();
|
||||
// Exactly ONE header, never both — see custom-model-hosts.ts's CustomModelAuthStyle
|
||||
// doc comment for why: sending both reliably HANGS some real servers.
|
||||
const style = host.authStyle ?? 'bearer';
|
||||
if (apiKey && style === 'bearer') headers.Authorization = `Bearer ${apiKey}`;
|
||||
if (apiKey && style === 'api-key') headers['api-key'] = apiKey;
|
||||
|
||||
const res = await fetch(`${host.baseUrl.replace(/\/+$/, '')}/v1/models`, {
|
||||
headers,
|
||||
signal: AbortSignal.timeout(DISCOVER_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const body = (await res.json()) as { data?: Array<{ id?: unknown }> };
|
||||
return (body.data ?? []).map((m) => m.id).filter((id): id is string => typeof id === 'string' && id.length > 0);
|
||||
}
|
||||
|
||||
export function registerCustomModelRoutes(app: FastifyInstance): void {
|
||||
app.get('/api/model-endpoints', async (req) =>
|
||||
isMultiUserMode() && !isAdmin(req) ? [] : readCustomModelHosts(CODEMAN_CONFIG_DIR)
|
||||
);
|
||||
|
||||
app.post('/api/model-endpoints', async (req, reply): Promise<ApiResponse<{ host: CustomModelHost }>> => {
|
||||
const denied = adminOnly(req, reply);
|
||||
if (denied) return denied;
|
||||
const host = parseBody(CustomModelHostSchema, req.body);
|
||||
if (isBlockedWebviewUrl(host.baseUrl)) {
|
||||
return createErrorResponse(ApiErrorCode.INVALID_INPUT, 'Endpoint base URL is not allowed');
|
||||
}
|
||||
const hosts = await readCustomModelHosts(CODEMAN_CONFIG_DIR);
|
||||
if (hosts.some((item) => item.id === host.id)) {
|
||||
return createErrorResponse(ApiErrorCode.ALREADY_EXISTS, 'Model endpoint already exists');
|
||||
}
|
||||
await writeCustomModelHosts(CODEMAN_CONFIG_DIR, [...hosts, host]);
|
||||
return { success: true, data: { host } };
|
||||
});
|
||||
|
||||
app.put('/api/model-endpoints/:id', async (req, reply): Promise<ApiResponse<{ host: CustomModelHost }>> => {
|
||||
const denied = adminOnly(req, reply);
|
||||
if (denied) return denied;
|
||||
const { id } = req.params as { id: string };
|
||||
const host = parseBody(CustomModelHostSchema, { ...(req.body as object), id });
|
||||
if (isBlockedWebviewUrl(host.baseUrl)) {
|
||||
return createErrorResponse(ApiErrorCode.INVALID_INPUT, 'Endpoint base URL is not allowed');
|
||||
}
|
||||
const hosts = await readCustomModelHosts(CODEMAN_CONFIG_DIR);
|
||||
const index = hosts.findIndex((item) => item.id === id);
|
||||
if (index === -1) return createErrorResponse(ApiErrorCode.NOT_FOUND, 'Model endpoint not found');
|
||||
const next = [...hosts];
|
||||
next[index] = host;
|
||||
await writeCustomModelHosts(CODEMAN_CONFIG_DIR, next);
|
||||
return { success: true, data: { host } };
|
||||
});
|
||||
|
||||
app.delete('/api/model-endpoints/:id', async (req, reply): Promise<ApiResponse<{ id: string }>> => {
|
||||
const denied = adminOnly(req, reply);
|
||||
if (denied) return denied;
|
||||
const { id } = req.params as { id: string };
|
||||
const hosts = await readCustomModelHosts(CODEMAN_CONFIG_DIR);
|
||||
await writeCustomModelHosts(
|
||||
CODEMAN_CONFIG_DIR,
|
||||
hosts.filter((item) => item.id !== id)
|
||||
);
|
||||
return { success: true, data: { id } };
|
||||
});
|
||||
|
||||
app.post(
|
||||
'/api/model-endpoints/:id/discover-models',
|
||||
async (req, reply): Promise<ApiResponse<{ models: string[] }>> => {
|
||||
const denied = adminOnly(req, reply);
|
||||
if (denied) return denied;
|
||||
const { id } = req.params as { id: string };
|
||||
const hosts = await readCustomModelHosts(CODEMAN_CONFIG_DIR);
|
||||
const index = hosts.findIndex((item) => item.id === id);
|
||||
if (index === -1) return createErrorResponse(ApiErrorCode.NOT_FOUND, 'Model endpoint not found');
|
||||
const host = hosts[index];
|
||||
if (isBlockedWebviewUrl(host.baseUrl)) {
|
||||
return createErrorResponse(ApiErrorCode.INVALID_INPUT, 'Endpoint base URL is not allowed');
|
||||
}
|
||||
try {
|
||||
const models = await discoverModels(host);
|
||||
const next = [...hosts];
|
||||
next[index] = { ...host, models, lastDiscoveredAt: new Date().toISOString() };
|
||||
await writeCustomModelHosts(CODEMAN_CONFIG_DIR, next);
|
||||
return { success: true, data: { models } };
|
||||
} catch (err) {
|
||||
return createErrorResponse(
|
||||
ApiErrorCode.OPERATION_FAILED,
|
||||
`Could not reach endpoint: ${err instanceof Error ? err.message : String(err)}`
|
||||
);
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -27,3 +27,4 @@ export { registerWsRoutes } from './ws-routes.js';
|
||||
export { registerVoiceRoutes } from './voice-routes.js';
|
||||
export { registerWebviewRoutes, tryWebviewRefererFallback } from './webview-routes.js';
|
||||
export { registerTabLayoutRoutes } from './tab-layout-routes.js';
|
||||
export { registerCustomModelRoutes } from './custom-model-routes.js';
|
||||
|
||||
@@ -8,7 +8,7 @@ import { FastifyInstance, type FastifyReply } from 'fastify';
|
||||
import { z } from 'zod';
|
||||
import { join, dirname, extname, basename } from 'node:path';
|
||||
import { homedir } from 'node:os';
|
||||
import { existsSync, statSync, mkdirSync, writeFileSync } from 'node:fs';
|
||||
import { existsSync, statSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { execFile } from 'node:child_process';
|
||||
import fs from 'node:fs/promises';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
@@ -51,7 +51,10 @@ import {
|
||||
SessionOrderUpdateSchema,
|
||||
SessionWaitQuerySchema,
|
||||
SessionWaitOutputQuerySchema,
|
||||
CustomModelSelectionSchema,
|
||||
} from '../schemas.js';
|
||||
import { readCustomModelHosts } from '../../custom-model-hosts.js';
|
||||
import { buildCustomModelInjection } from '../../custom-model-injection.js';
|
||||
import { ownerLayoutKey } from '../../tab-layout-persistence.js';
|
||||
import { TabLayoutValidationError } from '../../tab-layout.js';
|
||||
import {
|
||||
@@ -1161,6 +1164,88 @@ export function registerSessionRoutes(
|
||||
return { color: session.color };
|
||||
});
|
||||
|
||||
// ========== Custom Model Endpoint Profiles (deployment_plan.md) ==========
|
||||
//
|
||||
// Applies (or clears) a session's custom OpenAI-compatible endpoint selection and
|
||||
// RESTARTS the pane's CLI process — these harnesses read endpoint config at process
|
||||
// start, not per-turn, so a live hot-swap isn't possible (confirmed with the
|
||||
// maintainer). Endpoints come from the admin-configured custom-model-hosts store
|
||||
// (chunk 3's CRUD routes), never raw client-supplied env — that's what keeps this
|
||||
// route safe to let any session owner call for their own session, unlike the
|
||||
// generic envOverrides field the privilegedEnvKeys clamp exists to guard.
|
||||
app.post('/api/sessions/:id/custom-model', async (req) => {
|
||||
const { id } = req.params as { id: string };
|
||||
const body = parseBody(CustomModelSelectionSchema, req.body, 'Invalid request body');
|
||||
const session = findSessionOrFail(ctx, id, req);
|
||||
|
||||
if (session.isBusy()) {
|
||||
return createErrorResponse(ApiErrorCode.SESSION_BUSY, 'Session is busy');
|
||||
}
|
||||
|
||||
if ('clear' in body) {
|
||||
const previousConfigDir = session.setCustomModel(undefined);
|
||||
if (previousConfigDir) rmSync(previousConfigDir, { recursive: true, force: true });
|
||||
const restarted = await session.restartCli();
|
||||
persistAndBroadcastSession(ctx, session);
|
||||
return { customModel: session.customModel, restarted };
|
||||
}
|
||||
|
||||
const entry = getCli(session.mode);
|
||||
if (!entry) {
|
||||
return createErrorResponse(ApiErrorCode.INVALID_INPUT, `No CLI registry entry for mode ${session.mode}`);
|
||||
}
|
||||
if (entry.capabilities.customModelInjection.kind === 'unsupported') {
|
||||
return createErrorResponse(ApiErrorCode.OPERATION_FAILED, `${session.mode} has no known custom-model mechanism`);
|
||||
}
|
||||
|
||||
const hosts = await readCustomModelHosts(getDataDir());
|
||||
const endpoint = hosts.find((h) => h.id === body.endpointId);
|
||||
if (!endpoint) {
|
||||
return createErrorResponse(ApiErrorCode.NOT_FOUND, 'Model endpoint not found');
|
||||
}
|
||||
|
||||
const injection = buildCustomModelInjection(entry, endpoint, body.modelId);
|
||||
|
||||
let envOverrides: Record<string, string>;
|
||||
let envKeys: string[];
|
||||
let configDir: string | undefined;
|
||||
|
||||
if (injection.kind === 'env') {
|
||||
envOverrides = injection.envOverrides;
|
||||
envKeys = Object.keys(injection.envOverrides);
|
||||
} else if (injection.kind === 'configDir') {
|
||||
// Isolated per-session dir — never the user's real CLI config path.
|
||||
configDir = join(dataPath('custom-model-configs'), session.id);
|
||||
for (const file of injection.files) {
|
||||
const filePath = join(configDir, file.relPath);
|
||||
mkdirSync(dirname(filePath), { recursive: true });
|
||||
writeFileSync(filePath, file.content, 'utf8');
|
||||
}
|
||||
// extraEnv: vars the written config file REFERENCES by name (codex's `env_key`
|
||||
// convention) rather than embedding a literal value — must ride alongside
|
||||
// dirEnvVar or the config points at a credential that was never actually set.
|
||||
envOverrides = { [injection.dirEnvVar]: configDir, ...injection.extraEnv };
|
||||
envKeys = [injection.dirEnvVar, ...Object.keys(injection.extraEnv ?? {})];
|
||||
} else {
|
||||
// 'unsupported' is already handled above; this keeps the switch exhaustive.
|
||||
return createErrorResponse(ApiErrorCode.OPERATION_FAILED, `${session.mode} has no known custom-model mechanism`);
|
||||
}
|
||||
|
||||
const previousConfigDir = session.setCustomModel(
|
||||
{ endpointId: endpoint.id, modelId: body.modelId, label: endpoint.label, envKeys, configDir },
|
||||
envOverrides
|
||||
);
|
||||
// Clean up the OLD config dir on disk, unless the new one happens to reuse the same
|
||||
// path (same session, configDir kind again) — never delete the dir we just wrote.
|
||||
if (previousConfigDir && previousConfigDir !== configDir) {
|
||||
rmSync(previousConfigDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
const restarted = await session.restartCli();
|
||||
persistAndBroadcastSession(ctx, session);
|
||||
return { customModel: session.customModel, restarted };
|
||||
});
|
||||
|
||||
// ========== Delete Session ==========
|
||||
|
||||
app.delete('/api/sessions/:id', async (req) => {
|
||||
|
||||
@@ -741,6 +741,29 @@ export const RemoteHostSchema = z.object({
|
||||
commands: RemoteCommandOverridesSchema,
|
||||
});
|
||||
|
||||
// Custom Model Endpoint Profiles (deployment_plan.md) — a user-configured custom
|
||||
// OpenAI-compatible endpoint, local (llama.cpp) or cloud (Azure AI Foundry, etc.).
|
||||
export const CustomModelHostSchema = z.object({
|
||||
id: z.string().regex(/^[a-zA-Z0-9_-]+$/, 'Invalid endpoint id'),
|
||||
label: z.string().min(1).max(100),
|
||||
baseUrl: z.string().url().max(2048),
|
||||
apiKey: z.string().max(4096).optional(),
|
||||
// No 'both': live-tested against a real server, sending both auth header
|
||||
// conventions on one request reliably HANGS it — see custom-model-hosts.ts.
|
||||
authStyle: z.enum(['bearer', 'api-key']).optional(),
|
||||
models: z.array(z.string().max(200)).max(200).optional(),
|
||||
lastDiscoveredAt: z.string().max(64).optional(),
|
||||
});
|
||||
|
||||
/** POST /api/sessions/:id/custom-model — apply or clear a session's custom-model selection. */
|
||||
export const CustomModelSelectionSchema = z.union([
|
||||
z.object({
|
||||
endpointId: z.string().regex(/^[a-zA-Z0-9_-]+$/, 'Invalid endpoint id'),
|
||||
modelId: z.string().min(1).max(200),
|
||||
}),
|
||||
z.object({ clear: z.literal(true) }),
|
||||
]);
|
||||
|
||||
export const RemoteCaseLinkSchema = z.object({
|
||||
name: z.string().regex(/^[a-zA-Z0-9_-]+$/, 'Invalid case name format'),
|
||||
hostId: z.string().regex(/^[a-zA-Z0-9_-]+$/, 'Invalid remote host id'),
|
||||
@@ -1239,6 +1262,13 @@ export const SettingsUpdateSchema = z
|
||||
* stored profiles stay until DELETE /api/sessions/:id/intent.
|
||||
*/
|
||||
readMyMindEnabled: z.boolean().optional(),
|
||||
/**
|
||||
* Custom Model Endpoint Profiles (deployment_plan.md): the toolbar picker that lets a
|
||||
* session point at a user-configured custom OpenAI-compatible endpoint (local or
|
||||
* cloud) instead of its native cloud backend. SYNCED, default OFF — endpoint entry,
|
||||
* discovery, and the extra toolbar surface are all opt-in.
|
||||
*/
|
||||
customModelEndpointsEnabled: z.boolean().optional(),
|
||||
/**
|
||||
* Read My Mind predictor model override. Empty/absent = the AI-checker
|
||||
* default (opus: prediction quality is the product and it runs only on an
|
||||
|
||||
@@ -179,6 +179,7 @@ import {
|
||||
registerVoiceRoutes,
|
||||
registerWebviewRoutes,
|
||||
registerTabLayoutRoutes,
|
||||
registerCustomModelRoutes,
|
||||
tryWebviewRefererFallback,
|
||||
} from './routes/index.js';
|
||||
import { CronService } from '../cron/cron-service.js';
|
||||
@@ -1051,6 +1052,7 @@ export class WebServer extends EventEmitter {
|
||||
registerOrchestratorRoutes(this.app, ctx);
|
||||
registerWebviewRoutes(this.app, ctx, this.basePath);
|
||||
registerTabLayoutRoutes(this.app, ctx);
|
||||
registerCustomModelRoutes(this.app);
|
||||
|
||||
// Cron: build the service from the same context, recompute
|
||||
// due times for any persisted jobs, then expose it to its routes.
|
||||
|
||||
Reference in New Issue
Block a user