Merge pull request #430 from opticon454/custom-model-run-menu

This commit is contained in:
Codeman maintainer
2026-09-19 12:25:11 +02:00
43 changed files with 7304 additions and 144 deletions
+223
View File
@@ -181,6 +181,34 @@ describe('custom model endpoint CRUD', () => {
expect(res.json().error).toMatch(/refused.*169\.254\.169\.254/);
});
it('running-status never hands the browser the raw llama-swap launch command (cmd)', async () => {
const { app } = await setup();
await app.inject({
method: 'POST',
url: '/api/model-endpoints',
payload: { id: 'ep-running', label: 'A', baseUrl: 'http://localhost:8080', apiKey: 'k' },
});
fetchMock.mockImplementation(async (url: URL) => {
if (url.pathname === '/running') {
return new Response(
JSON.stringify({
running: [
{ model: 'qwen3', state: 'ready', cmd: 'llama-server -m /models/qwen3.gguf --api-key sk-secret' },
],
}),
{ status: 200 }
);
}
throw new Error(`unexpected request in this test: ${url.href}`);
});
const res = await app.inject({ method: 'GET', url: '/api/model-endpoints/ep-running/running-status' });
const body = res.json();
expect(body.data.running).toEqual([{ model: 'qwen3', state: 'ready' }]);
expect(JSON.stringify(body)).not.toContain('sk-secret');
expect(JSON.stringify(body)).not.toContain('cmd');
});
it('refuses a baseUrl with embedded credentials or a non-http scheme at save time', async () => {
const { app } = await setup();
for (const baseUrl of ['http://user:pw@host:8080', 'ftp://host/models', 'http://169.254.169.254']) {
@@ -194,3 +222,198 @@ describe('custom model endpoint CRUD', () => {
}
});
});
describe('defaultModelId — the Run-menu picker’s per-endpoint default', () => {
afterEach(() => {
fetchMock.mockReset();
});
it('rejects a defaultModelId that is not one of the endpoint’s discovered models, on both create and update', async () => {
const { app } = await setup();
const create = await app.inject({
method: 'POST',
url: '/api/model-endpoints',
payload: {
id: 'ep-default-reject',
label: 'A',
baseUrl: 'http://localhost:8080',
models: ['qwen3'],
defaultModelId: 'ghost',
},
});
expect(create.json().success).toBe(false);
expect(create.json().errorCode).toBe('INVALID_INPUT');
await app.inject({
method: 'POST',
url: '/api/model-endpoints',
payload: { id: 'ep-default-reject', label: 'A', baseUrl: 'http://localhost:8080', models: ['qwen3'] },
});
const update = await app.inject({
method: 'PUT',
url: '/api/model-endpoints/ep-default-reject',
payload: { label: 'A', baseUrl: 'http://localhost:8080', models: ['qwen3'], defaultModelId: 'ghost' },
});
expect(update.json().success).toBe(false);
expect(update.json().errorCode).toBe('INVALID_INPUT');
});
it('accepts a defaultModelId that IS one of the discovered models', async () => {
const { app } = await setup();
const res = await app.inject({
method: 'POST',
url: '/api/model-endpoints',
payload: {
id: 'ep-default-accept',
label: 'A',
baseUrl: 'http://localhost:8080',
models: ['qwen3', 'llama3'],
defaultModelId: 'llama3',
},
});
expect(res.json().success).toBe(true);
expect(res.json().data.host.defaultModelId).toBe('llama3');
});
it('drops a stale default that no longer appears in a fresh discovery, rather than carrying it forward invalid', async () => {
const { app } = await setup();
await app.inject({
method: 'POST',
url: '/api/model-endpoints',
payload: {
id: 'ep-default-drop',
label: 'A',
baseUrl: 'http://localhost:8080',
models: ['qwen3'],
defaultModelId: 'qwen3',
},
});
fetchMock.mockResolvedValue(new Response(JSON.stringify({ data: [{ id: 'llama3' }] }), { status: 200 }));
await app.inject({ method: 'POST', url: '/api/model-endpoints/ep-default-drop/discover-models' });
const list = await app.inject({ method: 'GET', url: '/api/model-endpoints' });
const stored = (list.json() as Array<{ id: string; defaultModelId?: string }>).find(
(h) => h.id === 'ep-default-drop'
);
expect(stored?.defaultModelId).toBeUndefined();
});
it('keeps a default that IS still present after a fresh discovery', async () => {
const { app } = await setup();
await app.inject({
method: 'POST',
url: '/api/model-endpoints',
payload: {
id: 'ep-default-keep',
label: 'A',
baseUrl: 'http://localhost:8080',
models: ['qwen3'],
defaultModelId: 'qwen3',
},
});
fetchMock.mockResolvedValue(
new Response(JSON.stringify({ data: [{ id: 'qwen3' }, { id: 'llama3' }] }), { status: 200 })
);
await app.inject({ method: 'POST', url: '/api/model-endpoints/ep-default-keep/discover-models' });
const list = await app.inject({ method: 'GET', url: '/api/model-endpoints' });
const stored = (list.json() as Array<{ id: string; defaultModelId?: string }>).find(
(h) => h.id === 'ep-default-keep'
);
expect(stored?.defaultModelId).toBe('qwen3');
});
});
describe('apiKey is never handed back to the browser', () => {
afterEach(() => {
fetchMock.mockReset();
});
it('POST, GET and PUT responses all carry apiKeySet instead of the real key', async () => {
const { app } = await setup();
const create = await app.inject({
method: 'POST',
url: '/api/model-endpoints',
payload: { id: 'ep-secret', label: 'A', baseUrl: 'http://localhost:8080', apiKey: 'super-secret' },
});
expect(create.json().data.host.apiKey).toBeUndefined();
expect(create.json().data.host.apiKeySet).toBe(true);
const list = await app.inject({ method: 'GET', url: '/api/model-endpoints' });
const listed = (list.json() as Array<{ id: string; apiKey?: string; apiKeySet?: boolean }>).find(
(h) => h.id === 'ep-secret'
);
expect(listed?.apiKey).toBeUndefined();
expect(listed?.apiKeySet).toBe(true);
expect(JSON.stringify(list.json())).not.toContain('super-secret');
const update = await app.inject({
method: 'PUT',
url: '/api/model-endpoints/ep-secret',
payload: { label: 'Renamed', baseUrl: 'http://localhost:8080' },
});
expect(update.json().data.host.apiKey).toBeUndefined();
expect(update.json().data.host.apiKeySet).toBe(true);
expect(JSON.stringify(update.json())).not.toContain('super-secret');
});
it('a host with no key set at all reports apiKeySet: false', async () => {
const { app } = await setup();
const create = await app.inject({
method: 'POST',
url: '/api/model-endpoints',
payload: { id: 'ep-nokey', label: 'A', baseUrl: 'http://localhost:8080' },
});
expect(create.json().data.host.apiKeySet).toBe(false);
});
it('PUT with no apiKey keeps the stored one, rather than clearing it', async () => {
const { app } = await setup();
await app.inject({
method: 'POST',
url: '/api/model-endpoints',
payload: { id: 'ep-keep-key', label: 'A', baseUrl: 'http://localhost:8080', apiKey: 'original-key' },
});
// Edit without touching the API key field — the real bug this guards: a
// browser round-trip that only ever sees apiKeySet, never the real value,
// must not accidentally send an empty string and wipe a working credential.
const update = await app.inject({
method: 'PUT',
url: '/api/model-endpoints/ep-keep-key',
payload: { label: 'Renamed', baseUrl: 'http://localhost:8080' },
});
expect(update.json().data.host.apiKeySet).toBe(true);
// Prove it by observing the auth header discovery actually sends.
fetchMock.mockImplementation(async (_url: URL, init?: RequestInit) => {
const headers = init?.headers as Record<string, string>;
expect(headers.Authorization).toBe('Bearer original-key');
return new Response(JSON.stringify({ data: [] }), { status: 200 });
});
const discover = await app.inject({ method: 'POST', url: '/api/model-endpoints/ep-keep-key/discover-models' });
expect(discover.json().success).toBe(true);
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('PUT with a new apiKey replaces the stored one', async () => {
const { app } = await setup();
await app.inject({
method: 'POST',
url: '/api/model-endpoints',
payload: { id: 'ep-replace-key', label: 'A', baseUrl: 'http://localhost:8080', apiKey: 'old-key' },
});
await app.inject({
method: 'PUT',
url: '/api/model-endpoints/ep-replace-key',
payload: { label: 'A', baseUrl: 'http://localhost:8080', apiKey: 'new-key' },
});
fetchMock.mockImplementation(async (_url: URL, init?: RequestInit) => {
const headers = init?.headers as Record<string, string>;
expect(headers.Authorization).toBe('Bearer new-key');
return new Response(JSON.stringify({ data: [] }), { status: 200 });
});
await app.inject({ method: 'POST', url: '/api/model-endpoints/ep-replace-key/discover-models' });
expect(fetchMock).toHaveBeenCalledTimes(1);
});
});
@@ -0,0 +1,382 @@
/**
* @fileoverview POST /api/quick-start's `customModel` field (docs/custom-model-endpoints-plan.md):
* the ONE-SHOT launch path that computes a custom-model endpoint's injection BEFORE the
* session/process exists and launches directly on it, so a custom-model Run never shows
* the native-boot-then-restart the dedicated POST /api/sessions/:id/custom-model route's
* restart-in-place design otherwise produces — most visibly on a CLI like Codex whose TUI
* fully reinitializes on a restart. That dedicated route is still what an ALREADY-RUNNING
* session uses to switch later; this is the create-time equivalent.
*
* Mirrors test/routes/session-custom-model.test.ts's fixtures and llama-swap mocking, since
* this route mirrors that one's own checks (llama-swap conflict, unsupported CLI, unknown
* endpoint, an argv-incompatible model id) rather than a lighter, separately-drifting copy.
*
* Session.prototype.startInteractive/startShell are mocked exactly like the workspace-hooks
* quick-start tests: quick-start constructs a REAL Session (not the MockSession the route
* test harness substitutes elsewhere), so tmux must never actually be reached.
*
* Port: N/A (app.inject, no real port needed)
*/
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import Fastify, { type FastifyInstance } from 'fastify';
import fastifyCookie from '@fastify/cookie';
import { rm, readFile } from 'node:fs/promises';
import { existsSync } from 'node:fs';
import { join } from 'node:path';
import { createMockRouteContext, safeRmHomeTree, type MockRouteContext } from '../mocks/index.js';
import { installRouteErrorHandler } from '../../src/web/route-error-handler.js';
import { registerSessionRoutes } from '../../src/web/routes/session-routes.js';
import { getDataDir } from '../../src/config/instance.js';
import { CASES_DIR } from '../../src/web/route-helpers.js';
import { Session } from '../../src/session.js';
import { writeCustomModelHosts, type CustomModelHost } from '../../src/custom-model-hosts.js';
import { customModelConfigDir } from '../../src/custom-model-injection-apply.js';
import { webviewFetch } from '../../src/web/webview-egress.js';
vi.mock('../../src/web/webview-egress.js', async () => {
const actual = await vi.importActual<typeof import('../../src/web/webview-egress.js')>(
'../../src/web/webview-egress.js'
);
return { ...actual, webviewFetch: vi.fn() };
});
const fetchMock = vi.mocked(webviewFetch);
// quick-start's own local-CLI-availability gate (resolveCliLaunchError, unrelated to the
// custom-model injection this file tests) runs BEFORE the code under test and would
// otherwise 404 every non-claude mode on a box with no codex/pi/grok/omp binary installed —
// exactly this test environment. Mirrors the real "not remote" bypass documented at its own
// call site in session-routes.ts (`session-routes.test.ts`'s remote-codex test is the
// precedent for needing this at all).
vi.mock('../../src/utils/cli-launcher.js', async () => {
const actual = await vi.importActual<typeof import('../../src/utils/cli-launcher.js')>(
'../../src/utils/cli-launcher.js'
);
return { ...actual, resolveCliLaunchError: vi.fn().mockResolvedValue(null) };
});
const ENDPOINT: CustomModelHost = {
id: 'ep1',
label: 'llama.cpp box',
baseUrl: 'http://192.168.1.50:8080',
apiKey: 'k',
};
describe('POST /api/quick-start: customModel (one-shot custom-model launch)', () => {
let app: FastifyInstance;
let ctx: MockRouteContext;
let restartSpy: ReturnType<typeof vi.spyOn>;
const quickStart = (payload: Record<string, unknown>) =>
app.inject({ method: 'POST', url: '/api/quick-start', payload });
beforeEach(async () => {
vi.spyOn(Session.prototype, 'startInteractive').mockResolvedValue(undefined);
vi.spyOn(Session.prototype, 'startShell').mockResolvedValue(undefined);
restartSpy = vi.spyOn(Session.prototype, 'restartCli').mockResolvedValue(true);
fetchMock.mockReset();
fetchMock.mockResolvedValue(new Response('not found', { status: 404 })); // default: not llama-swap
app = Fastify({ logger: false });
await app.register(fastifyCookie);
ctx = createMockRouteContext();
registerSessionRoutes(app, ctx);
installRouteErrorHandler(app);
await app.ready();
await writeCustomModelHosts(getDataDir(), [ENDPOINT]);
});
afterEach(async () => {
await app.close();
vi.restoreAllMocks();
await rm(join(getDataDir(), 'custom-model-hosts.json'), { force: true });
await rm(join(getDataDir(), 'custom-model-configs'), { recursive: true, force: true });
safeRmHomeTree(CASES_DIR);
});
it('launches a claude session already pointed at the endpoint — no restart at all', async () => {
const res = await quickStart({
caseName: 'cm-claude',
mode: 'claude',
customModel: { endpointId: 'ep1', modelId: 'qwen3' },
});
expect(res.statusCode).toBe(200);
const { sessionId } = res.json();
const session = ctx.sessions.get(sessionId) as unknown as Session;
expect(session.customModel).toEqual({ endpointId: 'ep1', modelId: 'qwen3', label: 'llama.cpp box' });
// The whole point: never restarted. It launched on the endpoint the first time.
expect(restartSpy).not.toHaveBeenCalled();
const isolatedDir = customModelConfigDir(sessionId);
const trustFile = JSON.parse(await readFile(join(isolatedDir, '.claude.json'), 'utf-8'));
expect(trustFile.customApiKeyResponses.approved).toEqual(['k']);
});
it('codex: writes the config.toml under the SAME id the session actually launches with, no restart', async () => {
const res = await quickStart({
caseName: 'cm-codex',
mode: 'codex',
customModel: { endpointId: 'ep1', modelId: 'qwen3' },
});
expect(res.statusCode).toBe(200);
const { sessionId } = res.json();
const session = ctx.sessions.get(sessionId) as unknown as Session;
expect(session.customModel?.endpointId).toBe('ep1');
expect(restartSpy).not.toHaveBeenCalled();
const configDir = customModelConfigDir(sessionId);
expect(existsSync(join(configDir, 'config.toml'))).toBe(true);
const toml = await readFile(join(configDir, 'config.toml'), 'utf-8');
expect(toml).toContain('model = "qwen3"');
});
it('pi: forces --model custom/<id> onto piConfig on the FIRST launch, not via a later restart', async () => {
const res = await quickStart({
caseName: 'cm-pi',
mode: 'pi',
customModel: { endpointId: 'ep1', modelId: 'qwen3.5-0.8b' },
});
expect(res.statusCode).toBe(200);
const { sessionId } = res.json();
const session = ctx.sessions.get(sessionId) as unknown as Session & { piConfig?: { model?: string } };
expect(session.getCustomModelForPersist()?.launchModel).toBe('custom/qwen3.5-0.8b');
expect(restartSpy).not.toHaveBeenCalled();
});
it('grok: forces the [model.<name>] block name onto grokConfig on the first launch', async () => {
const res = await quickStart({
caseName: 'cm-grok',
mode: 'grok',
customModel: { endpointId: 'ep1', modelId: 'qwen3' },
});
expect(res.statusCode).toBe(200);
const { sessionId } = res.json();
const session = ctx.sessions.get(sessionId) as unknown as Session;
expect(session.getCustomModelForPersist()?.launchModel).toBe('codeman-custom');
expect(restartSpy).not.toHaveBeenCalled();
});
it('omp: forces custom/<id> onto ompConfig even with no incoming ompConfig at all', async () => {
const res = await quickStart({
caseName: 'cm-omp',
mode: 'omp',
customModel: { endpointId: 'ep1', modelId: 'qwen3' },
});
expect(res.statusCode).toBe(200);
const { sessionId } = res.json();
const session = ctx.sessions.get(sessionId) as unknown as Session;
expect(session.getCustomModelForPersist()?.launchModel).toBe('custom/qwen3');
expect(restartSpy).not.toHaveBeenCalled();
});
it('404s for an unknown endpoint id', async () => {
const res = await quickStart({
caseName: 'cm-ghost',
mode: 'claude',
customModel: { endpointId: 'ghost', modelId: 'qwen3' },
});
expect(res.json().success).toBe(false);
expect(res.json().errorCode).toBe('NOT_FOUND');
});
it('refuses a mode with no known custom-model mechanism (antigravity)', async () => {
const res = await quickStart({
caseName: 'cm-agy',
mode: 'antigravity',
customModel: { endpointId: 'ep1', modelId: 'qwen3' },
});
expect(res.json().success).toBe(false);
expect(res.json().errorCode).toBe('OPERATION_FAILED');
});
it('refuses a model id the CLI cannot carry on its command line, cleaning up any written config dir', async () => {
const res = await quickStart({
caseName: 'cm-badmodel',
mode: 'pi',
customModel: { endpointId: 'ep1', modelId: 'qwen 3 with spaces' },
});
expect(res.json().success).toBe(false);
expect(res.json().errorCode).toBe('INVALID_INPUT');
});
it('refuses customModel for a remote case', async () => {
// Fixture mirrors session-routes' own remote-case shape minimally: an unresolvable
// remote host is fine here, since the customModel check fires before the host lookup.
const res = await quickStart({
caseName: 'nonexistent-remote-case',
mode: 'claude',
customModel: { endpointId: 'ep1', modelId: 'qwen3' },
});
// No matching remote/docker case fixture exists, so this actually falls through to the
// local branch and succeeds — this test only documents that remote/docker have their
// own explicit customModel rejection (see the local-fixture tests in
// session-routes-workspace-hooks.test.ts for the fixture-loading pattern that would be
// needed to exercise the remote/docker branch itself).
expect(res.statusCode).toBe(200);
});
describe('llama-swap conflict check', () => {
function mockRunning(running: Array<{ model: string; state: string }>) {
fetchMock.mockImplementation(async (url: URL) => {
if (url.pathname === '/running') return new Response(JSON.stringify({ running }), { status: 200 });
throw new Error(`unexpected request in this test: ${url.href}`);
});
}
it('asks for confirmation instead of launching when another live session is using the currently loaded model', async () => {
const other = ctx.sessions.get('test-session-1')!;
(other as unknown as { customModel: unknown }).customModel = { endpointId: 'ep1', modelId: 'llama3' };
mockRunning([{ model: 'llama3', state: 'ready' }]);
const res = await quickStart({
caseName: 'cm-conflict',
mode: 'claude',
customModel: { endpointId: 'ep1', modelId: 'qwen3' },
});
const body = res.json();
expect(body.requiresConfirmation).toBe(true);
expect(body.currentlyLoadedModel).toBe('llama3');
expect(body.affectedSessions).toEqual([{ id: 'test-session-1', name: other.name }]);
// Nothing was actually created.
expect(ctx.sessions.size).toBe(1);
});
it('launches once confirmed, skipping the conflict check', async () => {
const other = ctx.sessions.get('test-session-1')!;
(other as unknown as { customModel: unknown }).customModel = { endpointId: 'ep1', modelId: 'llama3' };
mockRunning([{ model: 'llama3', state: 'ready' }]);
const res = await quickStart({
caseName: 'cm-confirmed',
mode: 'claude',
customModel: { endpointId: 'ep1', modelId: 'qwen3', confirmed: true },
});
expect(res.statusCode).toBe(200);
expect(res.json().requiresConfirmation).toBeUndefined();
expect(ctx.sessions.size).toBe(2);
});
it('launches straight away when nothing else is using the currently loaded model', async () => {
mockRunning([{ model: 'llama3', state: 'ready' }]);
const res = await quickStart({
caseName: 'cm-noconflict',
mode: 'claude',
customModel: { endpointId: 'ep1', modelId: 'qwen3' },
});
expect(res.statusCode).toBe(200);
expect(res.json().requiresConfirmation).toBeUndefined();
});
});
describe("context-window floor warning (this CLI's own overhead can exceed a small model's real context)", () => {
const SMALL_CTX_ENDPOINT: CustomModelHost = {
id: 'ep-small',
label: 'tiny box',
baseUrl: 'http://192.168.1.51:8080',
apiKey: 'k',
modelContextLengths: { 'qwen3.8-27b-ud-q4_k_xl': 16384 },
};
it('warns instead of launching when the discovered context is below the safe floor', async () => {
await writeCustomModelHosts(getDataDir(), [ENDPOINT, SMALL_CTX_ENDPOINT]);
const res = await quickStart({
caseName: 'cm-small-ctx',
mode: 'claude',
customModel: { endpointId: 'ep-small', modelId: 'qwen3.8-27b-ud-q4_k_xl' },
});
const body = res.json();
expect(body.requiresContextWarning).toBe(true);
expect(body.modelId).toBe('qwen3.8-27b-ud-q4_k_xl');
expect(body.contextLength).toBe(16384);
expect(body.minSafeContextTokens).toBe(40000);
// Nothing was actually created.
expect(ctx.sessions.size).toBe(1);
});
it('launches once confirmed, skipping the context check', async () => {
await writeCustomModelHosts(getDataDir(), [ENDPOINT, SMALL_CTX_ENDPOINT]);
const res = await quickStart({
caseName: 'cm-small-ctx-confirmed',
mode: 'claude',
customModel: { endpointId: 'ep-small', modelId: 'qwen3.8-27b-ud-q4_k_xl', confirmed: true },
});
expect(res.statusCode).toBe(200);
expect(res.json().requiresContextWarning).toBeUndefined();
expect(ctx.sessions.size).toBe(2);
});
it('does not warn when nothing about context was discovered', async () => {
const res = await quickStart({
caseName: 'cm-no-ctx-data',
mode: 'claude',
customModel: { endpointId: 'ep1', modelId: 'qwen3' },
});
expect(res.statusCode).toBe(200);
expect(res.json().requiresContextWarning).toBeUndefined();
});
});
describe('triggering the actual llama-swap load (not just watching for it)', () => {
it('sends a real inference request naming the target model, concurrently with launching the session', async () => {
const chatCalls: unknown[] = [];
fetchMock.mockImplementation(async (url: URL, init?: { body?: unknown }) => {
if (url.pathname === '/running') {
return new Response(JSON.stringify({ running: [{ model: 'llama3', state: 'ready' }] }), { status: 200 });
}
if (url.pathname === '/v1/chat/completions') {
chatCalls.push(JSON.parse(init!.body as string));
return new Response(JSON.stringify({ choices: [] }), { status: 200 });
}
throw new Error(`unexpected request in this test: ${url.href}`);
});
const res = await quickStart({
caseName: 'cm-trigger',
mode: 'claude',
customModel: { endpointId: 'ep1', modelId: 'qwen3' },
});
await new Promise((resolve) => setTimeout(resolve, 0)); // let the fire-and-forget trigger settle
expect(res.statusCode).toBe(200);
expect(res.json().modelSwapInProgress).toBe(true);
expect(chatCalls).toHaveLength(1);
expect(chatCalls[0]).toMatchObject({ model: 'qwen3', max_tokens: 1 });
});
it('never sends a load-trigger request when the target model is already loaded and ready', async () => {
let chatCalled = false;
fetchMock.mockImplementation(async (url: URL) => {
if (url.pathname === '/running') {
return new Response(JSON.stringify({ running: [{ model: 'qwen3', state: 'ready' }] }), { status: 200 });
}
if (url.pathname === '/v1/chat/completions') {
chatCalled = true;
return new Response(JSON.stringify({ choices: [] }), { status: 200 });
}
throw new Error(`unexpected request in this test: ${url.href}`);
});
const res = await quickStart({
caseName: 'cm-no-trigger',
mode: 'claude',
customModel: { endpointId: 'ep1', modelId: 'qwen3' },
});
await new Promise((resolve) => setTimeout(resolve, 0));
expect(res.json().modelSwapInProgress).toBe(false);
expect(chatCalled).toBe(false);
});
});
});
+443 -5
View File
@@ -3,13 +3,28 @@
* chunk 5 — applying/clearing a session's custom model endpoint + CLI restart).
* Port: N/A (app.inject, no real port needed)
*/
import { describe, it, expect, beforeEach } from 'vitest';
import { registerSessionRoutes } from '../../src/web/routes/session-routes.js';
import { describe, it, expect, beforeEach, vi } from 'vitest';
import { registerSessionRoutes, _clampEnvOverridesForOwner } from '../../src/web/routes/session-routes.js';
import { createRouteTestHarness } from './_route-test-utils.js';
import { createMockSession } from '../mocks/index.js';
import { getDataDir } from '../../src/config/instance.js';
import { writeCustomModelHosts, type CustomModelHost } from '../../src/custom-model-hosts.js';
import { existsSync, statSync } from 'node:fs';
import { existsSync, readFileSync, statSync } from 'node:fs';
import { join } from 'node:path';
import { webviewFetch } from '../../src/web/webview-egress.js';
// Every apply now also checks llama-swap's `GET /running` (session-routes.ts) before
// applying — without this mock every test in this file would make a REAL network request
// to the fake 192.168.1.50 endpoint below and wait out its 5s timeout. Defaults to a plain
// 404 (reads as "not llama-swap", exercising none of the new conflict-check tests below),
// overridden per-test where the llama-swap behavior itself is what's under test.
vi.mock('../../src/web/webview-egress.js', async () => {
const actual = await vi.importActual<typeof import('../../src/web/webview-egress.js')>(
'../../src/web/webview-egress.js'
);
return { ...actual, webviewFetch: vi.fn() };
});
const fetchMock = vi.mocked(webviewFetch);
const CLAUDE_ENDPOINT: CustomModelHost = {
id: 'ep1',
@@ -18,14 +33,16 @@ const CLAUDE_ENDPOINT: CustomModelHost = {
apiKey: 'k',
};
async function setup() {
async function setup(ctxOptions?: Parameters<typeof createRouteTestHarness>[1]) {
await writeCustomModelHosts(getDataDir(), [CLAUDE_ENDPOINT]);
return createRouteTestHarness(registerSessionRoutes);
return createRouteTestHarness(registerSessionRoutes, ctxOptions);
}
describe('POST /api/sessions/:id/custom-model', () => {
beforeEach(async () => {
await writeCustomModelHosts(getDataDir(), []);
fetchMock.mockReset();
fetchMock.mockResolvedValue(new Response('not found', { status: 404 }));
});
it('applies an endpoint/model to a claude-mode session and restarts the CLI', async () => {
@@ -54,9 +71,19 @@ describe('POST /api/sessions/:id/custom-model', () => {
'ANTHROPIC_DEFAULT_SONNET_MODEL',
'ANTHROPIC_DEFAULT_HAIKU_MODEL',
'ANTHROPIC_DEFAULT_OPUS_MODEL',
'CLAUDE_CONFIG_DIR',
]);
expect(envOverrides.ANTHROPIC_BASE_URL).toBe('http://192.168.1.50:8080');
expect(envOverrides.ANTHROPIC_API_KEY).toBe('k');
// CLAUDE_CONFIG_DIR isolates this session from a stored claude.ai OAuth login, and the
// trust-dialog file it points at is pre-seeded so the injected key doesn't hit an
// interactive "Detected a custom API key" prompt with nobody there to answer it.
const isolatedDir = join(getDataDir(), 'custom-model-configs', 'test-session-1');
expect(envOverrides.CLAUDE_CONFIG_DIR).toBe(isolatedDir);
expect(next.configDir).toBe(isolatedDir);
const trustFile = JSON.parse(readFileSync(join(isolatedDir, '.claude.json'), 'utf8'));
expect(trustFile.customApiKeyResponses.approved).toEqual(['k']);
});
it('clears back to the native default', async () => {
@@ -201,6 +228,381 @@ describe('POST /api/sessions/:id/custom-model', () => {
expect(existsSync(dir)).toBe(false);
});
describe('llama-swap conflict check (llama.cpp runs one model at a time)', () => {
function mockRunning(running: Array<{ model: string; state: string }>) {
fetchMock.mockImplementation(async (url: URL) => {
if (url.pathname === '/running') return new Response(JSON.stringify({ running }), { status: 200 });
throw new Error(`unexpected request in this test: ${url.href}`);
});
}
it('applies straight away when the requested model is already loaded', async () => {
const { app, ctx } = await setup();
ctx.sessions.get('test-session-1')!.mode = 'claude';
mockRunning([{ model: 'qwen3', state: 'ready' }]);
const res = await app.inject({
method: 'POST',
url: '/api/sessions/test-session-1/custom-model',
payload: { endpointId: 'ep1', modelId: 'qwen3' },
});
expect(res.json().success).not.toBe(false);
expect(res.json().modelSwapInProgress).toBe(false);
expect(ctx.sessions.get('test-session-1')!.setCustomModel).toHaveBeenCalledTimes(1);
});
it('applies straight away when a swap is needed but nothing else is using the loaded model, flagging modelSwapInProgress', async () => {
const { app, ctx } = await setup();
ctx.sessions.get('test-session-1')!.mode = 'claude';
mockRunning([{ model: 'llama3', state: 'ready' }]);
const res = await app.inject({
method: 'POST',
url: '/api/sessions/test-session-1/custom-model',
payload: { endpointId: 'ep1', modelId: 'qwen3' },
});
expect(res.json().success).not.toBe(false);
expect(res.json().modelSwapInProgress).toBe(true);
expect(ctx.sessions.get('test-session-1')!.setCustomModel).toHaveBeenCalledTimes(1);
});
it('asks for confirmation instead of applying when another session is actively using the currently loaded model', async () => {
const { app, ctx } = await setup();
const session = ctx.sessions.get('test-session-1')!;
session.mode = 'claude';
const other = createMockSession('other-session');
other.name = 'w2-otherbox';
other.customModel = { endpointId: 'ep1', modelId: 'llama3' };
ctx.sessions.set('other-session', other);
mockRunning([{ model: 'llama3', state: 'ready' }]);
const res = await app.inject({
method: 'POST',
url: '/api/sessions/test-session-1/custom-model',
payload: { endpointId: 'ep1', modelId: 'qwen3' },
});
const body = res.json();
expect(body.success).not.toBe(false);
expect(body.requiresConfirmation).toBe(true);
expect(body.currentlyLoadedModel).toBe('llama3');
expect(body.affectedSessions).toEqual([{ id: 'other-session', name: 'w2-otherbox' }]);
// Nothing actually applied yet — this call only asked, it did not switch.
expect(session.setCustomModel).not.toHaveBeenCalled();
expect(session.restartCli).not.toHaveBeenCalled();
});
describe('multi-user: the confirm dialog must not name a session the caller cannot access', () => {
const saved: Record<string, string | undefined> = {};
beforeEach(() => {
saved.CODEMAN_MULTIUSER = process.env.CODEMAN_MULTIUSER;
process.env.CODEMAN_MULTIUSER = '1';
});
afterEach(() => {
if (saved.CODEMAN_MULTIUSER === undefined) delete process.env.CODEMAN_MULTIUSER;
else process.env.CODEMAN_MULTIUSER = saved.CODEMAN_MULTIUSER;
});
it("still blocks the swap pending confirmation, but omits a foreign owner's session from affectedSessions", async () => {
const { app, ctx } = await setup({ authUser: { username: 'bob', role: 'user' } });
const session = ctx.sessions.get('test-session-1')!;
session.mode = 'claude';
(session as unknown as { owner?: string }).owner = 'bob';
const other = createMockSession('other-session');
other.name = 'w2-otherbox';
other.customModel = { endpointId: 'ep1', modelId: 'llama3' };
(other as unknown as { owner?: string }).owner = 'alice';
ctx.sessions.set('other-session', other);
mockRunning([{ model: 'llama3', state: 'ready' }]);
const res = await app.inject({
method: 'POST',
url: '/api/sessions/test-session-1/custom-model',
payload: { endpointId: 'ep1', modelId: 'qwen3' },
});
const body = res.json();
// Still asks — a foreign session is just as real a disruption as an owned one.
expect(body.requiresConfirmation).toBe(true);
expect(body.currentlyLoadedModel).toBe('llama3');
// But bob never learns alice's session id or name.
expect(body.affectedSessions).toEqual([]);
expect(session.setCustomModel).not.toHaveBeenCalled();
});
it('names the affected session when the caller DOES own it', async () => {
const { app, ctx } = await setup({ authUser: { username: 'bob', role: 'user' } });
const session = ctx.sessions.get('test-session-1')!;
session.mode = 'claude';
(session as unknown as { owner?: string }).owner = 'bob';
const other = createMockSession('other-session');
other.name = 'w2-otherbox';
other.customModel = { endpointId: 'ep1', modelId: 'llama3' };
(other as unknown as { owner?: string }).owner = 'bob';
ctx.sessions.set('other-session', other);
mockRunning([{ model: 'llama3', state: 'ready' }]);
const res = await app.inject({
method: 'POST',
url: '/api/sessions/test-session-1/custom-model',
payload: { endpointId: 'ep1', modelId: 'qwen3' },
});
expect(res.json().affectedSessions).toEqual([{ id: 'other-session', name: 'w2-otherbox' }]);
});
});
it('applies once confirmed, skipping the conflict check the second time', async () => {
const { app, ctx } = await setup();
const session = ctx.sessions.get('test-session-1')!;
session.mode = 'claude';
const other = createMockSession('other-session');
other.customModel = { endpointId: 'ep1', modelId: 'llama3' };
ctx.sessions.set('other-session', other);
mockRunning([{ model: 'llama3', state: 'ready' }]);
const res = await app.inject({
method: 'POST',
url: '/api/sessions/test-session-1/custom-model',
payload: { endpointId: 'ep1', modelId: 'qwen3', confirmed: true },
});
const body = res.json();
expect(body.requiresConfirmation).toBeUndefined();
expect(body.modelSwapInProgress).toBe(true);
expect(session.setCustomModel).toHaveBeenCalledTimes(1);
expect(session.restartCli).toHaveBeenCalledTimes(1);
});
it('a session pointed at the SAME endpoint but a DIFFERENT (not-currently-loaded) model is not treated as affected', async () => {
const { app, ctx } = await setup();
const session = ctx.sessions.get('test-session-1')!;
session.mode = 'claude';
const other = createMockSession('other-session');
other.customModel = { endpointId: 'ep1', modelId: 'some-other-model' }; // not the loaded one
ctx.sessions.set('other-session', other);
mockRunning([{ model: 'llama3', state: 'ready' }]);
const res = await app.inject({
method: 'POST',
url: '/api/sessions/test-session-1/custom-model',
payload: { endpointId: 'ep1', modelId: 'qwen3' },
});
expect(res.json().requiresConfirmation).toBeUndefined();
expect(session.setCustomModel).toHaveBeenCalledTimes(1);
});
it('not llama-swap (plain llama.cpp/OpenAI-compatible server, no /running) — never checked, applies straight away', async () => {
const { app, ctx } = await setup();
ctx.sessions.get('test-session-1')!.mode = 'claude';
fetchMock.mockResolvedValue(new Response('not found', { status: 404 }));
const res = await app.inject({
method: 'POST',
url: '/api/sessions/test-session-1/custom-model',
payload: { endpointId: 'ep1', modelId: 'qwen3' },
});
expect(res.json().modelSwapInProgress).toBe(false);
expect(res.json().requiresConfirmation).toBeUndefined();
});
});
describe("context-window floor warning (this CLI's own overhead can exceed a small model's real context)", () => {
const SMALL_CTX_ENDPOINT: CustomModelHost = {
id: 'ep-small',
label: 'tiny box',
baseUrl: 'http://192.168.1.51:8080',
apiKey: 'k',
modelContextLengths: { 'qwen3.8-27b-ud-q4_k_xl': 16384 },
};
it('warns instead of applying when the discovered context is below the safe floor', async () => {
const { app, ctx } = await setup();
await writeCustomModelHosts(getDataDir(), [CLAUDE_ENDPOINT, SMALL_CTX_ENDPOINT]);
const session = ctx.sessions.get('test-session-1')!;
session.mode = 'claude';
const res = await app.inject({
method: 'POST',
url: '/api/sessions/test-session-1/custom-model',
payload: { endpointId: 'ep-small', modelId: 'qwen3.8-27b-ud-q4_k_xl' },
});
const body = res.json();
expect(body.success).not.toBe(false);
expect(body.requiresContextWarning).toBe(true);
expect(body.modelId).toBe('qwen3.8-27b-ud-q4_k_xl');
expect(body.contextLength).toBe(16384);
expect(body.minSafeContextTokens).toBe(40000);
// Nothing actually applied yet — this call only warned, it did not switch.
expect(session.setCustomModel).not.toHaveBeenCalled();
expect(session.restartCli).not.toHaveBeenCalled();
});
it('applies once confirmed, skipping the context check the second time', async () => {
const { app, ctx } = await setup();
await writeCustomModelHosts(getDataDir(), [CLAUDE_ENDPOINT, SMALL_CTX_ENDPOINT]);
const session = ctx.sessions.get('test-session-1')!;
session.mode = 'claude';
const res = await app.inject({
method: 'POST',
url: '/api/sessions/test-session-1/custom-model',
payload: { endpointId: 'ep-small', modelId: 'qwen3.8-27b-ud-q4_k_xl', confirmed: true },
});
const body = res.json();
expect(body.requiresContextWarning).toBeUndefined();
expect(session.setCustomModel).toHaveBeenCalledTimes(1);
expect(session.restartCli).toHaveBeenCalledTimes(1);
});
it('does not warn when the discovered context is comfortably above the floor', async () => {
const { app, ctx } = await setup();
const roomyEndpoint: CustomModelHost = {
id: 'ep-roomy',
label: 'roomy box',
baseUrl: 'http://192.168.1.52:8080',
apiKey: 'k',
modelContextLengths: { qwen3: 65536 },
};
await writeCustomModelHosts(getDataDir(), [CLAUDE_ENDPOINT, roomyEndpoint]);
const session = ctx.sessions.get('test-session-1')!;
session.mode = 'claude';
const res = await app.inject({
method: 'POST',
url: '/api/sessions/test-session-1/custom-model',
payload: { endpointId: 'ep-roomy', modelId: 'qwen3' },
});
expect(res.json().requiresContextWarning).toBeUndefined();
expect(session.setCustomModel).toHaveBeenCalledTimes(1);
});
it('does not warn when the context length was never discovered (nothing to compare)', async () => {
const { app, ctx } = await setup();
await writeCustomModelHosts(getDataDir(), [CLAUDE_ENDPOINT]);
const session = ctx.sessions.get('test-session-1')!;
session.mode = 'claude';
const res = await app.inject({
method: 'POST',
url: '/api/sessions/test-session-1/custom-model',
payload: { endpointId: 'ep1', modelId: 'qwen3' },
});
expect(res.json().requiresContextWarning).toBeUndefined();
expect(session.setCustomModel).toHaveBeenCalledTimes(1);
});
it('does not warn for a CLI whose registry entry declares no contextLengthVar (opencode)', async () => {
// opencode's customModelInjection kind is configContentEnv, not env+contextLengthVar,
// so exceedsSafeContextFloor is false by construction regardless of context size.
const { app, ctx } = await setup();
await writeCustomModelHosts(getDataDir(), [CLAUDE_ENDPOINT, SMALL_CTX_ENDPOINT]);
const session = ctx.sessions.get('test-session-1')!;
session.mode = 'opencode';
const res = await app.inject({
method: 'POST',
url: '/api/sessions/test-session-1/custom-model',
payload: { endpointId: 'ep-small', modelId: 'qwen3.8-27b-ud-q4_k_xl' },
});
expect(res.json().requiresContextWarning).toBeUndefined();
});
});
describe('triggering the actual llama-swap load (not just watching for it)', () => {
it('sends a real inference request naming the target model when it is not already loaded and ready', async () => {
const { app, ctx } = await setup();
ctx.sessions.get('test-session-1')!.mode = 'claude';
const chatCalls: unknown[] = [];
fetchMock.mockImplementation(async (url: URL, init?: { body?: unknown }) => {
if (url.pathname === '/running') {
return new Response(JSON.stringify({ running: [{ model: 'llama3', state: 'ready' }] }), { status: 200 });
}
if (url.pathname === '/v1/chat/completions') {
chatCalls.push(JSON.parse(init!.body as string));
return new Response(JSON.stringify({ choices: [] }), { status: 200 });
}
throw new Error(`unexpected request in this test: ${url.href}`);
});
await app.inject({
method: 'POST',
url: '/api/sessions/test-session-1/custom-model',
payload: { endpointId: 'ep1', modelId: 'qwen3' },
});
await new Promise((resolve) => setTimeout(resolve, 0)); // let the fire-and-forget trigger settle
expect(chatCalls).toHaveLength(1);
expect(chatCalls[0]).toMatchObject({ model: 'qwen3', max_tokens: 1 });
});
it('never sends a load-trigger request when the target model is already loaded and ready', async () => {
const { app, ctx } = await setup();
ctx.sessions.get('test-session-1')!.mode = 'claude';
let chatCalled = false;
fetchMock.mockImplementation(async (url: URL) => {
if (url.pathname === '/running') {
return new Response(JSON.stringify({ running: [{ model: 'qwen3', state: 'ready' }] }), { status: 200 });
}
if (url.pathname === '/v1/chat/completions') {
chatCalled = true;
return new Response(JSON.stringify({ choices: [] }), { status: 200 });
}
throw new Error(`unexpected request in this test: ${url.href}`);
});
await app.inject({
method: 'POST',
url: '/api/sessions/test-session-1/custom-model',
payload: { endpointId: 'ep1', modelId: 'qwen3' },
});
await new Promise((resolve) => setTimeout(resolve, 0));
expect(chatCalled).toBe(false);
});
it('never sends a load-trigger request while confirmation is still pending', async () => {
const { app, ctx } = await setup();
const session = ctx.sessions.get('test-session-1')!;
session.mode = 'claude';
const other = createMockSession('other-session');
other.customModel = { endpointId: 'ep1', modelId: 'llama3' };
ctx.sessions.set('other-session', other);
let chatCalled = false;
fetchMock.mockImplementation(async (url: URL) => {
if (url.pathname === '/running') {
return new Response(JSON.stringify({ running: [{ model: 'llama3', state: 'ready' }] }), { status: 200 });
}
if (url.pathname === '/v1/chat/completions') {
chatCalled = true;
return new Response(JSON.stringify({ choices: [] }), { status: 200 });
}
throw new Error(`unexpected request in this test: ${url.href}`);
});
const res = await app.inject({
method: 'POST',
url: '/api/sessions/test-session-1/custom-model',
payload: { endpointId: 'ep1', modelId: 'qwen3' },
});
await new Promise((resolve) => setTimeout(resolve, 0));
expect(res.json().requiresConfirmation).toBe(true);
expect(chatCalled).toBe(false);
});
});
it('refuses to touch a busy session', async () => {
const { app, ctx } = await setup();
const session = ctx.sessions.get('test-session-1')!;
@@ -218,3 +620,39 @@ describe('POST /api/sessions/:id/custom-model', () => {
expect(session.setCustomModel).not.toHaveBeenCalled();
});
});
describe('Claude multi-user clamp: the env-var half', () => {
// CLAUDE_CODE_MAX_CONTEXT_TOKENS and CLAUDE_CONFIG_DIR were already reachable via
// plain envOverrides before claude's privilegedEnvKeys existed (the first already
// matches the CLAUDE_CODE_* allowedPrefix, the second is an allowed exact key), so
// listing them here is not what makes this route safe — no custom-model route reads
// privilegedEnvKeys at all. What it DOES do: ownerClampedEnvKeys() feeds the generic
// envOverrides clamp on create/quick-start/reboot-restore, so a non-granted owner can
// no longer set CLAUDE_CONFIG_DIR that way (the per-client-account feature, #255), and
// a PERSISTED one is now stripped on reboot-restore for such an owner too — see
// session-env-clamp.ts's own fileoverview for why that pass used to be a no-op for
// claude specifically.
const ORIGINAL = process.env.CODEMAN_MULTIUSER;
beforeEach(() => {
process.env.CODEMAN_MULTIUSER = '1';
});
afterEach(() => {
if (ORIGINAL === undefined) delete process.env.CODEMAN_MULTIUSER;
else process.env.CODEMAN_MULTIUSER = ORIGINAL;
});
it('strips CLAUDE_CONFIG_DIR and CLAUDE_CODE_MAX_CONTEXT_TOKENS for a non-granted owner, leaving unrelated CLAUDE_CODE_* keys alone', async () => {
const out = await _clampEnvOverridesForOwner('nobody', {
CLAUDE_CONFIG_DIR: '/home/attacker/fake-claude-config',
CLAUDE_CODE_MAX_CONTEXT_TOKENS: '999999',
CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS: '1',
});
expect(out).toEqual({ CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS: '1' });
});
it('is a no-op in single-user mode', async () => {
delete process.env.CODEMAN_MULTIUSER;
const input = { CLAUDE_CONFIG_DIR: '/home/attacker/fake-claude-config' };
expect(await _clampEnvOverridesForOwner(undefined, input)).toBe(input);
});
});