mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-10-02 13:39:41 +02:00
feat(custom-model): Custom Model Endpoint Profiles (local or cloud, all harnesses)
Point any Codeman-supported harness (Claude, opencode, Codex, Gemini, Pi, Grok, DeepSeek, OMP) at a custom OpenAI-compatible endpoint instead of its native cloud backend, for a given session. Covers local hardware (llama.cpp, Ollama, vLLM, DGX Spark, Strix Halo) and cloud (Azure AI Foundry, OpenRouter). Off by default (customModelEndpointsEnabled, synced, default OFF). - Registry: capabilities.customModelInjection per CLI entry (env / configContentEnv / configDir / unsupported kinds) - Pure injection builder (custom-model-injection.ts) turning an endpoint + model id into the real env vars / config content per CLI - Endpoint store + CRUD routes (custom-model-hosts.ts, custom-model-routes.ts), discovery via GET /v1/models, SSRF-guarded - Session integration: Session.setCustomModel()/restartCli() (POST /api/sessions/:id/custom-model), reusing the existing respawn-pane -k primitive to restart the CLI process with new env - Multi-user hardening: every new redirect-capable env var added to its CLI's privilegedEnvKeys, closing a pre-existing gap where several were already reachable via the generic envOverrides field's prefix allowlist - Standalone scripts/test-local-llm-harnesses.mjs: spawns real CLI binaries against a real endpoint outside the web UI, independent of tmux/sessions - Mock-server contract tests (test/fixtures/mock-openai-server.ts) replaying every CLI's injected values through a real HTTP shape Real end-to-end validation against a live llama-swap server (inside a codeman/agent:llm-test Docker image with all 9 CLI binaries) found and fixed three real bugs before they shipped: - Codex's config.toml schema was wrong ([model].default table instead of a top-level model string + [model_providers.custom]); fixing it then surfaced a genuine, documented protocol incompatibility (Codex only speaks the Responses API since Feb 2026, which llama.cpp/llama-swap don't implement) - Claude Code's async session-title-generation call validates ANTHROPIC_DEFAULT_HAIKU_MODEL against its own internal model list and hangs the whole -p invocation on an unrecognized name; documented for chunk 6, worked around in the standalone script only (--bare is NOT safe for a real interactive session, which needs hooks) - The discovery route's authStyle: 'both' option (send both Authorization and api-key headers) reliably hung a real server; removed the option entirely rather than just changing the default Status: draft. Chunk 6 (frontend toolbar/settings UI) not yet built — see PR.md and deployment_plan.md for the full chunk breakdown and confidence table. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017HqNWfmtBU2KN29SvSVWB3
This commit is contained in:
Devvyn
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
a017e9a8e0
commit
41416566aa
@@ -0,0 +1,211 @@
|
||||
/**
|
||||
* @fileoverview Contract tests for Custom Model Endpoint Profiles
|
||||
* (deployment_plan.md chunk 7): for every CLI with a `customModelInjection`
|
||||
* capability, build the real injection via `buildCustomModelInjection()`,
|
||||
* then replay those exact values through an HTTP request shaped the way that
|
||||
* CLI is documented to send it, against the in-process mock server
|
||||
* (`test/fixtures/mock-openai-server.ts`). Asserts the mock received the
|
||||
* request at the injected base URL, with the injected API key in the
|
||||
* expected header, and the injected model id in the body.
|
||||
*
|
||||
* LIMITATION (stated here and in deployment_plan.md, not left implicit): this
|
||||
* proves "if the CLI honors its documented env/config contract, it will hit
|
||||
* the right endpoint with the right model." It does NOT prove the real CLI
|
||||
* binary actually reads that env var / config file the way its docs say —
|
||||
* that's still the job of `scripts/test-local-llm-harnesses.mjs` against a
|
||||
* real endpoint and real binaries. This suite catches regressions in
|
||||
* Codeman's own injection logic; it cannot catch a CLI changing its env-var
|
||||
* name in a future release.
|
||||
*
|
||||
* Port: N/A (mock server binds a random free port, not a fixed one)
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { getCli } from '../src/config/cli-registry/index.js';
|
||||
import { buildCustomModelInjection, type CustomModelEndpoint } from '../src/custom-model-injection.js';
|
||||
import { startMockOpenAiServer, type MockOpenAiServer } from './fixtures/mock-openai-server.js';
|
||||
|
||||
let mock: MockOpenAiServer;
|
||||
|
||||
beforeEach(async () => {
|
||||
mock = await startMockOpenAiServer();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await mock.close();
|
||||
});
|
||||
|
||||
function entryOrThrow(id: string) {
|
||||
const entry = getCli(id);
|
||||
if (!entry) throw new Error(`missing CLI registry entry: ${id}`);
|
||||
return entry;
|
||||
}
|
||||
|
||||
function endpointFor(mock: MockOpenAiServer): CustomModelEndpoint {
|
||||
return { id: 'ep1', label: 'mock', baseUrl: mock.baseUrl, apiKey: 'contract-test-key' };
|
||||
}
|
||||
|
||||
/** Replays an OpenAI-shaped chat-completions call using the given base URL/key/model. */
|
||||
async function callOpenAiCompat(baseUrl: string, apiKey: string, model: string) {
|
||||
return fetch(`${baseUrl}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json', authorization: `Bearer ${apiKey}` },
|
||||
body: JSON.stringify({ model, messages: [{ role: 'user', content: 'hello world' }] }),
|
||||
});
|
||||
}
|
||||
|
||||
describe('custom-model-injection contract (mock server)', () => {
|
||||
it('claude: ANTHROPIC_BASE_URL/API_KEY reach a real Anthropic-shaped /v1/messages call', async () => {
|
||||
const injection = buildCustomModelInjection(entryOrThrow('claude'), endpointFor(mock), 'qwen3');
|
||||
if (injection.kind !== 'env') throw new Error('unreachable');
|
||||
|
||||
await fetch(`${injection.envOverrides.ANTHROPIC_BASE_URL}/v1/messages`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json', 'x-api-key': injection.envOverrides.ANTHROPIC_API_KEY },
|
||||
body: JSON.stringify({
|
||||
model: injection.envOverrides.ANTHROPIC_DEFAULT_SONNET_MODEL,
|
||||
messages: [{ role: 'user', content: 'hello world' }],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(mock.requests).toHaveLength(1);
|
||||
expect(mock.requests[0].path).toBe('/v1/messages');
|
||||
expect(mock.requests[0].headers['x-api-key']).toBe('contract-test-key');
|
||||
expect((mock.requests[0].body as { model: string }).model).toBe('qwen3');
|
||||
});
|
||||
|
||||
it('opencode: OPENCODE_CONFIG_CONTENT decodes to a baseURL/apiKey that reach the mock', async () => {
|
||||
const injection = buildCustomModelInjection(entryOrThrow('opencode'), endpointFor(mock), 'qwen3');
|
||||
if (injection.kind !== 'env') throw new Error('unreachable');
|
||||
const config = JSON.parse(injection.envOverrides.OPENCODE_CONFIG_CONTENT);
|
||||
const { baseURL, apiKey } = config.provider.custom.options;
|
||||
expect(baseURL).toBe(`${mock.baseUrl}/v1`);
|
||||
|
||||
await callOpenAiCompat(baseURL, apiKey, 'qwen3');
|
||||
|
||||
expect(mock.requests[0].path).toBe('/v1/chat/completions');
|
||||
expect(mock.requests[0].headers.authorization).toBe('Bearer contract-test-key');
|
||||
expect((mock.requests[0].body as { model: string }).model).toBe('qwen3');
|
||||
});
|
||||
|
||||
it('codex: config.toml decodes to a base_url/model, and env_key/extraEnv reach the mock over /v1/responses', async () => {
|
||||
const injection = buildCustomModelInjection(entryOrThrow('codex'), endpointFor(mock), 'qwen3');
|
||||
if (injection.kind !== 'configDir') throw new Error('unreachable');
|
||||
const toml = injection.files[0].content;
|
||||
const baseUrl = /base_url = "([^"]+)"/.exec(toml)?.[1];
|
||||
const model = /^model = "([^"]+)"/m.exec(toml)?.[1];
|
||||
const envKeyName = /env_key = "([^"]+)"/.exec(toml)?.[1];
|
||||
expect(baseUrl).toBe(`${mock.baseUrl}/v1`);
|
||||
expect(model).toBe('qwen3');
|
||||
expect(toml).toContain('wire_api = "responses"');
|
||||
expect(toml).not.toContain('api_key ='); // never a literal TOML field
|
||||
expect(envKeyName).toBe('CODEMAN_CUSTOM_MODEL_API_KEY');
|
||||
expect(injection.extraEnv).toEqual({ CODEMAN_CUSTOM_MODEL_API_KEY: 'contract-test-key' });
|
||||
|
||||
// The real credential rides as an env var (env_key names it) — replay it, not a
|
||||
// value read from the file, since the file itself never carries the secret.
|
||||
const apiKey = injection.extraEnv!.CODEMAN_CUSTOM_MODEL_API_KEY;
|
||||
await fetch(`${baseUrl}/responses`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json', authorization: `Bearer ${apiKey}` },
|
||||
body: JSON.stringify({ model, input: 'hello world' }),
|
||||
});
|
||||
|
||||
expect(mock.requests[0].path).toBe('/v1/responses');
|
||||
expect(mock.requests[0].headers.authorization).toBe('Bearer contract-test-key');
|
||||
});
|
||||
|
||||
it('pi: models.json decodes to a baseUrl/apiKey that reach the mock', async () => {
|
||||
const injection = buildCustomModelInjection(entryOrThrow('pi'), endpointFor(mock), 'qwen3');
|
||||
if (injection.kind !== 'configDir') throw new Error('unreachable');
|
||||
const parsed = JSON.parse(injection.files[0].content);
|
||||
const { baseUrl, apiKey } = parsed.providers.custom;
|
||||
expect(baseUrl).toBe(`${mock.baseUrl}/v1`);
|
||||
|
||||
await callOpenAiCompat(baseUrl, apiKey, 'qwen3');
|
||||
|
||||
expect(mock.requests[0].path).toBe('/v1/chat/completions');
|
||||
expect(mock.requests[0].headers.authorization).toBe('Bearer contract-test-key');
|
||||
});
|
||||
|
||||
it('omp: models.yml decodes to a baseUrl/apiKey that reach the mock', async () => {
|
||||
const injection = buildCustomModelInjection(entryOrThrow('omp'), endpointFor(mock), 'qwen3');
|
||||
if (injection.kind !== 'configDir') throw new Error('unreachable');
|
||||
const yml = injection.files[0].content;
|
||||
const baseUrl = JSON.parse(/baseUrl: (".*")\n/.exec(yml)![1]);
|
||||
const apiKey = JSON.parse(/apiKey: (".*")\n/.exec(yml)![1]);
|
||||
expect(baseUrl).toBe(`${mock.baseUrl}/v1`);
|
||||
|
||||
await callOpenAiCompat(baseUrl, apiKey, 'qwen3');
|
||||
|
||||
expect(mock.requests[0].path).toBe('/v1/chat/completions');
|
||||
expect(mock.requests[0].headers.authorization).toBe('Bearer contract-test-key');
|
||||
});
|
||||
|
||||
// gemini/grok/deepseek's `env` kind passes the base URL through UNCHANGED (unlike
|
||||
// opencode/codex/pi/omp, which build a structured config and explicitly append /v1) —
|
||||
// matching Anthropic's own convention for claude's ANTHROPIC_BASE_URL, where the SDK
|
||||
// appends the path itself. Whether each of these THREE CLIs' own OpenAI-compatible
|
||||
// client expects the var to already include /v1 (the common OpenAI-SDK convention) or
|
||||
// appends it itself is genuinely CLI-specific and UNVERIFIED (see the confidence table
|
||||
// in deployment_plan.md) — these tests model the common OpenAI-SDK convention (base_url
|
||||
// ends in /v1) since that's the more likely behavior for an OpenAI-compatible client,
|
||||
// but that assumption should be corrected here the moment it's checked against a real
|
||||
// binary.
|
||||
|
||||
it('gemini: GOOGLE_GEMINI_BASE_URL/GEMINI_API_KEY reach the mock', async () => {
|
||||
const injection = buildCustomModelInjection(entryOrThrow('gemini'), endpointFor(mock), 'qwen3');
|
||||
if (injection.kind !== 'env') throw new Error('unreachable');
|
||||
|
||||
await callOpenAiCompat(
|
||||
`${injection.envOverrides.GOOGLE_GEMINI_BASE_URL}/v1`,
|
||||
injection.envOverrides.GEMINI_API_KEY,
|
||||
injection.envOverrides.GEMINI_MODEL
|
||||
);
|
||||
|
||||
expect(mock.requests[0].path).toBe('/v1/chat/completions');
|
||||
expect(mock.requests[0].headers.authorization).toBe('Bearer contract-test-key');
|
||||
expect((mock.requests[0].body as { model: string }).model).toBe('qwen3');
|
||||
});
|
||||
|
||||
it('grok: GROK_BASE_URL/XAI_API_KEY reach the mock', async () => {
|
||||
const injection = buildCustomModelInjection(entryOrThrow('grok'), endpointFor(mock), 'qwen3');
|
||||
if (injection.kind !== 'env') throw new Error('unreachable');
|
||||
|
||||
await callOpenAiCompat(
|
||||
`${injection.envOverrides.GROK_BASE_URL}/v1`,
|
||||
injection.envOverrides.XAI_API_KEY,
|
||||
injection.envOverrides.GROK_MODEL
|
||||
);
|
||||
|
||||
expect(mock.requests[0].path).toBe('/v1/chat/completions');
|
||||
expect(mock.requests[0].headers.authorization).toBe('Bearer contract-test-key');
|
||||
});
|
||||
|
||||
it('deepseek: DEEPSEEK_BASE_URL/DEEPSEEK_API_KEY reach the mock (base URL/key only, no model var)', async () => {
|
||||
const injection = buildCustomModelInjection(entryOrThrow('deepseek'), endpointFor(mock), 'qwen3');
|
||||
if (injection.kind !== 'env') throw new Error('unreachable');
|
||||
expect(Object.keys(injection.envOverrides).sort()).toEqual(['DEEPSEEK_API_KEY', 'DEEPSEEK_BASE_URL']);
|
||||
|
||||
await callOpenAiCompat(
|
||||
`${injection.envOverrides.DEEPSEEK_BASE_URL}/v1`,
|
||||
injection.envOverrides.DEEPSEEK_API_KEY,
|
||||
'qwen3'
|
||||
);
|
||||
|
||||
expect(mock.requests[0].path).toBe('/v1/chat/completions');
|
||||
expect(mock.requests[0].headers.authorization).toBe('Bearer contract-test-key');
|
||||
});
|
||||
|
||||
it('antigravity: unsupported, never reaches the mock', () => {
|
||||
const injection = buildCustomModelInjection(entryOrThrow('antigravity'), endpointFor(mock), 'qwen3');
|
||||
expect(injection).toEqual({ kind: 'unsupported' });
|
||||
expect(mock.requests).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('mock server also answers GET /v1/models for the discovery route', async () => {
|
||||
const res = await fetch(`${mock.baseUrl}/v1/models`);
|
||||
const body = await res.json();
|
||||
expect(body.data.map((m: { id: string }) => m.id)).toEqual(['qwen3', 'llama3']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,149 @@
|
||||
/**
|
||||
* @fileoverview Tests for the Custom Model Endpoint Profiles pure builder.
|
||||
* Uses the real CLI registry entries (getCli) rather than hand-rolled
|
||||
* fixtures, so a change to a real entry's customModelInjection declaration
|
||||
* is exercised here automatically instead of silently diverging.
|
||||
*
|
||||
* Port: N/A (no server needed)
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { getCli } from '../src/config/cli-registry/index.js';
|
||||
import { buildCustomModelInjection, withV1Suffix, type CustomModelEndpoint } from '../src/custom-model-injection.js';
|
||||
|
||||
const endpoint: CustomModelEndpoint = {
|
||||
id: 'ep1',
|
||||
label: 'llama.cpp box',
|
||||
baseUrl: 'http://192.168.1.50:8080',
|
||||
apiKey: 'my-key',
|
||||
};
|
||||
|
||||
function entryOrThrow(id: string) {
|
||||
const entry = getCli(id);
|
||||
if (!entry) throw new Error(`missing CLI registry entry: ${id}`);
|
||||
return entry;
|
||||
}
|
||||
|
||||
describe('withV1Suffix', () => {
|
||||
it('appends /v1 when missing', () => {
|
||||
expect(withV1Suffix('http://host:8080')).toBe('http://host:8080/v1');
|
||||
});
|
||||
|
||||
it('is idempotent when already present', () => {
|
||||
expect(withV1Suffix('http://host:8080/v1')).toBe('http://host:8080/v1');
|
||||
expect(withV1Suffix('http://host:8080/v1/')).toBe('http://host:8080/v1');
|
||||
});
|
||||
|
||||
it('strips a trailing slash with no /v1', () => {
|
||||
expect(withV1Suffix('http://host:8080/')).toBe('http://host:8080/v1');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildCustomModelInjection', () => {
|
||||
it('claude: env kind sets base URL, api key, and all three tier model vars', () => {
|
||||
const result = buildCustomModelInjection(entryOrThrow('claude'), endpoint, 'qwen3');
|
||||
expect(result.kind).toBe('env');
|
||||
if (result.kind !== 'env') throw new Error('unreachable');
|
||||
expect(result.envOverrides).toEqual({
|
||||
ANTHROPIC_BASE_URL: 'http://192.168.1.50:8080',
|
||||
ANTHROPIC_API_KEY: 'my-key',
|
||||
ANTHROPIC_DEFAULT_SONNET_MODEL: 'qwen3',
|
||||
ANTHROPIC_DEFAULT_HAIKU_MODEL: 'qwen3',
|
||||
ANTHROPIC_DEFAULT_OPUS_MODEL: 'qwen3',
|
||||
});
|
||||
});
|
||||
|
||||
it('claude: falls back to a dummy key when the endpoint has none', () => {
|
||||
const result = buildCustomModelInjection(entryOrThrow('claude'), { ...endpoint, apiKey: undefined }, 'qwen3');
|
||||
if (result.kind !== 'env') throw new Error('unreachable');
|
||||
expect(result.envOverrides.ANTHROPIC_API_KEY).toBe('local-dummy-key');
|
||||
});
|
||||
|
||||
it('opencode: configContentEnv carries a JSON blob in OPENCODE_CONFIG_CONTENT', () => {
|
||||
const result = buildCustomModelInjection(entryOrThrow('opencode'), endpoint, 'qwen3');
|
||||
expect(result.kind).toBe('env');
|
||||
if (result.kind !== 'env') throw new Error('unreachable');
|
||||
const parsed = JSON.parse(result.envOverrides.OPENCODE_CONFIG_CONTENT);
|
||||
expect(parsed.model).toBe('custom/qwen3');
|
||||
expect(parsed.provider.custom.options.baseURL).toBe('http://192.168.1.50:8080/v1');
|
||||
expect(parsed.provider.custom.options.apiKey).toBe('my-key');
|
||||
expect(parsed.provider.custom.models.qwen3).toEqual({});
|
||||
});
|
||||
|
||||
it('codex: configDir writes an isolated config.toml with model/base_url, and the key rides as extraEnv (never a literal TOML field)', () => {
|
||||
const result = buildCustomModelInjection(entryOrThrow('codex'), endpoint, 'qwen3');
|
||||
expect(result.kind).toBe('configDir');
|
||||
if (result.kind !== 'configDir') throw new Error('unreachable');
|
||||
expect(result.dirEnvVar).toBe('CODEX_HOME');
|
||||
expect(result.files).toHaveLength(1);
|
||||
expect(result.files[0].relPath).toBe('config.toml');
|
||||
expect(result.files[0].content).toContain('model = "qwen3"');
|
||||
expect(result.files[0].content).toContain('base_url = "http://192.168.1.50:8080/v1"');
|
||||
expect(result.files[0].content).toContain('wire_api = "responses"');
|
||||
expect(result.files[0].content).not.toContain('api_key ='); // never a literal TOML field
|
||||
expect(result.files[0].content).toContain('env_key = "CODEMAN_CUSTOM_MODEL_API_KEY"');
|
||||
expect(result.extraEnv).toEqual({ CODEMAN_CUSTOM_MODEL_API_KEY: 'my-key' });
|
||||
});
|
||||
|
||||
it('codex: escapes a quote in the model id so it cannot break out of the TOML string', () => {
|
||||
const result = buildCustomModelInjection(entryOrThrow('codex'), endpoint, 'weird"model');
|
||||
if (result.kind !== 'configDir') throw new Error('unreachable');
|
||||
expect(result.files[0].content).toContain('model = "weird\\"model"');
|
||||
});
|
||||
|
||||
it('pi: configDir writes models.json under agent/', () => {
|
||||
const result = buildCustomModelInjection(entryOrThrow('pi'), endpoint, 'qwen3');
|
||||
if (result.kind !== 'configDir') throw new Error('unreachable');
|
||||
expect(result.dirEnvVar).toBe('PI_CONFIG_DIR');
|
||||
expect(result.files[0].relPath).toBe('agent/models.json');
|
||||
const parsed = JSON.parse(result.files[0].content);
|
||||
expect(parsed.providers.custom.baseUrl).toBe('http://192.168.1.50:8080/v1');
|
||||
});
|
||||
|
||||
it('omp: configDir writes models.yml under agent/, redirected via PI_CONFIG_DIR', () => {
|
||||
const result = buildCustomModelInjection(entryOrThrow('omp'), endpoint, 'qwen3');
|
||||
if (result.kind !== 'configDir') throw new Error('unreachable');
|
||||
expect(result.dirEnvVar).toBe('PI_CONFIG_DIR');
|
||||
expect(result.files[0].relPath).toBe('agent/models.yml');
|
||||
expect(result.files[0].content).toContain('baseUrl: "http://192.168.1.50:8080/v1"');
|
||||
});
|
||||
|
||||
it('gemini: env kind sets GOOGLE_GEMINI_BASE_URL/GEMINI_API_KEY/GEMINI_MODEL', () => {
|
||||
const result = buildCustomModelInjection(entryOrThrow('gemini'), endpoint, 'qwen3');
|
||||
if (result.kind !== 'env') throw new Error('unreachable');
|
||||
expect(result.envOverrides).toEqual({
|
||||
GOOGLE_GEMINI_BASE_URL: 'http://192.168.1.50:8080',
|
||||
GEMINI_API_KEY: 'my-key',
|
||||
GEMINI_MODEL: 'qwen3',
|
||||
});
|
||||
});
|
||||
|
||||
it('grok: env kind sets GROK_BASE_URL/XAI_API_KEY/GROK_MODEL', () => {
|
||||
const result = buildCustomModelInjection(entryOrThrow('grok'), endpoint, 'qwen3');
|
||||
if (result.kind !== 'env') throw new Error('unreachable');
|
||||
expect(result.envOverrides).toEqual({
|
||||
GROK_BASE_URL: 'http://192.168.1.50:8080',
|
||||
XAI_API_KEY: 'my-key',
|
||||
GROK_MODEL: 'qwen3',
|
||||
});
|
||||
});
|
||||
|
||||
it('deepseek: env kind sets base URL/key only, no model var', () => {
|
||||
const result = buildCustomModelInjection(entryOrThrow('deepseek'), endpoint, 'qwen3');
|
||||
if (result.kind !== 'env') throw new Error('unreachable');
|
||||
expect(result.envOverrides).toEqual({
|
||||
DEEPSEEK_BASE_URL: 'http://192.168.1.50:8080',
|
||||
DEEPSEEK_API_KEY: 'my-key',
|
||||
});
|
||||
});
|
||||
|
||||
it('antigravity: unsupported', () => {
|
||||
const result = buildCustomModelInjection(entryOrThrow('antigravity'), endpoint, 'qwen3');
|
||||
expect(result).toEqual({ kind: 'unsupported' });
|
||||
});
|
||||
|
||||
it('shell: unsupported', () => {
|
||||
const result = buildCustomModelInjection(entryOrThrow('shell'), endpoint, 'qwen3');
|
||||
expect(result).toEqual({ kind: 'unsupported' });
|
||||
});
|
||||
});
|
||||
Vendored
+118
@@ -0,0 +1,118 @@
|
||||
/**
|
||||
* @fileoverview In-process fake OpenAI/Anthropic-compatible HTTP server for the
|
||||
* Custom Model Endpoint Profiles contract tests (deployment_plan.md chunk 7).
|
||||
*
|
||||
* No external deps — plain `node:http`. Captures every request it receives
|
||||
* (method, path, headers, parsed JSON body) so a test can assert the injected
|
||||
* base URL / API key / model actually reached the right place, with the right
|
||||
* auth header, in the shape a real llama.cpp/Azure/etc. endpoint would see it.
|
||||
*
|
||||
* Serves the request shapes this feature's recipes produce: OpenAI-style
|
||||
* `POST /v1/chat/completions` (opencode/pi/grok/omp/gemini's compat
|
||||
* endpoint), Anthropic-style `POST /v1/messages` (claude's ANTHROPIC_BASE_URL
|
||||
* traffic), OpenAI's newer `POST /v1/responses` (codex's actual wire protocol
|
||||
* as of Feb 2026 — it dropped chat-completions support), plus `GET /v1/models`
|
||||
* for the discovery route's own tests.
|
||||
*/
|
||||
|
||||
import { createServer, type IncomingMessage, type Server } from 'node:http';
|
||||
import { AddressInfo } from 'node:net';
|
||||
|
||||
export interface CapturedRequest {
|
||||
method: string;
|
||||
path: string;
|
||||
headers: Record<string, string | string[] | undefined>;
|
||||
body: unknown;
|
||||
}
|
||||
|
||||
export interface MockOpenAiServer {
|
||||
baseUrl: string;
|
||||
requests: CapturedRequest[];
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
function readJsonBody(req: IncomingMessage): Promise<unknown> {
|
||||
return new Promise((resolve) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on('data', (c) => chunks.push(c));
|
||||
req.on('end', () => {
|
||||
const raw = Buffer.concat(chunks).toString('utf8');
|
||||
if (!raw) return resolve(undefined);
|
||||
try {
|
||||
resolve(JSON.parse(raw));
|
||||
} catch {
|
||||
resolve(raw);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/** Starts the mock server on a random free port and resolves once it's listening. */
|
||||
export async function startMockOpenAiServer(): Promise<MockOpenAiServer> {
|
||||
const requests: CapturedRequest[] = [];
|
||||
|
||||
const server: Server = createServer((req, res) => {
|
||||
void (async () => {
|
||||
const body = await readJsonBody(req);
|
||||
const path = (req.url ?? '').split('?')[0];
|
||||
requests.push({ method: req.method ?? 'GET', path, headers: req.headers, body });
|
||||
|
||||
res.setHeader('content-type', 'application/json');
|
||||
|
||||
if (path === '/v1/models' && req.method === 'GET') {
|
||||
res.writeHead(200);
|
||||
res.end(JSON.stringify({ data: [{ id: 'qwen3' }, { id: 'llama3' }] }));
|
||||
return;
|
||||
}
|
||||
|
||||
if (path === '/v1/chat/completions' && req.method === 'POST') {
|
||||
res.writeHead(200);
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
id: 'mock-completion',
|
||||
choices: [{ index: 0, message: { role: 'assistant', content: 'hello world' } }],
|
||||
})
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (path === '/v1/messages' && req.method === 'POST') {
|
||||
res.writeHead(200);
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
id: 'mock-message',
|
||||
role: 'assistant',
|
||||
content: [{ type: 'text', text: 'hello world' }],
|
||||
})
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Codex's real wire protocol (verified against a live binary: it dropped
|
||||
// wire_api="chat" support in Feb 2026, so its config.toml always says
|
||||
// wire_api="responses") — a different shape from OpenAI's chat-completions.
|
||||
if (path === '/v1/responses' && req.method === 'POST') {
|
||||
res.writeHead(200);
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
id: 'mock-response',
|
||||
output: [{ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'hello world' }] }],
|
||||
})
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
res.writeHead(404);
|
||||
res.end(JSON.stringify({ error: 'not found in mock server', path }));
|
||||
})();
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
|
||||
const { port } = server.address() as AddressInfo;
|
||||
|
||||
return {
|
||||
baseUrl: `http://127.0.0.1:${port}`,
|
||||
requests,
|
||||
close: () => new Promise<void>((resolve, reject) => server.close((err) => (err ? reject(err) : resolve()))),
|
||||
};
|
||||
}
|
||||
@@ -318,6 +318,22 @@ export class MockSession extends EventEmitter {
|
||||
this.color = c;
|
||||
});
|
||||
|
||||
/** Custom Model Endpoint Profiles (deployment_plan.md) */
|
||||
customModel: { endpointId: string; modelId: string; label?: string } | undefined = undefined;
|
||||
private _mockCustomModelConfigDir: string | undefined;
|
||||
setCustomModel = vi.fn(
|
||||
(
|
||||
next: { endpointId: string; modelId: string; label?: string; envKeys: string[]; configDir?: string } | undefined,
|
||||
_envOverrides?: Record<string, string>
|
||||
): string | undefined => {
|
||||
const previous = this._mockCustomModelConfigDir;
|
||||
this._mockCustomModelConfigDir = next?.configDir;
|
||||
this.customModel = next ? { endpointId: next.endpointId, modelId: next.modelId, label: next.label } : undefined;
|
||||
return previous;
|
||||
}
|
||||
);
|
||||
restartCli = vi.fn(async () => true);
|
||||
|
||||
/** Stub for sendInput */
|
||||
sendInput = vi.fn();
|
||||
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
/**
|
||||
* @fileoverview Route tests for Custom Model Endpoint Profiles CRUD + discovery.
|
||||
* Port: N/A (app.inject, no real port needed)
|
||||
*/
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest';
|
||||
import { registerCustomModelRoutes } from '../../src/web/routes/custom-model-routes.js';
|
||||
import { createRouteTestHarness } from './_route-test-utils.js';
|
||||
|
||||
async function setup() {
|
||||
return createRouteTestHarness(registerCustomModelRoutes);
|
||||
}
|
||||
|
||||
describe('custom model endpoint CRUD', () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it('starts empty', async () => {
|
||||
const { app } = await setup();
|
||||
const res = await app.inject({ method: 'GET', url: '/api/model-endpoints' });
|
||||
expect(res.json()).toEqual([]);
|
||||
});
|
||||
|
||||
it('creates, lists, updates, and deletes an endpoint', async () => {
|
||||
const { app } = await setup();
|
||||
|
||||
const create = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/model-endpoints',
|
||||
payload: { id: 'ep1', label: 'llama.cpp box', baseUrl: 'http://192.168.1.50:8080' },
|
||||
});
|
||||
expect(create.statusCode).toBe(200);
|
||||
expect(create.json().data.host.id).toBe('ep1');
|
||||
|
||||
const list = await app.inject({ method: 'GET', url: '/api/model-endpoints' });
|
||||
expect(list.json()).toHaveLength(1);
|
||||
|
||||
const update = await app.inject({
|
||||
method: 'PUT',
|
||||
url: '/api/model-endpoints/ep1',
|
||||
payload: { label: 'Renamed', baseUrl: 'http://192.168.1.50:8080' },
|
||||
});
|
||||
expect(update.statusCode).toBe(200);
|
||||
expect(update.json().data.host.label).toBe('Renamed');
|
||||
|
||||
const del = await app.inject({ method: 'DELETE', url: '/api/model-endpoints/ep1' });
|
||||
expect(del.statusCode).toBe(200);
|
||||
|
||||
const listAfter = await app.inject({ method: 'GET', url: '/api/model-endpoints' });
|
||||
expect(listAfter.json()).toEqual([]);
|
||||
});
|
||||
|
||||
it('rejects a duplicate id on create', async () => {
|
||||
const { app } = await setup();
|
||||
const payload = { id: 'dup', label: 'A', baseUrl: 'http://localhost:8080' };
|
||||
await app.inject({ method: 'POST', url: '/api/model-endpoints', payload });
|
||||
const second = await app.inject({ method: 'POST', url: '/api/model-endpoints', payload });
|
||||
expect(second.json().success).toBe(false);
|
||||
expect(second.json().errorCode).toBe('ALREADY_EXISTS');
|
||||
});
|
||||
|
||||
it('404s updating/deleting an id that does not exist', async () => {
|
||||
const { app } = await setup();
|
||||
const update = await app.inject({
|
||||
method: 'PUT',
|
||||
url: '/api/model-endpoints/ghost',
|
||||
payload: { label: 'A', baseUrl: 'http://localhost:8080' },
|
||||
});
|
||||
expect(update.json().errorCode).toBe('NOT_FOUND');
|
||||
});
|
||||
|
||||
it('rejects a link-local/cloud-metadata base URL', async () => {
|
||||
const { app } = await setup();
|
||||
const res = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/model-endpoints',
|
||||
payload: { id: 'meta', label: 'A', baseUrl: 'http://169.254.169.254/' },
|
||||
});
|
||||
expect(res.json().success).toBe(false);
|
||||
expect(res.json().errorCode).toBe('INVALID_INPUT');
|
||||
});
|
||||
|
||||
it('discovers models via GET /v1/models and stores the result', async () => {
|
||||
const { app } = await setup();
|
||||
await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/model-endpoints',
|
||||
payload: { id: 'ep1', label: 'A', baseUrl: 'http://localhost:8080', apiKey: 'k' },
|
||||
});
|
||||
|
||||
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
|
||||
expect(url).toBe('http://localhost:8080/v1/models');
|
||||
const headers = init?.headers as Record<string, string>;
|
||||
// Exactly ONE auth header — never both (a real server hung when sent both).
|
||||
expect(headers.Authorization).toBe('Bearer k');
|
||||
expect(headers['api-key']).toBeUndefined();
|
||||
return new Response(JSON.stringify({ data: [{ id: 'qwen3' }, { id: 'llama3' }] }), { status: 200 });
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const res = await app.inject({ method: 'POST', url: '/api/model-endpoints/ep1/discover-models' });
|
||||
expect(res.json().data.models).toEqual(['qwen3', 'llama3']);
|
||||
|
||||
// Data dir is shared across this WHOLE test file (one temp HOME per file, not per
|
||||
// test — test/setup.ts), so find by id rather than assuming index 0.
|
||||
const list = await app.inject({ method: 'GET', url: '/api/model-endpoints' });
|
||||
const stored = (list.json() as Array<{ id: string }>).find((h) => h.id === 'ep1');
|
||||
expect(stored?.models).toEqual(['qwen3', 'llama3']);
|
||||
expect(stored?.lastDiscoveredAt).toBeTruthy();
|
||||
});
|
||||
|
||||
it('discovers models with authStyle "api-key" using only that header, never Authorization', async () => {
|
||||
const { app } = await setup();
|
||||
await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/model-endpoints',
|
||||
payload: { id: 'ep-azure', label: 'A', baseUrl: 'http://localhost:8080', apiKey: 'k', authStyle: 'api-key' },
|
||||
});
|
||||
|
||||
const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => {
|
||||
const headers = init?.headers as Record<string, string>;
|
||||
expect(headers['api-key']).toBe('k');
|
||||
expect(headers.Authorization).toBeUndefined();
|
||||
return new Response(JSON.stringify({ data: [] }), { status: 200 });
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
await app.inject({ method: 'POST', url: '/api/model-endpoints/ep-azure/discover-models' });
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('reports a clear error when the endpoint is unreachable', async () => {
|
||||
const { app } = await setup();
|
||||
await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/model-endpoints',
|
||||
payload: { id: 'ep-err', label: 'A', baseUrl: 'http://localhost:8080' },
|
||||
});
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(async () => {
|
||||
throw new Error('connect ECONNREFUSED');
|
||||
})
|
||||
);
|
||||
|
||||
const res = await app.inject({ method: 'POST', url: '/api/model-endpoints/ep-err/discover-models' });
|
||||
expect(res.json().success).toBe(false);
|
||||
expect(res.json().errorCode).toBe('OPERATION_FAILED');
|
||||
expect(res.json().error).toContain('ECONNREFUSED');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* @fileoverview Tests for POST /api/sessions/:id/custom-model (deployment_plan.md
|
||||
* chunk 5 — applying/clearing a session's custom model endpoint + CLI restart).
|
||||
* Port: N/A (app.inject, no real port needed)
|
||||
*/
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { registerSessionRoutes } from '../../src/web/routes/session-routes.js';
|
||||
import { createRouteTestHarness } from './_route-test-utils.js';
|
||||
import { getDataDir } from '../../src/config/instance.js';
|
||||
import { writeCustomModelHosts, type CustomModelHost } from '../../src/custom-model-hosts.js';
|
||||
|
||||
const CLAUDE_ENDPOINT: CustomModelHost = {
|
||||
id: 'ep1',
|
||||
label: 'llama.cpp box',
|
||||
baseUrl: 'http://192.168.1.50:8080',
|
||||
apiKey: 'k',
|
||||
};
|
||||
|
||||
async function setup() {
|
||||
await writeCustomModelHosts(getDataDir(), [CLAUDE_ENDPOINT]);
|
||||
return createRouteTestHarness(registerSessionRoutes);
|
||||
}
|
||||
|
||||
describe('POST /api/sessions/:id/custom-model', () => {
|
||||
beforeEach(async () => {
|
||||
await writeCustomModelHosts(getDataDir(), []);
|
||||
});
|
||||
|
||||
it('applies an endpoint/model to a claude-mode session and restarts the CLI', async () => {
|
||||
const { app, ctx } = await setup();
|
||||
const session = ctx.sessions.get('test-session-1')!;
|
||||
session.mode = 'claude';
|
||||
|
||||
const res = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/sessions/test-session-1/custom-model',
|
||||
payload: { endpointId: 'ep1', modelId: 'qwen3' },
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = res.json();
|
||||
expect(body.customModel).toEqual({ endpointId: 'ep1', modelId: 'qwen3', label: 'llama.cpp box' });
|
||||
expect(body.restarted).toBe(true);
|
||||
expect(session.setCustomModel).toHaveBeenCalledTimes(1);
|
||||
expect(session.restartCli).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Verify the actual injected env vars via setCustomModel's captured call args.
|
||||
const [next, envOverrides] = session.setCustomModel.mock.calls[0];
|
||||
expect(next.envKeys).toEqual([
|
||||
'ANTHROPIC_BASE_URL',
|
||||
'ANTHROPIC_API_KEY',
|
||||
'ANTHROPIC_DEFAULT_SONNET_MODEL',
|
||||
'ANTHROPIC_DEFAULT_HAIKU_MODEL',
|
||||
'ANTHROPIC_DEFAULT_OPUS_MODEL',
|
||||
]);
|
||||
expect(envOverrides.ANTHROPIC_BASE_URL).toBe('http://192.168.1.50:8080');
|
||||
expect(envOverrides.ANTHROPIC_API_KEY).toBe('k');
|
||||
});
|
||||
|
||||
it('clears back to the native default', async () => {
|
||||
const { app, ctx } = await setup();
|
||||
const session = ctx.sessions.get('test-session-1')!;
|
||||
session.mode = 'claude';
|
||||
|
||||
const res = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/sessions/test-session-1/custom-model',
|
||||
payload: { clear: true },
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.json().customModel).toBeUndefined();
|
||||
expect(session.setCustomModel).toHaveBeenCalledWith(undefined);
|
||||
expect(session.restartCli).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('404s for an unknown endpoint id', async () => {
|
||||
const { app, ctx } = await setup();
|
||||
ctx.sessions.get('test-session-1')!.mode = 'claude';
|
||||
|
||||
const res = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/sessions/test-session-1/custom-model',
|
||||
payload: { endpointId: 'ghost', modelId: 'qwen3' },
|
||||
});
|
||||
|
||||
expect(res.json().success).toBe(false);
|
||||
expect(res.json().errorCode).toBe('NOT_FOUND');
|
||||
});
|
||||
|
||||
it('refuses a mode with no known custom-model mechanism (antigravity)', async () => {
|
||||
const { app, ctx } = await setup();
|
||||
ctx.sessions.get('test-session-1')!.mode = 'antigravity';
|
||||
|
||||
const res = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/sessions/test-session-1/custom-model',
|
||||
payload: { endpointId: 'ep1', modelId: 'qwen3' },
|
||||
});
|
||||
|
||||
expect(res.json().success).toBe(false);
|
||||
expect(res.json().errorCode).toBe('OPERATION_FAILED');
|
||||
});
|
||||
|
||||
it('refuses to touch a busy session', async () => {
|
||||
const { app, ctx } = await setup();
|
||||
const session = ctx.sessions.get('test-session-1')!;
|
||||
session.mode = 'claude';
|
||||
session.isBusy = () => true;
|
||||
|
||||
const res = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/sessions/test-session-1/custom-model',
|
||||
payload: { endpointId: 'ep1', modelId: 'qwen3' },
|
||||
});
|
||||
|
||||
expect(res.json().success).toBe(false);
|
||||
expect(res.json().errorCode).toBe('SESSION_BUSY');
|
||||
expect(session.setCustomModel).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user