mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-09-30 12:39:42 +02:00
Point any Codeman-supported harness (Claude, opencode, Codex, Gemini, Pi, Grok, DeepSeek, OMP) at a custom OpenAI-compatible endpoint instead of its native cloud backend, for a given session. Covers local hardware (llama.cpp, Ollama, vLLM, DGX Spark, Strix Halo) and cloud (Azure AI Foundry, OpenRouter). Off by default (customModelEndpointsEnabled, synced, default OFF). - Registry: capabilities.customModelInjection per CLI entry (env / configContentEnv / configDir / unsupported kinds) - Pure injection builder (custom-model-injection.ts) turning an endpoint + model id into the real env vars / config content per CLI - Endpoint store + CRUD routes (custom-model-hosts.ts, custom-model-routes.ts), discovery via GET /v1/models, SSRF-guarded - Session integration: Session.setCustomModel()/restartCli() (POST /api/sessions/:id/custom-model), reusing the existing respawn-pane -k primitive to restart the CLI process with new env - Multi-user hardening: every new redirect-capable env var added to its CLI's privilegedEnvKeys, closing a pre-existing gap where several were already reachable via the generic envOverrides field's prefix allowlist - Standalone scripts/test-local-llm-harnesses.mjs: spawns real CLI binaries against a real endpoint outside the web UI, independent of tmux/sessions - Mock-server contract tests (test/fixtures/mock-openai-server.ts) replaying every CLI's injected values through a real HTTP shape Real end-to-end validation against a live llama-swap server (inside a codeman/agent:llm-test Docker image with all 9 CLI binaries) found and fixed three real bugs before they shipped: - Codex's config.toml schema was wrong ([model].default table instead of a top-level model string + [model_providers.custom]); fixing it then surfaced a genuine, documented protocol incompatibility (Codex only speaks the Responses API since Feb 2026, which llama.cpp/llama-swap don't implement) - Claude Code's async session-title-generation call validates ANTHROPIC_DEFAULT_HAIKU_MODEL against its own internal model list and hangs the whole -p invocation on an unrecognized name; documented for chunk 6, worked around in the standalone script only (--bare is NOT safe for a real interactive session, which needs hooks) - The discovery route's authStyle: 'both' option (send both Authorization and api-key headers) reliably hung a real server; removed the option entirely rather than just changing the default Status: draft. Chunk 6 (frontend toolbar/settings UI) not yet built — see PR.md and deployment_plan.md for the full chunk breakdown and confidence table. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017HqNWfmtBU2KN29SvSVWB3
150 lines
6.6 KiB
TypeScript
150 lines
6.6 KiB
TypeScript
/**
|
|
* @fileoverview Tests for the Custom Model Endpoint Profiles pure builder.
|
|
* Uses the real CLI registry entries (getCli) rather than hand-rolled
|
|
* fixtures, so a change to a real entry's customModelInjection declaration
|
|
* is exercised here automatically instead of silently diverging.
|
|
*
|
|
* Port: N/A (no server needed)
|
|
*/
|
|
|
|
import { describe, it, expect } from 'vitest';
|
|
import { getCli } from '../src/config/cli-registry/index.js';
|
|
import { buildCustomModelInjection, withV1Suffix, type CustomModelEndpoint } from '../src/custom-model-injection.js';
|
|
|
|
const endpoint: CustomModelEndpoint = {
|
|
id: 'ep1',
|
|
label: 'llama.cpp box',
|
|
baseUrl: 'http://192.168.1.50:8080',
|
|
apiKey: 'my-key',
|
|
};
|
|
|
|
function entryOrThrow(id: string) {
|
|
const entry = getCli(id);
|
|
if (!entry) throw new Error(`missing CLI registry entry: ${id}`);
|
|
return entry;
|
|
}
|
|
|
|
describe('withV1Suffix', () => {
|
|
it('appends /v1 when missing', () => {
|
|
expect(withV1Suffix('http://host:8080')).toBe('http://host:8080/v1');
|
|
});
|
|
|
|
it('is idempotent when already present', () => {
|
|
expect(withV1Suffix('http://host:8080/v1')).toBe('http://host:8080/v1');
|
|
expect(withV1Suffix('http://host:8080/v1/')).toBe('http://host:8080/v1');
|
|
});
|
|
|
|
it('strips a trailing slash with no /v1', () => {
|
|
expect(withV1Suffix('http://host:8080/')).toBe('http://host:8080/v1');
|
|
});
|
|
});
|
|
|
|
describe('buildCustomModelInjection', () => {
|
|
it('claude: env kind sets base URL, api key, and all three tier model vars', () => {
|
|
const result = buildCustomModelInjection(entryOrThrow('claude'), endpoint, 'qwen3');
|
|
expect(result.kind).toBe('env');
|
|
if (result.kind !== 'env') throw new Error('unreachable');
|
|
expect(result.envOverrides).toEqual({
|
|
ANTHROPIC_BASE_URL: 'http://192.168.1.50:8080',
|
|
ANTHROPIC_API_KEY: 'my-key',
|
|
ANTHROPIC_DEFAULT_SONNET_MODEL: 'qwen3',
|
|
ANTHROPIC_DEFAULT_HAIKU_MODEL: 'qwen3',
|
|
ANTHROPIC_DEFAULT_OPUS_MODEL: 'qwen3',
|
|
});
|
|
});
|
|
|
|
it('claude: falls back to a dummy key when the endpoint has none', () => {
|
|
const result = buildCustomModelInjection(entryOrThrow('claude'), { ...endpoint, apiKey: undefined }, 'qwen3');
|
|
if (result.kind !== 'env') throw new Error('unreachable');
|
|
expect(result.envOverrides.ANTHROPIC_API_KEY).toBe('local-dummy-key');
|
|
});
|
|
|
|
it('opencode: configContentEnv carries a JSON blob in OPENCODE_CONFIG_CONTENT', () => {
|
|
const result = buildCustomModelInjection(entryOrThrow('opencode'), endpoint, 'qwen3');
|
|
expect(result.kind).toBe('env');
|
|
if (result.kind !== 'env') throw new Error('unreachable');
|
|
const parsed = JSON.parse(result.envOverrides.OPENCODE_CONFIG_CONTENT);
|
|
expect(parsed.model).toBe('custom/qwen3');
|
|
expect(parsed.provider.custom.options.baseURL).toBe('http://192.168.1.50:8080/v1');
|
|
expect(parsed.provider.custom.options.apiKey).toBe('my-key');
|
|
expect(parsed.provider.custom.models.qwen3).toEqual({});
|
|
});
|
|
|
|
it('codex: configDir writes an isolated config.toml with model/base_url, and the key rides as extraEnv (never a literal TOML field)', () => {
|
|
const result = buildCustomModelInjection(entryOrThrow('codex'), endpoint, 'qwen3');
|
|
expect(result.kind).toBe('configDir');
|
|
if (result.kind !== 'configDir') throw new Error('unreachable');
|
|
expect(result.dirEnvVar).toBe('CODEX_HOME');
|
|
expect(result.files).toHaveLength(1);
|
|
expect(result.files[0].relPath).toBe('config.toml');
|
|
expect(result.files[0].content).toContain('model = "qwen3"');
|
|
expect(result.files[0].content).toContain('base_url = "http://192.168.1.50:8080/v1"');
|
|
expect(result.files[0].content).toContain('wire_api = "responses"');
|
|
expect(result.files[0].content).not.toContain('api_key ='); // never a literal TOML field
|
|
expect(result.files[0].content).toContain('env_key = "CODEMAN_CUSTOM_MODEL_API_KEY"');
|
|
expect(result.extraEnv).toEqual({ CODEMAN_CUSTOM_MODEL_API_KEY: 'my-key' });
|
|
});
|
|
|
|
it('codex: escapes a quote in the model id so it cannot break out of the TOML string', () => {
|
|
const result = buildCustomModelInjection(entryOrThrow('codex'), endpoint, 'weird"model');
|
|
if (result.kind !== 'configDir') throw new Error('unreachable');
|
|
expect(result.files[0].content).toContain('model = "weird\\"model"');
|
|
});
|
|
|
|
it('pi: configDir writes models.json under agent/', () => {
|
|
const result = buildCustomModelInjection(entryOrThrow('pi'), endpoint, 'qwen3');
|
|
if (result.kind !== 'configDir') throw new Error('unreachable');
|
|
expect(result.dirEnvVar).toBe('PI_CONFIG_DIR');
|
|
expect(result.files[0].relPath).toBe('agent/models.json');
|
|
const parsed = JSON.parse(result.files[0].content);
|
|
expect(parsed.providers.custom.baseUrl).toBe('http://192.168.1.50:8080/v1');
|
|
});
|
|
|
|
it('omp: configDir writes models.yml under agent/, redirected via PI_CONFIG_DIR', () => {
|
|
const result = buildCustomModelInjection(entryOrThrow('omp'), endpoint, 'qwen3');
|
|
if (result.kind !== 'configDir') throw new Error('unreachable');
|
|
expect(result.dirEnvVar).toBe('PI_CONFIG_DIR');
|
|
expect(result.files[0].relPath).toBe('agent/models.yml');
|
|
expect(result.files[0].content).toContain('baseUrl: "http://192.168.1.50:8080/v1"');
|
|
});
|
|
|
|
it('gemini: env kind sets GOOGLE_GEMINI_BASE_URL/GEMINI_API_KEY/GEMINI_MODEL', () => {
|
|
const result = buildCustomModelInjection(entryOrThrow('gemini'), endpoint, 'qwen3');
|
|
if (result.kind !== 'env') throw new Error('unreachable');
|
|
expect(result.envOverrides).toEqual({
|
|
GOOGLE_GEMINI_BASE_URL: 'http://192.168.1.50:8080',
|
|
GEMINI_API_KEY: 'my-key',
|
|
GEMINI_MODEL: 'qwen3',
|
|
});
|
|
});
|
|
|
|
it('grok: env kind sets GROK_BASE_URL/XAI_API_KEY/GROK_MODEL', () => {
|
|
const result = buildCustomModelInjection(entryOrThrow('grok'), endpoint, 'qwen3');
|
|
if (result.kind !== 'env') throw new Error('unreachable');
|
|
expect(result.envOverrides).toEqual({
|
|
GROK_BASE_URL: 'http://192.168.1.50:8080',
|
|
XAI_API_KEY: 'my-key',
|
|
GROK_MODEL: 'qwen3',
|
|
});
|
|
});
|
|
|
|
it('deepseek: env kind sets base URL/key only, no model var', () => {
|
|
const result = buildCustomModelInjection(entryOrThrow('deepseek'), endpoint, 'qwen3');
|
|
if (result.kind !== 'env') throw new Error('unreachable');
|
|
expect(result.envOverrides).toEqual({
|
|
DEEPSEEK_BASE_URL: 'http://192.168.1.50:8080',
|
|
DEEPSEEK_API_KEY: 'my-key',
|
|
});
|
|
});
|
|
|
|
it('antigravity: unsupported', () => {
|
|
const result = buildCustomModelInjection(entryOrThrow('antigravity'), endpoint, 'qwen3');
|
|
expect(result).toEqual({ kind: 'unsupported' });
|
|
});
|
|
|
|
it('shell: unsupported', () => {
|
|
const result = buildCustomModelInjection(entryOrThrow('shell'), endpoint, 'qwen3');
|
|
expect(result).toEqual({ kind: 'unsupported' });
|
|
});
|
|
});
|