Files
Codeman/test/custom-model-injection.test.ts
T
DevvynandClaude Sonnet 5 61779745aa test(custom-model): make the harness smoke test dynamic, verify all 9 CLIs end-to-end
Rewrites scripts/test-local-llm-harnesses.mjs -> .ts to read the live CLI
registry (enabledClis()) and call the real production
buildCustomModelInjection()/applyConfigDirInjection() instead of keeping a
second hand-maintained copy of every CLI's env/config shape. A future
registry change (new CLI, edited env var, fixed config template) is now
picked up automatically with zero edits to this script; only the one-shot
invocation flags (info the registry genuinely doesn't model) stay in a
small hand-maintained ONE_SHOT table, and a registry CLI with no entry
there reports UNKNOWN rather than being silently skipped.

Extracted src/custom-model-injection-apply.ts (applyConfigDirInjection/
removeConfigDir) so the production route and this script share one
implementation instead of two.

Full end-to-end run against a real llama-swap server, inside a
codeman/agent:llm-test Docker image with all 9 CLI binaries:

- claude, opencode, pi, grok, omp: PASS, real "hello world" replies
- codex: confirmed FAIL for a real protocol reason, not a bug — it only
  speaks the Responses API since Feb 2026, which llama.cpp/llama-swap
  don't implement
- gemini: confirmed FAIL, unresolved after real investigation — an
  undocumented GATEWAY AuthType gemini-cli selects once
  GOOGLE_GEMINI_BASE_URL is set rejects every auth-key format/override
  tried
- deepseek: reaches the server (env vars are read) but gets a consistent
  HTTP_404; root cause not identified, documented as best-effort/unknown
- antigravity: SKIP, no known mechanism (unchanged)

Two real bugs found and fixed along the way (grok, pi/omp registry
entries in stock.ts): grok's original recipe (env vars) was flat-out
wrong, not just unverified — the real mechanism is a config.toml
[model.<name>] block redirected via GROK_HOME. pi/omp's PI_CONFIG_DIR
does nothing for either (grepped pi's entire bundled source — the string
appears nowhere); the real redirect is the child process's own HOME, and
both need `models` as an array of {id} objects, not an object keyed by
id (silently loaded zero models otherwise).

deployment_plan.md, PR.md, docs/custom-model-endpoints.md, and CLAUDE.md
updated with the final confidence table reflecting all of the above.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017HqNWfmtBU2KN29SvSVWB3
2026-09-13 17:42:35 +08:00

159 lines
7.5 KiB
TypeScript

/**
* @fileoverview Tests for the Custom Model Endpoint Profiles pure builder.
* Uses the real CLI registry entries (getCli) rather than hand-rolled
* fixtures, so a change to a real entry's customModelInjection declaration
* is exercised here automatically instead of silently diverging.
*
* Port: N/A (no server needed)
*/
import { describe, it, expect } from 'vitest';
import { getCli } from '../src/config/cli-registry/index.js';
import { buildCustomModelInjection, withV1Suffix, type CustomModelEndpoint } from '../src/custom-model-injection.js';
const endpoint: CustomModelEndpoint = {
id: 'ep1',
label: 'llama.cpp box',
baseUrl: 'http://192.168.1.50:8080',
apiKey: 'my-key',
};
function entryOrThrow(id: string) {
const entry = getCli(id);
if (!entry) throw new Error(`missing CLI registry entry: ${id}`);
return entry;
}
describe('withV1Suffix', () => {
it('appends /v1 when missing', () => {
expect(withV1Suffix('http://host:8080')).toBe('http://host:8080/v1');
});
it('is idempotent when already present', () => {
expect(withV1Suffix('http://host:8080/v1')).toBe('http://host:8080/v1');
expect(withV1Suffix('http://host:8080/v1/')).toBe('http://host:8080/v1');
});
it('strips a trailing slash with no /v1', () => {
expect(withV1Suffix('http://host:8080/')).toBe('http://host:8080/v1');
});
});
describe('buildCustomModelInjection', () => {
it('claude: env kind sets base URL, api key, and all three tier model vars', () => {
const result = buildCustomModelInjection(entryOrThrow('claude'), endpoint, 'qwen3');
expect(result.kind).toBe('env');
if (result.kind !== 'env') throw new Error('unreachable');
expect(result.envOverrides).toEqual({
ANTHROPIC_BASE_URL: 'http://192.168.1.50:8080',
ANTHROPIC_API_KEY: 'my-key',
ANTHROPIC_DEFAULT_SONNET_MODEL: 'qwen3',
ANTHROPIC_DEFAULT_HAIKU_MODEL: 'qwen3',
ANTHROPIC_DEFAULT_OPUS_MODEL: 'qwen3',
});
});
it('claude: falls back to a dummy key when the endpoint has none', () => {
const result = buildCustomModelInjection(entryOrThrow('claude'), { ...endpoint, apiKey: undefined }, 'qwen3');
if (result.kind !== 'env') throw new Error('unreachable');
expect(result.envOverrides.ANTHROPIC_API_KEY).toBe('local-dummy-key');
});
it('opencode: configContentEnv carries a JSON blob in OPENCODE_CONFIG_CONTENT', () => {
const result = buildCustomModelInjection(entryOrThrow('opencode'), endpoint, 'qwen3');
expect(result.kind).toBe('env');
if (result.kind !== 'env') throw new Error('unreachable');
const parsed = JSON.parse(result.envOverrides.OPENCODE_CONFIG_CONTENT);
expect(parsed.model).toBe('custom/qwen3');
expect(parsed.provider.custom.options.baseURL).toBe('http://192.168.1.50:8080/v1');
expect(parsed.provider.custom.options.apiKey).toBe('my-key');
expect(parsed.provider.custom.models.qwen3).toEqual({});
});
it('codex: configDir writes an isolated config.toml with model/base_url, and the key rides as extraEnv (never a literal TOML field)', () => {
const result = buildCustomModelInjection(entryOrThrow('codex'), endpoint, 'qwen3');
expect(result.kind).toBe('configDir');
if (result.kind !== 'configDir') throw new Error('unreachable');
expect(result.dirEnvVar).toBe('CODEX_HOME');
expect(result.files).toHaveLength(1);
expect(result.files[0].relPath).toBe('config.toml');
expect(result.files[0].content).toContain('model = "qwen3"');
expect(result.files[0].content).toContain('base_url = "http://192.168.1.50:8080/v1"');
expect(result.files[0].content).toContain('wire_api = "responses"');
expect(result.files[0].content).not.toContain('api_key ='); // never a literal TOML field
expect(result.files[0].content).toContain('env_key = "CODEMAN_CUSTOM_MODEL_API_KEY"');
expect(result.extraEnv).toEqual({ CODEMAN_CUSTOM_MODEL_API_KEY: 'my-key' });
});
it('codex: escapes a quote in the model id so it cannot break out of the TOML string', () => {
const result = buildCustomModelInjection(entryOrThrow('codex'), endpoint, 'weird"model');
if (result.kind !== 'configDir') throw new Error('unreachable');
expect(result.files[0].content).toContain('model = "weird\\"model"');
});
it('pi: configDir writes .pi/agent/models.json, redirected via HOME (verified live — PI_CONFIG_DIR does nothing for pi)', () => {
const result = buildCustomModelInjection(entryOrThrow('pi'), endpoint, 'qwen3');
if (result.kind !== 'configDir') throw new Error('unreachable');
expect(result.dirEnvVar).toBe('HOME');
expect(result.files[0].relPath).toBe('.pi/agent/models.json');
const parsed = JSON.parse(result.files[0].content);
expect(parsed.providers.custom.baseUrl).toBe('http://192.168.1.50:8080/v1');
expect(parsed.providers.custom.authHeader).toBe(true);
expect(parsed.providers.custom.models).toEqual([{ id: 'qwen3' }]); // array, NOT keyed by id
});
it('omp: configDir writes .omp/agent/models.yml, redirected via HOME (verified live end-to-end)', () => {
const result = buildCustomModelInjection(entryOrThrow('omp'), endpoint, 'qwen3');
if (result.kind !== 'configDir') throw new Error('unreachable');
expect(result.dirEnvVar).toBe('HOME');
expect(result.files[0].relPath).toBe('.omp/agent/models.yml');
expect(result.files[0].content).toContain('baseUrl: "http://192.168.1.50:8080/v1"');
expect(result.files[0].content).toContain('authHeader: true');
expect(result.files[0].content).toContain('- id: "qwen3"');
});
it('gemini: env kind sets GOOGLE_GEMINI_BASE_URL/GEMINI_API_KEY/GEMINI_MODEL', () => {
const result = buildCustomModelInjection(entryOrThrow('gemini'), endpoint, 'qwen3');
if (result.kind !== 'env') throw new Error('unreachable');
expect(result.envOverrides).toEqual({
GOOGLE_GEMINI_BASE_URL: 'http://192.168.1.50:8080',
GEMINI_API_KEY: 'my-key',
GEMINI_MODEL: 'qwen3',
});
});
it('grok: configDir writes a config.toml [model.<name>] block, key rides as extraEnv (XAI_API_KEY)', () => {
const result = buildCustomModelInjection(entryOrThrow('grok'), endpoint, 'qwen3');
expect(result.kind).toBe('configDir');
if (result.kind !== 'configDir') throw new Error('unreachable');
expect(result.dirEnvVar).toBe('GROK_HOME');
expect(result.files).toHaveLength(1);
expect(result.files[0].relPath).toBe('config.toml');
expect(result.files[0].content).toContain('model = "qwen3"');
expect(result.files[0].content).toContain('base_url = "http://192.168.1.50:8080/v1"');
expect(result.files[0].content).toContain('api_backend = "chat_completions"');
expect(result.files[0].content).toContain('env_key = "XAI_API_KEY"');
expect(result.files[0].content).not.toContain('api_key ='); // never a literal TOML field
expect(result.extraEnv).toEqual({ XAI_API_KEY: 'my-key' });
});
it('deepseek: env kind sets base URL/key only, no model var', () => {
const result = buildCustomModelInjection(entryOrThrow('deepseek'), endpoint, 'qwen3');
if (result.kind !== 'env') throw new Error('unreachable');
expect(result.envOverrides).toEqual({
DEEPSEEK_BASE_URL: 'http://192.168.1.50:8080',
DEEPSEEK_API_KEY: 'my-key',
});
});
it('antigravity: unsupported', () => {
const result = buildCustomModelInjection(entryOrThrow('antigravity'), endpoint, 'qwen3');
expect(result).toEqual({ kind: 'unsupported' });
});
it('shell: unsupported', () => {
const result = buildCustomModelInjection(entryOrThrow('shell'), endpoint, 'qwen3');
expect(result).toEqual({ kind: 'unsupported' });
});
});