Codeman/test/advisor-model.test.ts

/**
 * @fileoverview Tests for Claude Code's advisor tool (code.claude.com/docs/en/advisor)
 * carried as a per-session `advisorModel`.
 *
 * The advisor rides the launch's ONE `--settings` JSON object as the `advisorModel` key,
 * never the `--advisor` flag: the flag exits at launch on a pairing the CLI refuses
 * (`claude --advisor haiku` prints "cannot be used as an advisor" and exits 1), while the
 * settings key degrades to "no advisor". Verified against Claude Code 2.1.289 with
 * `claude -p --settings '{"advisorModel":"opus"}' /advisor` → "Advisor: Opus 5.5".
 *
 * `--settings` is extracted through a REAL shell, as in statusline-cli-flag.test.ts, so the
 * assertions see exactly what a spawned pane would.
 */

import { describe, it, expect, vi, afterEach } from 'vitest';
import { execFileSync } from 'node:child_process';
import { buildAdvisorSettings, buildInteractiveArgs } from '../src/session-cli-builder.js';
import { buildSpawnCommand } from '../src/tmux-manager.js';
import { isAdvisorModel, ADVISOR_MODEL_ALIASES } from '../src/types.js';
import { Session } from '../src/session.js';
import {
  CreateSessionSchema,
  QuickStartSchema,
  RalphLoopStartSchema,
  SettingsUpdateSchema,
} from '../src/web/schemas.js';

const EXPORTER_CMD = 'curl -sfk -X POST "$CODEMAN_API_URL/api/status-telemetry" --data @- 2>/dev/null || true';

/**
 * What the direct-PTY fallback hands `pty.spawn`. The file and argv are recorded, then a
 * harmless stand-in runs instead: the real `claude` must never start from a test, and the
 * stand-in is a real process so `Session.stop()` has a real pid to signal.
 */
const ptySpawns = vi.hoisted(() => [] as Array<{ file: string; args: string[] }>);
vi.mock('node-pty', async (importOriginal) => {
  const real = await importOriginal<typeof import('node-pty')>();
  return {
    ...real,
    spawn: (file: string, args: string[] | string, options: import('node-pty').IPtyForkOptions) => {
      ptySpawns.push({ file, args: Array.isArray(args) ? args : [args] });
      return real.spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], options);
    },
  };
});

function extractSettingsJson(cmd: string): unknown {
  const idx = cmd.indexOf('--settings ');
  expect(idx).toBeGreaterThan(-1);
  const out = execFileSync('bash', ['-c', `set -- ${cmd.slice(idx)}; printf '%s' "$2"`]).toString();
  return JSON.parse(out);
}

describe('isAdvisorModel', () => {
  it('accepts the documented aliases', () => {
    for (const alias of ADVISOR_MODEL_ALIASES) expect(isAdvisorModel(alias)).toBe(true);
  });

  it('accepts full model ids in the advisor-capable families', () => {
    expect(isAdvisorModel('claude-opus-5-5')).toBe(true);
    expect(isAdvisorModel('claude-fable-5-1')).toBe(true);
    expect(isAdvisorModel('claude-sonnet-5-5')).toBe(true);
  });

  it('rejects haiku, which can call an advisor but never act as one', () => {
    expect(isAdvisorModel('haiku')).toBe(false);
    expect(isAdvisorModel('claude-haiku-4-5-20251001')).toBe(false);
  });

  it('rejects anything that could break out of the quoted --settings argument', () => {
    for (const bad of [
      '',
      'OPUS',
      'opus[1m]',
      "opus'; rm -rf /; '",
      'opus"}',
      'claude-opus-5-5 --dangerously-skip-permissions',
      `claude-opus-${'5-'.repeat(40)}5`,
      undefined,
      null,
      42,
    ]) {
      expect(isAdvisorModel(bad)).toBe(false);
    }
  });
});

describe('buildAdvisorSettings', () => {
  it('returns the settings key for a valid model and nothing otherwise', () => {
    expect(buildAdvisorSettings('opus')).toEqual({ advisorModel: 'opus' });
    expect(buildAdvisorSettings(undefined)).toEqual({});
    expect(buildAdvisorSettings('haiku')).toEqual({});
  });
});

describe('buildSpawnCommand advisorModel (tmux launch, claude mode)', () => {
  it('rides --settings as the advisorModel key, never the --advisor flag', () => {
    const cmd = buildSpawnCommand({ mode: 'claude', sessionId: 'sid-1', advisorModel: 'opus' });
    expect(cmd).not.toContain('--advisor');
    expect(cmd.match(/--settings/g)).toHaveLength(1);
    expect(extractSettingsJson(cmd)).toEqual({ advisorModel: 'opus' });
  });

  it('merges ultracode, advisor and the statusLine exporter into ONE --settings object', () => {
    const cmd = buildSpawnCommand({
      mode: 'claude',
      sessionId: 'sid-1',
      effort: 'ultracode',
      advisorModel: 'fable',
      statusLineCommand: EXPORTER_CMD,
    });
    expect(cmd.match(/--settings/g)).toHaveLength(1);
    expect(extractSettingsJson(cmd)).toEqual({
      ultracode: true,
      advisorModel: 'fable',
      statusLine: { type: 'command', command: EXPORTER_CMD },
    });
  });

  it('keeps a regular --effort flag beside the advisor settings', () => {
    const cmd = buildSpawnCommand({ mode: 'claude', sessionId: 'sid-1', effort: 'high', advisorModel: 'sonnet' });
    expect(cmd).toContain("--effort 'high'");
    expect(extractSettingsJson(cmd)).toEqual({ advisorModel: 'sonnet' });
  });

  it('carries the advisor on the resume variant too', () => {
    const cmd = buildSpawnCommand({
      mode: 'claude',
      sessionId: 'sid-1',
      resumeSessionId: '11111111-2222-3333-4444-555555555555',
      advisorModel: 'opus',
    });
    expect(cmd).toContain('--resume');
    expect(extractSettingsJson(cmd)).toEqual({ advisorModel: 'opus' });
  });

  it('leaves the command byte-identical when no advisor (or an invalid one) is set', () => {
    for (const base of [
      { mode: 'claude', sessionId: 'sid-1' },
      { mode: 'claude', sessionId: 'sid-1', effort: 'ultracode' as const, statusLineCommand: EXPORTER_CMD },
    ]) {
      const without = buildSpawnCommand(base);
      expect(buildSpawnCommand({ ...base, advisorModel: undefined })).toBe(without);
      expect(buildSpawnCommand({ ...base, advisorModel: 'haiku' })).toBe(without);
    }
  });

  it('is inert for a CLI that has no --settings carrier', () => {
    const cmd = buildSpawnCommand({ mode: 'codex', sessionId: 'sid-1', advisorModel: 'opus' });
    expect(cmd).not.toContain('advisorModel');
  });
});

describe('buildInteractiveArgs advisorModel (direct-PTY fallback)', () => {
  const settingsOf = (args: string[]) => {
    expect(args.filter((a) => a === '--settings')).toHaveLength(1);
    return JSON.parse(args[args.indexOf('--settings') + 1]);
  };

  it('adds a --settings object holding only the advisor', () => {
    const args = buildInteractiveArgs('sid', 'normal', undefined, undefined, undefined, undefined, null, 'opus');
    expect(args).not.toContain('--advisor');
    expect(settingsOf(args)).toEqual({ advisorModel: 'opus' });
  });

  it("folds the advisor into ultracode's --settings object", () => {
    const args = buildInteractiveArgs('sid', 'normal', undefined, undefined, 'ultracode', undefined, null, 'fable');
    expect(settingsOf(args)).toEqual({ ultracode: true, advisorModel: 'fable' });
  });

  it('keeps --effort beside it for a regular level', () => {
    const args = buildInteractiveArgs('sid', 'normal', undefined, undefined, 'max', undefined, null, 'sonnet');
    expect(args).toEqual(expect.arrayContaining(['--effort', 'max']));
    expect(settingsOf(args)).toEqual({ advisorModel: 'sonnet' });
  });

  it('is unchanged without an advisor', () => {
    expect(buildInteractiveArgs('sid', 'normal', undefined, undefined, 'high', undefined, null, undefined)).toEqual(
      buildInteractiveArgs('sid', 'normal', undefined, undefined, 'high', undefined, null)
    );
  });
});

/**
 * #514 (per-session `model` → `--model`) and #530 landed together. A claude session carrying
 * both must launch with `--model <id>` AND the advisor folded into the one `--settings` JSON,
 * whatever the effort, on the tmux template and on the direct-PTY fallback alike. Recovery of
 * the same pair is pinned in test/session-model-recovery.test.ts.
 */
describe('a per-session model together with an advisor', () => {
  const MODEL = 'claude-fable-5-1';
  const cases = [
    ['ultracode', { ultracode: true, advisorModel: 'opus' }],
    ['high', { advisorModel: 'opus' }],
    [undefined, { advisorModel: 'opus' }],
  ] as const;

  it.each(cases)('tmux template carries --model and the merged --settings, effort %s', (effort, settings) => {
    for (const resumeSessionId of [undefined, '11111111-2222-3333-4444-555555555555']) {
      const cmd = buildSpawnCommand({
        mode: 'claude',
        sessionId: 'sid-1',
        model: MODEL,
        effort,
        advisorModel: 'opus',
        resumeSessionId,
        claudeCliVersion: null,
      });
      // The resume variant renders `resume || new`, so the model appears once per branch.
      expect(cmd).toContain(`--model "${MODEL}"`);
      expect(cmd).not.toContain('--advisor ');
      const settingsFlags = cmd.match(/--settings /g) ?? [];
      expect(settingsFlags.length).toBe(resumeSessionId ? 2 : 1);
      expect(extractSettingsJson(cmd)).toEqual(settings);
      if (effort === 'high') expect(cmd).toContain("--effort 'high'");
    }
  });

  it('keeps the statusLine exporter in the same object beside the model', () => {
    const cmd = buildSpawnCommand({
      mode: 'claude',
      sessionId: 'sid-1',
      model: MODEL,
      effort: 'ultracode',
      advisorModel: 'fable',
      statusLineCommand: EXPORTER_CMD,
      claudeCliVersion: null,
    });
    expect(cmd).toContain(`--model "${MODEL}"`);
    expect(cmd.match(/--settings /g)).toHaveLength(1);
    expect(extractSettingsJson(cmd)).toEqual({
      ultracode: true,
      advisorModel: 'fable',
      statusLine: { type: 'command', command: EXPORTER_CMD },
    });
  });

  it.each(cases)('direct-PTY args carry --model and the merged --settings, effort %s', (effort, settings) => {
    const args = buildInteractiveArgs('sid', 'normal', MODEL, undefined, effort, undefined, null, 'opus');
    expect(args[args.indexOf('--model') + 1]).toBe(MODEL);
    expect(args.filter((a) => a === '--settings')).toHaveLength(1);
    expect(JSON.parse(args[args.indexOf('--settings') + 1])).toEqual(settings);
    if (effort === 'high') expect(args).toEqual(expect.arrayContaining(['--effort', 'high']));
  });

  describe('a real Session on the direct-PTY fallback', () => {
    const live: Session[] = [];
    afterEach(async () => {
      for (const s of live.splice(0)) await s.stop();
      ptySpawns.length = 0;
    });

    it.each(cases)('hands pty.spawn both, effort %s', async (effort, settings) => {
      const session = new Session({
        workingDir: '/tmp',
        mode: 'claude',
        useMux: false,
        model: MODEL,
        advisorModel: 'opus',
        effort,
      });
      live.push(session);
      await session.startInteractive();

      expect(ptySpawns).toHaveLength(1);
      const { args } = ptySpawns[0];
      expect(args[args.indexOf('--model') + 1]).toBe(MODEL);
      expect(args.filter((a) => a === '--settings')).toHaveLength(1);
      expect(JSON.parse(args[args.indexOf('--settings') + 1])).toEqual(settings);
    });
  });
});

describe('Session advisorModel', () => {
  it('stores a valid advisor and persists it through toState()', () => {
    const session = new Session({ workingDir: '/tmp', advisorModel: 'opus' });
    expect(session.toState().advisorModel).toBe('opus');
  });

  it('drops an invalid value instead of forwarding it to the launch', () => {
    expect(new Session({ workingDir: '/tmp', advisorModel: 'haiku' }).toState().advisorModel).toBeUndefined();
    expect(new Session({ workingDir: '/tmp' }).toState().advisorModel).toBeUndefined();
  });
});

describe('advisorModel request validation', () => {
  it.each([
    ['CreateSessionSchema', CreateSessionSchema, { workingDir: '/tmp' }],
    ['QuickStartSchema', QuickStartSchema, {}],
    ['RalphLoopStartSchema', RalphLoopStartSchema, { taskDescription: 'x' }],
  ] as const)('%s accepts advisor models and rejects the rest', (_name, schema, base) => {
    expect(schema.safeParse({ ...base, advisorModel: 'opus' }).success).toBe(true);
    expect(schema.safeParse({ ...base, advisorModel: 'claude-fable-5-1' }).success).toBe(true);
    expect(schema.safeParse({ ...base }).success).toBe(true);
    expect(schema.safeParse({ ...base, advisorModel: 'haiku' }).success).toBe(false);
    expect(schema.safeParse({ ...base, advisorModel: "opus'" }).success).toBe(false);
  });

  it('SettingsUpdateSchema takes claudeAdvisorModel, with "" meaning the CLI default', () => {
    expect(SettingsUpdateSchema.safeParse({ claudeAdvisorModel: '' }).success).toBe(true);
    expect(SettingsUpdateSchema.safeParse({ claudeAdvisorModel: 'fable' }).success).toBe(true);
    expect(SettingsUpdateSchema.safeParse({ claudeAdvisorModel: 'haiku' }).success).toBe(false);
  });
});