old-docker-actions/ai-code-review
2
0
Fork 0
Code Pull Requests 1 Actions Releases 2 Activity

test(ai-code-review): 更新 CLI 呼叫與前置驗證測試

Browse Source
This commit is contained in:
jiantw83
2026-06-27 13:30:45 +00:00
parent 08a72a8c7f
commit 4fcb240208
3 changed files with 115 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/test/config.test.js
+22 -13
View File
@@ -3,7 +3,7 @@ import assert from 'node:assert/strict';
import { getLLMConfig, getOpenCodeHttpsAgent } from '../config.js'; import { getLLMConfig, getOpenCodeHttpsAgent } from '../config.js';
const ENV_KEYS = [ const ENV_KEYS = [
'OPENCODE_BASE_URL', 'OPENCODE_MODEL', 'OPENCODE_PROVIDER', 'AI_ASSISTANT_CLI', 'MODEL', 'OPENCODE_MODEL',
]; ];
let saved = {}; let saved = {};
@@ -20,26 +20,35 @@ afterEach(() => {
describe('getLLMConfig', () => { describe('getLLMConfig', () => {
it('returns null provider when no env vars set', () => { it('returns null provider when no env vars set', () => {
const cfg = getLLMConfig(); const cfg = getLLMConfig({ commandExistsFn: () => false });
assert.equal(cfg.provider, null); assert.equal(cfg.provider, null);
assert.deepEqual(cfg.apiKeys, []); assert.deepEqual(cfg.apiKeys, []);
assert.equal(cfg.command, null);
}); });
it('detects opencode server with defaults', () => { it('detects the first installed assistant CLI with defaults', () => {
process.env.OPENCODE_BASE_URL = 'http://opencode.local:4096'; const cfg = getLLMConfig({ commandExistsFn: command => command === 'claude' });
const cfg = getLLMConfig(); assert.equal(cfg.provider, 'claude');
assert.equal(cfg.provider, 'opencode'); assert.deepEqual(cfg.apiKeys, ['claude']);
assert.deepEqual(cfg.apiKeys, ['opencode']); assert.equal(cfg.baseURL, null);
assert.equal(cfg.baseURL, 'http://opencode.local:4096'); assert.equal(cfg.command, 'claude');
assert.equal(cfg.model, 'gemini-2.5-flash'); assert.equal(cfg.model, 'sonnet');
}); });
it('detects opencode server with custom model', () => { it('uses MODEL for the selected assistant CLI', () => {
process.env.OPENCODE_BASE_URL = 'http://opencode.local:4096'; process.env.MODEL = 'gpt-5-mini';
const cfg = getLLMConfig({ commandExistsFn: command => command === 'codex' });
assert.equal(cfg.provider, 'codex');
assert.equal(cfg.command, 'codex');
assert.equal(cfg.model, 'gpt-5-mini');
});
it('can force a CLI with AI_ASSISTANT_CLI', () => {
process.env.AI_ASSISTANT_CLI = 'opencode';
process.env.OPENCODE_MODEL = 'google/gemini-2.5-pro'; process.env.OPENCODE_MODEL = 'google/gemini-2.5-pro';
const cfg = getLLMConfig(); const cfg = getLLMConfig({ commandExistsFn: command => command === 'codex' || command === 'opencode' });
assert.equal(cfg.provider, 'opencode'); assert.equal(cfg.provider, 'opencode');
assert.equal(cfg.baseURL, 'http://opencode.local:4096'); assert.equal(cfg.command, 'opencode');
assert.equal(cfg.model, 'google/gemini-2.5-pro'); assert.equal(cfg.model, 'google/gemini-2.5-pro');
}); });
app/test/llm.test.js
+57 -94
View File
@@ -1,133 +1,96 @@
import { describe, it, beforeEach, afterEach, mock } from 'node:test'; import { describe, it, beforeEach, afterEach, mock } from 'node:test';
import assert from 'node:assert/strict'; import assert from 'node:assert/strict';
import axios from 'axios'; import { mkdtemp, writeFile, chmod, rm, readFile } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { extractBalancedJSON, extractJSONText } from '../llm.js'; import { extractBalancedJSON, extractJSONText } from '../llm.js';
const ENV_KEYS = [ const ENV_KEYS = [
'OPENCODE_BASE_URL', 'OPENCODE_MODEL', 'OPENCODE_PROVIDER', 'OPENCODE_RETRY_ATTEMPTS', 'AI_ASSISTANT_CLI', 'MODEL', 'OPENCODE_MODEL', 'PATH', 'AI_ASSISTANT_TIMEOUT_MS', 'AI_ASSISTANT_MAX_BUFFER',
'FAKE_AI_STDOUT', 'FAKE_AI_STDERR', 'FAKE_AI_EXIT', 'FAKE_AI_STDIN_PATH', 'FAKE_AI_ARGS_PATH',
]; ];
let saved = {}; let saved = {};
let tempDir;
beforeEach(() => { beforeEach(() => {
saved = {}; saved = {};
for (const k of ENV_KEYS) { saved[k] = process.env[k]; delete process.env[k]; } for (const k of ENV_KEYS) { saved[k] = process.env[k]; delete process.env[k]; }
tempDir = null;
}); });
afterEach(() => { afterEach(async () => {
for (const k of ENV_KEYS) { for (const k of ENV_KEYS) {
if (saved[k] === undefined) delete process.env[k]; if (saved[k] === undefined) delete process.env[k];
else process.env[k] = saved[k]; else process.env[k] = saved[k];
} }
if (tempDir) await rm(tempDir, { recursive: true, force: true });
mock.restoreAll(); mock.restoreAll();
}); });
function mockOpenCodeResponse(content) { async function installFakeCLI(command = 'codex') {
let calls = 0; tempDir = await mkdtemp(join(tmpdir(), 'ai-cli-test-'));
mock.method(axios, 'post', async () => { const script = join(tempDir, command);
calls += 1; await writeFile(script, `#!/bin/sh
if (calls === 1) return { data: { id: 'ses_test' } }; if [ -n "$FAKE_AI_ARGS_PATH" ]; then printf '%s\\n' "$*" > "$FAKE_AI_ARGS_PATH"; fi
return { data: { parts: [{ type: 'text', text: content }] } }; if [ -n "$FAKE_AI_STDIN_PATH" ]; then cat > "$FAKE_AI_STDIN_PATH"; else cat >/dev/null; fi
}); if [ -n "$FAKE_AI_STDERR" ]; then printf '%s' "$FAKE_AI_STDERR" >&2; fi
if [ -n "$FAKE_AI_STDOUT" ]; then printf '%s' "$FAKE_AI_STDOUT"; fi
exit "\${FAKE_AI_EXIT:-0}"
`);
await chmod(script, 0o755);
process.env.PATH = `${tempDir}:${saved.PATH || ''}`;
return { stdinPath: join(tempDir, 'stdin.txt'), argsPath: join(tempDir, 'args.txt') };
} }
describe('chat - OpenCode', async () => { describe('chat - assistant CLI', async () => {
const { chat } = await import('../llm.js'); const { chat } = await import('../llm.js');
it('uses OpenCode server session API', async () => { it('runs the detected CLI with MODEL and sends the prompts through stdin', async () => {
process.env.OPENCODE_BASE_URL = 'http://opencode.local:4096'; const { stdinPath, argsPath } = await installFakeCLI('codex');
process.env.OPENCODE_PROVIDER = 'google'; process.env.MODEL = 'gpt-5-mini';
process.env.OPENCODE_MODEL = 'gemini-2.5-flash'; process.env.FAKE_AI_STDOUT = 'cli response';
const calls = []; process.env.FAKE_AI_STDIN_PATH = stdinPath;
mock.method(axios, 'post', async (url, payload, opts) => { process.env.FAKE_AI_ARGS_PATH = argsPath;
calls.push({ url, payload, headers: opts.headers });
if (url.endsWith('/session')) return { data: { id: 'ses_test' } };
return { data: { parts: [{ type: 'text', text: 'opencode response' }] } };
});
const result = await chat('sys', 'user'); const result = await chat('sys', 'user');
assert.equal(result, 'opencode response'); assert.equal(result, 'cli response');
assert.equal(calls[0].url, 'http://opencode.local:4096/session'); assert.match(await readFile(argsPath, 'utf8'), /exec --model gpt-5-mini/);
assert.deepEqual(calls[0].payload.model, { providerID: 'google', id: 'gemini-2.5-flash' }); const prompt = await readFile(stdinPath, 'utf8');
assert.equal(calls[1].url, 'http://opencode.local:4096/session/ses_test/message'); assert.match(prompt, /<system>\nsys\n<\/system>/);
assert.deepEqual(calls[1].payload.model, { providerID: 'google', modelID: 'gemini-2.5-flash' }); assert.match(prompt, /<user>\nuser\n<\/user>/);
assert.equal(calls[1].payload.system, 'sys');
assert.deepEqual(calls[1].payload.parts, [{ type: 'text', text: 'user' }]);
assert.equal(calls[1].headers['Authorization'], undefined);
}); });
it('passes an insecure https agent to OpenCode by default', async () => { it('can force opencode with AI_ASSISTANT_CLI', async () => {
process.env.OPENCODE_BASE_URL = 'https://opencode.local:4096'; const { argsPath } = await installFakeCLI('opencode');
const agents = []; process.env.AI_ASSISTANT_CLI = 'opencode';
mock.method(axios, 'post', async (url, _payload, opts) => { process.env.MODEL = 'google/gemini-2.5-pro';
agents.push(opts.httpsAgent); process.env.FAKE_AI_STDOUT = 'ok';
if (url.endsWith('/session')) return { data: { id: 'ses_test' } }; process.env.FAKE_AI_ARGS_PATH = argsPath;
return { data: { parts: [{ type: 'text', text: 'ok' }] } };
});
await chat('sys', 'user');
assert.equal(agents.length, 2);
assert.equal(agents[0].options.rejectUnauthorized, false);
assert.equal(agents[1].options.rejectUnauthorized, false);
});
it('extracts text from OpenCode message parts', async () => {
process.env.OPENCODE_BASE_URL = 'http://opencode.local:4096';
let calls = 0;
mock.method(axios, 'post', async () => {
calls += 1;
if (calls === 1) return { data: { id: 'ses_test' } };
return { data: { parts: [{ type: 'text', text: 'hello' }, { type: 'text', text: ' world' }] } };
});
const result = await chat('sys', 'user'); const result = await chat('sys', 'user');
assert.equal(result, 'hello world'); assert.equal(result, 'ok');
assert.match(await readFile(argsPath, 'utf8'), /run --model google\/gemini-2.5-pro/);
}); });
it('throws an error when OpenCode fails instead of exiting the process', async () => { it('throws an error when the CLI fails instead of exiting the process', async () => {
process.env.OPENCODE_BASE_URL = 'http://opencode.local:4096'; await installFakeCLI('codex');
process.env.OPENCODE_RETRY_ATTEMPTS = '1'; process.env.FAKE_AI_EXIT = '2';
mock.method(axios, 'post', async () => { process.env.FAKE_AI_STDERR = 'provider overloaded';
const err = new Error('Request failed with status code 500');
err.response = { status: 500, data: { error: 'provider overloaded' } };
throw err;
});
const exitMock = mock.method(process, 'exit', () => { throw new Error('exit should not be called'); }); const exitMock = mock.method(process, 'exit', () => { throw new Error('exit should not be called'); });
await assert.rejects(() => chat('sys', 'user'), /HTTP 500.*provider overloaded/); await assert.rejects(() => chat('sys', 'user'), /provider overloaded/);
assert.equal(exitMock.mock.calls.length, 0); assert.equal(exitMock.mock.calls.length, 0);
}); });
it('retries transient OpenCode failures before returning content', async () => {
process.env.OPENCODE_BASE_URL = 'http://opencode.local:4096';
process.env.OPENCODE_RETRY_ATTEMPTS = '2';
let messageAttempts = 0;
mock.method(axios, 'post', async (url) => {
if (url.endsWith('/session')) return { data: { id: 'ses_test' } };
messageAttempts += 1;
if (messageAttempts === 1) {
const err = new Error('Request failed with status code 500');
err.response = { status: 500, data: { error: 'temporary failure' } };
throw err;
}
return { data: { parts: [{ type: 'text', text: 'ok after retry' }] } };
});
const result = await chat('sys', 'user');
assert.equal(result, 'ok after retry');
assert.equal(messageAttempts, 2);
});
}); });
describe('chatJSON', async () => { describe('chatJSON', async () => {
const { chatJSON } = await import('../llm.js'); const { chatJSON } = await import('../llm.js');
it('parses plain JSON response', async () => { it('parses plain JSON response', async () => {
process.env.OPENCODE_BASE_URL = 'http://opencode.local:4096'; await installFakeCLI('codex');
mockOpenCodeResponse('[{"level":"critical"}]'); process.env.FAKE_AI_STDOUT = '[{"level":"critical"}]';
const result = await chatJSON('sys', 'user'); const result = await chatJSON('sys', 'user');
@@ -135,8 +98,8 @@ describe('chatJSON', async () => {
}); });
it('strips markdown code block before parsing', async () => { it('strips markdown code block before parsing', async () => {
process.env.OPENCODE_BASE_URL = 'http://opencode.local:4096'; await installFakeCLI('codex');
mockOpenCodeResponse('```json\n[{"level":"info"}]\n```'); process.env.FAKE_AI_STDOUT = '```json\n[{"level":"info"}]\n```';
const result = await chatJSON('sys', 'user'); const result = await chatJSON('sys', 'user');
@@ -144,8 +107,8 @@ describe('chatJSON', async () => {
}); });
it('extracts JSON array from surrounding prose', async () => { it('extracts JSON array from surrounding prose', async () => {
process.env.OPENCODE_BASE_URL = 'http://opencode.local:4096'; await installFakeCLI('codex');
mockOpenCodeResponse('**Reviewing findings**\n\n[{"level":"warning","suggestion":"x"}]\n\nDone.'); process.env.FAKE_AI_STDOUT = '**Reviewing findings**\n\n[{"level":"warning","suggestion":"x"}]\n\nDone.';
const result = await chatJSON('sys', 'user'); const result = await chatJSON('sys', 'user');
@@ -153,8 +116,8 @@ describe('chatJSON', async () => {
}); });
it('extracts JSON object from surrounding prose', async () => { it('extracts JSON object from surrounding prose', async () => {
process.env.OPENCODE_BASE_URL = 'http://opencode.local:4096'; await installFakeCLI('codex');
mockOpenCodeResponse('**Begin Combine**\n{"merged_text":"repo block\\n\\nsource block"}'); process.env.FAKE_AI_STDOUT = '**Begin Combine**\n{"merged_text":"repo block\\n\\nsource block"}';
const result = await chatJSON('sys', 'user'); const result = await chatJSON('sys', 'user');
@@ -162,8 +125,8 @@ describe('chatJSON', async () => {
}); });
it('returns [] when JSON is invalid', async () => { it('returns [] when JSON is invalid', async () => {
process.env.OPENCODE_BASE_URL = 'http://opencode.local:4096'; await installFakeCLI('codex');
mockOpenCodeResponse('not json'); process.env.FAKE_AI_STDOUT = 'not json';
const result = await chatJSON('sys', 'user'); const result = await chatJSON('sys', 'user');
app/test/preflight.test.js
+36 -60
View File
@@ -1,21 +1,38 @@
import { describe, it, afterEach, mock } from 'node:test'; import { describe, it, afterEach, mock } from 'node:test';
import assert from 'node:assert/strict'; import assert from 'node:assert/strict';
import axios from 'axios'; import axios from 'axios';
import { mkdtemp, writeFile, chmod, rm } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { checkRequiredEnv, verifyGiteaToken, verifyCommentToken, verifyLLM, runPreflight } from '../preflight.js'; import { checkRequiredEnv, verifyGiteaToken, verifyCommentToken, verifyLLM, runPreflight } from '../preflight.js';
const LLM_ENV_KEYS = [ const LLM_ENV_KEYS = [
'OPENCODE_BASE_URL', 'OPENCODE_MODEL', 'OPENCODE_PROVIDER', 'AI_ASSISTANT_CLI', 'MODEL', 'OPENCODE_MODEL', 'PATH',
]; ];
const ORIGINAL_PATH = process.env.PATH;
function clearLLMEnv() { function clearLLMEnv() {
for (const k of LLM_ENV_KEYS) delete process.env[k]; for (const k of LLM_ENV_KEYS) delete process.env[k];
} }
afterEach(() => { let tempDir;
afterEach(async () => {
mock.restoreAll(); mock.restoreAll();
clearLLMEnv(); clearLLMEnv();
process.env.PATH = ORIGINAL_PATH;
if (tempDir) await rm(tempDir, { recursive: true, force: true });
tempDir = null;
}); });
async function installFakeCLI(command = 'codex') {
tempDir = await mkdtemp(join(tmpdir(), 'preflight-cli-test-'));
const script = join(tempDir, command);
await writeFile(script, '#!/bin/sh\nexit 0\n');
await chmod(script, 0o755);
process.env.PATH = tempDir;
}
describe('checkRequiredEnv', () => { describe('checkRequiredEnv', () => {
it('reports all three missing when nothing provided', () => { it('reports all three missing when nothing provided', () => {
const result = checkRequiredEnv({ token: '', repo: '', pr: '' }); const result = checkRequiredEnv({ token: '', repo: '', pr: '' });
@@ -102,81 +119,40 @@ describe('verifyCommentToken', () => {
}); });
describe('verifyLLM', () => { describe('verifyLLM', () => {
it('fails when OpenCode is not configured', async () => { it('fails when no supported assistant CLI is detected', async () => {
clearLLMEnv(); clearLLMEnv();
process.env.AI_ASSISTANT_CLI = 'no-such-ai-cli';
process.env.PATH = '';
const result = await verifyLLM(); const result = await verifyLLM();
assert.equal(result.ok, false); assert.equal(result.ok, false);
assert.match(result.error, /OPENCODE_BASE_URL/); assert.match(result.error, /AI 助理 CLI/);
}); });
it('checks OpenCode server provider and model', async () => { it('passes when a supported assistant CLI is detected', async () => {
clearLLMEnv(); clearLLMEnv();
process.env.OPENCODE_BASE_URL = 'http://opencode.local:4096'; await installFakeCLI('codex');
process.env.OPENCODE_PROVIDER = 'google'; process.env.AI_ASSISTANT_CLI = 'codex';
process.env.OPENCODE_MODEL = 'gemini-2.5-flash'; process.env.MODEL = 'gpt-5-mini';
const urls = [];
mock.method(axios, 'get', async (url) => {
urls.push(url);
if (url.endsWith('/global/health')) return { data: { healthy: true, version: '1.17.7' } };
return { data: { providers: [{ id: 'google', models: { 'gemini-2.5-flash': { id: 'gemini-2.5-flash' } } }] } };
});
const result = await verifyLLM(); const result = await verifyLLM();
assert.equal(result.ok, true); assert.equal(result.ok, true);
assert.equal(result.provider, 'opencode'); assert.equal(result.provider, 'codex');
assert.deepEqual(urls, ['http://opencode.local:4096/global/health', 'http://opencode.local:4096/config/providers']); assert.equal(result.command, 'codex');
assert.equal(result.model, 'gpt-5-mini');
}); });
it('fails when configured provider is missing', async () => { it('fails when a requested CLI is not installed', async () => {
clearLLMEnv(); clearLLMEnv();
process.env.OPENCODE_BASE_URL = 'http://opencode.local:4096'; process.env.AI_ASSISTANT_CLI = 'missing-cli';
process.env.OPENCODE_PROVIDER = 'google'; process.env.PATH = '';
mock.method(axios, 'get', async (url) => {
if (url.endsWith('/global/health')) return { data: { healthy: true } };
return { data: { providers: [{ id: 'anthropic', models: {} }] } };
});
const result = await verifyLLM(); const result = await verifyLLM();
assert.equal(result.ok, false); assert.equal(result.ok, false);
assert.match(result.error, /未設定 provider=google/); assert.match(result.error, /AI 助理 CLI/);
});
it('fails when configured model is missing', async () => {
clearLLMEnv();
process.env.OPENCODE_BASE_URL = 'http://opencode.local:4096';
process.env.OPENCODE_PROVIDER = 'google';
process.env.OPENCODE_MODEL = 'gemini-2.5-pro';
mock.method(axios, 'get', async (url) => {
if (url.endsWith('/global/health')) return { data: { healthy: true } };
return { data: { providers: [{ id: 'google', models: { 'gemini-2.5-flash': { id: 'gemini-2.5-flash' } } }] } };
});
const result = await verifyLLM();
assert.equal(result.ok, false);
assert.match(result.error, /未列出 model=gemini-2.5-pro/);
});
it('passes an insecure https agent by default', async () => {
clearLLMEnv();
process.env.OPENCODE_BASE_URL = 'https://opencode.local:4096';
const agents = [];
mock.method(axios, 'get', async (url, opts) => {
agents.push(opts.httpsAgent);
if (url.endsWith('/global/health')) return { data: { healthy: true } };
return { data: { providers: [{ id: 'google', models: { 'gemini-2.5-flash': { id: 'gemini-2.5-flash' } } }] } };
});
const result = await verifyLLM();
assert.equal(result.ok, true);
assert.equal(agents.length, 2);
assert.equal(agents[0].options.rejectUnauthorized, false);
assert.equal(agents[1].options.rejectUnauthorized, false);
}); });
}); });
@@ -188,7 +164,7 @@ describe('runPreflight', () => {
verifyToken: async () => ({ ok: true }), verifyToken: async () => ({ ok: true }),
verifyComment: async () => ({ ok: true }), verifyComment: async () => ({ ok: true }),
verifyRemote: () => ({ ok: true }), verifyRemote: () => ({ ok: true }),
verifyLLMFn: async () => ({ ok: true, provider: 'opencode' }), verifyLLMFn: async () => ({ ok: true, provider: 'codex' }),
...overrides, ...overrides,
}; };
} }
@@ -239,7 +215,7 @@ describe('runPreflight', () => {
it('returns false when LLM verification fails', async () => { it('returns false when LLM verification fails', async () => {
const result = await runPreflight('/ws', makeDeps({ const result = await runPreflight('/ws', makeDeps({
verifyLLMFn: async () => ({ ok: false, error: 'OpenCode server 驗證失敗' }), verifyLLMFn: async () => ({ ok: false, error: 'AI 助理 CLI 驗證失敗' }),
})); }));
assert.equal(result, false); assert.equal(result, false);
}); });