Saas-voc-server/scripts/benchmark-jd-voc-image-review.ts

import { FmodeGeminiImageReviewProvider } from '../src/modules/listing-ai/image-review/gemini-image-review.provider.js';

const baseUrl = (process.env.LAB_URL ?? 'http://127.0.0.1:4410').replace(/\/+$/, '');
const provider = new FmodeGeminiImageReviewProvider({
  baseUrl: process.env.FMODE_LLM_BASE_URL ?? '', token: process.env.FMODE_LLM_API_KEY ?? '', timeoutMs: 45_000,
});
const cases = [
  ['10020722928820', ['去皮', '削皮', '土豆']],
  ['10020518260748', ['开水器', '开水机', '烧水']],
  ['10020422869649', ['消毒柜', '茶具']],
  ['10020670145859', ['烤箱', '烤炉']],
  ['10020770480513', ['置物架', '货架', '储物架']],
] as const;
const rows: Array<{ productId: string; matched: boolean; facts: number; inferences: number; latencyMs: number; error: string | null }> = [];
for (const [productId, expected] of cases) {
  try {
    const response = await fetch(`${baseUrl}/lab/products/${productId}`, { signal: AbortSignal.timeout(30_000) });
    if (!response.ok) throw new Error(`source_http_${response.status}`);
    const payload = await response.json() as { source?: { images?: Array<{ url?: string }>; title?: string } };
    const source = payload.source;
    const urls = (source?.images ?? []).map((image) => image.url ?? '').filter(Boolean).slice(0, 3);
    if (!source || !urls.length) throw new Error('image_asset_missing');
    const result = await provider.analyze(urls, source as never);
    const haystack = result.visibleFacts.map((fact) => `${fact.field} ${fact.value}`).join(' ');
    rows.push({ productId, matched: expected.some((term) => haystack.includes(term)), facts: result.visibleFacts.length, inferences: result.visualInferences.length, latencyMs: result.latencyMs, error: null });
  } catch (error) { rows.push({ productId, matched: false, facts: 0, inferences: 0, latencyMs: 0, error: error instanceof Error ? error.message.slice(0, 80) : 'image_review_failed' }); }
}
const matched = rows.filter((row) => row.matched).length;
const report = { generatedAt: new Date().toISOString(), model: 'gemini-3.1-flash-image-preview', requested: rows.length, completed: rows.filter((row) => row.error === null).length, matched, accuracy: matched / rows.length, averageMs: rows.filter((row) => row.error === null).reduce((sum, row) => sum + row.latencyMs, 0) / Math.max(1, rows.filter((row) => row.error === null).length), rows: rows.map(({ productId, matched: rowMatched, facts, inferences, latencyMs, error }) => ({ productId, matched: rowMatched, facts, inferences, latencyMs, error })) };
console.log(JSON.stringify(report, null, 2));
if (rows.some((row) => row.error !== null)) process.exitCode = 2;