Saas-voc-server/scripts/verify-listing-rollout.ts

import 'dotenv/config';
import { z } from 'zod';
import { ParseRestClient } from '../src/db/parse-rest.client.js';
import { VOC_PARSE_CLASSES } from '../src/db/parse-rest.schema.js';
import type { JdVocScoreResult, ListingDimension, ListingScoreJob, ListingScoreJobItem, ListingScoreResult, ListingSourceSnapshot } from '../src/modules/listing-ai/domain.js';
import { LISTING_DIMENSION_MAX, LISTING_RUBRIC_VERSION } from '../src/modules/listing-ai/scoring/rule-engine.js';
import { LISTING_AI_RUBRIC_VERSION } from '../src/modules/listing-ai/scoring/ai-rubric.js';
import { isListingV7Score } from '../src/modules/listing-ai/scoring/score-status.js';

const CODEX_GROUNDED_MODEL = 'codex-grounded-evaluator-v1';

interface Stored<T> { workspaceId: string; productId?: string; jobId?: string; idempotencyKey?: string; slot?: 'rule_precheck' | 'formal_ai' | 'jd_voc_rules' | 'jd_voc_hybrid_ai'; model?: string; payload: T }

async function main() {
  const env = z.object({ PARSE_SERVER_URL: z.url(), PARSE_APP_ID: z.string().min(1), PARSE_MASTER_KEY: z.string().min(1), SAAS_DEFAULT_WORKSPACE_ID: z.string().default('demashi') }).parse(process.env);
  const client = new ParseRestClient({ serverUrl: env.PARSE_SERVER_URL, appId: env.PARSE_APP_ID, masterKey: env.PARSE_MASTER_KEY });
  const workspaceId = env.SAAS_DEFAULT_WORKSPACE_ID;
  const [sourceRows, allResultRows, itemRows, jobRows] = await Promise.all([
    client.findAll<Stored<ListingSourceSnapshot>>(VOC_PARSE_CLASSES.listingSourceSnapshot, { workspaceId, platform: 'jd', isCurrent: true, catalogIncluded: true, catalogCohort: 'listing-jd-v3-formal-625' }),
    client.findAll<Stored<ListingScoreResult | JdVocScoreResult>>(VOC_PARSE_CLASSES.listingCurrentScore, { workspaceId }),
    client.findAll<Stored<ListingScoreJobItem>>(VOC_PARSE_CLASSES.listingScoreItem, { workspaceId }),
    client.findAll<Stored<ListingScoreJob> & { idempotencyKey: string }>(VOC_PARSE_CLASSES.listingScoreJob, { workspaceId }),
  ]);
  const resultRows = allResultRows.filter((row) => !String(row.payload.scoreKind ?? '').startsWith('jd_voc_')) as Array<Stored<ListingScoreResult> & { objectId: string; createdAt: string; updatedAt: string }>;
  const jdVocRows = allResultRows.filter((row) => String(row.payload.scoreKind ?? '').startsWith('jd_voc_')) as Array<Stored<JdVocScoreResult> & { objectId: string; createdAt: string; updatedAt: string }>;
  const schemas = await client.schemas();
  const legacyScoreSchemaPresent = schemas.some((schema) => schema.className === 'VocListingScoreResult');
  const slotKeys = allResultRows.map((row) => `${row.workspaceId}|${row.productId ?? row.payload.productId}|${row.slot ?? (row.payload.scoreKind === 'hybrid_ai' ? 'formal_ai' : row.payload.scoreKind === 'jd_voc_hybrid_ai' ? 'jd_voc_hybrid_ai' : row.payload.scoreKind === 'jd_voc_rules' ? 'jd_voc_rules' : 'rule_precheck')}`);
  const duplicateCurrentSlots = slotKeys.length - new Set(slotKeys).size;
  const latestSources = new Map<string, ListingSourceSnapshot>();
  for (const row of sourceRows) { const value = row.payload; const current = latestSources.get(value.productId); if (!current || value.syncedAt > current.syncedAt) latestSources.set(value.productId, value); }
  const jdVocInvalid = jdVocRows.filter((row) => row.payload.rubricVersion !== 'jd-voc-v0.5' || row.payload.dimensions.length !== 6 || row.payload.sourceHash.length !== 64 || !row.payload.executionKey || !row.payload.inputFingerprint).length;
  const jdVocStale = jdVocRows.filter((row) => latestSources.get(row.payload.productId)?.sourceHash !== row.payload.sourceHash).length;
  const jdVocAiEvidenceInvalid = jdVocRows.filter((row) => row.payload.scoreKind === 'jd_voc_hybrid_ai').filter((row) => row.payload.aiReview?.assessments.some((assessment) => (assessment.score ?? 0) > 0 && !assessment.evidenceIds.length)).length;
  const jdVocHybridRows = jdVocRows.filter((row) => row.payload.scoreKind === 'jd_voc_hybrid_ai');
  const jdVocAiLatencies = jdVocHybridRows.map((row) => row.payload.aiReview?.latencyMs).filter((value): value is number => typeof value === 'number');
  const jdVocAiUsage = jdVocHybridRows.reduce((sum,row)=>({promptTokens:sum.promptTokens+(row.payload.aiReview?.usage?.promptTokens??0),completionTokens:sum.completionTokens+(row.payload.aiReview?.usage?.completionTokens??0),totalTokens:sum.totalTokens+(row.payload.aiReview?.usage?.totalTokens??0)}),{promptTokens:0,completionTokens:0,totalTokens:0});
  const jdVocAiEstimatedPublicCostUsd = Math.round((jdVocAiUsage.promptTokens * 0.15 / 1_000_000 + jdVocAiUsage.completionTokens * 0.60 / 1_000_000) * 1_000_000) / 1_000_000;
  const jdVocImageLeak = jdVocRows.filter((row) => row.payload.imageReview && /https?:\/\//i.test(JSON.stringify(row.payload.imageReview))).length;
  const ruleResults = new Map<string, ListingScoreResult>();
  const aiResults = new Map<string, ListingScoreResult>();
  for (const row of resultRows) {
    const value = row.payload;
    if (!isListingV7Score(value)) continue;
    const target = row.slot === 'formal_ai' || value.scoreKind === 'hybrid_ai' ? aiResults : ruleResults;
    const current = target.get(value.productId);
    if (!current || value.createdAt > current.createdAt) target.set(value.productId, value);
  }
  const results = [...ruleResults.values()];
  const orphanResults = results.filter((result) => !latestSources.has(result.productId));
  const missingResults = [...latestSources.values()].filter((source) => !ruleResults.has(source.productId));
  const invalidDimensions = results.filter((result) => result.dimensions.length !== 5);
  const staleResults = results.filter((result) => latestSources.get(result.productId)?.sourceHash !== result.sourceHash);
  const invalidWeights = results.filter((result) => result.dimensions.some((item) => item.maxScore !== LISTING_DIMENSION_MAX[item.dimension]));
  const knownScores = results.map((result) => result.knownOverallScore).filter((value): value is number => typeof value === 'number');
  const staleAiResults = [...aiResults.values()].filter((result) => latestSources.get(result.productId)?.sourceHash !== result.sourceHash);
  const invalidAiResults = [...aiResults.values()].filter((result) => result.rubricVersion !== LISTING_AI_RUBRIC_VERSION || result.scoreKind !== 'hybrid_ai' || result.aiStatus !== 'completed' || !result.model || result.model === 'listing-showcase-ai');
  const simulatedScores = [...aiResults.values()].filter((result) => result.model === 'listing-v7-demo-simulation');
  const codexScores = [...aiResults.values()].filter((result) => result.model === CODEX_GROUNDED_MODEL);
  const invalidFormalScores = codexScores.filter((result) => result.overallScore === null || result.knownOverallMaxScore !== 100 || result.dimensions.some((dimension) => dimension.score === null));
  const formalNumericScores = codexScores.map((result) => result.overallScore).filter((value): value is number => value !== null);
  const simulatedAverage = simulatedScores.length ? Math.round(simulatedScores.reduce((sum, result) => sum + (result.overallScore ?? 0), 0) / simulatedScores.length * 10) / 10 : null;
  const simulatedMinimum = simulatedScores.length ? Math.min(...simulatedScores.map((result) => result.overallScore ?? Number.POSITIVE_INFINITY)) : null;
  const simulatedMaximum = simulatedScores.length ? Math.max(...simulatedScores.map((result) => result.overallScore ?? Number.NEGATIVE_INFINITY)) : null;
  const simulatedMinimumCount = simulatedScores.filter((result) => result.overallScore === 73).length;
  const simulatedMaximumCount = simulatedScores.filter((result) => result.overallScore === 100).length;
  const dimensions = Object.keys(LISTING_DIMENSION_MAX) as ListingDimension[];
  const dimensionPartial = Object.fromEntries(dimensions.map((dimension) => [dimension, results.filter((result) => result.dimensions.find((item) => item.dimension === dimension)?.status === 'partial').length]));
  const knownMaxDistribution = results.reduce<Record<string, number>>((output, result) => { const key = String(result.knownOverallMaxScore ?? 'legacy'); output[key] = (output[key] ?? 0) + 1; return output; }, {});
  const compliance = results.reduce<Record<string, number>>((output, result) => { const key = result.compliance?.status ?? 'legacy'; output[key] = (output[key] ?? 0) + 1; return output; }, {});
  const normalizers = [...latestSources.values()].reduce<Record<string, number>>((output, source) => { const key = source.normalizerVersion ?? 'legacy'; output[key] = (output[key] ?? 0) + 1; return output; }, {});
  const latestJob = jobRows.map((row) => row.payload).filter((job) => job.rubricVersion === LISTING_RUBRIC_VERSION && job.total === 625).sort((a, b) => b.requestedAt.localeCompare(a.requestedAt))[0] ?? null;
  const latestJobItems = latestJob ? itemRows.map((row) => row.payload).filter((item) => item.jobId === latestJob.id) : [];
  const ruleOutcomes = (ruleId: string) => results.reduce<Record<string, number>>((output, result) => {
    const evidence = result.dimensions.flatMap((dimension) => dimension.evidence).find((item) => item.ruleId === ruleId);
    const key = evidence?.outcome ?? 'missing';
    output[key] = (output[key] ?? 0) + 1;
    return output;
  }, {});
  const partialWithErrorCode = latestJobItems.filter((item) => item.status === 'partial' && item.errorCode !== null).length;
  const failedWithoutErrorCode = latestJobItems.filter((item) => item.status === 'failed' && !item.errorCode).length;
  const titleLengthOutcomes = ruleOutcomes('title.length');
  const titleCategoryOutcomes = ruleOutcomes('title.category_in_first_15');
  const hardFailRate = (outcomes: Record<string, number>) => {
    const decided = (outcomes.pass ?? 0) + (outcomes.fail ?? 0);
    return decided ? Math.round((outcomes.fail ?? 0) / decided * 10_000) / 10_000 : 0;
  };
  const systemicHardFailGate = {
    maximumAllowedRate: 0.5,
    titleLength: hardFailRate(titleLengthOutcomes),
    titleCategoryInFirst15: hardFailRate(titleCategoryOutcomes),
  };
  const report = {
    workspaceId, rubricVersion: `${LISTING_RUBRIC_VERSION} / ${LISTING_AI_RUBRIC_VERSION}`, weights: LISTING_DIMENSION_MAX, sources: latestSources.size, normalizers,
    currentScoreRows: allResultRows.length, legacyCurrentScoreRows: resultRows.length, jdVocCurrentScoreRows: jdVocRows.length,
    jdVocRuleScores: jdVocRows.filter((row) => row.payload.scoreKind === 'jd_voc_rules').length,
    jdVocHybridScores: jdVocRows.filter((row) => row.payload.scoreKind === 'jd_voc_hybrid_ai').length,
    jdVocImageShadowScores: jdVocRows.filter((row) => row.payload.imageReview?.status === 'shadow_completed').length,
    jdVocInvalid, jdVocStale, jdVocAiEvidenceInvalid, jdVocImageLeak,
    jdVocAiAverageLatencyMs: jdVocAiLatencies.length ? Math.round(jdVocAiLatencies.reduce((sum,value)=>sum+value,0)/jdVocAiLatencies.length) : null,
    jdVocAiUsage,
    jdVocAiEstimatedPublicCostUsd,
    jdVocAiEstimatedPublicCostCny: Math.round(jdVocAiEstimatedPublicCostUsd * Number(process.env.FMODE_USD_TO_CNY_RATE ?? 6.8) * 10_000) / 10_000,
    jdVocAiPricingBasis: 'OpenAI public gpt-4o-mini token rates; excludes Fmode gateway markup',
    duplicateCurrentSlots, legacyScoreSchemaPresent,
    ruleScores: ruleResults.size, rulePartialResults: results.filter((result) => result.overallScore === null).length,
    formalScores: aiResults.size, codexScores: codexScores.length, codexAverage: formalNumericScores.length ? Math.round(formalNumericScores.reduce((sum, value) => sum + value, 0) / formalNumericScores.length * 10) / 10 : null, simulatedScores: simulatedScores.length, simulatedAverage, simulatedMinimum, simulatedMaximum, simulatedMinimumCount, simulatedMaximumCount, invalidAiResults: invalidAiResults.length, invalidFormalScores: invalidFormalScores.length, staleAiResults: staleAiResults.length,
    averageKnownScore: knownScores.length ? Math.round(knownScores.reduce((sum, value) => sum + value, 0) / knownScores.length * 10) / 10 : null,
    knownMaxDistribution, dimensionPartial, compliance,
    ruleOutcomes: {
      titleLength: titleLengthOutcomes,
      titleCategoryInFirst15: titleCategoryOutcomes,
      imagePrimary: ruleOutcomes('images.primary'),
    },
    systemicHardFailGate,
    dataReadiness: {
      categoryRuleMissing: [...latestSources.values()].filter((source) => !source.categoryContext?.ruleVersion).length,
      vocEvidenceMissing: [...latestSources.values()].filter((source) => !source.vocEvidence?.length).length,
      descriptionStructureMissing: [...latestSources.values()].filter((source) => !source.descriptionStructure?.observed).length,
    },
    missingResults: missingResults.length, orphanResults: orphanResults.length, staleResults: staleResults.length, invalidDimensions: invalidDimensions.length, invalidWeights: invalidWeights.length,
    latestJob: latestJob ? { id: latestJob.id, status: latestJob.status, total: latestJob.total, processed: latestJob.processed, succeeded: latestJob.succeeded, partial: latestJob.partial, blocked: latestJob.blocked, failed: latestJob.failed } : null,
    latestJobItems: latestJobItems.length,
    jobItemStatusSemantics: { partialWithErrorCode, failedWithoutErrorCode },
  };
  console.log(JSON.stringify(report, null, 2));
  if (latestSources.size !== 625 || normalizers['jd-listing-v5'] !== 625 || resultRows.length !== 1250 || jdVocRows.filter((row) => row.payload.scoreKind === 'jd_voc_rules').length < 100 || jdVocRows.filter((row) => row.payload.scoreKind === 'jd_voc_hybrid_ai').length < 5 || jdVocRows.filter((row) => row.payload.imageReview?.status === 'shadow_completed').length < 5 || jdVocInvalid || jdVocStale || jdVocAiEvidenceInvalid || jdVocImageLeak || duplicateCurrentSlots || legacyScoreSchemaPresent || ruleResults.size !== 625 || aiResults.size !== 625 || codexScores.length !== 625 || simulatedScores.length !== 0 || invalidAiResults.length || invalidFormalScores.length || staleAiResults.length || missingResults.length || orphanResults.length || staleResults.length || invalidDimensions.length || invalidWeights.length || (latestJob !== null && latestJob.processed !== latestJob.total) || partialWithErrorCode || failedWithoutErrorCode || systemicHardFailGate.titleLength > systemicHardFailGate.maximumAllowedRate || systemicHardFailGate.titleCategoryInFirst15 > systemicHardFailGate.maximumAllowedRate) process.exitCode = 2;
}

main().catch((error) => { console.error(`[verify-listing-rollout] ${error instanceof Error ? error.message : error}`); process.exitCode = 1; });