import 'dotenv/config'; import { z } from 'zod'; import { ParseRestClient } from '../src/db/parse-rest.client.js'; import { VOC_PARSE_CLASSES } from '../src/db/parse-rest.schema.js'; import type { JdVocScoreResult, ListingDimension, ListingScoreJob, ListingScoreJobItem, ListingScoreResult, ListingSourceSnapshot } from '../src/modules/listing-ai/domain.js'; import { LISTING_DIMENSION_MAX, LISTING_RUBRIC_VERSION } from '../src/modules/listing-ai/scoring/rule-engine.js'; import { LISTING_AI_RUBRIC_VERSION } from '../src/modules/listing-ai/scoring/ai-rubric.js'; import { isListingV7Score } from '../src/modules/listing-ai/scoring/score-status.js'; const CODEX_GROUNDED_MODEL = 'codex-grounded-evaluator-v1'; interface Stored { workspaceId: string; productId?: string; jobId?: string; idempotencyKey?: string; slot?: 'rule_precheck' | 'formal_ai' | 'jd_voc_rules' | 'jd_voc_hybrid_ai'; model?: string; payload: T } async function main() { const env = z.object({ PARSE_SERVER_URL: z.url(), PARSE_APP_ID: z.string().min(1), PARSE_MASTER_KEY: z.string().min(1), SAAS_DEFAULT_WORKSPACE_ID: z.string().default('demashi') }).parse(process.env); const client = new ParseRestClient({ serverUrl: env.PARSE_SERVER_URL, appId: env.PARSE_APP_ID, masterKey: env.PARSE_MASTER_KEY }); const workspaceId = env.SAAS_DEFAULT_WORKSPACE_ID; const [sourceRows, allResultRows, itemRows, jobRows] = await Promise.all([ client.findAll>(VOC_PARSE_CLASSES.listingSourceSnapshot, { workspaceId, platform: 'jd', isCurrent: true, catalogIncluded: true, catalogCohort: 'listing-jd-v3-formal-625' }), client.findAll>(VOC_PARSE_CLASSES.listingCurrentScore, { workspaceId }), client.findAll>(VOC_PARSE_CLASSES.listingScoreItem, { workspaceId }), client.findAll & { idempotencyKey: string }>(VOC_PARSE_CLASSES.listingScoreJob, { workspaceId }), ]); const resultRows = allResultRows.filter((row) => !String(row.payload.scoreKind ?? '').startsWith('jd_voc_')) as Array & { objectId: string; createdAt: string; updatedAt: string }>; const jdVocRows = allResultRows.filter((row) => String(row.payload.scoreKind ?? '').startsWith('jd_voc_')) as Array & { objectId: string; createdAt: string; updatedAt: string }>; const schemas = await client.schemas(); const legacyScoreSchemaPresent = schemas.some((schema) => schema.className === 'VocListingScoreResult'); const slotKeys = allResultRows.map((row) => `${row.workspaceId}|${row.productId ?? row.payload.productId}|${row.slot ?? (row.payload.scoreKind === 'hybrid_ai' ? 'formal_ai' : row.payload.scoreKind === 'jd_voc_hybrid_ai' ? 'jd_voc_hybrid_ai' : row.payload.scoreKind === 'jd_voc_rules' ? 'jd_voc_rules' : 'rule_precheck')}`); const duplicateCurrentSlots = slotKeys.length - new Set(slotKeys).size; const latestSources = new Map(); for (const row of sourceRows) { const value = row.payload; const current = latestSources.get(value.productId); if (!current || value.syncedAt > current.syncedAt) latestSources.set(value.productId, value); } const jdVocInvalid = jdVocRows.filter((row) => row.payload.rubricVersion !== 'jd-voc-v0.5' || row.payload.dimensions.length !== 6 || row.payload.sourceHash.length !== 64 || !row.payload.executionKey || !row.payload.inputFingerprint).length; const jdVocStale = jdVocRows.filter((row) => latestSources.get(row.payload.productId)?.sourceHash !== row.payload.sourceHash).length; const jdVocAiEvidenceInvalid = jdVocRows.filter((row) => row.payload.scoreKind === 'jd_voc_hybrid_ai').filter((row) => row.payload.aiReview?.assessments.some((assessment) => (assessment.score ?? 0) > 0 && !assessment.evidenceIds.length)).length; const jdVocHybridRows = jdVocRows.filter((row) => row.payload.scoreKind === 'jd_voc_hybrid_ai'); const jdVocAiLatencies = jdVocHybridRows.map((row) => row.payload.aiReview?.latencyMs).filter((value): value is number => typeof value === 'number'); const jdVocAiUsage = jdVocHybridRows.reduce((sum,row)=>({promptTokens:sum.promptTokens+(row.payload.aiReview?.usage?.promptTokens??0),completionTokens:sum.completionTokens+(row.payload.aiReview?.usage?.completionTokens??0),totalTokens:sum.totalTokens+(row.payload.aiReview?.usage?.totalTokens??0)}),{promptTokens:0,completionTokens:0,totalTokens:0}); const jdVocAiEstimatedPublicCostUsd = Math.round((jdVocAiUsage.promptTokens * 0.15 / 1_000_000 + jdVocAiUsage.completionTokens * 0.60 / 1_000_000) * 1_000_000) / 1_000_000; const jdVocImageLeak = jdVocRows.filter((row) => row.payload.imageReview && /https?:\/\//i.test(JSON.stringify(row.payload.imageReview))).length; const ruleResults = new Map(); const aiResults = new Map(); for (const row of resultRows) { const value = row.payload; if (!isListingV7Score(value)) continue; const target = row.slot === 'formal_ai' || value.scoreKind === 'hybrid_ai' ? aiResults : ruleResults; const current = target.get(value.productId); if (!current || value.createdAt > current.createdAt) target.set(value.productId, value); } const results = [...ruleResults.values()]; const orphanResults = results.filter((result) => !latestSources.has(result.productId)); const missingResults = [...latestSources.values()].filter((source) => !ruleResults.has(source.productId)); const invalidDimensions = results.filter((result) => result.dimensions.length !== 5); const staleResults = results.filter((result) => latestSources.get(result.productId)?.sourceHash !== result.sourceHash); const invalidWeights = results.filter((result) => result.dimensions.some((item) => item.maxScore !== LISTING_DIMENSION_MAX[item.dimension])); const knownScores = results.map((result) => result.knownOverallScore).filter((value): value is number => typeof value === 'number'); const staleAiResults = [...aiResults.values()].filter((result) => latestSources.get(result.productId)?.sourceHash !== result.sourceHash); const invalidAiResults = [...aiResults.values()].filter((result) => result.rubricVersion !== LISTING_AI_RUBRIC_VERSION || result.scoreKind !== 'hybrid_ai' || result.aiStatus !== 'completed' || !result.model || result.model === 'listing-showcase-ai'); const simulatedScores = [...aiResults.values()].filter((result) => result.model === 'listing-v7-demo-simulation'); const codexScores = [...aiResults.values()].filter((result) => result.model === CODEX_GROUNDED_MODEL); const invalidFormalScores = codexScores.filter((result) => result.overallScore === null || result.knownOverallMaxScore !== 100 || result.dimensions.some((dimension) => dimension.score === null)); const formalNumericScores = codexScores.map((result) => result.overallScore).filter((value): value is number => value !== null); const simulatedAverage = simulatedScores.length ? Math.round(simulatedScores.reduce((sum, result) => sum + (result.overallScore ?? 0), 0) / simulatedScores.length * 10) / 10 : null; const simulatedMinimum = simulatedScores.length ? Math.min(...simulatedScores.map((result) => result.overallScore ?? Number.POSITIVE_INFINITY)) : null; const simulatedMaximum = simulatedScores.length ? Math.max(...simulatedScores.map((result) => result.overallScore ?? Number.NEGATIVE_INFINITY)) : null; const simulatedMinimumCount = simulatedScores.filter((result) => result.overallScore === 73).length; const simulatedMaximumCount = simulatedScores.filter((result) => result.overallScore === 100).length; const dimensions = Object.keys(LISTING_DIMENSION_MAX) as ListingDimension[]; const dimensionPartial = Object.fromEntries(dimensions.map((dimension) => [dimension, results.filter((result) => result.dimensions.find((item) => item.dimension === dimension)?.status === 'partial').length])); const knownMaxDistribution = results.reduce>((output, result) => { const key = String(result.knownOverallMaxScore ?? 'legacy'); output[key] = (output[key] ?? 0) + 1; return output; }, {}); const compliance = results.reduce>((output, result) => { const key = result.compliance?.status ?? 'legacy'; output[key] = (output[key] ?? 0) + 1; return output; }, {}); const normalizers = [...latestSources.values()].reduce>((output, source) => { const key = source.normalizerVersion ?? 'legacy'; output[key] = (output[key] ?? 0) + 1; return output; }, {}); const latestJob = jobRows.map((row) => row.payload).filter((job) => job.rubricVersion === LISTING_RUBRIC_VERSION && job.total === 625).sort((a, b) => b.requestedAt.localeCompare(a.requestedAt))[0] ?? null; const latestJobItems = latestJob ? itemRows.map((row) => row.payload).filter((item) => item.jobId === latestJob.id) : []; const ruleOutcomes = (ruleId: string) => results.reduce>((output, result) => { const evidence = result.dimensions.flatMap((dimension) => dimension.evidence).find((item) => item.ruleId === ruleId); const key = evidence?.outcome ?? 'missing'; output[key] = (output[key] ?? 0) + 1; return output; }, {}); const partialWithErrorCode = latestJobItems.filter((item) => item.status === 'partial' && item.errorCode !== null).length; const failedWithoutErrorCode = latestJobItems.filter((item) => item.status === 'failed' && !item.errorCode).length; const titleLengthOutcomes = ruleOutcomes('title.length'); const titleCategoryOutcomes = ruleOutcomes('title.category_in_first_15'); const hardFailRate = (outcomes: Record) => { const decided = (outcomes.pass ?? 0) + (outcomes.fail ?? 0); return decided ? Math.round((outcomes.fail ?? 0) / decided * 10_000) / 10_000 : 0; }; const systemicHardFailGate = { maximumAllowedRate: 0.5, titleLength: hardFailRate(titleLengthOutcomes), titleCategoryInFirst15: hardFailRate(titleCategoryOutcomes), }; const report = { workspaceId, rubricVersion: `${LISTING_RUBRIC_VERSION} / ${LISTING_AI_RUBRIC_VERSION}`, weights: LISTING_DIMENSION_MAX, sources: latestSources.size, normalizers, currentScoreRows: allResultRows.length, legacyCurrentScoreRows: resultRows.length, jdVocCurrentScoreRows: jdVocRows.length, jdVocRuleScores: jdVocRows.filter((row) => row.payload.scoreKind === 'jd_voc_rules').length, jdVocHybridScores: jdVocRows.filter((row) => row.payload.scoreKind === 'jd_voc_hybrid_ai').length, jdVocImageShadowScores: jdVocRows.filter((row) => row.payload.imageReview?.status === 'shadow_completed').length, jdVocInvalid, jdVocStale, jdVocAiEvidenceInvalid, jdVocImageLeak, jdVocAiAverageLatencyMs: jdVocAiLatencies.length ? Math.round(jdVocAiLatencies.reduce((sum,value)=>sum+value,0)/jdVocAiLatencies.length) : null, jdVocAiUsage, jdVocAiEstimatedPublicCostUsd, jdVocAiEstimatedPublicCostCny: Math.round(jdVocAiEstimatedPublicCostUsd * Number(process.env.FMODE_USD_TO_CNY_RATE ?? 6.8) * 10_000) / 10_000, jdVocAiPricingBasis: 'OpenAI public gpt-4o-mini token rates; excludes Fmode gateway markup', duplicateCurrentSlots, legacyScoreSchemaPresent, ruleScores: ruleResults.size, rulePartialResults: results.filter((result) => result.overallScore === null).length, formalScores: aiResults.size, codexScores: codexScores.length, codexAverage: formalNumericScores.length ? Math.round(formalNumericScores.reduce((sum, value) => sum + value, 0) / formalNumericScores.length * 10) / 10 : null, simulatedScores: simulatedScores.length, simulatedAverage, simulatedMinimum, simulatedMaximum, simulatedMinimumCount, simulatedMaximumCount, invalidAiResults: invalidAiResults.length, invalidFormalScores: invalidFormalScores.length, staleAiResults: staleAiResults.length, averageKnownScore: knownScores.length ? Math.round(knownScores.reduce((sum, value) => sum + value, 0) / knownScores.length * 10) / 10 : null, knownMaxDistribution, dimensionPartial, compliance, ruleOutcomes: { titleLength: titleLengthOutcomes, titleCategoryInFirst15: titleCategoryOutcomes, imagePrimary: ruleOutcomes('images.primary'), }, systemicHardFailGate, dataReadiness: { categoryRuleMissing: [...latestSources.values()].filter((source) => !source.categoryContext?.ruleVersion).length, vocEvidenceMissing: [...latestSources.values()].filter((source) => !source.vocEvidence?.length).length, descriptionStructureMissing: [...latestSources.values()].filter((source) => !source.descriptionStructure?.observed).length, }, missingResults: missingResults.length, orphanResults: orphanResults.length, staleResults: staleResults.length, invalidDimensions: invalidDimensions.length, invalidWeights: invalidWeights.length, latestJob: latestJob ? { id: latestJob.id, status: latestJob.status, total: latestJob.total, processed: latestJob.processed, succeeded: latestJob.succeeded, partial: latestJob.partial, blocked: latestJob.blocked, failed: latestJob.failed } : null, latestJobItems: latestJobItems.length, jobItemStatusSemantics: { partialWithErrorCode, failedWithoutErrorCode }, }; console.log(JSON.stringify(report, null, 2)); if (latestSources.size !== 625 || normalizers['jd-listing-v5'] !== 625 || resultRows.length !== 1250 || jdVocRows.filter((row) => row.payload.scoreKind === 'jd_voc_rules').length < 100 || jdVocRows.filter((row) => row.payload.scoreKind === 'jd_voc_hybrid_ai').length < 5 || jdVocRows.filter((row) => row.payload.imageReview?.status === 'shadow_completed').length < 5 || jdVocInvalid || jdVocStale || jdVocAiEvidenceInvalid || jdVocImageLeak || duplicateCurrentSlots || legacyScoreSchemaPresent || ruleResults.size !== 625 || aiResults.size !== 625 || codexScores.length !== 625 || simulatedScores.length !== 0 || invalidAiResults.length || invalidFormalScores.length || staleAiResults.length || missingResults.length || orphanResults.length || staleResults.length || invalidDimensions.length || invalidWeights.length || (latestJob !== null && latestJob.processed !== latestJob.total) || partialWithErrorCode || failedWithoutErrorCode || systemicHardFailGate.titleLength > systemicHardFailGate.maximumAllowedRate || systemicHardFailGate.titleCategoryInFirst15 > systemicHardFailGate.maximumAllowedRate) process.exitCode = 2; } main().catch((error) => { console.error(`[verify-listing-rollout] ${error instanceof Error ? error.message : error}`); process.exitCode = 1; });