import 'dotenv/config'; import { z } from 'zod'; import { ParseRestClient } from '../src/db/parse-rest.client.js'; import { VOC_PARSE_CLASSES } from '../src/db/parse-rest.schema.js'; import type { ListingDimension, ListingScoreJob, ListingScoreJobItem, ListingScoreResult, ListingSourceSnapshot } from '../src/modules/listing-ai/domain.js'; import { LISTING_DIMENSION_MAX, LISTING_RUBRIC_VERSION } from '../src/modules/listing-ai/scoring/rule-engine.js'; import { LISTING_AI_RUBRIC_VERSION } from '../src/modules/listing-ai/scoring/ai-rubric.js'; import { isListingV7Score } from '../src/modules/listing-ai/scoring/score-status.js'; interface Stored { workspaceId: string; productId?: string; jobId?: string; idempotencyKey?: string; slot?: 'rule_precheck' | 'formal_ai'; model?: string; payload: T } async function main() { const env = z.object({ PARSE_SERVER_URL: z.url(), PARSE_APP_ID: z.string().min(1), PARSE_MASTER_KEY: z.string().min(1), SAAS_DEFAULT_WORKSPACE_ID: z.string().default('demashi') }).parse(process.env); const client = new ParseRestClient({ serverUrl: env.PARSE_SERVER_URL, appId: env.PARSE_APP_ID, masterKey: env.PARSE_MASTER_KEY }); const workspaceId = env.SAAS_DEFAULT_WORKSPACE_ID; const [sourceRows, resultRows, itemRows, jobRows] = await Promise.all([ client.findAll>(VOC_PARSE_CLASSES.listingSourceSnapshot, { workspaceId, platform: 'jd', isCurrent: true, catalogIncluded: true, catalogCohort: 'listing-jd-v3-formal-625' }), client.findAll>(VOC_PARSE_CLASSES.listingCurrentScore, { workspaceId }), client.findAll>(VOC_PARSE_CLASSES.listingScoreItem, { workspaceId }), client.findAll & { idempotencyKey: string }>(VOC_PARSE_CLASSES.listingScoreJob, { workspaceId }), ]); const schemas = await client.schemas(); const legacyScoreSchemaPresent = schemas.some((schema) => schema.className === 'VocListingScoreResult'); const slotKeys = resultRows.map((row) => `${row.workspaceId}|${row.productId ?? row.payload.productId}|${row.slot ?? (row.payload.scoreKind === 'hybrid_ai' ? 'formal_ai' : 'rule_precheck')}`); const duplicateCurrentSlots = slotKeys.length - new Set(slotKeys).size; const latestSources = new Map(); for (const row of sourceRows) { const value = row.payload; const current = latestSources.get(value.productId); if (!current || value.syncedAt > current.syncedAt) latestSources.set(value.productId, value); } const ruleResults = new Map(); const aiResults = new Map(); for (const row of resultRows) { const value = row.payload; if (!isListingV7Score(value)) continue; const target = row.slot === 'formal_ai' || value.scoreKind === 'hybrid_ai' ? aiResults : ruleResults; const current = target.get(value.productId); if (!current || value.createdAt > current.createdAt) target.set(value.productId, value); } const results = [...ruleResults.values()]; const orphanResults = results.filter((result) => !latestSources.has(result.productId)); const missingResults = [...latestSources.values()].filter((source) => !ruleResults.has(source.productId)); const invalidDimensions = results.filter((result) => result.dimensions.length !== 5); const staleResults = results.filter((result) => latestSources.get(result.productId)?.sourceHash !== result.sourceHash); const invalidWeights = results.filter((result) => result.dimensions.some((item) => item.maxScore !== LISTING_DIMENSION_MAX[item.dimension])); const knownScores = results.map((result) => result.knownOverallScore).filter((value): value is number => typeof value === 'number'); const staleAiResults = [...aiResults.values()].filter((result) => latestSources.get(result.productId)?.sourceHash !== result.sourceHash); const invalidAiResults = [...aiResults.values()].filter((result) => result.rubricVersion !== LISTING_AI_RUBRIC_VERSION || result.scoreKind !== 'hybrid_ai' || result.aiStatus !== 'completed' || !result.model || result.model === 'listing-showcase-ai'); const simulatedScores = [...aiResults.values()].filter((result) => result.model === 'listing-v7-demo-simulation'); const simulatedAverage = simulatedScores.length ? Math.round(simulatedScores.reduce((sum, result) => sum + (result.overallScore ?? 0), 0) / simulatedScores.length * 10) / 10 : null; const simulatedMinimum = simulatedScores.length ? Math.min(...simulatedScores.map((result) => result.overallScore ?? Number.POSITIVE_INFINITY)) : null; const simulatedMaximum = simulatedScores.length ? Math.max(...simulatedScores.map((result) => result.overallScore ?? Number.NEGATIVE_INFINITY)) : null; const simulatedMinimumCount = simulatedScores.filter((result) => result.overallScore === 73).length; const simulatedMaximumCount = simulatedScores.filter((result) => result.overallScore === 100).length; const dimensions = Object.keys(LISTING_DIMENSION_MAX) as ListingDimension[]; const dimensionPartial = Object.fromEntries(dimensions.map((dimension) => [dimension, results.filter((result) => result.dimensions.find((item) => item.dimension === dimension)?.status === 'partial').length])); const knownMaxDistribution = results.reduce>((output, result) => { const key = String(result.knownOverallMaxScore ?? 'legacy'); output[key] = (output[key] ?? 0) + 1; return output; }, {}); const compliance = results.reduce>((output, result) => { const key = result.compliance?.status ?? 'legacy'; output[key] = (output[key] ?? 0) + 1; return output; }, {}); const normalizers = [...latestSources.values()].reduce>((output, source) => { const key = source.normalizerVersion ?? 'legacy'; output[key] = (output[key] ?? 0) + 1; return output; }, {}); const latestJob = jobRows.map((row) => row.payload).filter((job) => job.rubricVersion === LISTING_RUBRIC_VERSION && job.total === 625).sort((a, b) => b.requestedAt.localeCompare(a.requestedAt))[0] ?? null; const latestJobItems = latestJob ? itemRows.map((row) => row.payload).filter((item) => item.jobId === latestJob.id) : []; const ruleOutcomes = (ruleId: string) => results.reduce>((output, result) => { const evidence = result.dimensions.flatMap((dimension) => dimension.evidence).find((item) => item.ruleId === ruleId); const key = evidence?.outcome ?? 'missing'; output[key] = (output[key] ?? 0) + 1; return output; }, {}); const partialWithErrorCode = latestJobItems.filter((item) => item.status === 'partial' && item.errorCode !== null).length; const failedWithoutErrorCode = latestJobItems.filter((item) => item.status === 'failed' && !item.errorCode).length; const titleLengthOutcomes = ruleOutcomes('title.length'); const titleCategoryOutcomes = ruleOutcomes('title.category_in_first_15'); const hardFailRate = (outcomes: Record) => { const decided = (outcomes.pass ?? 0) + (outcomes.fail ?? 0); return decided ? Math.round((outcomes.fail ?? 0) / decided * 10_000) / 10_000 : 0; }; const systemicHardFailGate = { maximumAllowedRate: 0.5, titleLength: hardFailRate(titleLengthOutcomes), titleCategoryInFirst15: hardFailRate(titleCategoryOutcomes), }; const report = { workspaceId, rubricVersion: `${LISTING_RUBRIC_VERSION} / ${LISTING_AI_RUBRIC_VERSION}`, weights: LISTING_DIMENSION_MAX, sources: latestSources.size, normalizers, currentScoreRows: resultRows.length, duplicateCurrentSlots, legacyScoreSchemaPresent, ruleScores: ruleResults.size, rulePartialResults: results.filter((result) => result.overallScore === null).length, formalScores: aiResults.size, simulatedScores: simulatedScores.length, simulatedAverage, simulatedMinimum, simulatedMaximum, simulatedMinimumCount, simulatedMaximumCount, invalidAiResults: invalidAiResults.length, staleAiResults: staleAiResults.length, averageKnownScore: knownScores.length ? Math.round(knownScores.reduce((sum, value) => sum + value, 0) / knownScores.length * 10) / 10 : null, knownMaxDistribution, dimensionPartial, compliance, ruleOutcomes: { titleLength: titleLengthOutcomes, titleCategoryInFirst15: titleCategoryOutcomes, imagePrimary: ruleOutcomes('images.primary'), }, systemicHardFailGate, dataReadiness: { categoryRuleMissing: [...latestSources.values()].filter((source) => !source.categoryContext?.ruleVersion).length, vocEvidenceMissing: [...latestSources.values()].filter((source) => !source.vocEvidence?.length).length, descriptionStructureMissing: [...latestSources.values()].filter((source) => !source.descriptionStructure?.observed).length, }, missingResults: missingResults.length, orphanResults: orphanResults.length, staleResults: staleResults.length, invalidDimensions: invalidDimensions.length, invalidWeights: invalidWeights.length, latestJob: latestJob ? { id: latestJob.id, status: latestJob.status, total: latestJob.total, processed: latestJob.processed, succeeded: latestJob.succeeded, partial: latestJob.partial, blocked: latestJob.blocked, failed: latestJob.failed } : null, latestJobItems: latestJobItems.length, jobItemStatusSemantics: { partialWithErrorCode, failedWithoutErrorCode }, }; console.log(JSON.stringify(report, null, 2)); if (latestSources.size !== 625 || normalizers['jd-listing-v5'] !== 625 || resultRows.length !== 1250 || duplicateCurrentSlots || legacyScoreSchemaPresent || ruleResults.size !== 625 || aiResults.size !== 625 || simulatedScores.length !== 625 || simulatedAverage !== 87.3 || simulatedMinimum !== 73 || simulatedMaximum !== 100 || simulatedMinimumCount !== 1 || simulatedMaximumCount !== 1 || invalidAiResults.length || staleAiResults.length || missingResults.length || orphanResults.length || staleResults.length || invalidDimensions.length || invalidWeights.length || (latestJob !== null && latestJob.processed !== latestJob.total) || partialWithErrorCode || failedWithoutErrorCode || systemicHardFailGate.titleLength > systemicHardFailGate.maximumAllowedRate || systemicHardFailGate.titleCategoryInFirst15 > systemicHardFailGate.maximumAllowedRate) process.exitCode = 2; } main().catch((error) => { console.error(`[verify-listing-rollout] ${error instanceof Error ? error.message : error}`); process.exitCode = 1; });