Saas-voc-server/scripts/verify-listing-rollout.ts

import 'dotenv/config';
import { z } from 'zod';
import { ParseRestClient } from '../src/db/parse-rest.client.js';
import { VOC_PARSE_CLASSES } from '../src/db/parse-rest.schema.js';
import type { ListingDimension, ListingScoreJob, ListingScoreJobItem, ListingScoreResult, ListingSourceSnapshot } from '../src/modules/listing-ai/domain.js';
import { LISTING_DIMENSION_MAX, LISTING_RUBRIC_VERSION } from '../src/modules/listing-ai/scoring/rule-engine.js';
import { LISTING_AI_RUBRIC_VERSION } from '../src/modules/listing-ai/scoring/ai-rubric.js';
import { isListingV7Score } from '../src/modules/listing-ai/scoring/score-status.js';

interface Stored<T> { workspaceId: string; productId?: string; jobId?: string; idempotencyKey?: string; slot?: 'rule_precheck' | 'formal_ai'; model?: string; payload: T }

async function main() {
  const env = z.object({ PARSE_SERVER_URL: z.url(), PARSE_APP_ID: z.string().min(1), PARSE_MASTER_KEY: z.string().min(1), SAAS_DEFAULT_WORKSPACE_ID: z.string().default('demashi') }).parse(process.env);
  const client = new ParseRestClient({ serverUrl: env.PARSE_SERVER_URL, appId: env.PARSE_APP_ID, masterKey: env.PARSE_MASTER_KEY });
  const workspaceId = env.SAAS_DEFAULT_WORKSPACE_ID;
  const [sourceRows, resultRows, itemRows, jobRows] = await Promise.all([
    client.findAll<Stored<ListingSourceSnapshot>>(VOC_PARSE_CLASSES.listingSourceSnapshot, { workspaceId, platform: 'jd', isCurrent: true, catalogIncluded: true, catalogCohort: 'listing-jd-v3-formal-625' }),
    client.findAll<Stored<ListingScoreResult>>(VOC_PARSE_CLASSES.listingCurrentScore, { workspaceId }),
    client.findAll<Stored<ListingScoreJobItem>>(VOC_PARSE_CLASSES.listingScoreItem, { workspaceId }),
    client.findAll<Stored<ListingScoreJob> & { idempotencyKey: string }>(VOC_PARSE_CLASSES.listingScoreJob, { workspaceId }),
  ]);
  const schemas = await client.schemas();
  const legacyScoreSchemaPresent = schemas.some((schema) => schema.className === 'VocListingScoreResult');
  const slotKeys = resultRows.map((row) => `${row.workspaceId}|${row.productId ?? row.payload.productId}|${row.slot ?? (row.payload.scoreKind === 'hybrid_ai' ? 'formal_ai' : 'rule_precheck')}`);
  const duplicateCurrentSlots = slotKeys.length - new Set(slotKeys).size;
  const latestSources = new Map<string, ListingSourceSnapshot>();
  for (const row of sourceRows) { const value = row.payload; const current = latestSources.get(value.productId); if (!current || value.syncedAt > current.syncedAt) latestSources.set(value.productId, value); }
  const ruleResults = new Map<string, ListingScoreResult>();
  const aiResults = new Map<string, ListingScoreResult>();
  for (const row of resultRows) {
    const value = row.payload;
    if (!isListingV7Score(value)) continue;
    const target = row.slot === 'formal_ai' || value.scoreKind === 'hybrid_ai' ? aiResults : ruleResults;
    const current = target.get(value.productId);
    if (!current || value.createdAt > current.createdAt) target.set(value.productId, value);
  }
  const results = [...ruleResults.values()];
  const orphanResults = results.filter((result) => !latestSources.has(result.productId));
  const missingResults = [...latestSources.values()].filter((source) => !ruleResults.has(source.productId));
  const invalidDimensions = results.filter((result) => result.dimensions.length !== 5);
  const staleResults = results.filter((result) => latestSources.get(result.productId)?.sourceHash !== result.sourceHash);
  const invalidWeights = results.filter((result) => result.dimensions.some((item) => item.maxScore !== LISTING_DIMENSION_MAX[item.dimension]));
  const knownScores = results.map((result) => result.knownOverallScore).filter((value): value is number => typeof value === 'number');
  const staleAiResults = [...aiResults.values()].filter((result) => latestSources.get(result.productId)?.sourceHash !== result.sourceHash);
  const invalidAiResults = [...aiResults.values()].filter((result) => result.rubricVersion !== LISTING_AI_RUBRIC_VERSION || result.scoreKind !== 'hybrid_ai' || result.aiStatus !== 'completed' || !result.model || result.model === 'listing-showcase-ai');
  const simulatedScores = [...aiResults.values()].filter((result) => result.model === 'listing-v7-demo-simulation');
  const simulatedAverage = simulatedScores.length ? Math.round(simulatedScores.reduce((sum, result) => sum + (result.overallScore ?? 0), 0) / simulatedScores.length * 10) / 10 : null;
  const simulatedMinimum = simulatedScores.length ? Math.min(...simulatedScores.map((result) => result.overallScore ?? Number.POSITIVE_INFINITY)) : null;
  const simulatedMaximum = simulatedScores.length ? Math.max(...simulatedScores.map((result) => result.overallScore ?? Number.NEGATIVE_INFINITY)) : null;
  const simulatedMinimumCount = simulatedScores.filter((result) => result.overallScore === 73).length;
  const simulatedMaximumCount = simulatedScores.filter((result) => result.overallScore === 100).length;
  const dimensions = Object.keys(LISTING_DIMENSION_MAX) as ListingDimension[];
  const dimensionPartial = Object.fromEntries(dimensions.map((dimension) => [dimension, results.filter((result) => result.dimensions.find((item) => item.dimension === dimension)?.status === 'partial').length]));
  const knownMaxDistribution = results.reduce<Record<string, number>>((output, result) => { const key = String(result.knownOverallMaxScore ?? 'legacy'); output[key] = (output[key] ?? 0) + 1; return output; }, {});
  const compliance = results.reduce<Record<string, number>>((output, result) => { const key = result.compliance?.status ?? 'legacy'; output[key] = (output[key] ?? 0) + 1; return output; }, {});
  const normalizers = [...latestSources.values()].reduce<Record<string, number>>((output, source) => { const key = source.normalizerVersion ?? 'legacy'; output[key] = (output[key] ?? 0) + 1; return output; }, {});
  const latestJob = jobRows.map((row) => row.payload).filter((job) => job.rubricVersion === LISTING_RUBRIC_VERSION && job.total === 625).sort((a, b) => b.requestedAt.localeCompare(a.requestedAt))[0] ?? null;
  const latestJobItems = latestJob ? itemRows.map((row) => row.payload).filter((item) => item.jobId === latestJob.id) : [];
  const ruleOutcomes = (ruleId: string) => results.reduce<Record<string, number>>((output, result) => {
    const evidence = result.dimensions.flatMap((dimension) => dimension.evidence).find((item) => item.ruleId === ruleId);
    const key = evidence?.outcome ?? 'missing';
    output[key] = (output[key] ?? 0) + 1;
    return output;
  }, {});
  const partialWithErrorCode = latestJobItems.filter((item) => item.status === 'partial' && item.errorCode !== null).length;
  const failedWithoutErrorCode = latestJobItems.filter((item) => item.status === 'failed' && !item.errorCode).length;
  const titleLengthOutcomes = ruleOutcomes('title.length');
  const titleCategoryOutcomes = ruleOutcomes('title.category_in_first_15');
  const hardFailRate = (outcomes: Record<string, number>) => {
    const decided = (outcomes.pass ?? 0) + (outcomes.fail ?? 0);
    return decided ? Math.round((outcomes.fail ?? 0) / decided * 10_000) / 10_000 : 0;
  };
  const systemicHardFailGate = {
    maximumAllowedRate: 0.5,
    titleLength: hardFailRate(titleLengthOutcomes),
    titleCategoryInFirst15: hardFailRate(titleCategoryOutcomes),
  };
  const report = {
    workspaceId, rubricVersion: `${LISTING_RUBRIC_VERSION} / ${LISTING_AI_RUBRIC_VERSION}`, weights: LISTING_DIMENSION_MAX, sources: latestSources.size, normalizers,
    currentScoreRows: resultRows.length, duplicateCurrentSlots, legacyScoreSchemaPresent,
    ruleScores: ruleResults.size, rulePartialResults: results.filter((result) => result.overallScore === null).length,
    formalScores: aiResults.size, simulatedScores: simulatedScores.length, simulatedAverage, simulatedMinimum, simulatedMaximum, simulatedMinimumCount, simulatedMaximumCount, invalidAiResults: invalidAiResults.length, staleAiResults: staleAiResults.length,
    averageKnownScore: knownScores.length ? Math.round(knownScores.reduce((sum, value) => sum + value, 0) / knownScores.length * 10) / 10 : null,
    knownMaxDistribution, dimensionPartial, compliance,
    ruleOutcomes: {
      titleLength: titleLengthOutcomes,
      titleCategoryInFirst15: titleCategoryOutcomes,
      imagePrimary: ruleOutcomes('images.primary'),
    },
    systemicHardFailGate,
    dataReadiness: {
      categoryRuleMissing: [...latestSources.values()].filter((source) => !source.categoryContext?.ruleVersion).length,
      vocEvidenceMissing: [...latestSources.values()].filter((source) => !source.vocEvidence?.length).length,
      descriptionStructureMissing: [...latestSources.values()].filter((source) => !source.descriptionStructure?.observed).length,
    },
    missingResults: missingResults.length, orphanResults: orphanResults.length, staleResults: staleResults.length, invalidDimensions: invalidDimensions.length, invalidWeights: invalidWeights.length,
    latestJob: latestJob ? { id: latestJob.id, status: latestJob.status, total: latestJob.total, processed: latestJob.processed, succeeded: latestJob.succeeded, partial: latestJob.partial, blocked: latestJob.blocked, failed: latestJob.failed } : null,
    latestJobItems: latestJobItems.length,
    jobItemStatusSemantics: { partialWithErrorCode, failedWithoutErrorCode },
  };
  console.log(JSON.stringify(report, null, 2));
  if (latestSources.size !== 625 || normalizers['jd-listing-v5'] !== 625 || resultRows.length !== 1250 || duplicateCurrentSlots || legacyScoreSchemaPresent || ruleResults.size !== 625 || aiResults.size !== 625 || simulatedScores.length !== 625 || simulatedAverage !== 87.3 || simulatedMinimum !== 73 || simulatedMaximum !== 100 || simulatedMinimumCount !== 1 || simulatedMaximumCount !== 1 || invalidAiResults.length || staleAiResults.length || missingResults.length || orphanResults.length || staleResults.length || invalidDimensions.length || invalidWeights.length || (latestJob !== null && latestJob.processed !== latestJob.total) || partialWithErrorCode || failedWithoutErrorCode || systemicHardFailGate.titleLength > systemicHardFailGate.maximumAllowedRate || systemicHardFailGate.titleCategoryInFirst15 > systemicHardFailGate.maximumAllowedRate) process.exitCode = 2;
}

main().catch((error) => { console.error(`[verify-listing-rollout] ${error instanceof Error ? error.message : error}`); process.exitCode = 1; });