Saas-voc-server/scripts/enrich-local-competitors.ts

import 'dotenv/config';
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { dirname, resolve } from 'node:path';
import type {
  DomesticDataset,
  DomesticProduct,
  DomesticProductRelation,
  DomesticReview,
} from '../src/types/domestic-dataset.js';
import { adaptJdReviewResponse, JD_PRODUCT_COMMENTS_PATH } from '../src/modules/domestic-voc/adapters/jd-review.adapter.js';
import { adaptJdSearchResponse, JD_PRODUCT_SEARCH_PATH } from '../src/modules/domestic-voc/adapters/jd-search.adapter.js';
import { FmodeVocEcommerceClient } from '../src/modules/domestic-voc/upstream/fmode-client.js';

interface BrandCategoryPair {
  key: string;
  brand: string;
  category: string;
  baseRelations: DomesticProductRelation[];
}

interface SearchResult {
  pair: BrandCategoryPair;
  products: DomesticProduct[];
  ok: boolean;
}

function defaultDatasetPath(): string {
  return resolve(process.cwd(), '..', '..', 'Saas-voc', 'src', 'assets', 'data', 'demashi-summary.json');
}

function positiveInteger(value: string | undefined, fallback: number): number {
  const parsed = Number(value);
  return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
}

async function mapLimit<T, R>(items: T[], concurrency: number, worker: (item: T, index: number) => Promise<R>): Promise<R[]> {
  const results = new Array<R>(items.length);
  let cursor = 0;
  async function run(): Promise<void> {
    while (cursor < items.length) {
      const index = cursor;
      cursor += 1;
      results[index] = await worker(items[index] as T, index);
    }
  }
  await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => run()));
  return results;
}

function buildPairs(relations: DomesticProductRelation[]): BrandCategoryPair[] {
  const grouped = new Map<string, DomesticProductRelation[]>();
  for (const relation of relations) {
    if (!relation.competitorBrand || !relation.category) continue;
    const key = `${relation.competitorBrand}\u0000${relation.category}`;
    const rows = grouped.get(key) ?? [];
    rows.push(relation);
    grouped.set(key, rows);
  }
  return [...grouped.entries()]
    .map(([key, rows]) => ({
      key,
      brand: rows[0]?.competitorBrand ?? '',
      category: rows[0]?.category ?? '',
      baseRelations: rows,
    }))
    .sort((left, right) => right.baseRelations.length - left.baseRelations.length || left.key.localeCompare(right.key, 'zh-CN'));
}

function relevanceScore(product: DomesticProduct, pair: BrandCategoryPair): number {
  const normalize = (value: string) => value.toLowerCase().replace(/^pop/i, '').replace(/[^a-z0-9\u3400-\u9fff]/g, '');
  const brand = normalize(pair.brand);
  const title = normalize(product.title);
  const shop = normalize(product.market?.shopName ?? '');
  let score = 0;
  if (brand && title.includes(brand)) score += 20;
  if (brand && shop.includes(brand)) score += 10;
  if (product.market?.monthSalesText) score += 4;
  if (product.market?.salesText) score += 2;
  if (product.market?.currentPrice) score += 1;
  if (/德玛仕|demashi/i.test(product.title)) score -= 100;
  return score;
}

function mergeProduct(existing: DomesticProduct | undefined, discovered: DomesticProduct): DomesticProduct {
  if (!existing) return discovered;
  return {
    ...existing,
    ...discovered,
    productId: existing.productId,
    productKey: existing.productKey,
    asin: existing.asin || discovered.asin,
    role: 'competitor',
    relationCount: existing.relationCount,
  };
}

async function main(): Promise<void> {
  const baseUrl = process.env.FMODE_BASE_URL?.trim();
  const apiKey = process.env.FMODE_API_KEY?.trim();
  if (!baseUrl || !apiKey) throw new Error('Company ecommerce gateway configuration is missing');

  const inputPath = resolve(process.env.COMPETITOR_BASE_DATASET_PATH || defaultDatasetPath());
  const outputPath = resolve(process.env.LOCAL_ENRICHED_DATASET_PATH || resolve(process.cwd(), 'logs', 'local-enriched-dataset.json'));
  const resultsPerPair = positiveInteger(process.env.COMPETITOR_RESULTS_PER_PAIR, 3);
  const reviewProductsPerPair = positiveInteger(process.env.COMPETITOR_REVIEW_PRODUCTS_PER_PAIR, 1);
  const concurrency = positiveInteger(process.env.COMPETITOR_ENRICH_CONCURRENCY, 3);
  const dataset = JSON.parse(await readFile(inputPath, 'utf8')) as DomesticDataset;
  dataset.reviews = Array.isArray(dataset.reviews) ? dataset.reviews : [];
  const collectedAt = new Date().toISOString();
  const baseRelations = dataset.relations.map((relation) => ({ ...relation, discoverySource: 'workbook' as const }));
  const pairs = buildPairs(baseRelations);
  const client = new FmodeVocEcommerceClient({
    baseUrl,
    apiKey,
    timeoutMs: positiveInteger(process.env.FMODE_TIMEOUT_MS, 30_000),
    retries: positiveInteger(process.env.FMODE_RETRIES, 2),
  });

  console.log(`[competitor-enrich] searching ${pairs.length} brand/category combinations`);
  const searches = await mapLimit(pairs, concurrency, async (pair): Promise<SearchResult> => {
    const keyword = `${pair.brand} ${pair.category}`;
    try {
      const response = await client.request<unknown>(JD_PRODUCT_SEARCH_PATH, { params: { keyword, page: 1 } });
      const products = adaptJdSearchResponse(response, { brand: pair.brand, category: pair.category, keyword, collectedAt })
        .sort((left, right) => relevanceScore(right, pair) - relevanceScore(left, pair))
        .filter((product) => relevanceScore(product, pair) >= 10)
        .slice(0, resultsPerPair);
      console.log(`[competitor-enrich] ${pair.brand} / ${pair.category}: ${products.length} products`);
      return { pair, products, ok: true };
    } catch (error) {
      const status = typeof error === 'object' && error && 'status' in error ? String(error.status ?? '') : '';
      console.warn(`[competitor-enrich] ${pair.brand} / ${pair.category}: request failed${status ? ` (${status})` : ''}`);
      return { pair, products: [], ok: false };
    }
  });

  const productsByKey = new Map(dataset.products.map((product) => [product.productKey, product]));
  const relationsByKey = new Map<string, DomesticProductRelation>(baseRelations.map((relation) => [relation.relationKey, relation]));
  const discoveredProductKeys = new Set<string>();
  for (const search of searches) {
    for (const product of search.products) {
      discoveredProductKeys.add(product.productKey);
      productsByKey.set(product.productKey, mergeProduct(productsByKey.get(product.productKey), product));
      const ownRelations = new Map(search.pair.baseRelations.map((relation) => [relation.ownProductKey, relation]));
      for (const relation of ownRelations.values()) {
        const relationKey = `${relation.ownProductKey}:${product.productKey}`;
        if (relationsByKey.has(relationKey)) continue;
        relationsByKey.set(relationKey, {
          relationKey,
          ownProductKey: relation.ownProductKey,
          ownProductId: relation.ownProductId,
          competitorProductKey: product.productKey,
          competitorProductId: product.productId,
          competitorBrand: search.pair.brand,
          category: search.pair.category,
          discoverySource: 'brand_category_search',
          ...(product.market?.searchKeyword ? { searchKeyword: product.market.searchKeyword } : {}),
          discoveredAt: collectedAt,
        });
      }
    }
  }

  const reviewTargets = [...new Map(searches.flatMap((search) => search.products.slice(0, reviewProductsPerPair))
    .map((product) => [product.productKey, product])).values()];
  console.log(`[competitor-enrich] collecting reviews for ${reviewTargets.length} representative products`);
  const reviewPages = await mapLimit(reviewTargets, concurrency, async (product) => {
    try {
      const response = await client.request<unknown>(JD_PRODUCT_COMMENTS_PATH, { params: { itemId: product.productId, page: 1 } });
      const reviews = adaptJdReviewResponse(response, product.productId, 1).reviews;
      console.log(`[competitor-enrich] ${product.productId}: ${reviews.length} reviews`);
      return { product, reviews, ok: true };
    } catch (error) {
      const status = typeof error === 'object' && error && 'status' in error ? String(error.status ?? '') : '';
      console.warn(`[competitor-enrich] ${product.productId}: review request failed${status ? ` (${status})` : ''}`);
      return { product, reviews: [] as DomesticReview[], ok: false };
    }
  });

  const reviewByKey = new Map(dataset.reviews.map((review) => [`${review.productId}:${review.reviewId}`, review]));
  for (const page of reviewPages) {
    for (const review of page.reviews) reviewByKey.set(`${review.productId}:${review.reviewId}`, review);
  }
  const relations = [...relationsByKey.values()];
  const relationCounts = new Map<string, Set<string>>();
  for (const relation of relations) {
    const ownSet = relationCounts.get(relation.ownProductKey) ?? new Set<string>();
    ownSet.add(relation.competitorProductKey);
    relationCounts.set(relation.ownProductKey, ownSet);
    const competitorSet = relationCounts.get(relation.competitorProductKey) ?? new Set<string>();
    competitorSet.add(relation.ownProductKey);
    relationCounts.set(relation.competitorProductKey, competitorSet);
  }
  const products = [...productsByKey.values()].map((product) => ({
    ...product,
    relationCount: relationCounts.get(product.productKey)?.size ?? product.relationCount,
  }));
  const relationsByOwnId = new Map<string, DomesticProductRelation[]>();
  for (const relation of relations) {
    const rows = relationsByOwnId.get(relation.ownProductId) ?? [];
    rows.push(relation);
    relationsByOwnId.set(relation.ownProductId, rows);
  }
  const mappingGroups = dataset.mappingGroups.map((group) => ({
    ...group,
    competitors: relationsByOwnId.get(group.ownProductId) ?? group.competitors,
  }));
  const reviews = [...reviewByKey.values()];
  const successfulQueries = searches.filter((search) => search.ok).length;
  const successfulReviewQueries = reviewPages.filter((page) => page.ok).length;
  const output: DomesticDataset = {
    ...dataset,
    generatedAt: collectedAt,
    products,
    relations,
    mappingGroups,
    reviews,
    summary: {
      ...dataset.summary,
      relations: relations.length,
      uniqueCompetitorProducts: products.filter((product) => product.role === 'competitor').length,
      reviewCount: reviews.length,
    },
    enrichment: {
      status: successfulQueries === pairs.length && successfulReviewQueries === reviewTargets.length ? 'complete' : 'partial',
      queryCount: pairs.length,
      successfulQueries,
      discoveredProducts: discoveredProductKeys.size,
      reviewedProducts: new Set(reviews.map((review) => review.productId)).size,
      collectedReviews: reviews.length,
      collectedAt,
    },
  };
  await mkdir(dirname(outputPath), { recursive: true });
  await writeFile(outputPath, `${JSON.stringify(output, null, 2)}\n`, 'utf8');
  console.log(`[competitor-enrich] wrote ${output.products.length} products, ${output.relations.length} relations, and ${output.reviews.length} reviews`);
  console.log(`[competitor-enrich] output ${outputPath}`);
}

main().catch((error) => {
  console.error('[competitor-enrich] failed', error instanceof Error ? error.message : error);
  process.exitCode = 1;
});