import 'dotenv/config'; import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { dirname, resolve } from 'node:path'; import type { DomesticDataset, DomesticProduct, DomesticProductRelation, DomesticReview, } from '../src/types/domestic-dataset.js'; import { adaptJdReviewResponse, JD_PRODUCT_COMMENTS_PATH } from '../src/modules/domestic-voc/adapters/jd-review.adapter.js'; import { adaptJdSearchResponse, JD_PRODUCT_SEARCH_PATH } from '../src/modules/domestic-voc/adapters/jd-search.adapter.js'; import { FmodeVocEcommerceClient } from '../src/modules/domestic-voc/upstream/fmode-client.js'; interface BrandCategoryPair { key: string; brand: string; category: string; baseRelations: DomesticProductRelation[]; } interface SearchResult { pair: BrandCategoryPair; products: DomesticProduct[]; ok: boolean; } function defaultDatasetPath(): string { return resolve(process.cwd(), '..', '..', 'Saas-voc', 'src', 'assets', 'data', 'demashi-summary.json'); } function positiveInteger(value: string | undefined, fallback: number): number { const parsed = Number(value); return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; } async function mapLimit(items: T[], concurrency: number, worker: (item: T, index: number) => Promise): Promise { const results = new Array(items.length); let cursor = 0; async function run(): Promise { while (cursor < items.length) { const index = cursor; cursor += 1; results[index] = await worker(items[index] as T, index); } } await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => run())); return results; } function buildPairs(relations: DomesticProductRelation[]): BrandCategoryPair[] { const grouped = new Map(); for (const relation of relations) { if (!relation.competitorBrand || !relation.category) continue; const key = `${relation.competitorBrand}\u0000${relation.category}`; const rows = grouped.get(key) ?? []; rows.push(relation); grouped.set(key, rows); } return [...grouped.entries()] .map(([key, rows]) => ({ key, brand: rows[0]?.competitorBrand ?? '', category: rows[0]?.category ?? '', baseRelations: rows, })) .sort((left, right) => right.baseRelations.length - left.baseRelations.length || left.key.localeCompare(right.key, 'zh-CN')); } function relevanceScore(product: DomesticProduct, pair: BrandCategoryPair): number { const normalize = (value: string) => value.toLowerCase().replace(/^pop/i, '').replace(/[^a-z0-9\u3400-\u9fff]/g, ''); const brand = normalize(pair.brand); const title = normalize(product.title); const shop = normalize(product.market?.shopName ?? ''); let score = 0; if (brand && title.includes(brand)) score += 20; if (brand && shop.includes(brand)) score += 10; if (product.market?.monthSalesText) score += 4; if (product.market?.salesText) score += 2; if (product.market?.currentPrice) score += 1; if (/德玛仕|demashi/i.test(product.title)) score -= 100; return score; } function mergeProduct(existing: DomesticProduct | undefined, discovered: DomesticProduct): DomesticProduct { if (!existing) return discovered; return { ...existing, ...discovered, productId: existing.productId, productKey: existing.productKey, asin: existing.asin || discovered.asin, role: 'competitor', relationCount: existing.relationCount, }; } async function main(): Promise { const baseUrl = process.env.FMODE_BASE_URL?.trim(); const apiKey = process.env.FMODE_API_KEY?.trim(); if (!baseUrl || !apiKey) throw new Error('Company ecommerce gateway configuration is missing'); const inputPath = resolve(process.env.COMPETITOR_BASE_DATASET_PATH || defaultDatasetPath()); const outputPath = resolve(process.env.LOCAL_ENRICHED_DATASET_PATH || resolve(process.cwd(), 'logs', 'local-enriched-dataset.json')); const resultsPerPair = positiveInteger(process.env.COMPETITOR_RESULTS_PER_PAIR, 3); const reviewProductsPerPair = positiveInteger(process.env.COMPETITOR_REVIEW_PRODUCTS_PER_PAIR, 1); const concurrency = positiveInteger(process.env.COMPETITOR_ENRICH_CONCURRENCY, 3); const dataset = JSON.parse(await readFile(inputPath, 'utf8')) as DomesticDataset; dataset.reviews = Array.isArray(dataset.reviews) ? dataset.reviews : []; const collectedAt = new Date().toISOString(); const baseRelations = dataset.relations.map((relation) => ({ ...relation, discoverySource: 'workbook' as const })); const pairs = buildPairs(baseRelations); const client = new FmodeVocEcommerceClient({ baseUrl, apiKey, timeoutMs: positiveInteger(process.env.FMODE_TIMEOUT_MS, 30_000), retries: positiveInteger(process.env.FMODE_RETRIES, 2), }); console.log(`[competitor-enrich] searching ${pairs.length} brand/category combinations`); const searches = await mapLimit(pairs, concurrency, async (pair): Promise => { const keyword = `${pair.brand} ${pair.category}`; try { const response = await client.request(JD_PRODUCT_SEARCH_PATH, { params: { keyword, page: 1 } }); const products = adaptJdSearchResponse(response, { brand: pair.brand, category: pair.category, keyword, collectedAt }) .sort((left, right) => relevanceScore(right, pair) - relevanceScore(left, pair)) .filter((product) => relevanceScore(product, pair) >= 10) .slice(0, resultsPerPair); console.log(`[competitor-enrich] ${pair.brand} / ${pair.category}: ${products.length} products`); return { pair, products, ok: true }; } catch (error) { const status = typeof error === 'object' && error && 'status' in error ? String(error.status ?? '') : ''; console.warn(`[competitor-enrich] ${pair.brand} / ${pair.category}: request failed${status ? ` (${status})` : ''}`); return { pair, products: [], ok: false }; } }); const productsByKey = new Map(dataset.products.map((product) => [product.productKey, product])); const relationsByKey = new Map(baseRelations.map((relation) => [relation.relationKey, relation])); const discoveredProductKeys = new Set(); for (const search of searches) { for (const product of search.products) { discoveredProductKeys.add(product.productKey); productsByKey.set(product.productKey, mergeProduct(productsByKey.get(product.productKey), product)); const ownRelations = new Map(search.pair.baseRelations.map((relation) => [relation.ownProductKey, relation])); for (const relation of ownRelations.values()) { const relationKey = `${relation.ownProductKey}:${product.productKey}`; if (relationsByKey.has(relationKey)) continue; relationsByKey.set(relationKey, { relationKey, ownProductKey: relation.ownProductKey, ownProductId: relation.ownProductId, competitorProductKey: product.productKey, competitorProductId: product.productId, competitorBrand: search.pair.brand, category: search.pair.category, discoverySource: 'brand_category_search', ...(product.market?.searchKeyword ? { searchKeyword: product.market.searchKeyword } : {}), discoveredAt: collectedAt, }); } } } const reviewTargets = [...new Map(searches.flatMap((search) => search.products.slice(0, reviewProductsPerPair)) .map((product) => [product.productKey, product])).values()]; console.log(`[competitor-enrich] collecting reviews for ${reviewTargets.length} representative products`); const reviewPages = await mapLimit(reviewTargets, concurrency, async (product) => { try { const response = await client.request(JD_PRODUCT_COMMENTS_PATH, { params: { itemId: product.productId, page: 1 } }); const reviews = adaptJdReviewResponse(response, product.productId, 1).reviews; console.log(`[competitor-enrich] ${product.productId}: ${reviews.length} reviews`); return { product, reviews, ok: true }; } catch (error) { const status = typeof error === 'object' && error && 'status' in error ? String(error.status ?? '') : ''; console.warn(`[competitor-enrich] ${product.productId}: review request failed${status ? ` (${status})` : ''}`); return { product, reviews: [] as DomesticReview[], ok: false }; } }); const reviewByKey = new Map(dataset.reviews.map((review) => [`${review.productId}:${review.reviewId}`, review])); for (const page of reviewPages) { for (const review of page.reviews) reviewByKey.set(`${review.productId}:${review.reviewId}`, review); } const relations = [...relationsByKey.values()]; const relationCounts = new Map>(); for (const relation of relations) { const ownSet = relationCounts.get(relation.ownProductKey) ?? new Set(); ownSet.add(relation.competitorProductKey); relationCounts.set(relation.ownProductKey, ownSet); const competitorSet = relationCounts.get(relation.competitorProductKey) ?? new Set(); competitorSet.add(relation.ownProductKey); relationCounts.set(relation.competitorProductKey, competitorSet); } const products = [...productsByKey.values()].map((product) => ({ ...product, relationCount: relationCounts.get(product.productKey)?.size ?? product.relationCount, })); const relationsByOwnId = new Map(); for (const relation of relations) { const rows = relationsByOwnId.get(relation.ownProductId) ?? []; rows.push(relation); relationsByOwnId.set(relation.ownProductId, rows); } const mappingGroups = dataset.mappingGroups.map((group) => ({ ...group, competitors: relationsByOwnId.get(group.ownProductId) ?? group.competitors, })); const reviews = [...reviewByKey.values()]; const successfulQueries = searches.filter((search) => search.ok).length; const successfulReviewQueries = reviewPages.filter((page) => page.ok).length; const output: DomesticDataset = { ...dataset, generatedAt: collectedAt, products, relations, mappingGroups, reviews, summary: { ...dataset.summary, relations: relations.length, uniqueCompetitorProducts: products.filter((product) => product.role === 'competitor').length, reviewCount: reviews.length, }, enrichment: { status: successfulQueries === pairs.length && successfulReviewQueries === reviewTargets.length ? 'complete' : 'partial', queryCount: pairs.length, successfulQueries, discoveredProducts: discoveredProductKeys.size, reviewedProducts: new Set(reviews.map((review) => review.productId)).size, collectedReviews: reviews.length, collectedAt, }, }; await mkdir(dirname(outputPath), { recursive: true }); await writeFile(outputPath, `${JSON.stringify(output, null, 2)}\n`, 'utf8'); console.log(`[competitor-enrich] wrote ${output.products.length} products, ${output.relations.length} relations, and ${output.reviews.length} reviews`); console.log(`[competitor-enrich] output ${outputPath}`); } main().catch((error) => { console.error('[competitor-enrich] failed', error instanceof Error ? error.message : error); process.exitCode = 1; });