fmode-voc-projects/scripts/tools/collect-amazon.js

#!/usr/bin/env node
/**
 * Amazon (Sorftime) 深度采集模块
 *
 * 对单个产品：
 *   1) 多关键词 × 多页 ProductQuery → 去重合并产品池
 *   2) 对 Top N 产品：ProductRequest 取详情（BSR、评分分布、Buybox、上线日、Photos）
 *   3) 对 Top M 产品：ProductReviewsQuery 取真实评论
 *
 * 导出：collectAmazonForProduct(product) → {by_keyword, product_details, reviews_by_asin}
 */

const https = require('https');

function httpRequest(options, bodyData = null, maxAttempts = 3) {
    return new Promise(async (resolve, reject) => {
        for (let attempt = 1; attempt <= maxAttempts; attempt++) {
            try {
                const result = await new Promise((res, rej) => {
                    const req = https.request(options, (response) => {
                        const chunks = [];
                        response.on('data', (c) => chunks.push(c));
                        response.on('end', () => res({ status: response.statusCode, body: Buffer.concat(chunks).toString('utf-8') }));
                    });
                    req.on('error', rej);
                    req.on('timeout', () => { req.destroy(); rej(new Error('timeout')); });
                    if (bodyData) req.write(bodyData);
                    req.end();
                });
                if (result.status === 200) return resolve(result);
                if ([400, 429, 500, 502, 503, 504].includes(result.status) && attempt < maxAttempts) {
                    await new Promise((r) => setTimeout(r, 1500 * Math.pow(1.8, attempt - 1)));
                    continue;
                }
                return resolve(result);
            } catch (e) {
                if (attempt === maxAttempts) return reject(e);
                await new Promise((r) => setTimeout(r, 2000 * attempt));
            }
        }
    });
}

async function sorftimeCall(apiPath, body = {}, domain = 1) {
    const bodyStr = JSON.stringify({ path: apiPath, method: 'POST', body, query: { domain } });
    const { status, body: respBody } = await httpRequest({
        hostname: 'server-msq.fmode.cn',
        path: '/api/sorftime/forward',
        method: 'POST',
        headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(bodyStr) },
        timeout: 60000,
    }, bodyStr);
    if (status !== 200) return { _error: { status, body: respBody.slice(0, 300) } };
    try {
        const j = JSON.parse(respBody);
        return j?.Data ?? j?.data ?? j;
    } catch (e) {
        return { _error: { parse: e.message } };
    }
}

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

// ============================================================
// 1) 多页关键词产品查询（Sorftime ProductQuery）
// ============================================================
async function fetchProductsForKeyword(kw, pages = 3) {
    console.log(`    [amz] 🔎 "${kw}" · ${pages}页 ProductQuery`);
    const all = [];
    for (let page = 1; page <= pages; page++) {
        const r = await sorftimeCall('/api/ProductQuery', {
            Page: page, Query: '1', QueryType: '7', Pattern: kw,
        });
        if (r?._error) {
            console.log(`      ↻ p${page} 失败: ${JSON.stringify(r._error).slice(0, 120)}`);
            break;
        }
        const products = r?.Products || [];
        console.log(`      · p${page}: ${products.length} products (total pages=${r?.PageCount || '?'})`);
        all.push(...products);
        if (!products.length || page >= (r?.PageCount || 0)) break;
        await sleep(500);
    }
    return all;
}

// ============================================================
// 2) 产品详情（ProductRequest）— 含 BSR/评分数/Buybox/上线日期/Photo 等
// ============================================================
async function fetchProductDetail(asin) {
    const r = await sorftimeCall('/api/ProductRequest', {
        ASIN: asin, Trend: 1, QueryTrendStartDt: '', QueryTrendEndDt: '',
    });
    if (r?._error) return { _error: r._error };
    return r;
}

// ============================================================
// 3) 产品评论（ProductReviewsQuery）
// ============================================================
async function fetchProductReviews(asin) {
    const r = await sorftimeCall('/api/ProductReviewsQuery', { ASIN: asin });
    if (r?._error) return { _error: r._error };
    return r;
}

// ============================================================
// 主入口
// ============================================================
async function collectAmazonForProduct(product) {
    const {
        keywords_en: keywords,
        sorftime_pages = 3,
        sorftime_top_detail = 6,
        sorftime_top_reviews = 3,
    } = product;

    console.log(`\n  ━━━ Amazon for "${product.name}" (${keywords.length} keywords) ━━━`);

    const by_keyword = {};
    const asinPool = new Map(); // asin → product summary (取 SalesVolume 最高的)

    for (const kw of keywords) {
        try {
            const products = await fetchProductsForKeyword(kw, sorftime_pages);
            by_keyword[kw] = {
                keyword: kw,
                top_products: { Products: products },
                total: products.length,
            };
            products.forEach((p) => {
                if (!p.Asin) return;
                const cur = asinPool.get(p.Asin);
                if (!cur || (p.ListingSalesVolumeOfMonth || 0) > (cur.ListingSalesVolumeOfMonth || 0)) {
                    asinPool.set(p.Asin, { ...p, _from_keyword: kw });
                }
            });
            await sleep(800);
        } catch (e) {
            console.log(`    [amz:${kw}] ❌ ${e.message}`);
            by_keyword[kw] = { _error: e.message };
        }
    }

    // 按月销量排序得到 top 产品
    const topByVolume = Array.from(asinPool.values())
        .sort((a, b) => (b.ListingSalesVolumeOfMonth || 0) - (a.ListingSalesVolumeOfMonth || 0));

    console.log(`\n  [amz] 合并去重后 ${topByVolume.length} 款 · 抓 Top ${sorftime_top_detail} 详情 + Top ${sorftime_top_reviews} 评论`);

    // 抓详情
    const product_details = {};
    for (const p of topByVolume.slice(0, sorftime_top_detail)) {
        console.log(`    [amz/detail] ${p.Asin} · ${(p.Brand || '').slice(0, 20)} · $${((p.SalesPrice || p.Price || 0) / 100).toFixed(2)}`);
        const detail = await fetchProductDetail(p.Asin);
        if (!detail?._error) product_details[p.Asin] = detail;
        else console.log(`      ❌ ${JSON.stringify(detail._error).slice(0, 80)}`);
        await sleep(500);
    }

    // 抓评论
    const reviews_by_asin = {};
    for (const p of topByVolume.slice(0, sorftime_top_reviews)) {
        console.log(`    [amz/reviews] ${p.Asin}`);
        const reviews = await fetchProductReviews(p.Asin);
        if (!reviews?._error) reviews_by_asin[p.Asin] = reviews;
        else console.log(`      ❌ ${JSON.stringify(reviews._error).slice(0, 80)}`);
        await sleep(500);
    }

    return {
        by_keyword,
        product_details,
        reviews_by_asin,
        merged_products: topByVolume,
    };
}

module.exports = { collectAmazonForProduct, fetchProductsForKeyword, fetchProductDetail, fetchProductReviews };