|
|
@@ -0,0 +1,309 @@
|
|
|
+import { createHash } from 'node:crypto';
|
|
|
+import { readFile } from 'node:fs/promises';
|
|
|
+import { basename } from 'node:path';
|
|
|
+import ExcelJS from 'exceljs';
|
|
|
+import type { DomesticDataset, DomesticMetricSummary, DomesticProductRelation } from '../../../types/domestic-dataset.js';
|
|
|
+import type { ImportDataset } from '../services/dataset-import.service.js';
|
|
|
+
|
|
|
+const METRIC_SHEET = '德玛仕产品基础数据';
|
|
|
+const RELATION_SHEET = '竟对品牌及编码';
|
|
|
+
|
|
|
+type Row = unknown[];
|
|
|
+type HeaderIndex = Map<string, number>;
|
|
|
+
|
|
|
+interface MetricPoint {
|
|
|
+ date: string;
|
|
|
+ gmv: number;
|
|
|
+ soldUnits: number;
|
|
|
+ transactionOrders: number;
|
|
|
+ transactionCustomers: number;
|
|
|
+ impressions: number;
|
|
|
+ clicks: number;
|
|
|
+ views: number;
|
|
|
+ visitors: number;
|
|
|
+ cartUnits: number;
|
|
|
+ orderAmount: number;
|
|
|
+ orderUnits: number;
|
|
|
+ orderCount: number;
|
|
|
+ refundAmount: number;
|
|
|
+ refundUnits: number;
|
|
|
+ refundOrders: number;
|
|
|
+}
|
|
|
+
|
|
|
+type MutableProduct = ImportDataset['products'][number];
|
|
|
+type MutableDailyMetric = DomesticDataset['dailyTotals'][number];
|
|
|
+
|
|
|
+export async function normalizeDemashiWorkbook(inputPath: string): Promise<ImportDataset> {
|
|
|
+ const file = await readFile(inputPath);
|
|
|
+ const sheets = await readWorkbookRows(inputPath);
|
|
|
+ const metricRows = sheets.get(METRIC_SHEET);
|
|
|
+ const relationRows = sheets.get(RELATION_SHEET);
|
|
|
+ if (!metricRows) throw new Error(`Missing worksheet: ${METRIC_SHEET}`);
|
|
|
+ if (!relationRows) throw new Error(`Missing worksheet: ${RELATION_SHEET}`);
|
|
|
+ const metricHeaderIndex = findHeaderRow(metricRows, ['时间', 'SKU', 'SKU名称']);
|
|
|
+ const relationHeaderIndex = findHeaderRow(relationRows, ['类目', '德玛仕sku']);
|
|
|
+ const metricHeaders = indexHeaders(metricRows[metricHeaderIndex]!);
|
|
|
+ const relationHeaders = indexHeaders(relationRows[relationHeaderIndex]!);
|
|
|
+ const products = new Map<string, MutableProduct>();
|
|
|
+ const dailyTotals = new Map<string, MutableDailyMetric>();
|
|
|
+ let metricRowCount = 0;
|
|
|
+
|
|
|
+ for (const row of metricRows.slice(metricHeaderIndex + 1)) {
|
|
|
+ const productId = asId(read(row, metricHeaders, 'SKU'));
|
|
|
+ const date = asDate(read(row, metricHeaders, '时间'));
|
|
|
+ if (!productId || !date) continue;
|
|
|
+ metricRowCount += 1;
|
|
|
+ const productKey = `jd:${productId}`;
|
|
|
+ const metric = metricPoint(row, metricHeaders, date);
|
|
|
+ const product = products.get(productKey) ?? createProduct(row, metricHeaders, productId, productKey);
|
|
|
+ product.trend.push(metric);
|
|
|
+ addMetric(product.summary, metric);
|
|
|
+ products.set(productKey, product);
|
|
|
+
|
|
|
+ const day = dailyTotals.get(date) ?? { date, ...emptySummary() };
|
|
|
+ addMetric(day, metric);
|
|
|
+ dailyTotals.set(date, day);
|
|
|
+ }
|
|
|
+
|
|
|
+ for (const product of products.values()) {
|
|
|
+ product.trend.sort((left, right) => left.date.localeCompare(right.date));
|
|
|
+ finalizeSummary(product.summary);
|
|
|
+ }
|
|
|
+
|
|
|
+ const mappingGroups: DomesticDataset['mappingGroups'] = [];
|
|
|
+ const relations: DomesticProductRelation[] = [];
|
|
|
+ const competitorIds = new Set<string>();
|
|
|
+ const orphanMappings: DomesticDataset['quality']['orphanMappings'] = [];
|
|
|
+ const mappingsWithoutCompetitor: DomesticDataset['quality']['mappingsWithoutCompetitor'] = [];
|
|
|
+ const brandWithoutProductId: DomesticDataset['quality']['brandWithoutProductId'] = [];
|
|
|
+
|
|
|
+ for (const row of relationRows.slice(relationHeaderIndex + 1)) {
|
|
|
+ const ownProductId = asId(read(row, relationHeaders, '德玛仕sku'));
|
|
|
+ if (!ownProductId) continue;
|
|
|
+ const ownProductKey = `jd:${ownProductId}`;
|
|
|
+ const category = cleanText(read(row, relationHeaders, '类目'));
|
|
|
+ const model = cleanText(read(row, relationHeaders, '德玛仕型号'));
|
|
|
+ const competitors: DomesticProductRelation[] = [];
|
|
|
+ for (let slot = 1; slot <= 3; slot += 1) {
|
|
|
+ const brand = cleanText(read(row, relationHeaders, `竟品${slot}品牌`));
|
|
|
+ const competitorProductId = asId(read(row, relationHeaders, `竟品sku${slot}`));
|
|
|
+ if (brand && !competitorProductId) {
|
|
|
+ brandWithoutProductId.push({ ownProductId, slot, brand });
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if (!competitorProductId) continue;
|
|
|
+ const competitorProductKey = `jd:${competitorProductId}`;
|
|
|
+ const relation: DomesticProductRelation = {
|
|
|
+ relationKey: `${ownProductKey}:${competitorProductKey}`,
|
|
|
+ ownProductKey,
|
|
|
+ ownProductId,
|
|
|
+ competitorProductKey,
|
|
|
+ competitorProductId,
|
|
|
+ competitorBrand: brand,
|
|
|
+ category,
|
|
|
+ };
|
|
|
+ competitors.push(relation);
|
|
|
+ relations.push(relation);
|
|
|
+ competitorIds.add(competitorProductId);
|
|
|
+ }
|
|
|
+ const ownProduct = products.get(ownProductKey);
|
|
|
+ if (ownProduct) {
|
|
|
+ ownProduct.model = model || ownProduct.model;
|
|
|
+ ownProduct.relationCount = competitors.length;
|
|
|
+ } else {
|
|
|
+ orphanMappings.push({ ownProductId, model, category });
|
|
|
+ }
|
|
|
+ if (!competitors.length) mappingsWithoutCompetitor.push({ ownProductId, model, category });
|
|
|
+ mappingGroups.push({ ownProductId, ownProductKey, model, category, competitors });
|
|
|
+ }
|
|
|
+
|
|
|
+ const productList = [...products.values()].sort((left, right) => right.summary.gmv - left.summary.gmv);
|
|
|
+ const daily = [...dailyTotals.values()]
|
|
|
+ .sort((left, right) => left.date.localeCompare(right.date))
|
|
|
+ .map((metric) => finalizeSummary(metric));
|
|
|
+ const dates = daily.map((metric) => metric.date);
|
|
|
+
|
|
|
+ return {
|
|
|
+ schemaVersion: 1,
|
|
|
+ generatedAt: new Date().toISOString(),
|
|
|
+ caseName: '德玛仕',
|
|
|
+ platform: 'jd',
|
|
|
+ source: {
|
|
|
+ sourceFile: basename(inputPath),
|
|
|
+ sourceHash: createHash('sha256').update(file).digest('hex'),
|
|
|
+ sheets: [METRIC_SHEET, RELATION_SHEET],
|
|
|
+ dateRange: { start: dates[0] ?? '', end: dates.at(-1) ?? '' },
|
|
|
+ },
|
|
|
+ summary: {
|
|
|
+ metricRows: metricRowCount,
|
|
|
+ metricProducts: productList.length,
|
|
|
+ mappingRows: mappingGroups.length,
|
|
|
+ relations: relations.length,
|
|
|
+ uniqueCompetitorProducts: competitorIds.size,
|
|
|
+ category2Count: new Set(productList.map((product) => product.category2).filter(Boolean)).size,
|
|
|
+ category3Count: new Set(productList.map((product) => product.category3).filter(Boolean)).size,
|
|
|
+ reviewCount: 0,
|
|
|
+ },
|
|
|
+ dailyTotals: daily,
|
|
|
+ products: productList,
|
|
|
+ mappingGroups,
|
|
|
+ relations,
|
|
|
+ reviews: [],
|
|
|
+ quality: { orphanMappings, mappingsWithoutCompetitor, brandWithoutProductId },
|
|
|
+ };
|
|
|
+}
|
|
|
+
|
|
|
+function createProduct(row: Row, headers: HeaderIndex, productId: string, productKey: string): MutableProduct {
|
|
|
+ return {
|
|
|
+ platform: 'jd',
|
|
|
+ productId,
|
|
|
+ productKey,
|
|
|
+ asin: productId,
|
|
|
+ role: 'own',
|
|
|
+ brand: '德玛仕',
|
|
|
+ title: cleanText(read(row, headers, 'SKU名称')),
|
|
|
+ model: '',
|
|
|
+ category1: cleanText(read(row, headers, '一级类目')),
|
|
|
+ category2: cleanText(read(row, headers, '二级类目')),
|
|
|
+ category3: cleanText(read(row, headers, '三级类目')),
|
|
|
+ source: 'excel',
|
|
|
+ relationCount: 0,
|
|
|
+ summary: emptySummary(),
|
|
|
+ trend: [],
|
|
|
+ };
|
|
|
+}
|
|
|
+
|
|
|
+function metricPoint(row: Row, headers: HeaderIndex, date: string): MetricPoint {
|
|
|
+ return {
|
|
|
+ date,
|
|
|
+ gmv: asNumber(read(row, headers, '成交金额')),
|
|
|
+ soldUnits: asNumber(read(row, headers, '成交商品件数')),
|
|
|
+ transactionOrders: asNumber(read(row, headers, '成交单量')),
|
|
|
+ transactionCustomers: asNumber(read(row, headers, '成交客户数')),
|
|
|
+ impressions: asNumber(read(row, headers, '搜索曝光次数')),
|
|
|
+ clicks: asNumber(read(row, headers, '搜索点击次数')),
|
|
|
+ views: asNumber(read(row, headers, '商品浏览量')),
|
|
|
+ visitors: asNumber(read(row, headers, '商品访客数')),
|
|
|
+ cartUnits: asNumber(read(row, headers, '加购商品件数')),
|
|
|
+ orderAmount: asNumber(read(row, headers, '下单金额')),
|
|
|
+ orderUnits: asNumber(read(row, headers, '下单商品件数')),
|
|
|
+ orderCount: asNumber(read(row, headers, '下单单量')),
|
|
|
+ refundAmount: asNumber(read(row, headers, '取消及售后退款金额')),
|
|
|
+ refundUnits: asNumber(read(row, headers, '取消及售后退款商品件数')),
|
|
|
+ refundOrders: asNumber(read(row, headers, '取消及售后退款单量')),
|
|
|
+ };
|
|
|
+}
|
|
|
+
|
|
|
+function emptySummary(): DomesticMetricSummary {
|
|
|
+ return {
|
|
|
+ gmv: 0, soldUnits: 0, transactionOrders: 0, transactionCustomers: 0,
|
|
|
+ impressions: 0, clicks: 0, views: 0, visitors: 0, cartUnits: 0,
|
|
|
+ orderAmount: 0, orderUnits: 0, orderCount: 0, refundAmount: 0,
|
|
|
+ refundUnits: 0, refundOrders: 0, conversionRate: 0, clickThroughRate: 0,
|
|
|
+ averageUnitPrice: 0, refundToGmvRate: 0,
|
|
|
+ };
|
|
|
+}
|
|
|
+
|
|
|
+function addMetric(target: DomesticMetricSummary, metric: MetricPoint): void {
|
|
|
+ target.gmv += metric.gmv;
|
|
|
+ target.soldUnits += metric.soldUnits;
|
|
|
+ target.transactionOrders += metric.transactionOrders;
|
|
|
+ target.transactionCustomers += metric.transactionCustomers;
|
|
|
+ target.impressions += metric.impressions;
|
|
|
+ target.clicks += metric.clicks;
|
|
|
+ target.views += metric.views;
|
|
|
+ target.visitors += metric.visitors;
|
|
|
+ target.cartUnits += metric.cartUnits;
|
|
|
+ target.orderAmount += metric.orderAmount;
|
|
|
+ target.orderUnits += metric.orderUnits;
|
|
|
+ target.orderCount += metric.orderCount;
|
|
|
+ target.refundAmount += metric.refundAmount;
|
|
|
+ target.refundUnits += metric.refundUnits;
|
|
|
+ target.refundOrders += metric.refundOrders;
|
|
|
+}
|
|
|
+
|
|
|
+function finalizeSummary<T extends DomesticMetricSummary>(summary: T): T {
|
|
|
+ summary.conversionRate = divide(summary.transactionCustomers, summary.visitors);
|
|
|
+ summary.clickThroughRate = divide(summary.clicks, summary.impressions);
|
|
|
+ summary.averageUnitPrice = divide(summary.gmv, summary.soldUnits);
|
|
|
+ summary.refundToGmvRate = divide(summary.refundAmount, summary.gmv);
|
|
|
+ return summary;
|
|
|
+}
|
|
|
+
|
|
|
+function findHeaderRow(rows: Row[], required: string[]): number {
|
|
|
+ const index = rows.findIndex((row) => required.every((header) => row.some((cell) => cleanText(cell) === header)));
|
|
|
+ if (index < 0) throw new Error(`Header row not found: ${required.join(', ')}`);
|
|
|
+ return index;
|
|
|
+}
|
|
|
+
|
|
|
+function indexHeaders(row: Row): HeaderIndex {
|
|
|
+ return new Map(row
|
|
|
+ .map((value, index): [string, number] => [cleanText(value), index])
|
|
|
+ .filter(([name]) => Boolean(name)));
|
|
|
+}
|
|
|
+
|
|
|
+function read(row: Row, headers: HeaderIndex, name: string): unknown {
|
|
|
+ const index = headers.get(name);
|
|
|
+ return index === undefined ? null : row[index];
|
|
|
+}
|
|
|
+
|
|
|
+function cleanText(value: unknown): string {
|
|
|
+ return value === null || value === undefined ? '' : String(value).trim();
|
|
|
+}
|
|
|
+
|
|
|
+function asId(value: unknown): string {
|
|
|
+ if (value === null || value === undefined || value === '') return '';
|
|
|
+ if (typeof value === 'number' && Number.isFinite(value)) return Math.trunc(value).toString();
|
|
|
+ return cleanText(value).replace(/\.0$/, '');
|
|
|
+}
|
|
|
+
|
|
|
+function asNumber(value: unknown): number {
|
|
|
+ if (typeof value === 'number') return Number.isFinite(value) ? value : 0;
|
|
|
+ const parsed = Number(cleanText(value).replace(/,/g, ''));
|
|
|
+ return Number.isFinite(parsed) ? parsed : 0;
|
|
|
+}
|
|
|
+
|
|
|
+function asDate(value: unknown): string {
|
|
|
+ if (value instanceof Date && !Number.isNaN(value.getTime())) return value.toISOString().slice(0, 10);
|
|
|
+ if (typeof value === 'number') {
|
|
|
+ const date = new Date(Date.UTC(1899, 11, 30) + Math.round(value * 86_400_000));
|
|
|
+ if (!Number.isNaN(date.getTime())) return date.toISOString().slice(0, 10);
|
|
|
+ }
|
|
|
+ const text = cleanText(value);
|
|
|
+ if (!text) return '';
|
|
|
+ const timestamp = Date.parse(text);
|
|
|
+ return Number.isNaN(timestamp) ? text.slice(0, 10) : new Date(timestamp).toISOString().slice(0, 10);
|
|
|
+}
|
|
|
+
|
|
|
+async function readWorkbookRows(inputPath: string): Promise<Map<string, Row[]>> {
|
|
|
+ const sheets = new Map<string, Row[]>();
|
|
|
+ const reader = new ExcelJS.stream.xlsx.WorkbookReader(inputPath, {
|
|
|
+ worksheets: 'emit',
|
|
|
+ sharedStrings: 'cache',
|
|
|
+ hyperlinks: 'ignore',
|
|
|
+ styles: 'ignore',
|
|
|
+ });
|
|
|
+ for await (const worksheet of reader) {
|
|
|
+ const sheetName = (worksheet as typeof worksheet & { name: string }).name;
|
|
|
+ if (sheetName !== METRIC_SHEET && sheetName !== RELATION_SHEET) continue;
|
|
|
+ const rows: Row[] = [];
|
|
|
+ for await (const row of worksheet) {
|
|
|
+ const values = Array.isArray(row.values) ? row.values.slice(1) : [];
|
|
|
+ rows.push(values.map((value) => cellValue(value as ExcelJS.CellValue)));
|
|
|
+ }
|
|
|
+ sheets.set(sheetName, rows);
|
|
|
+ }
|
|
|
+ return sheets;
|
|
|
+}
|
|
|
+
|
|
|
+function cellValue(value: ExcelJS.CellValue): unknown {
|
|
|
+ if (value === null || value === undefined || typeof value !== 'object' || value instanceof Date) return value;
|
|
|
+ if ('result' in value) return value.result ?? null;
|
|
|
+ if ('richText' in value) return value.richText.map((part) => part.text).join('');
|
|
|
+ if ('text' in value) return value.text;
|
|
|
+ return String(value);
|
|
|
+}
|
|
|
+
|
|
|
+function divide(numerator: number, denominator: number): number {
|
|
|
+ return denominator ? numerator / denominator : 0;
|
|
|
+}
|
gangvy