|
|
@@ -1,12 +1,6 @@
|
|
|
-const fs = require('fs');
|
|
|
-const path = require('path');
|
|
|
+'use strict';
|
|
|
|
|
|
-const DEFAULT_TIMEOUT_MS = 8000;
|
|
|
-const DEFAULT_LIMIT = 10;
|
|
|
-
|
|
|
-function ensureDir(dirPath) {
|
|
|
- fs.mkdirSync(dirPath, { recursive: true });
|
|
|
-}
|
|
|
+const { processMedia, replaceMediaUrls } = require('../media/media-pipeline');
|
|
|
|
|
|
function safeName(value, fallback = 'asset') {
|
|
|
const cleaned = String(value || '')
|
|
|
@@ -17,142 +11,37 @@ function safeName(value, fallback = 'asset') {
|
|
|
return cleaned || fallback;
|
|
|
}
|
|
|
|
|
|
-function extFromContentType(contentType = '') {
|
|
|
- const value = String(contentType || '').split(';')[0].trim().toLowerCase();
|
|
|
- if (value === 'image/jpeg' || value === 'image/jpg') return '.jpg';
|
|
|
- if (value === 'image/png') return '.png';
|
|
|
- if (value === 'image/webp') return '.webp';
|
|
|
- if (value === 'image/gif') return '.gif';
|
|
|
- if (value === 'image/svg+xml') return '.svg';
|
|
|
- return '';
|
|
|
-}
|
|
|
-
|
|
|
-function extFromUrl(url = '') {
|
|
|
- try {
|
|
|
- const parsed = new URL(String(url));
|
|
|
- const ext = path.extname(parsed.pathname).toLowerCase();
|
|
|
- if (['.jpg', '.jpeg', '.png', '.webp', '.gif', '.svg'].includes(ext)) return ext === '.jpeg' ? '.jpg' : ext;
|
|
|
- } catch {
|
|
|
- // Ignore invalid URLs; caller will decide whether to skip.
|
|
|
- }
|
|
|
- return '';
|
|
|
-}
|
|
|
-
|
|
|
function parseDataUrl(value = '') {
|
|
|
const match = String(value).match(/^data:([^,]*),(.*)$/s);
|
|
|
if (!match) return null;
|
|
|
const meta = match[1] || '';
|
|
|
- const [rawContentType] = meta.split(';');
|
|
|
- const contentType = rawContentType || 'application/octet-stream';
|
|
|
- const isBase64 = meta.split(';').some(part => part.toLowerCase() === 'base64');
|
|
|
+ const contentType = meta.split(';')[0] || 'application/octet-stream';
|
|
|
const body = match[2] || '';
|
|
|
return {
|
|
|
contentType,
|
|
|
- buffer: isBase64 ? Buffer.from(body, 'base64') : Buffer.from(decodeURIComponent(body), 'utf8')
|
|
|
+ buffer: meta.split(';').some(part => part.toLowerCase() === 'base64')
|
|
|
+ ? Buffer.from(body, 'base64')
|
|
|
+ : Buffer.from(decodeURIComponent(body), 'utf8')
|
|
|
};
|
|
|
}
|
|
|
|
|
|
-async function fetchWithTimeout(url, timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
|
- const controller = typeof AbortController !== 'undefined' ? new AbortController() : null;
|
|
|
- const timeout = controller ? setTimeout(() => controller.abort(), timeoutMs) : null;
|
|
|
- try {
|
|
|
- const response = await fetch(url, {
|
|
|
- signal: controller ? controller.signal : undefined,
|
|
|
- headers: {
|
|
|
- Accept: 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
|
|
|
- Referer: 'https://www.xiaohongshu.com/',
|
|
|
- Origin: 'https://www.xiaohongshu.com',
|
|
|
- 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36'
|
|
|
- }
|
|
|
- });
|
|
|
- const arrayBuffer = await response.arrayBuffer();
|
|
|
- return {
|
|
|
- ok: response.ok,
|
|
|
- status: response.status,
|
|
|
- contentType: response.headers.get('content-type') || '',
|
|
|
- buffer: Buffer.from(arrayBuffer)
|
|
|
- };
|
|
|
- } finally {
|
|
|
- if (timeout) clearTimeout(timeout);
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-async function readImageSource(url, timeoutMs) {
|
|
|
- if (!url) return null;
|
|
|
- const data = parseDataUrl(url);
|
|
|
- if (data) return data;
|
|
|
- if (!/^https?:\/\//i.test(String(url))) return null;
|
|
|
- const fetched = await fetchWithTimeout(url, timeoutMs);
|
|
|
- if (!fetched.ok) {
|
|
|
- const error = new Error(`image request failed with HTTP ${fetched.status}`);
|
|
|
- error.httpStatus = fetched.status;
|
|
|
- throw error;
|
|
|
- }
|
|
|
+// Compatibility adapter for callers of the former cover-only helper. All
|
|
|
+// callers now share the all-media, local-only manifest pipeline.
|
|
|
+async function cacheEvidenceAssets({ notes = [], outputDir, usageTraceId, platform = 'xiaohongshu', timeoutMs } = {}) {
|
|
|
+ const result = await processMedia({ notes }, {
|
|
|
+ outputDir,
|
|
|
+ usageTraceId,
|
|
|
+ platform,
|
|
|
+ ...(timeoutMs ? { timeoutMs } : {})
|
|
|
+ });
|
|
|
+ const normalized = replaceMediaUrls(notes, result.replacements);
|
|
|
+ normalized.forEach((note, index) => {
|
|
|
+ if (notes[index] && note && typeof note === 'object') Object.assign(notes[index], note);
|
|
|
+ });
|
|
|
return {
|
|
|
- contentType: fetched.contentType,
|
|
|
- buffer: fetched.buffer
|
|
|
+ ...result,
|
|
|
+ files: [...result.files, result.manifestFile].filter(Boolean)
|
|
|
};
|
|
|
}
|
|
|
|
|
|
-function firstImageUrl(note = {}) {
|
|
|
- return note.coverUrl || (Array.isArray(note.imageUrls) ? note.imageUrls[0] : '') || '';
|
|
|
-}
|
|
|
-
|
|
|
-function normalizeLimit(value) {
|
|
|
- if (value === undefined || value === null || value === '') return DEFAULT_LIMIT;
|
|
|
- const parsed = Number(value);
|
|
|
- if (!Number.isFinite(parsed)) return DEFAULT_LIMIT;
|
|
|
- return Math.max(0, parsed);
|
|
|
-}
|
|
|
-
|
|
|
-async function cacheEvidenceAssets({ notes = [], outputDir, limit = DEFAULT_LIMIT, timeoutMs = DEFAULT_TIMEOUT_MS } = {}) {
|
|
|
- const warnings = [];
|
|
|
- const files = [];
|
|
|
- if (!outputDir || !Array.isArray(notes) || !notes.length) {
|
|
|
- return { warnings, files };
|
|
|
- }
|
|
|
-
|
|
|
- const assetsDir = path.join(outputDir, 'assets');
|
|
|
- ensureDir(assetsDir);
|
|
|
- const maxItems = normalizeLimit(limit);
|
|
|
-
|
|
|
- for (const note of notes.slice(0, maxItems)) {
|
|
|
- const imageUrl = firstImageUrl(note);
|
|
|
- if (!imageUrl) continue;
|
|
|
- const noteId = safeName(note.id || note.noteId || note.title, 'note');
|
|
|
- try {
|
|
|
- const source = await readImageSource(imageUrl, timeoutMs);
|
|
|
- if (!source || !source.buffer || !source.buffer.length) {
|
|
|
- warnings.push({
|
|
|
- stage: 'asset_cache',
|
|
|
- noteId: note.id || '',
|
|
|
- message: '未识别到可缓存的图片地址'
|
|
|
- });
|
|
|
- continue;
|
|
|
- }
|
|
|
- const ext = extFromContentType(source.contentType) || extFromUrl(imageUrl) || '.jpg';
|
|
|
- const filename = `${noteId}-cover${ext}`;
|
|
|
- const absolutePath = path.join(assetsDir, filename);
|
|
|
- fs.writeFileSync(absolutePath, source.buffer);
|
|
|
- const relativePath = path.posix.join('assets', filename);
|
|
|
- note.localCoverPath = relativePath;
|
|
|
- note.localCoverAbsolutePath = absolutePath;
|
|
|
- note.coverOriginalUrl = note.coverOriginalUrl || imageUrl;
|
|
|
- files.push(absolutePath);
|
|
|
- } catch (error) {
|
|
|
- warnings.push({
|
|
|
- stage: 'asset_cache',
|
|
|
- noteId: note.id || '',
|
|
|
- message: error.message || String(error)
|
|
|
- });
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- return { warnings, files };
|
|
|
-}
|
|
|
-
|
|
|
-module.exports = {
|
|
|
- cacheEvidenceAssets,
|
|
|
- parseDataUrl,
|
|
|
- safeName
|
|
|
-};
|
|
|
+module.exports = { cacheEvidenceAssets, parseDataUrl, safeName };
|