|
@@ -42,13 +42,14 @@ import { parseArgs } from 'node:util';
|
|
|
import {
|
|
import {
|
|
|
TENANT_DEFAULT, SKILL_ROOT, buildIdempotencyKey, deriveAssetUrl, ensureDir, firstText, imageUrlsFrom,
|
|
TENANT_DEFAULT, SKILL_ROOT, buildIdempotencyKey, deriveAssetUrl, ensureDir, firstText, imageUrlsFrom,
|
|
|
learnAliases, learnTags, loadDictionary, newRunId, normalizeMaterialAsset, outputsDir,
|
|
learnAliases, learnTags, loadDictionary, newRunId, normalizeMaterialAsset, outputsDir,
|
|
|
- readJson, reorderAssets, resolvePython, saveDictionary, sortAssets, truncate, uniq, writeJson,
|
|
|
|
|
|
|
+ readJson, reorderAssets, resolvePython, saveDictionary, sha1, sortAssets, truncate, uniq, writeJson,
|
|
|
} from './lib.mjs';
|
|
} from './lib.mjs';
|
|
|
import { readImageInfo, IMAGE_EXTENSIONS, VIDEO_EXTENSIONS, DOC_EXTENSIONS, DECK_EXTENSIONS, TEXT_EXTENSIONS } from './image-size.mjs';
|
|
import { readImageInfo, IMAGE_EXTENSIONS, VIDEO_EXTENSIONS, DOC_EXTENSIONS, DECK_EXTENSIONS, TEXT_EXTENSIONS } from './image-size.mjs';
|
|
|
import { applyGroupOverrides, groupMaterials, DEFAULT_ORDERING_RULE } from './grouping.mjs';
|
|
import { applyGroupOverrides, groupMaterials, DEFAULT_ORDERING_RULE } from './grouping.mjs';
|
|
|
import { classifyCase } from './classify.mjs';
|
|
import { classifyCase } from './classify.mjs';
|
|
|
import { processVideo } from './listen-bridge.mjs';
|
|
import { processVideo } from './listen-bridge.mjs';
|
|
|
import { analyzeImages } from './vision-bridge.mjs';
|
|
import { analyzeImages } from './vision-bridge.mjs';
|
|
|
|
|
+import { uploadMaterial, StorageUnavailableError } from './storage-bridge.mjs';
|
|
|
import { CloudError, submitCase } from './cloud-client.mjs';
|
|
import { CloudError, submitCase } from './cloud-client.mjs';
|
|
|
import { submitViaParse, probeParse } from './parse-direct.mjs';
|
|
import { submitViaParse, probeParse } from './parse-direct.mjs';
|
|
|
|
|
|
|
@@ -314,6 +315,71 @@ export async function runPlan(options = {}) {
|
|
|
// ingest:plan + 图片理解 + 分类打标 → 案例包
|
|
// ingest:plan + 图片理解 + 分类打标 → 案例包
|
|
|
// ---------------------------------------------------------------------------
|
|
// ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
+/**
|
|
|
|
|
+ * 解析单个素材的公网 URL(materialAssets[].url 的唯一来源)。
|
|
|
|
|
+ *
|
|
|
|
|
+ * 优先级(任务书硬要求):
|
|
|
|
|
+ * ① item.url 已存在(文档自带的远程 URL)→ 原样用,**不触发上传**;
|
|
|
|
|
+ * ② 本地文件且 upload 开 → 真上传(fmode-storage 第 0 级免密钥通道)拿公网 URL;
|
|
|
|
|
+ * ③ --asset-base-url 派生(保留旧行为,显式回退);
|
|
|
|
|
+ * ④ 都没有 → 空字符串(由调用方补 warning)。
|
|
|
|
|
+ *
|
|
|
|
|
+ * 本函数**不抛错、不打印**:上传失败以 `error` 返回,由调用方降级并汇总告警,
|
|
|
|
|
+ * 保证「单张失败不拖垮整个 ingest」。
|
|
|
|
|
+ *
|
|
|
|
|
+ * @returns {Promise<{url:string, upload:object|null, fallback:boolean, error:Error|null, reason:string}>}
|
|
|
|
|
+ */
|
|
|
|
|
+
|
|
|
|
|
+// 文件内容指纹:避免两个不同文件落成同一个 object key 互相覆盖(读不到则退化为 0)。
|
|
|
|
|
+function contentFingerprint(filePath) {
|
|
|
|
|
+ try {
|
|
|
|
|
+ const buf = fs.readFileSync(filePath);
|
|
|
|
|
+ return `${buf.length}-${sha1(buf.toString('base64')).slice(0, 12)}`;
|
|
|
|
|
+ } catch {
|
|
|
|
|
+ return '0';
|
|
|
|
|
+ }
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+export async function resolveAssetUrl(item, options = {}) {
|
|
|
|
|
+ const remote = item && item.url ? String(item.url) : '';
|
|
|
|
|
+ if (remote) return { url: remote, upload: null, fallback: false, error: null, reason: 'remote-url' };
|
|
|
|
|
+
|
|
|
|
|
+ const localPath = (item && item.localPath) || '';
|
|
|
|
|
+ let uploadError = null;
|
|
|
|
|
+ // key 区分度:优先 显式 slug / 来源引用 / 标题;都没有时用「文件内容指纹」兜底。
|
|
|
|
|
+ // 否则两个不同案例都会落成 case-1.png → 同日撞 key 互相覆盖(真实事故:两张不同图同 key)。
|
|
|
|
|
+ let slugSeed = options.caseSlug;
|
|
|
|
|
+ if (!slugSeed) {
|
|
|
|
|
+ slugSeed = localPath ? `c${contentFingerprint(localPath)}` : 'case';
|
|
|
|
|
+ }
|
|
|
|
|
+ if (localPath && options.upload) {
|
|
|
|
|
+ try {
|
|
|
|
|
+ const upload = await uploadMaterial(localPath, {
|
|
|
|
|
+ tenantId: options.tenantId,
|
|
|
|
|
+ caseSlug: slugSeed,
|
|
|
|
|
+ order: options.order,
|
|
|
|
|
+ });
|
|
|
|
|
+ if (upload && upload.url) {
|
|
|
|
|
+ return { url: upload.url, upload, fallback: false, error: null, reason: 'uploaded' };
|
|
|
|
|
+ }
|
|
|
|
|
+ uploadError = new Error('上传回执未给出公网 URL');
|
|
|
|
|
+ } catch (error) {
|
|
|
|
|
+ // 上传失败:**记下原因**并继续走回退链,绝不吞掉(调用方要据此告警)
|
|
|
|
|
+ uploadError = error;
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ const derived = deriveAssetUrl(localPath, options.assetBaseUrl);
|
|
|
|
|
+ if (derived) return { url: derived, upload: null, fallback: true, error: uploadError, reason: 'derived' };
|
|
|
|
|
+ return {
|
|
|
|
|
+ url: '',
|
|
|
|
|
+ upload: null,
|
|
|
|
|
+ fallback: false,
|
|
|
|
|
+ error: uploadError,
|
|
|
|
|
+ reason: localPath ? 'no-url' : 'no-source',
|
|
|
|
|
+ };
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
export async function runIngest(options = {}) {
|
|
export async function runIngest(options = {}) {
|
|
|
const plan = await runPlan(options);
|
|
const plan = await runPlan(options);
|
|
|
const outDir = options.outDir;
|
|
const outDir = options.outDir;
|
|
@@ -405,6 +471,11 @@ export async function runIngest(options = {}) {
|
|
|
const groupInfoList = [];
|
|
const groupInfoList = [];
|
|
|
const assetProblems = [];
|
|
const assetProblems = [];
|
|
|
let missingUrlCount = 0;
|
|
let missingUrlCount = 0;
|
|
|
|
|
+ let uploadFailedCount = 0;
|
|
|
|
|
+ let storageUnavailable = false;
|
|
|
|
|
+ const fallbackItems = [];
|
|
|
|
|
+ // 上传真正发生才需要 caseSlug;全走本地文件时没有标题也不影响。
|
|
|
|
|
+ const caseSlugSeed = firstText(options.sourceRef, options.title);
|
|
|
|
|
|
|
|
for (const group of enrichedGroups) {
|
|
for (const group of enrichedGroups) {
|
|
|
groupInfoList.push({
|
|
groupInfoList.push({
|
|
@@ -414,14 +485,36 @@ export async function runIngest(options = {}) {
|
|
|
orderingRule: group.orderingRule,
|
|
orderingRule: group.orderingRule,
|
|
|
});
|
|
});
|
|
|
for (const item of sortAssets(group.items)) {
|
|
for (const item of sortAssets(group.items)) {
|
|
|
- // 本地文件没有公网 URL 时,按 --asset-base-url 派生;没给基础地址则留空并警告
|
|
|
|
|
- const derivedUrl = item.url || deriveAssetUrl(item.localPath, options.assetBaseUrl);
|
|
|
|
|
- if (!derivedUrl && item.localPath) missingUrlCount += 1;
|
|
|
|
|
|
|
+ // URL 优先级:① 文档自带远程 url → ② --upload 真上传 → ③ --asset-base-url 派生 → ④ 留空 + warning
|
|
|
|
|
+ const resolved = await resolveAssetUrl(item, {
|
|
|
|
|
+ // 默认开(与 CLI 一致):只有显式 upload:false / --no-upload 才关
|
|
|
|
|
+ upload: options.upload !== false,
|
|
|
|
|
+ tenantId: plan.tenantId,
|
|
|
|
|
+ // 这里**不要**用 'case' 兜底:留空让 resolveAssetUrl 用文件内容指纹保证 key 区分度
|
|
|
|
|
+ caseSlug: item.slug || caseSlugSeed || '',
|
|
|
|
|
+ order: materialAssets.length + 1,
|
|
|
|
|
+ assetBaseUrl: options.assetBaseUrl,
|
|
|
|
|
+ });
|
|
|
|
|
+ const assetUrl = resolved.url;
|
|
|
|
|
+ const upload = resolved.upload;
|
|
|
|
|
+ const uploadError = resolved.error;
|
|
|
|
|
+ const usedFallback = resolved.fallback;
|
|
|
|
|
+ if (uploadError) {
|
|
|
|
|
+ // 上传失败绝不能拖垮整个 ingest:单张降级、明确告警,其余素材照常
|
|
|
|
|
+ uploadFailedCount += 1;
|
|
|
|
|
+ if (uploadError instanceof StorageUnavailableError) storageUnavailable = true;
|
|
|
|
|
+ const reason = uploadError && uploadError.message ? uploadError.message : String(uploadError);
|
|
|
|
|
+ warnings.push(`上传失败(${path.basename(item.localPath)}):${reason};该素材将无公网 URL`);
|
|
|
|
|
+ }
|
|
|
|
|
+ if (!assetUrl && item.localPath) {
|
|
|
|
|
+ missingUrlCount += 1;
|
|
|
|
|
+ if (!uploadError) fallbackItems.push(path.basename(item.localPath));
|
|
|
|
|
+ }
|
|
|
const { asset, problems } = normalizeMaterialAsset({
|
|
const { asset, problems } = normalizeMaterialAsset({
|
|
|
order: item.order,
|
|
order: item.order,
|
|
|
groupId: group.groupId,
|
|
groupId: group.groupId,
|
|
|
kind: item.kind,
|
|
kind: item.kind,
|
|
|
- url: derivedUrl,
|
|
|
|
|
|
|
+ url: assetUrl,
|
|
|
localPath: item.localPath || '',
|
|
localPath: item.localPath || '',
|
|
|
label: item.label || '',
|
|
label: item.label || '',
|
|
|
ocrText: item.ocrText || '',
|
|
ocrText: item.ocrText || '',
|
|
@@ -430,12 +523,23 @@ export async function runIngest(options = {}) {
|
|
|
role: item.role || 'material',
|
|
role: item.role || 'material',
|
|
|
}, materialAssets.length);
|
|
}, materialAssets.length);
|
|
|
assetProblems.push(...problems);
|
|
assetProblems.push(...problems);
|
|
|
- materialAssets.push(asset);
|
|
|
|
|
|
|
+ materialAssets.push({
|
|
|
|
|
+ ...asset,
|
|
|
|
|
+ ...(upload ? { uploadKey: upload.key, uploadBytes: upload.bytes, uploadVia: 'cloud-function' } : {}),
|
|
|
|
|
+ isFallbackUrl: usedFallback || undefined,
|
|
|
|
|
+ });
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
+ if (uploadFailedCount) {
|
|
|
|
|
+ warnings.push(`${uploadFailedCount} 张上传失败:这些素材的 url 已降级(本地派生前缀/留空),检索侧拿不到它们的图。`
|
|
|
|
|
+ + (storageUnavailable ? '(fmode-storage 不可用:安装兄弟技能或改用 --no-upload + --asset-base-url)' : ''));
|
|
|
|
|
+ }
|
|
|
for (const problem of assetProblems) warnings.push(problem);
|
|
for (const problem of assetProblems) warnings.push(problem);
|
|
|
if (missingUrlCount) {
|
|
if (missingUrlCount) {
|
|
|
- warnings.push(`${missingUrlCount} 个素材只有本地路径、没有 URL:用 --asset-base-url <CDN前缀> 派生,或由归档方补齐 url 后再 submit(检索侧只看 url)。`);
|
|
|
|
|
|
|
+ const detail = options.upload === false
|
|
|
|
|
+ ? '(--no-upload 已关闭上传;可改用 --asset-base-url <CDN前缀> 派生)'
|
|
|
|
|
+ : `已尝试上传但未拿到公网地址(${uniq(fallbackItems).slice(0, 5).join('、') || '见上方上传失败原因'})。请确认 fmode-storage 可用(免密钥通道)或改用 --asset-base-url <CDN前缀> 派生(检索侧只看 url)`;
|
|
|
|
|
+ warnings.push(`${missingUrlCount} 个素材只有本地路径、没有 URL:${detail}。`);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// 全局重编号:materialAssets 已按「分组顺序 → 组内 order」构建,
|
|
// 全局重编号:materialAssets 已按「分组顺序 → 组内 order」构建,
|
|
@@ -896,11 +1000,13 @@ function parseCli(argv) {
|
|
|
args: rest,
|
|
args: rest,
|
|
|
options: {
|
|
options: {
|
|
|
input: { type: 'string', multiple: true, default: [] },
|
|
input: { type: 'string', multiple: true, default: [] },
|
|
|
|
|
+ files: { type: 'string', multiple: true, default: [] },
|
|
|
image: { type: 'string', multiple: true, default: [] },
|
|
image: { type: 'string', multiple: true, default: [] },
|
|
|
doc: { type: 'string', multiple: true, default: [] },
|
|
doc: { type: 'string', multiple: true, default: [] },
|
|
|
deck: { type: 'string', multiple: true, default: [] },
|
|
deck: { type: 'string', multiple: true, default: [] },
|
|
|
video: { type: 'string', multiple: true, default: [] },
|
|
video: { type: 'string', multiple: true, default: [] },
|
|
|
'out-dir': { type: 'string' },
|
|
'out-dir': { type: 'string' },
|
|
|
|
|
+ 'out': { type: 'string' },
|
|
|
'source-type': { type: 'string' },
|
|
'source-type': { type: 'string' },
|
|
|
'source-ref': { type: 'string' },
|
|
'source-ref': { type: 'string' },
|
|
|
authorization: { type: 'string', default: 'pending' },
|
|
authorization: { type: 'string', default: 'pending' },
|
|
@@ -918,6 +1024,8 @@ function parseCli(argv) {
|
|
|
objection: { type: 'string', multiple: true, default: [] },
|
|
objection: { type: 'string', multiple: true, default: [] },
|
|
|
'idempotency-key': { type: 'string' },
|
|
'idempotency-key': { type: 'string' },
|
|
|
'asset-base-url': { type: 'string' },
|
|
'asset-base-url': { type: 'string' },
|
|
|
|
|
+ upload: { type: 'boolean', default: true },
|
|
|
|
|
+ 'no-upload': { type: 'boolean', default: false },
|
|
|
'submitted-by': { type: 'string' },
|
|
'submitted-by': { type: 'string' },
|
|
|
'vision-model': { type: 'string' },
|
|
'vision-model': { type: 'string' },
|
|
|
'skip-vision': { type: 'boolean', default: false },
|
|
'skip-vision': { type: 'boolean', default: false },
|
|
@@ -946,6 +1054,7 @@ function parseCli(argv) {
|
|
|
function collectInputs(values, positionals) {
|
|
function collectInputs(values, positionals) {
|
|
|
return [
|
|
return [
|
|
|
...values.input,
|
|
...values.input,
|
|
|
|
|
+ ...values.files,
|
|
|
...values.image,
|
|
...values.image,
|
|
|
...values.doc,
|
|
...values.doc,
|
|
|
...values.deck,
|
|
...values.deck,
|
|
@@ -1005,7 +1114,7 @@ async function cliMain(argv) {
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// plan / ingest
|
|
// plan / ingest
|
|
|
- const outDir = values['out-dir']
|
|
|
|
|
|
|
+ const outDir = values['out-dir'] || values.out
|
|
|
|| path.join(outputsDir(), `${values['source-ref'] ? `${slug(values['source-ref'])}-` : ''}${newRunId()}`);
|
|
|| path.join(outputsDir(), `${values['source-ref'] ? `${slug(values['source-ref'])}-` : ''}${newRunId()}`);
|
|
|
|
|
|
|
|
const options = {
|
|
const options = {
|
|
@@ -1020,6 +1129,8 @@ async function cliMain(argv) {
|
|
|
usageSuggestion: values['usage-suggestion'],
|
|
usageSuggestion: values['usage-suggestion'],
|
|
|
idempotencyKey: values['idempotency-key'],
|
|
idempotencyKey: values['idempotency-key'],
|
|
|
assetBaseUrl: values['asset-base-url'] || process.env.CASE_ASSET_BASE_URL || '',
|
|
assetBaseUrl: values['asset-base-url'] || process.env.CASE_ASSET_BASE_URL || '',
|
|
|
|
|
+ // 默认开:真正上传拿公网 URL;--no-upload 关闭(离线自测用),此时回落到 --asset-base-url 派生
|
|
|
|
|
+ upload: values['no-upload'] ? false : values.upload !== false,
|
|
|
submittedBy: values['submitted-by'],
|
|
submittedBy: values['submitted-by'],
|
|
|
visionModel: values['vision-model'],
|
|
visionModel: values['vision-model'],
|
|
|
skipVision: values['skip-vision'],
|
|
skipVision: values['skip-vision'],
|
|
@@ -1117,6 +1228,7 @@ const HELP_TEXT = `skill-case-get —— 案例采集 / 拆解 / 归档
|
|
|
--deck deck.pptx PPT
|
|
--deck deck.pptx PPT
|
|
|
--video clip.mp4 视频(抽帧 + 转写)
|
|
--video clip.mp4 视频(抽帧 + 转写)
|
|
|
--input <file|dir> 自动按扩展名分流
|
|
--input <file|dir> 自动按扩展名分流
|
|
|
|
|
+ --files <file|dir> 同 --input(别名)
|
|
|
|
|
|
|
|
采集选项:
|
|
采集选项:
|
|
|
--authorization authorized|pending|denied 授权状态(硬门:仅 authorized 可入库,默认 pending)
|
|
--authorization authorized|pending|denied 授权状态(硬门:仅 authorized 可入库,默认 pending)
|
|
@@ -1126,9 +1238,13 @@ const HELP_TEXT = `skill-case-get —— 案例采集 / 拆解 / 归档
|
|
|
--school-canonical / --product-line / --major / --stage / --country
|
|
--school-canonical / --product-line / --major / --stage / --country
|
|
|
--highlight / --scenario / --objection 显式补标签(可重复)
|
|
--highlight / --scenario / --objection 显式补标签(可重复)
|
|
|
--idempotency-key <key> 幂等键(默认按内容指纹生成)
|
|
--idempotency-key <key> 幂等键(默认按内容指纹生成)
|
|
|
- --asset-base-url <base> 本地素材 URL 前缀(CDN / Parse Files),用于派生 materialAssets[].url
|
|
|
|
|
- (也可用环境变量 CASE_ASSET_BASE_URL)
|
|
|
|
|
|
|
+ --asset-base-url <base> 本地素材 URL 前缀(CDN / Parse Files / 对象存储目录)**回退用**
|
|
|
|
|
+ (或环境变量 CASE_ASSET_BASE_URL)
|
|
|
|
|
+ --upload 真正上传本地素材到对象存储,拿公网 URL(默认开)
|
|
|
|
|
+ --no-upload 关闭上传;url 只按 --asset-base-url 派生(离线自测用)
|
|
|
|
|
+ object key 规范:case-library/<YYYYMMDD>/<caseSlug>-<order>.<ext>
|
|
|
--out-dir <dir> 产物目录(默认 <cwd>/case-get-outputs/<runId>)
|
|
--out-dir <dir> 产物目录(默认 <cwd>/case-get-outputs/<runId>)
|
|
|
|
|
+ --out <dir> 同 --out-dir(别名)
|
|
|
--skip-vision / --skip-video / --no-learn 关闭图片理解 / 视频处理 / 字典自进化
|
|
--skip-vision / --skip-video / --no-learn 关闭图片理解 / 视频处理 / 字典自进化
|
|
|
--submitted-by <userId> 提交人
|
|
--submitted-by <userId> 提交人
|
|
|
|
|
|