openclaw-voc-skill/scripts/deploy/upload-skills.js

#!/usr/bin/env node
/**
 * OpenClaw Skills CDN 上传脚本
 * 
 * 功能：
 *   1. 将所有 skills 上传到七牛云 CDN，生成下载 URL
 *   2. 将每个 skill 的元数据写入 Parse 数据库 Skill 表
 *   3. 生成 README.md（包含所有 skill 简介 + 下载链接）并上传到 CDN
 * 
 * 使用方法：
 *   node scripts/deploy/upload-skills.js
 * 
 * 前置条件：
 *   1. npm install qiniu
 *   2. 配置下方 QINIU_ACCESS_KEY 和 QINIU_SECRET_KEY
 * 
 * 参考：hongben-server/modules/fmode-tikhub-server/upload.sh
 */

const qiniu = require('qiniu');
const fs = require('fs');
const path = require('path');

// ============================================
// 七牛云配置
// ============================================
const QINIU_ACCESS_KEY = process.env.QINIU_AK || 'EXsA-z_n4LGmWrwC088bygcGJtAditnWQe2nH-ZE';
const QINIU_SECRET_KEY = process.env.QINIU_SK || 'HWTL92OL-Tup0-8ex8A9jnG3OaJzTxlF4OwiiDsX';
const BUCKET = 'nova-repos';
const CDN_DOMAIN = 'https://repos.fmode.cn';
const CDN_PREFIX = 'x/openclaw-skills';

// ============================================
// Parse 数据库配置
// ============================================
const PARSE_BASE = 'https://server.fmode.cn/parse';
const APP_ID = 'ncloudmaster';
const MASTER_KEY = process.env.PARSE_MASTER_KEY || 'SnkK12*&sunq2#@20!';

// Skills 源目录（部署目录）
const DEPLOYED_SKILLS_DIR = path.join(process.env.USERPROFILE || process.env.HOME, '.openclaw', 'skills');

// 要上传的技能目录列表（如果为空，则上传所有 skills）
const SKILL_FILTER = []; // 例如 ['tiktok-video-search', 'jimeng-img-v4'] 指定只上传某些

// ============================================
// 技能包分组定义
// ============================================
const SKILL_PACKAGES = {
    'voc-core': {
        label: 'VOC 核心套件',
        description: 'Amazon 品类洞察 + 评论分析 + 竞品分析 + 数据合成 + 社媒VOC + 工作坊',
        categories: ['voc', 'review-analysis', 'competitor-analysis', 'synthesis', 'social-voc', 'workshop'],
        status: 'stable',   // stable | testing | hidden
        emoji: '🔍'
    },
    'social-media': {
        label: '社媒数据套件',
        description: 'TikTok / Instagram / 抖音数据采集',
        categories: ['social-media', 'douyin'],
        status: 'stable',
        emoji: '📱'
    },
    'ai-creation': {
        label: 'AI 创作套件',
        description: '即梦AI图片视频生成 + 一键成片（测试中）',
        categories: ['jimeng', 'video-creation'],
        status: 'testing',
        emoji: '🎨'
    },
    '_other': {
        label: '其他/测试',
        description: '测试技能，不对外提供',
        categories: ['other'],
        status: 'hidden',
        emoji: '🧪'
    }
};

function getPackageForCategory(category) {
    for (const [pkgId, pkg] of Object.entries(SKILL_PACKAGES)) {
        if (pkg.categories.includes(category || 'other')) return pkgId;
    }
    return '_other';
}

// ============================================
// 七牛上传
// ============================================

function getUploadToken(bucket, key) {
    const mac = new qiniu.auth.digest.Mac(QINIU_ACCESS_KEY, QINIU_SECRET_KEY);
    const putPolicy = new qiniu.rs.PutPolicy({
        scope: `${bucket}:${key}`,
        expires: 3600
    });
    return putPolicy.uploadToken(mac);
}

function uploadFile(localFile, cdnKey) {
    return new Promise((resolve, reject) => {
        const token = getUploadToken(BUCKET, cdnKey);
        const formUploader = new qiniu.form_up.FormUploader(new qiniu.conf.Config({
            zone: qiniu.zone.Zone_z2
        }));
        const putExtra = new qiniu.form_up.PutExtra();

        formUploader.putFile(token, cdnKey, localFile, putExtra, (err, body, info) => {
            if (err) {
                reject(err);
                return;
            }
            if (info.statusCode === 200) {
                resolve({ key: body.key, url: `${CDN_DOMAIN}/${body.key}` });
            } else {
                reject(new Error(`Upload failed: ${info.statusCode} ${JSON.stringify(body)}`));
            }
        });
    });
}

function uploadBuffer(buffer, cdnKey) {
    return new Promise((resolve, reject) => {
        const token = getUploadToken(BUCKET, cdnKey);
        const formUploader = new qiniu.form_up.FormUploader(new qiniu.conf.Config({
            zone: qiniu.zone.Zone_z2
        }));
        const putExtra = new qiniu.form_up.PutExtra();

        formUploader.put(token, cdnKey, buffer, putExtra, (err, body, info) => {
            if (err) {
                reject(err);
                return;
            }
            if (info.statusCode === 200) {
                resolve({ key: body.key, url: `${CDN_DOMAIN}/${body.key}` });
            } else {
                reject(new Error(`Upload failed: ${info.statusCode} ${JSON.stringify(body)}`));
            }
        });
    });
}

// ============================================
// Parse REST API helpers
// ============================================

const PARSE_HEADERS = {
    'X-Parse-Application-Id': APP_ID,
    'X-Parse-Master-Key': MASTER_KEY,
    'Content-Type': 'application/json'
};

async function ensureSkillClass() {
    const resp = await fetch(`${PARSE_BASE}/schemas/Skill`, {
        headers: PARSE_HEADERS
    });
    if (resp.ok) return;
    // Class 不存在，创建
    const createResp = await fetch(`${PARSE_BASE}/schemas/Skill`, {
        method: 'POST',
        headers: PARSE_HEADERS,
        body: JSON.stringify({
            className: 'Skill',
            fields: {
                name:        { type: 'String' },
                displayName: { type: 'String' },
                description: { type: 'String' },
                category:    { type: 'String' },
                version:     { type: 'String' },
                author:      { type: 'String' },
                cdnUrls:     { type: 'Object' },
                cdnPrefix:   { type: 'String' }
            }
        })
    });
    const result = await createResp.json();
    if (createResp.ok) {
        console.log('  ✅ Skill 表已创建');
    } else {
        console.log('  ⚠️ 创建 Skill 表:', result.error || JSON.stringify(result));
    }
}

async function parseQuery(className, where) {
    const qs = new URLSearchParams({
        where: JSON.stringify(where),
        limit: '1'
    });
    const resp = await fetch(`${PARSE_BASE}/classes/${className}?${qs}`, {
        headers: PARSE_HEADERS
    });
    const data = await resp.json();
    return data.results || [];
}

async function parseCreate(className, body) {
    const resp = await fetch(`${PARSE_BASE}/classes/${className}`, {
        method: 'POST',
        headers: PARSE_HEADERS,
        body: JSON.stringify(body)
    });
    return resp.json();
}

async function parseUpdate(className, objectId, body) {
    const resp = await fetch(`${PARSE_BASE}/classes/${className}/${objectId}`, {
        method: 'PUT',
        headers: PARSE_HEADERS,
        body: JSON.stringify(body)
    });
    return resp.json();
}

async function upsertSkill(skillData) {
    const existing = await parseQuery('Skill', { name: skillData.name });
    if (existing.length > 0) {
        const result = await parseUpdate('Skill', existing[0].objectId, skillData);
        return { action: 'updated', objectId: existing[0].objectId, ...result };
    } else {
        const result = await parseCreate('Skill', skillData);
        return { action: 'created', ...result };
    }
}

// ============================================
// Skill metadata extraction
// ============================================

function parseSkillMdFrontmatter(filePath) {
    try {
        const content = fs.readFileSync(filePath, 'utf8');
        const match = content.match(/^---\n([\s\S]*?)\n---/);
        if (!match) return {};
        const fm = {};
        match[1].split('\n').forEach(line => {
            const idx = line.indexOf(':');
            if (idx > 0) {
                const key = line.substring(0, idx).trim();
                const val = line.substring(idx + 1).trim();
                fm[key] = val;
            }
        });
        return fm;
    } catch (e) {
        return {};
    }
}

function parseApiConfig(filePath) {
    try {
        return JSON.parse(fs.readFileSync(filePath, 'utf8'));
    } catch (e) {
        return {};
    }
}

function extractSkillMeta(skillDir, skillName) {
    const skillMdPath = path.join(skillDir, 'SKILL.md');
    const apiConfigPath = path.join(skillDir, 'api-config.json');

    const fm = parseSkillMdFrontmatter(skillMdPath);
    const cfg = fs.existsSync(apiConfigPath) ? parseApiConfig(apiConfigPath) : {};

    return {
        name: skillName,
        displayName: cfg.displayName || fm.name || skillName,
        description: cfg.description || fm.description || '',
        category: cfg.category || '',
        version: cfg.version || fm.version || '1.0.0',
        author: fm.author || 'nkkj-BrainHack'
    };
}

// ============================================
// Collect skills
// ============================================

function collectSkills() {
    const skills = [];
    const dir = DEPLOYED_SKILLS_DIR;

    if (!fs.existsSync(dir)) {
        console.error(`Skills 目录不存在: ${dir}`);
        process.exit(1);
    }

    const entries = fs.readdirSync(dir).filter(d => {
        const fullPath = path.join(dir, d);
        if (!fs.statSync(fullPath).isDirectory()) return false;
        if (SKILL_FILTER.length > 0 && !SKILL_FILTER.includes(d)) return false;
        return fs.existsSync(path.join(fullPath, 'SKILL.md'));
    });

    for (const skillName of entries) {
        const skillDir = path.join(dir, skillName);
        const files = [];

        for (const fileName of ['SKILL.md', 'api-config.json']) {
            const filePath = path.join(skillDir, fileName);
            if (fs.existsSync(filePath)) {
                files.push({
                    localPath: filePath,
                    cdnKey: `${CDN_PREFIX}/${skillName}/${fileName}`,
                    fileName
                });
            }
        }

        if (files.length > 0) {
            const meta = extractSkillMeta(skillDir, skillName);
            skills.push({ name: skillName, files, meta });
        }
    }

    return skills;
}

// ============================================
// Generate README.md
// ============================================

function generateReadme(skillEntries) {
    const now = new Date().toISOString().replace('T', ' ').substring(0, 19);

    // 按技能包分组
    const pkgMap = {};
    for (const entry of skillEntries) {
        const pkgId = getPackageForCategory(entry.meta.category);
        if (!pkgMap[pkgId]) pkgMap[pkgId] = [];
        pkgMap[pkgId].push(entry);
    }

    // 统计可见技能数
    let visibleCount = 0;
    for (const [pkgId, entries] of Object.entries(pkgMap)) {
        if (SKILL_PACKAGES[pkgId]?.status !== 'hidden') visibleCount += entries.length;
    }

    const lines = [
        '# OpenClaw Skills 技能市场',
        '',
        `> 自动生成于 ${now}，共 ${visibleCount} 个技能（分 ${Object.keys(pkgMap).filter(k => SKILL_PACKAGES[k]?.status !== 'hidden').length} 个套件）`,
        '',
    ];

    // 套件总览表
    lines.push('## 📦 套件总览');
    lines.push('');
    lines.push('| 套件 | 说明 | 技能数 | 状态 | 安装清单 |');
    lines.push('|------|------|--------|------|---------|');
    for (const [pkgId, pkg] of Object.entries(SKILL_PACKAGES)) {
        if (pkg.status === 'hidden') continue;
        const count = (pkgMap[pkgId] || []).length;
        const statusBadge = pkg.status === 'stable' ? '✅ 正式' : '🧪 测试中';
        const installUrl = `${CDN_DOMAIN}/${CDN_PREFIX}/packages/${pkgId}.json`;
        lines.push(`| ${pkg.emoji} **${pkg.label}** | ${pkg.description} | ${count} | ${statusBadge} | [${pkgId}.json](${installUrl}) |`);
    }
    lines.push('');
    lines.push('---');
    lines.push('');

    // 每个套件详细内容
    for (const [pkgId, pkg] of Object.entries(SKILL_PACKAGES)) {
        if (pkg.status === 'hidden') continue;
        const entries = pkgMap[pkgId] || [];
        if (entries.length === 0) continue;

        const statusNote = pkg.status === 'testing' ? ' ⚠️ 测试中，功能可能变更' : '';
        lines.push(`## ${pkg.emoji} ${pkg.label}${statusNote}`);
        lines.push('');
        lines.push(`> ${pkg.description}`);
        lines.push('>');
        const installUrl = `${CDN_DOMAIN}/${CDN_PREFIX}/packages/${pkgId}.json`;
        lines.push(`> 📥 一键安装清单：[\`${pkgId}.json\`](${installUrl})`);
        lines.push('');

        // 按 category 子分组
        const catMap = {};
        for (const entry of entries) {
            const cat = entry.meta.category || '其他';
            if (!catMap[cat]) catMap[cat] = [];
            catMap[cat].push(entry);
        }

        for (const [cat, catEntries] of Object.entries(catMap).sort()) {
            lines.push(`### ${cat}`);
            lines.push('');
            lines.push('| 技能名 | 说明 | 版本 | 下载 |');
            lines.push('|--------|------|------|------|');

            for (const entry of catEntries) {
                const m = entry.meta;
                const desc = m.description.length > 60 ? m.description.substring(0, 60) + '...' : m.description;
                const skillMdUrl = entry.cdnUrls['SKILL.md'] || '';
                const apiCfgUrl = entry.cdnUrls['api-config.json'] || '';
                const links = [];
                if (skillMdUrl) links.push(`[SKILL.md](${skillMdUrl})`);
                if (apiCfgUrl) links.push(`[api-config.json](${apiCfgUrl})`);
                lines.push(`| **${m.displayName}** (\`${m.name}\`) | ${desc} | ${m.version} | ${links.join(' / ')} |`);
            }
            lines.push('');
        }
    }

    lines.push('---');
    lines.push('');
    lines.push('## 快速使用');
    lines.push('');
    lines.push('每个 Skill 包含两个文件：');
    lines.push('- **SKILL.md** — 技能说明文档（功能、入参、调用链、注意事项）');
    lines.push('- **api-config.json** — API 配置（endpoint、参数 schema、错误处理）');
    lines.push('');
    lines.push('下载后放入 `~/.openclaw/skills/<skill-name>/` 即可被 OpenClaw Agent 识别和调用。');
    lines.push('');
    lines.push('### 按套件安装');
    lines.push('');
    lines.push('下载对应套件的 `.json` 安装清单，包含该套件所有技能的 CDN 下载地址。');
    lines.push('');

    return lines.join('\n');
}

// 生成每个技能包的 install.json 清单
function generatePackageManifests(skillEntries) {
    const pkgMap = {};
    for (const entry of skillEntries) {
        const pkgId = getPackageForCategory(entry.meta.category);
        if (!pkgMap[pkgId]) pkgMap[pkgId] = [];
        pkgMap[pkgId].push(entry);
    }

    const manifests = {};
    for (const [pkgId, pkg] of Object.entries(SKILL_PACKAGES)) {
        if (pkg.status === 'hidden') continue;
        const entries = pkgMap[pkgId] || [];
        if (entries.length === 0) continue;

        manifests[pkgId] = {
            package: pkgId,
            label: pkg.label,
            description: pkg.description,
            status: pkg.status,
            skillCount: entries.length,
            generatedAt: new Date().toISOString(),
            skills: entries.map(e => ({
                name: e.meta.name,
                displayName: e.meta.displayName,
                category: e.meta.category,
                version: e.meta.version,
                files: e.cdnUrls
            }))
        };
    }
    return manifests;
}

// ============================================
// Main
// ============================================

async function main() {
    console.log('==================================================');
    console.log('   OpenClaw Skills CDN 上传 + Parse 同步脚本');
    console.log('==================================================');
    console.log('');

    // 检查凭证
    if (QINIU_ACCESS_KEY === 'YOUR_ACCESS_KEY_HERE' || QINIU_SECRET_KEY === 'YOUR_SECRET_KEY_HERE') {
        console.error('❌ 请先配置七牛云凭证！');
        console.error('   方式1: 编辑脚本中的 QINIU_ACCESS_KEY / QINIU_SECRET_KEY');
        console.error('   方式2: 设置环境变量 QINIU_AK / QINIU_SK');
        process.exit(1);
    }

    // 收集技能
    const skills = collectSkills();
    console.log(`📦 找到 ${skills.length} 个技能待上传`);
    console.log(`📂 源目录: ${DEPLOYED_SKILLS_DIR}`);
    console.log(`☁️  Bucket: ${BUCKET}`);
    console.log(`🔗 CDN前缀: ${CDN_PREFIX}`);
    console.log('');

    let totalFiles = 0;
    skills.forEach(s => { totalFiles += s.files.length; });
    console.log(`📄 总计 ${totalFiles} 个文件`);
    console.log('');

    // ── Step 1: 上传文件到七牛 ──
    console.log('── Step 1: 上传文件到七牛云 ──');
    let successCount = 0;
    let failCount = 0;
    const results = [];

    for (const skill of skills) {
        process.stdout.write(`  📦 ${skill.name} ... `);
        const skillResults = [];
        const cdnUrls = {};

        for (const file of skill.files) {
            try {
                const result = await uploadFile(file.localPath, file.cdnKey);
                skillResults.push({ ...result, fileName: file.fileName, status: 'ok' });
                cdnUrls[file.fileName] = result.url;
                successCount++;
            } catch (err) {
                skillResults.push({ fileName: file.fileName, status: 'fail', error: err.message });
                failCount++;
            }
        }

        const allOk = skillResults.every(r => r.status === 'ok');
        console.log(allOk ? '✅' : '⚠️');
        results.push({ name: skill.name, files: skillResults, meta: skill.meta, cdnUrls });
    }

    console.log('');
    console.log(`   上传完成！成功: ${successCount}  失败: ${failCount}`);
    console.log('');

    // ── Step 2: 写入 Parse Skill 表 ──
    console.log('── Step 2: 同步到 Parse 数据库 (Skill 表) ──');
    await ensureSkillClass();
    let dbSuccess = 0;
    let dbFail = 0;

    for (const entry of results) {
        if (Object.keys(entry.cdnUrls).length === 0) continue;
        process.stdout.write(`  💾 ${entry.name} ... `);
        try {
            // Parse 不允许嵌套 key 含 '.'，将文件名中的 '.' 替换为 '_'
            const safeCdnUrls = {};
            for (const [k, v] of Object.entries(entry.cdnUrls)) {
                safeCdnUrls[k.replace(/\./g, '_')] = v;
            }
            const skillData = {
                name: entry.meta.name,
                displayName: entry.meta.displayName,
                description: entry.meta.description,
                category: entry.meta.category,
                version: entry.meta.version,
                author: entry.meta.author,
                cdnUrls: safeCdnUrls,
                cdnPrefix: `${CDN_DOMAIN}/${CDN_PREFIX}/${entry.name}`
            };
            const result = await upsertSkill(skillData);
            console.log(`${result.action === 'created' ? '🆕 创建' : '🔄 更新'} (${result.objectId})`);
            dbSuccess++;
        } catch (err) {
            console.log(`❌ ${err.message}`);
            dbFail++;
        }
    }

    console.log('');
    console.log(`   数据库同步完成！成功: ${dbSuccess}  失败: ${dbFail}`);
    console.log('');

    // ── Step 3: 生成 README.md + 技能包清单 并上传 ──
    console.log('── Step 3: 生成 README.md + 技能包清单 并上传 ──');
    let readmeUrl = '';
    try {
        const readme = generateReadme(results);
        const readmeKey = `${CDN_PREFIX}/README.md`;
        const readmeResult = await uploadBuffer(Buffer.from(readme, 'utf8'), readmeKey);
        readmeUrl = readmeResult.url;
        console.log(`  ✅ README.md 已上传: ${readmeUrl}`);
    } catch (err) {
        console.log(`  ❌ README.md 上传失败: ${err.message}`);
    }

    // 上传每个技能包的 install.json
    const manifests = generatePackageManifests(results);
    const packageUrls = {};
    for (const [pkgId, manifest] of Object.entries(manifests)) {
        try {
            const key = `${CDN_PREFIX}/packages/${pkgId}.json`;
            const buf = Buffer.from(JSON.stringify(manifest, null, 2), 'utf8');
            const res = await uploadBuffer(buf, key);
            packageUrls[pkgId] = res.url;
            const pkg = SKILL_PACKAGES[pkgId];
            const statusTag = pkg.status === 'testing' ? ' 🧪' : ' ✅';
            console.log(`  ${statusTag} ${pkg.label} (${manifest.skillCount}个技能): ${res.url}`);
        } catch (err) {
            console.log(`  ❌ ${pkgId}.json 上传失败: ${err.message}`);
        }
    }
    console.log('');

    // ── 输出 URL 汇总 ──
    console.log('📋 技能包下载地址：');
    console.log('');
    for (const [pkgId, url] of Object.entries(packageUrls)) {
        const pkg = SKILL_PACKAGES[pkgId];
        const statusTag = pkg.status === 'testing' ? '🧪' : '✅';
        console.log(`  ${statusTag} ${pkg.label}`);
        console.log(`    👉 ${url}`);
    }
    console.log('');

    // 保存 URL 列表到文件（按技能包分组）
    const urlMap = { _packages: packageUrls };
    if (readmeUrl) urlMap['_README'] = readmeUrl;
    for (const entry of results) {
        if (Object.keys(entry.cdnUrls).length > 0) {
            urlMap[entry.name] = entry.cdnUrls;
        }
    }
    const outputPath = path.resolve(__dirname, '..', 'skills-cdn-urls.json');
    fs.writeFileSync(outputPath, JSON.stringify(urlMap, null, 2), 'utf8');
    console.log(`💾 URL 列表已保存到: ${outputPath}`);
    console.log('');

    console.log('==================================================');
    console.log('📌 技能市场索引：');
    console.log(`  README:    ${CDN_DOMAIN}/${CDN_PREFIX}/README.md`);
    for (const [pkgId, url] of Object.entries(packageUrls)) {
        const pkg = SKILL_PACKAGES[pkgId];
        console.log(`  ${pkg.emoji} ${pkg.label}: ${url}`);
    }
    console.log('==================================================');
}

main().catch(err => {
    console.error('❌ 上传出错:', err);
    process.exit(1);
});