openclaw-voc-skill/scripts/deploy/upload-skills.js

#!/usr/bin/env node
/**
 * OpenClaw Skills CDN 上传脚本
 * 
 * 功能：
 *   1. 将所有 skills 上传到七牛云 CDN，生成下载 URL
 *   2. 将每个 skill 的元数据写入 Parse 数据库 Skill 表
 *   3. 生成 README.md（包含所有 skill 简介 + 下载链接）并上传到 CDN
 * 
 * 使用方法：
 *   node scripts/deploy/upload-skills.js
 * 
 * 前置条件：
 *   1. npm install qiniu
 *   2. 配置下方 QINIU_ACCESS_KEY 和 QINIU_SECRET_KEY
 * 
 * 参考：hongben-server/modules/fmode-tikhub-server/upload.sh
 */

const qiniu = require('qiniu');
const fs = require('fs');
const path = require('path');

// ============================================
// 七牛云配置
// ============================================
const QINIU_ACCESS_KEY = process.env.QINIU_AK || 'EXsA-z_n4LGmWrwC088bygcGJtAditnWQe2nH-ZE';
const QINIU_SECRET_KEY = process.env.QINIU_SK || 'HWTL92OL-Tup0-8ex8A9jnG3OaJzTxlF4OwiiDsX';
const BUCKET = 'nova-repos';
const CDN_DOMAIN = 'https://repos.fmode.cn';
const CDN_PREFIX = 'x/openclaw-skills';

// ============================================
// Parse 数据库配置
// ============================================
const PARSE_BASE = 'https://server.fmode.cn/parse';
const APP_ID = 'ncloudmaster';
const MASTER_KEY = process.env.PARSE_MASTER_KEY || 'SnkK12*&sunq2#@20!';

// Skills 源目录（部署目录）
const DEPLOYED_SKILLS_DIR = path.join(process.env.USERPROFILE || process.env.HOME, '.openclaw', 'skills');

// 要上传的技能目录列表（如果为空，则上传所有 skills）
const SKILL_FILTER = []; // 例如 ['tiktok-video-search', 'jimeng-img-v4'] 指定只上传某些

// ============================================
// 七牛上传
// ============================================

function getUploadToken(bucket, key) {
    const mac = new qiniu.auth.digest.Mac(QINIU_ACCESS_KEY, QINIU_SECRET_KEY);
    const putPolicy = new qiniu.rs.PutPolicy({
        scope: `${bucket}:${key}`,
        expires: 3600
    });
    return putPolicy.uploadToken(mac);
}

function uploadFile(localFile, cdnKey) {
    return new Promise((resolve, reject) => {
        const token = getUploadToken(BUCKET, cdnKey);
        const formUploader = new qiniu.form_up.FormUploader(new qiniu.conf.Config({
            zone: qiniu.zone.Zone_z2
        }));
        const putExtra = new qiniu.form_up.PutExtra();

        formUploader.putFile(token, cdnKey, localFile, putExtra, (err, body, info) => {
            if (err) {
                reject(err);
                return;
            }
            if (info.statusCode === 200) {
                resolve({ key: body.key, url: `${CDN_DOMAIN}/${body.key}` });
            } else {
                reject(new Error(`Upload failed: ${info.statusCode} ${JSON.stringify(body)}`));
            }
        });
    });
}

function uploadBuffer(buffer, cdnKey) {
    return new Promise((resolve, reject) => {
        const token = getUploadToken(BUCKET, cdnKey);
        const formUploader = new qiniu.form_up.FormUploader(new qiniu.conf.Config({
            zone: qiniu.zone.Zone_z2
        }));
        const putExtra = new qiniu.form_up.PutExtra();

        formUploader.put(token, cdnKey, buffer, putExtra, (err, body, info) => {
            if (err) {
                reject(err);
                return;
            }
            if (info.statusCode === 200) {
                resolve({ key: body.key, url: `${CDN_DOMAIN}/${body.key}` });
            } else {
                reject(new Error(`Upload failed: ${info.statusCode} ${JSON.stringify(body)}`));
            }
        });
    });
}

// ============================================
// Parse REST API helpers
// ============================================

const PARSE_HEADERS = {
    'X-Parse-Application-Id': APP_ID,
    'X-Parse-Master-Key': MASTER_KEY,
    'Content-Type': 'application/json'
};

async function ensureSkillClass() {
    const resp = await fetch(`${PARSE_BASE}/schemas/Skill`, {
        headers: PARSE_HEADERS
    });
    if (resp.ok) return;
    // Class 不存在，创建
    const createResp = await fetch(`${PARSE_BASE}/schemas/Skill`, {
        method: 'POST',
        headers: PARSE_HEADERS,
        body: JSON.stringify({
            className: 'Skill',
            fields: {
                name:        { type: 'String' },
                displayName: { type: 'String' },
                description: { type: 'String' },
                category:    { type: 'String' },
                version:     { type: 'String' },
                author:      { type: 'String' },
                cdnUrls:     { type: 'Object' },
                cdnPrefix:   { type: 'String' }
            }
        })
    });
    const result = await createResp.json();
    if (createResp.ok) {
        console.log('  ✅ Skill 表已创建');
    } else {
        console.log('  ⚠️ 创建 Skill 表:', result.error || JSON.stringify(result));
    }
}

async function parseQuery(className, where) {
    const qs = new URLSearchParams({
        where: JSON.stringify(where),
        limit: '1'
    });
    const resp = await fetch(`${PARSE_BASE}/classes/${className}?${qs}`, {
        headers: PARSE_HEADERS
    });
    const data = await resp.json();
    return data.results || [];
}

async function parseCreate(className, body) {
    const resp = await fetch(`${PARSE_BASE}/classes/${className}`, {
        method: 'POST',
        headers: PARSE_HEADERS,
        body: JSON.stringify(body)
    });
    return resp.json();
}

async function parseUpdate(className, objectId, body) {
    const resp = await fetch(`${PARSE_BASE}/classes/${className}/${objectId}`, {
        method: 'PUT',
        headers: PARSE_HEADERS,
        body: JSON.stringify(body)
    });
    return resp.json();
}

async function upsertSkill(skillData) {
    const existing = await parseQuery('Skill', { name: skillData.name });
    if (existing.length > 0) {
        const result = await parseUpdate('Skill', existing[0].objectId, skillData);
        return { action: 'updated', objectId: existing[0].objectId, ...result };
    } else {
        const result = await parseCreate('Skill', skillData);
        return { action: 'created', ...result };
    }
}

// ============================================
// Skill metadata extraction
// ============================================

function parseSkillMdFrontmatter(filePath) {
    try {
        const content = fs.readFileSync(filePath, 'utf8');
        const match = content.match(/^---\n([\s\S]*?)\n---/);
        if (!match) return {};
        const fm = {};
        match[1].split('\n').forEach(line => {
            const idx = line.indexOf(':');
            if (idx > 0) {
                const key = line.substring(0, idx).trim();
                const val = line.substring(idx + 1).trim();
                fm[key] = val;
            }
        });
        return fm;
    } catch (e) {
        return {};
    }
}

function parseApiConfig(filePath) {
    try {
        return JSON.parse(fs.readFileSync(filePath, 'utf8'));
    } catch (e) {
        return {};
    }
}

function extractSkillMeta(skillDir, skillName) {
    const skillMdPath = path.join(skillDir, 'SKILL.md');
    const apiConfigPath = path.join(skillDir, 'api-config.json');

    const fm = parseSkillMdFrontmatter(skillMdPath);
    const cfg = fs.existsSync(apiConfigPath) ? parseApiConfig(apiConfigPath) : {};

    return {
        name: skillName,
        displayName: cfg.displayName || fm.name || skillName,
        description: cfg.description || fm.description || '',
        category: cfg.category || '',
        version: cfg.version || fm.version || '1.0.0',
        author: fm.author || 'nkkj-BrainHack'
    };
}

// ============================================
// Collect skills
// ============================================

function collectSkills() {
    const skills = [];
    const dir = DEPLOYED_SKILLS_DIR;

    if (!fs.existsSync(dir)) {
        console.error(`Skills 目录不存在: ${dir}`);
        process.exit(1);
    }

    const entries = fs.readdirSync(dir).filter(d => {
        const fullPath = path.join(dir, d);
        if (!fs.statSync(fullPath).isDirectory()) return false;
        if (SKILL_FILTER.length > 0 && !SKILL_FILTER.includes(d)) return false;
        return fs.existsSync(path.join(fullPath, 'SKILL.md'));
    });

    for (const skillName of entries) {
        const skillDir = path.join(dir, skillName);
        const files = [];

        for (const fileName of ['SKILL.md', 'api-config.json']) {
            const filePath = path.join(skillDir, fileName);
            if (fs.existsSync(filePath)) {
                files.push({
                    localPath: filePath,
                    cdnKey: `${CDN_PREFIX}/${skillName}/${fileName}`,
                    fileName
                });
            }
        }

        if (files.length > 0) {
            const meta = extractSkillMeta(skillDir, skillName);
            skills.push({ name: skillName, files, meta });
        }
    }

    return skills;
}

// ============================================
// Generate README.md
// ============================================

function generateReadme(skillEntries) {
    const now = new Date().toISOString().replace('T', ' ').substring(0, 19);
    const lines = [
        '# OpenClaw Skills 目录',
        '',
        `> 自动生成于 ${now}，共 ${skillEntries.length} 个技能`,
        '',
        '---',
        ''
    ];

    // 按 category 分组
    const grouped = {};
    for (const entry of skillEntries) {
        const cat = entry.meta.category || '其他';
        if (!grouped[cat]) grouped[cat] = [];
        grouped[cat].push(entry);
    }

    for (const [cat, entries] of Object.entries(grouped).sort()) {
        lines.push(`## ${cat}`);
        lines.push('');
        lines.push('| 技能名 | 说明 | 版本 | 下载 |');
        lines.push('|--------|------|------|------|');

        for (const entry of entries) {
            const m = entry.meta;
            const desc = m.description.length > 60 ? m.description.substring(0, 60) + '...' : m.description;
            const skillMdUrl = entry.cdnUrls['SKILL.md'] || '';
            const apiCfgUrl = entry.cdnUrls['api-config.json'] || '';
            const links = [];
            if (skillMdUrl) links.push(`[SKILL.md](${skillMdUrl})`);
            if (apiCfgUrl) links.push(`[api-config.json](${apiCfgUrl})`);
            lines.push(`| **${m.displayName}** (\`${m.name}\`) | ${desc} | ${m.version} | ${links.join(' / ')} |`);
        }
        lines.push('');
    }

    lines.push('---');
    lines.push('');
    lines.push('## 快速使用');
    lines.push('');
    lines.push('每个 Skill 包含两个文件：');
    lines.push('- **SKILL.md** — 技能说明文档（功能、入参、调用链、注意事项）');
    lines.push('- **api-config.json** — API 配置（endpoint、参数 schema、错误处理）');
    lines.push('');
    lines.push('下载后放入 `~/.openclaw/skills/<skill-name>/` 即可被 OpenClaw Agent 识别和调用。');
    lines.push('');

    return lines.join('\n');
}

// ============================================
// Main
// ============================================

async function main() {
    console.log('==================================================');
    console.log('   OpenClaw Skills CDN 上传 + Parse 同步脚本');
    console.log('==================================================');
    console.log('');

    // 检查凭证
    if (QINIU_ACCESS_KEY === 'YOUR_ACCESS_KEY_HERE' || QINIU_SECRET_KEY === 'YOUR_SECRET_KEY_HERE') {
        console.error('❌ 请先配置七牛云凭证！');
        console.error('   方式1: 编辑脚本中的 QINIU_ACCESS_KEY / QINIU_SECRET_KEY');
        console.error('   方式2: 设置环境变量 QINIU_AK / QINIU_SK');
        process.exit(1);
    }

    // 收集技能
    const skills = collectSkills();
    console.log(`📦 找到 ${skills.length} 个技能待上传`);
    console.log(`📂 源目录: ${DEPLOYED_SKILLS_DIR}`);
    console.log(`☁️  Bucket: ${BUCKET}`);
    console.log(`🔗 CDN前缀: ${CDN_PREFIX}`);
    console.log('');

    let totalFiles = 0;
    skills.forEach(s => { totalFiles += s.files.length; });
    console.log(`📄 总计 ${totalFiles} 个文件`);
    console.log('');

    // ── Step 1: 上传文件到七牛 ──
    console.log('── Step 1: 上传文件到七牛云 ──');
    let successCount = 0;
    let failCount = 0;
    const results = [];

    for (const skill of skills) {
        process.stdout.write(`  📦 ${skill.name} ... `);
        const skillResults = [];
        const cdnUrls = {};

        for (const file of skill.files) {
            try {
                const result = await uploadFile(file.localPath, file.cdnKey);
                skillResults.push({ ...result, fileName: file.fileName, status: 'ok' });
                cdnUrls[file.fileName] = result.url;
                successCount++;
            } catch (err) {
                skillResults.push({ fileName: file.fileName, status: 'fail', error: err.message });
                failCount++;
            }
        }

        const allOk = skillResults.every(r => r.status === 'ok');
        console.log(allOk ? '✅' : '⚠️');
        results.push({ name: skill.name, files: skillResults, meta: skill.meta, cdnUrls });
    }

    console.log('');
    console.log(`   上传完成！成功: ${successCount}  失败: ${failCount}`);
    console.log('');

    // ── Step 2: 写入 Parse Skill 表 ──
    console.log('── Step 2: 同步到 Parse 数据库 (Skill 表) ──');
    await ensureSkillClass();
    let dbSuccess = 0;
    let dbFail = 0;

    for (const entry of results) {
        if (Object.keys(entry.cdnUrls).length === 0) continue;
        process.stdout.write(`  💾 ${entry.name} ... `);
        try {
            // Parse 不允许嵌套 key 含 '.'，将文件名中的 '.' 替换为 '_'
            const safeCdnUrls = {};
            for (const [k, v] of Object.entries(entry.cdnUrls)) {
                safeCdnUrls[k.replace(/\./g, '_')] = v;
            }
            const skillData = {
                name: entry.meta.name,
                displayName: entry.meta.displayName,
                description: entry.meta.description,
                category: entry.meta.category,
                version: entry.meta.version,
                author: entry.meta.author,
                cdnUrls: safeCdnUrls,
                cdnPrefix: `${CDN_DOMAIN}/${CDN_PREFIX}/${entry.name}`
            };
            const result = await upsertSkill(skillData);
            console.log(`${result.action === 'created' ? '🆕 创建' : '🔄 更新'} (${result.objectId})`);
            dbSuccess++;
        } catch (err) {
            console.log(`❌ ${err.message}`);
            dbFail++;
        }
    }

    console.log('');
    console.log(`   数据库同步完成！成功: ${dbSuccess}  失败: ${dbFail}`);
    console.log('');

    // ── Step 3: 生成 README.md 并上传（包含在技能文件夹内）──
    console.log('── Step 3: 生成 README.md 并上传 ──');
    let readmeUrl = '';
    try {
        const readme = generateReadme(results);
        const readmeKey = `${CDN_PREFIX}/README.md`;
        const readmeResult = await uploadBuffer(Buffer.from(readme, 'utf8'), readmeKey);
        readmeUrl = readmeResult.url;
        console.log(`  ✅ README.md 已上传: ${readmeUrl}`);
    } catch (err) {
        console.log(`  ❌ README.md 上传失败: ${err.message}`);
    }
    console.log('');

    // ── 输出 URL 汇总（README 包含在内）──
    console.log('📋 CDN 下载地址列表：');
    console.log('');

    const urlMap = {};

    // README 放在最前面
    if (readmeUrl) {
        urlMap['_README'] = { 'README.md': readmeUrl };
        console.log(`  README.md`);
        console.log(`    👉 ${readmeUrl}`);
    }

    for (const entry of results) {
        if (Object.keys(entry.cdnUrls).length > 0) {
            urlMap[entry.name] = entry.cdnUrls;
            for (const [fn, url] of Object.entries(entry.cdnUrls)) {
                console.log(`  ${entry.name}/${fn}`);
                console.log(`    👉 ${url}`);
            }
        }
    }

    // 保存 URL 列表到文件
    const outputPath = path.resolve(__dirname, '..', 'skills-cdn-urls.json');
    fs.writeFileSync(outputPath, JSON.stringify(urlMap, null, 2), 'utf8');
    console.log('');
    console.log(`💾 URL 列表已保存到: ${outputPath}`);
    console.log('');

    console.log('==================================================');
    console.log('📌 技能索引（README 已包含在文件夹内）：');
    console.log(`  README: ${CDN_DOMAIN}/${CDN_PREFIX}/README.md`);
    console.log(`  CDN:    ${CDN_DOMAIN}/${CDN_PREFIX}/`);
    console.log('==================================================');
}

main().catch(err => {
    console.error('❌ 上传出错:', err);
    process.exit(1);
});