openclaw-voc-skill/scripts/deploy/package-and-upload.js

#!/usr/bin/env node
/**
 * OpenClaw Skills 打包 + 上传脚本
 * 
 * 功能：
 *   1. 按套件分组，将 skills 打包成 zip（含 install.js + skills/ 目录）
 *   2. 上传 zip 到七牛云 CDN
 *   3. 输出下载链接
 * 
 * 使用：node scripts/deploy/package-and-upload.js
 * 前置：先运行 deploy-to-openclaw.ps1 确保 ~/.openclaw/skills 是最新的
 */

const fs = require('fs');
const path = require('path');
const os = require('os');
const { execSync } = require('child_process');
const qiniu = require('qiniu');

// ============================================
// 七牛云配置（同 upload-skills.js）
// ============================================
const QINIU_ACCESS_KEY = process.env.QINIU_AK || 'EXsA-z_n4LGmWrwC088bygcGJtAditnWQe2nH-ZE';
const QINIU_SECRET_KEY = process.env.QINIU_SK || 'HWTL92OL-Tup0-8ex8A9jnG3OaJzTxlF4OwiiDsX';
const BUCKET = 'nova-repos';
const CDN_DOMAIN = 'https://repos.fmode.cn';
const CDN_PREFIX = 'x/openclaw-skills/packages';

// ============================================
// 路径
// ============================================
const SKILLS_DIR = path.join(os.homedir(), '.openclaw', 'skills');
const INSTALL_JS = path.join(__dirname, '..', '..', 'dist', 'install.js');
const SET_VOC_TOKEN_JS = path.join(__dirname, '..', 'tools', 'set-voc-token.js');
const INSTALL_WORKSHOP_JS = path.join(__dirname, 'install-workshop.js');
const INSTALL_ALL_JS = path.join(__dirname, 'install-all.js');
const CREDENTIALS_TEMPLATE = path.join(__dirname, '..', '..', '__config', 'voc-credentials.template.json');
const DIST_DIR = path.join(__dirname, '..', '..', 'dist');
const TEMP_BASE = path.join(os.tmpdir(), 'openclaw-pkg');
const WORKSHOP_DIR = path.join(os.homedir(), '.openclaw', 'workspace');
const MEMORY_TEMPLATES_DIR = path.join(WORKSHOP_DIR, 'memory');

// OpenClaw 系统文件（不能打包，否则覆盖用户配置）
const SYSTEM_MD_FILES = new Set([
    'AGENTS.md', 'BOOTSTRAP.md', 'HEARTBEAT.md', 'IDENTITY.md',
    'SOUL.md', 'TOOLS.md', 'USER.md'
]);

// Workshop playbook 白名单（只打包这些 .md）
const WORKSHOP_MD_WHITELIST = new Set([
    'workshop-voc-playbook.md',
    'workshop-global-rules.md',
    'workshop-session-0-intake.md',
    'workshop-session-1-category.md',
    'workshop-session-2-brand.md',
    'workshop-session-3-voc.md',
    'workshop-session-4-deep.md',
    'workshop-session-5-report.md',
    'product-analysis-playbook.md',
    'synthesis-prompts.md',
    'voc-report-schema-spec.md'
]);

// Memory template 白名单（排除运行时数据）
const MEMORY_TEMPLATE_WHITELIST = new Set([
    'stage-1-output.json',
    'stage-2-output.json',
    'stage-2-competitor-output.json',
    'stage-3-output.json',
    'stage-4-output.json',
    'stage-5-output.json',
    'workshop-progress.json',
    'brand-context.json',
    'calibration-notes.json',
    'analysis-request.json'
]);

// 测试技能黑名单
const SKILL_BLACKLIST = new Set(['test-billing', 'test-skill']);

// ============================================
// 套件定义（技能名列表）
// ============================================
const PACKAGES = {
    'voc-core': {
        label: '🔍 VOC 核心套件',
        description: 'Amazon 品类洞察 + 评论分析 + 竞品分析 + 数据合成 + 社媒VOC + 工作坊',
        skills: [
            // voc
            'asin-reverse-keywords', 'asin-sales-volume', 'category-products', 'category-tree',
            'keyword-product-ranking', 'keyword-search', 'keyword-search-trend',
            'product-detail-query', 'product-monitor', 'product-reviews-query', 'product-search', 'similar-products',
            // review-analysis
            'review-batch-collection', 'review-highlight-extraction', 'review-keyword-cloud',
            'review-pain-point-extraction', 'review-sentiment-analysis',
            // competitor-analysis
            'competitor-bsr-tracking', 'competitor-discovery', 'competitor-pricing-analysis', 'competitor-product-comparison',
            // synthesis
            'brand-profile', 'category-landscape', 'html-report-generator', 'product-deep-analysis', 'user-persona', 'voc-proposal',
            // social-voc
            'instagram-brand-voc', 'social-trend-analysis', 'tiktok-brand-voc', 'tiktok-category-voc',
            // workshop
            'brand-context-builder'
        ]
    },
    'social-media': {
        label: '📱 社媒数据套件',
        description: 'TikTok / Instagram / 抖音数据采集',
        skills: [
            // social-media
            'instagram-search', 'instagram-user-info', 'instagram-user-posts',
            'tiktok-hashtag-detail', 'tiktok-hashtag-videos', 'tiktok-user-posts',
            'tiktok-user-profile', 'tiktok-user-search', 'tiktok-video-comments',
            'tiktok-video-detail', 'tiktok-video-search',
            // douyin
            'douyin-comment-replies', 'douyin-general-search', 'douyin-hashtag-search',
            'douyin-user-profile', 'douyin-user-search', 'douyin-video-comments', 'douyin-video-detail'
        ]
    }
};

// ============================================
// 七牛上传
// ============================================
function uploadFile(localFile, cdnKey) {
    return new Promise((resolve, reject) => {
        const mac = new qiniu.auth.digest.Mac(QINIU_ACCESS_KEY, QINIU_SECRET_KEY);
        const putPolicy = new qiniu.rs.PutPolicy({ scope: `${BUCKET}:${cdnKey}`, expires: 3600 });
        const token = putPolicy.uploadToken(mac);
        const formUploader = new qiniu.form_up.FormUploader(new qiniu.conf.Config({ zone: qiniu.zone.Zone_z2 }));
        const putExtra = new qiniu.form_up.PutExtra();

        formUploader.putFile(token, cdnKey, localFile, putExtra, (err, body, info) => {
            if (err) return reject(err);
            if (info.statusCode === 200) {
                resolve({ key: body.key, url: `${CDN_DOMAIN}/${body.key}` });
            } else {
                reject(new Error(`Upload failed: ${info.statusCode} ${JSON.stringify(body)}`));
            }
        });
    });
}

// ============================================
// 打包逻辑
// ============================================
function buildPackage(pkgId, pkg) {
    const pkgDir = path.join(TEMP_BASE, pkgId);
    const skillsDir = path.join(pkgDir, 'skills');

    // 清理 + 创建目录
    if (fs.existsSync(pkgDir)) fs.rmSync(pkgDir, { recursive: true, force: true });
    fs.mkdirSync(skillsDir, { recursive: true });

    // 复制 install.js
    if (fs.existsSync(INSTALL_JS)) {
        fs.copyFileSync(INSTALL_JS, path.join(pkgDir, 'install.js'));
    }

    // 复制 set-voc-token.js（Token 设置工具）
    if (fs.existsSync(SET_VOC_TOKEN_JS)) {
        fs.copyFileSync(SET_VOC_TOKEN_JS, path.join(pkgDir, 'set-voc-token.js'));
    }

    // 生成 README.txt
    const readme = [
        `OpenClaw ${pkg.label}`,
        `${'='.repeat(50)}`,
        '',
        pkg.description,
        '',
        `包含 ${pkg.skills.length} 个技能`,
        '',
        '安装方法：',
        '  1. 确保已安装 OpenClaw',
        '  2. 解压本压缩包',
        '  3. 在解压目录运行: node install.js',
        '  4. 设置 Token: node set-voc-token.js <your-session-token>',
        '  5. 重启 OpenClaw gateway',
        '',
        '其他命令：',
        '  node install.js --list          # 列出所有技能',
        '  node install.js --dry-run       # 预览模式',
        '  node install.js --only a,b,c    # 只安装指定技能',
        '  node install.js --skip a,b,c    # 跳过指定技能',
        '  node set-voc-token.js <token>   # 设置/更新 VOC Token',
        '',
        '技能列表：',
        ...pkg.skills.map(s => `  - ${s}`),
        ''
    ].join('\n');
    fs.writeFileSync(path.join(pkgDir, 'README.txt'), readme, 'utf8');

    // 复制每个技能
    let copied = 0;
    let missing = 0;
    for (const skillName of pkg.skills) {
        const srcDir = path.join(SKILLS_DIR, skillName);
        if (!fs.existsSync(srcDir)) {
            console.log(`    ⚠️  跳过 ${skillName}（未部署）`);
            missing++;
            continue;
        }

        const destDir = path.join(skillsDir, skillName);
        fs.mkdirSync(destDir, { recursive: true });

        const files = fs.readdirSync(srcDir).filter(f => fs.statSync(path.join(srcDir, f)).isFile());
        for (const f of files) {
            fs.copyFileSync(path.join(srcDir, f), path.join(destDir, f));
        }
        copied++;
    }

    console.log(`    ✅ 复制 ${copied} 个技能（跳过 ${missing} 个）`);

    // 压缩
    const zipPath = path.join(DIST_DIR, `${pkgId}.zip`);
    if (fs.existsSync(zipPath)) fs.unlinkSync(zipPath);

    // 使用 PowerShell Compress-Archive
    const absZipPath = path.resolve(zipPath);
    const absPkgDir = path.resolve(pkgDir);
    execSync(
        `powershell -Command "Compress-Archive -Path '${absPkgDir}\\*' -DestinationPath '${absZipPath}' -Force"`,
        { stdio: 'inherit' }
    );

    const zipSize = fs.statSync(zipPath).size;
    console.log(`    📦 ${path.basename(zipPath)} (${(zipSize / 1024).toFixed(0)} KB)`);

    return zipPath;
}

// ============================================
// Workshop 打包逻辑
// ============================================
function buildWorkshopPackage() {
    const pkgDir = path.join(TEMP_BASE, 'workshop');
    const memoryDir = path.join(pkgDir, 'memory-templates');

    if (fs.existsSync(pkgDir)) fs.rmSync(pkgDir, { recursive: true, force: true });
    fs.mkdirSync(memoryDir, { recursive: true });

    // 复制安装脚本
    if (fs.existsSync(INSTALL_WORKSHOP_JS)) {
        fs.copyFileSync(INSTALL_WORKSHOP_JS, path.join(pkgDir, 'install-workshop.js'));
    }

    let copied = 0;

    // 复制 workshop 文件（仅白名单 playbooks）
    if (fs.existsSync(WORKSHOP_DIR)) {
        const mdFiles = fs.readdirSync(WORKSHOP_DIR).filter(f =>
            f.endsWith('.md') && WORKSHOP_MD_WHITELIST.has(f) && fs.statSync(path.join(WORKSHOP_DIR, f)).isFile()
        );
        for (const f of mdFiles) {
            fs.copyFileSync(path.join(WORKSHOP_DIR, f), path.join(pkgDir, f));
            copied++;
        }
    }

    // 复制 memory templates（仅白名单模板）
    if (fs.existsSync(MEMORY_TEMPLATES_DIR)) {
        const jsonFiles = fs.readdirSync(MEMORY_TEMPLATES_DIR).filter(f =>
            f.endsWith('.json') && MEMORY_TEMPLATE_WHITELIST.has(f) && fs.statSync(path.join(MEMORY_TEMPLATES_DIR, f)).isFile()
        );
        for (const f of jsonFiles) {
            fs.copyFileSync(path.join(MEMORY_TEMPLATES_DIR, f), path.join(memoryDir, f));
            copied++;
        }
    }

    // 生成 README.txt
    const readme = [
        'OpenClaw 🦐 VOC虾工作坊 v4.1',
        '==================================================',
        '',
        'VOC 工作坊完整流程文件（Side Quest 架构）',
        '',
        '架构：6个Session + 17个Side Quest，主线感知 + 支线深耕',
        '',
        '安装方法：',
        '  1. 解压本压缩包',
        '  2. 在解压目录运行: node install-workshop.js',
        '  3. 重启 OpenClaw gateway',
        '',
        '其他命令：',
        '  node install-workshop.js --dry-run  # 预览模式',
        '',
        '文件说明：',
        '  workshop-voc-playbook.md       ← 入口路由',
        '  workshop-global-rules.md       ← 全局规则 + Side Quest Protocol',
        '  workshop-session-0-intake.md   ← Session 0: 品牌建档',
        '  workshop-session-1-category.md ← Session 1: 品类 + 4 Side Quests',
        '  workshop-session-2-brand.md    ← Session 2: 品牌 + 3 Side Quests',
        '  workshop-session-3-voc.md      ← Session 3: VOC + 3 Side Quests',
        '  workshop-session-4-deep.md     ← Session 4: 深入 + 5 Side Quests',
        '  workshop-session-5-report.md   ← Session 5: 报告 + 2 Side Quests',
        '  memory-templates/*.json        ← 记忆模板（9个）',
        ''
    ].join('\n');
    fs.writeFileSync(path.join(pkgDir, 'README.txt'), readme, 'utf8');

    console.log(`    ✅ 复制 ${copied} 个文件`);

    // 压缩
    const zipPath = path.join(DIST_DIR, 'workshop.zip');
    if (fs.existsSync(zipPath)) fs.unlinkSync(zipPath);
    const absZipPath = path.resolve(zipPath);
    const absPkgDir = path.resolve(pkgDir);
    execSync(
        `powershell -Command "Compress-Archive -Path '${absPkgDir}\\*' -DestinationPath '${absZipPath}' -Force"`,
        { stdio: 'inherit' }
    );

    const zipSize = fs.statSync(zipPath).size;
    console.log(`    📦 workshop.zip (${(zipSize / 1024).toFixed(0)} KB)`);

    return zipPath;
}

// ============================================
// 全量包打包逻辑
// ============================================
function buildAllPackage() {
    const pkgDir = path.join(TEMP_BASE, 'openclaw-all');
    const skillsDir = path.join(pkgDir, 'skills');
    const workshopDir = path.join(pkgDir, 'workshop');
    const memoryDir = path.join(workshopDir, 'memory-templates');

    if (fs.existsSync(pkgDir)) fs.rmSync(pkgDir, { recursive: true, force: true });
    fs.mkdirSync(memoryDir, { recursive: true });

    // 复制 install-all.js
    if (fs.existsSync(INSTALL_ALL_JS)) {
        fs.copyFileSync(INSTALL_ALL_JS, path.join(pkgDir, 'install-all.js'));
    }

    // 复制 set-voc-token.js
    if (fs.existsSync(SET_VOC_TOKEN_JS)) {
        fs.copyFileSync(SET_VOC_TOKEN_JS, path.join(pkgDir, 'set-voc-token.js'));
    }

    // 复制凭证模板
    if (fs.existsSync(CREDENTIALS_TEMPLATE)) {
        fs.copyFileSync(CREDENTIALS_TEMPLATE, path.join(pkgDir, 'voc-credentials.template.json'));
    }

    let skillCount = 0;
    let workshopCount = 0;

    // 复制发布套件中的技能（排除测试技能和未发布的即梦/视频创作技能）
    const RELEASED_SKILLS = new Set([
        ...PACKAGES['voc-core'].skills,
        ...PACKAGES['social-media'].skills
    ]);
    if (fs.existsSync(SKILLS_DIR)) {
        const allSkills = fs.readdirSync(SKILLS_DIR).filter(d => {
            const p = path.join(SKILLS_DIR, d);
            return fs.statSync(p).isDirectory() && fs.existsSync(path.join(p, 'SKILL.md'))
                && !SKILL_BLACKLIST.has(d) && RELEASED_SKILLS.has(d);
        });
        for (const skill of allSkills) {
            const srcDir = path.join(SKILLS_DIR, skill);
            const destDir = path.join(skillsDir, skill);
            fs.mkdirSync(destDir, { recursive: true });
            const files = fs.readdirSync(srcDir).filter(f => fs.statSync(path.join(srcDir, f)).isFile());
            for (const f of files) {
                fs.copyFileSync(path.join(srcDir, f), path.join(destDir, f));
            }
            skillCount++;
        }
    }
    console.log(`    ✅ 复制 ${skillCount} 个技能`);

    // 复制 workshop 文件（仅白名单）
    if (fs.existsSync(WORKSHOP_DIR)) {
        const mdFiles = fs.readdirSync(WORKSHOP_DIR).filter(f =>
            f.endsWith('.md') && WORKSHOP_MD_WHITELIST.has(f) && fs.statSync(path.join(WORKSHOP_DIR, f)).isFile()
        );
        for (const f of mdFiles) {
            fs.copyFileSync(path.join(WORKSHOP_DIR, f), path.join(workshopDir, f));
            workshopCount++;
        }
    }
    if (fs.existsSync(MEMORY_TEMPLATES_DIR)) {
        const jsonFiles = fs.readdirSync(MEMORY_TEMPLATES_DIR).filter(f =>
            f.endsWith('.json') && MEMORY_TEMPLATE_WHITELIST.has(f) && fs.statSync(path.join(MEMORY_TEMPLATES_DIR, f)).isFile()
        );
        for (const f of jsonFiles) {
            fs.copyFileSync(path.join(MEMORY_TEMPLATES_DIR, f), path.join(memoryDir, f));
            workshopCount++;
        }
    }
    console.log(`    ✅ 复制 ${workshopCount} 个 workshop 文件`);

    // README.txt
    const readme = [
        'OpenClaw 全量安装包 v4.1',
        '==================================================',
        '',
        `包含 ${skillCount} 个技能 + VOC虾工作坊 (${workshopCount}个文件) + Token 工具`,
        '',
        '安装方法：',
        '  1. 解压本压缩包',
        '  2. 在解压目录运行: node install-all.js',
        '  3. 设置 Token:',
        '     a. 打开 https://app.fmode.cn/dev/apig-pay/?apigid=Vo3ROWEvDy&fun_id=HOkkX72PMF',
        '     b. 登录并充值',
        '     c. 获取 session token (r:xxxx...)',
        '     d. 运行: node ~/.openclaw/tools/set-voc-token.js <your-token>',
        '  4. 重启 OpenClaw gateway',
        '',
        '其他命令：',
        '  node install-all.js --list            # 列出包含内容',
        '  node install-all.js --dry-run          # 预览模式',
        '  node install-all.js --skills-only      # 只装 Skills',
        '  node install-all.js --workshop-only    # 只装 Workshop',
        '  node set-voc-token.js <token>          # 设置/更新 Token',
        ''
    ].join('\n');
    fs.writeFileSync(path.join(pkgDir, 'README.txt'), readme, 'utf8');

    // 压缩
    const zipPath = path.join(DIST_DIR, 'openclaw-all.zip');
    if (fs.existsSync(zipPath)) fs.unlinkSync(zipPath);
    const absZipPath = path.resolve(zipPath);
    const absPkgDir = path.resolve(pkgDir);
    execSync(
        `powershell -Command "Compress-Archive -Path '${absPkgDir}\\*' -DestinationPath '${absZipPath}' -Force"`,
        { stdio: 'inherit' }
    );

    const zipSize = fs.statSync(zipPath).size;
    console.log(`    📦 openclaw-all.zip (${(zipSize / 1024).toFixed(0)} KB)`);

    return zipPath;
}

// ============================================
// Main
// ============================================
async function main() {
    console.log('');
    console.log('==================================================');
    console.log('  OpenClaw Skills + Workshop 打包 + CDN 上传 v4.1');
    console.log('==================================================');
    console.log('');

    // 前置检查
    if (!fs.existsSync(SKILLS_DIR)) {
        console.error(`❌ Skills 目录不存在: ${SKILLS_DIR}`);
        console.error('   请先运行 deploy-to-openclaw.ps1');
        process.exit(1);
    }
    if (!fs.existsSync(INSTALL_JS)) {
        console.error(`❌ install.js 不存在: ${INSTALL_JS}`);
        process.exit(1);
    }

    fs.mkdirSync(DIST_DIR, { recursive: true });
    fs.mkdirSync(TEMP_BASE, { recursive: true });

    // Step 1: 打包 Skills
    console.log('── Step 1: 打包 Skills ──');
    const zipFiles = {};
    for (const [pkgId, pkg] of Object.entries(PACKAGES)) {
        console.log(`  ${pkg.label} (${pkg.skills.length} 个技能)`);
        zipFiles[pkgId] = buildPackage(pkgId, pkg);
    }
    console.log('');

    // Step 1b: 打包 Workshop
    console.log('── Step 1b: 打包 Workshop ──');
    if (fs.existsSync(WORKSHOP_DIR)) {
        console.log('  🦐 VOC虾工作坊');
        zipFiles['workshop'] = buildWorkshopPackage();
    } else {
        console.log('  ⚠️  Workshop 目录不存在，跳过');
    }
    console.log('');

    // Step 1c: 打包全量包
    console.log('── Step 1c: 打包全量包 ──');
    console.log('  📦 openclaw-all (Skills + Workshop + Token工具)');
    zipFiles['openclaw-all'] = buildAllPackage();
    console.log('');

    // Step 2: 上传到七牛
    console.log('── Step 2: 上传到七牛云 CDN ──');
    const downloadUrls = {};
    for (const [pkgId, zipPath] of Object.entries(zipFiles)) {
        const pkg = PACKAGES[pkgId];
        const cdnKey = `${CDN_PREFIX}/${pkgId}.zip`;
        const label = pkgId === 'workshop' ? '🦐 VOC虾工作坊' : pkgId === 'openclaw-all' ? '📦 全量包' : pkg.label;
        process.stdout.write(`  ${label} ... `);
        try {
            const result = await uploadFile(zipPath, cdnKey);
            downloadUrls[pkgId] = result.url;
            console.log(`✅ ${result.url}`);
        } catch (err) {
            console.log(`❌ ${err.message}`);
        }
    }
    console.log('');

    // 输出结果
    console.log('==================================================');
    console.log('  📥 下载链接');
    console.log('==================================================');
    console.log('');
    for (const [pkgId, url] of Object.entries(downloadUrls)) {
        if (pkgId === 'workshop') {
            console.log('  🦐 VOC虾工作坊');
        } else if (pkgId === 'openclaw-all') {
            console.log('  📦 全量包');
        } else {
            const pkg = PACKAGES[pkgId];
            if (pkg) console.log(`  ${pkg.label}`);
        }
        console.log(`  👉 ${url}`);
        console.log('');
    }

    // 保存到文件
    const outputPath = path.join(DIST_DIR, 'download-urls.json');
    fs.writeFileSync(outputPath, JSON.stringify(downloadUrls, null, 2), 'utf8');
    console.log(`  💾 链接已保存到: ${outputPath}`);
    console.log('');

    // 清理临时目录
    if (fs.existsSync(TEMP_BASE)) fs.rmSync(TEMP_BASE, { recursive: true, force: true });
}

main().catch(err => {
    console.error('❌ 出错:', err);
    process.exit(1);
});