lami/lami-base-v1/fetch_url_to_md.py

"""
将单个网页 HTML 正文转为 Markdown，并把正文中的图片下载到输出目录下的 images/。

用法示例：
  pip install requests beautifulsoup4 html2text
  python fetch_url_to_md.py --url "https://example.com/page"
  python fetch_url_to_md.py -u "https://example.com" -o doc/out --name my-page
不传 --url 时，仍使用脚本内默认示例 URL（乐享 wiki）。
"""
import argparse
import requests
from bs4 import BeautifulSoup
import html2text
import re
import os
from urllib.parse import urljoin, unquote

def fetch_page_content(url):
    """获取页面内容"""
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8"
    }
    response = requests.get(url, headers=headers, timeout=30)
    response.encoding = 'utf-8'
    return response.text

def download_image(img_url, base_url, download_folder="images"):
    """下载图片并返回相对路径"""
    if not img_url or img_url.startswith('data:'):
        return None

    try:
        img_url = urljoin(base_url, img_url)

        if not os.path.exists(download_folder):
            os.makedirs(download_folder)

        img_name = re.sub(r'[^\w\-_.]', '_', unquote(os.path.basename(img_url.split('?')[0])))
        if not img_name or len(img_name) > 100:
            img_name = str(hash(img_url)) + '.png'

        img_path = os.path.join(download_folder, img_name)

        if os.path.exists(img_path):
            return img_path

        img_response = requests.get(img_url, headers={"User-Agent": "Mozilla/5.0"}, timeout=15)
        if img_response.status_code == 200:
            with open(img_path, 'wb') as f:
                f.write(img_response.content)
            return img_path
    except Exception as e:
        print(f"下载图片失败 {img_url}: {e}")
    return None

def extract_content_with_images(html_content, base_url, download_folder="images"):
    """提取页面内容，下载所有图片并替换路径"""
    soup = BeautifulSoup(html_content, 'html.parser')

    content_area = soup.find('div', class_='content')
    if not content_area:
        content_area = soup.find('div', class_='main-content') or soup.find('article') or soup.find('main') or soup.body

    folder_name = os.path.basename(download_folder)
    
    for img in content_area.find_all('img'):
        img_url = img.get('src') or img.get('data-src')
        if img_url:
            local_path = download_image(img_url, base_url, download_folder)
            if local_path:
                img['src'] = f"{folder_name}/{os.path.basename(local_path)}"
                print(f"  下载图片: {img_url} -> {folder_name}/{os.path.basename(local_path)}")

    return content_area if content_area else soup.body

def html_to_markdown(content_element):
    """将HTML内容转换为Markdown"""
    h = html2text.HTML2Text()
    h.ignore_links = False
    h.ignore_images = False
    h.ignore_tables = False
    h.body_width = 0
    h.mark_code = True
    h.single_line_break = True
    h.unicode_snob = True
    h.skip_internal_links = False
    h.inline_links = True
    h.default_image_alt = "图片"

    html_str = str(content_element)
    html_str = re.sub(r'\sclass="[^"]*"', '', html_str)
    html_str = re.sub(r'\sstyle="[^"]*"', '', html_str)

    markdown = h.handle(html_str)
    markdown = re.sub(r'\n{3,}', '\n\n', markdown)
    markdown = re.sub(r'!\[image\]\([^)]+\)', lambda m: re.sub(r'\\+', '/', m.group(0)), markdown)
    markdown = re.sub(r'!\[\]\((?!images/)', '![图片](images/', markdown)

    return markdown

def extract_title(html_content):
    """提取页面标题"""
    soup = BeautifulSoup(html_content, 'html.parser')
    title_tag = soup.find('title') or soup.find('h1')
    if title_tag:
        return title_tag.get_text().strip()
    return "未命名文档"

def url_to_filename(url):
    """从URL提取标题用于文件名"""
    match = re.search(r'#([\w\-%]+)', url)
    if match:
        return unquote(match.group(1))
    return "document"

def save_markdown(content, title, output_path):
    """保存Markdown文件"""
    with open(output_path, 'w', encoding='utf-8') as f:
        f.write(content)
    print(f"文档已保存: {output_path}")

def _safe_filename(name: str) -> str:
    """Windows 等系统下可用的文件名（不含路径分隔符）。"""
    name = name.strip() or "document"
    for c in '<>:"/\\|?*':
        name = name.replace(c, "_")
    return name[:200] if len(name) > 200 else name


def url_to_markdown(url, output_dir="output", filename_base=None):
    """主函数：将 URL 对应页面正文转为 Markdown。"""
    print(f"正在获取页面: {url}")

    html_content = fetch_page_content(url)
    title = extract_title(html_content)
    print(f"页面标题: {title}")

    print("正在提取内容并下载图片...")
    content_element = extract_content_with_images(html_content, url, os.path.join(output_dir, "images"))

    print("正在转换为Markdown...")
    markdown_content = html_to_markdown(content_element)

    title_tag = f"# {title}\n\n" if title else ""
    full_content = title_tag + markdown_content

    os.makedirs(output_dir, exist_ok=True)
    base = filename_base if filename_base else url_to_filename(url)
    filename = _safe_filename(base) + ".md"
    output_path = os.path.join(output_dir, filename)

    save_markdown(full_content, title, output_path)

    return output_path


if __name__ == "__main__":
    default_url = "https://lexiang.tencent.com/wiki/api/#获取-appkey-和-appsecret"

    parser = argparse.ArgumentParser(description="抓取单个网页并保存为 Markdown（含图片到 images/）")
    parser.add_argument(
        "-u",
        "--url",
        default=None,
        help=f"要抓取的页面 URL（省略则使用内置示例：乐享 wiki）",
    )
    parser.add_argument(
        "-o",
        "--output",
        default="lexiang_output",
        help="输出目录（会创建 images/ 子目录），默认 lexiang_output；若指定了 --url 未指定 -o 则用 output",
    )
    parser.add_argument(
        "-n",
        "--name",
        default=None,
        help="输出 .md 文件名（不含扩展名）；默认从 URL 的 # 锚点或 document",
    )
    args = parser.parse_args()

    target_url = args.url or default_url
    out_dir = args.output
    if args.url and args.output == "lexiang_output":
        out_dir = "output"

    output_file = url_to_markdown(target_url, out_dir, filename_base=args.name)
    print(f"\n转换完成！输出文件: {output_file}")