agent-yuyang001/scripts/fetch-wx-sogou.py

#!/usr/bin/env python3
"""搜狗微信搜索抓取文章摘要与正文缓存"""
import re, subprocess

url = "https://weixin.sogou.com/weixin?type=2&query=Harness+%E5%B7%A5%E7%A8%8B+%E8%87%AA%E6%88%91%E6%94%B9%E8%BF%9B%E4%B8%8D%E4%BB%8E%E6%A8%A1%E5%9E%8B%E5%BC%80%E5%A7%8B"
r = subprocess.run(["curl", "-sL", url,
                    "-H", "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
                    "--max-time", "25", "-o", "/tmp/sogou.html"], capture_output=True, timeout=40)
raw = open("/tmp/sogou.html", encoding="utf-8", errors="ignore").read()
print("len:", len(raw))
items = re.findall(r"<h3>(.*?)</h3>", raw, re.S)
for it in items[:3]:
    txt = re.sub(r"<[^>]+>", "", it).strip()
    print("·", txt[:80])
# 摘要
sps = re.findall(r'class="txt-info"[^>]*>(.*?)</p>', raw, re.S)
for s in sps[:3]:
    t = re.sub(r"<[^>]+>", "", s).strip()
    print("摘要:", t[:220])
# 找目标文章的跳转链接(搜狗转链)
links = re.findall(r'href="(/link\?url=[^"]+)"', raw)
print("links:", len(links))
if links:
    # 抓第一个link(通常就是最新相关文章) 跟踪跳转拿正文
    follow = "https://weixin.sogou.com" + links[0].replace("&amp;", "&")
    r2 = subprocess.run(["curl", "-sL", "-v", follow,
                         "-H", "User-Agent: Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) MicroMessenger/8.0.49",
                         "-H", "Referer: https://weixin.sogou.com/",
                         "--max-time", "25", "-o", "/tmp/sogou-art.html"], capture_output=True, timeout=40, text=True)
    import os
    if not os.path.exists("/tmp/sogou-art.html"):
        print("抓取失败, curl stderr尾部:", (r2.stderr or "")[-300:])
    else:
        art = open("/tmp/sogou-art.html", encoding="utf-8", errors="ignore").read()
        print("art len:", len(art))
        print("含js_content:", "js_content" in art, "| 含环境异常:", "环境异常" in art)
        m = re.search(r'id="js_content"[^>]*>(.*)$', art, re.S)
        if m:
            seg = m.group(1)
            text = re.sub(r"<style[^>]*>.*?</style>", "", seg, flags=re.S)
            text = re.sub(r"<[^>]+>", "\n", text)
            text = re.sub(r"\n\s*\n+", "\n", text).strip()
            print("正文长度:", len(text))
            open("/tmp/wx-article.txt", "w", encoding="utf-8").write(text)
            print(text[:1000])