|
|
@@ -0,0 +1,45 @@
|
|
|
+#!/usr/bin/env python3
|
|
|
+"""搜狗微信搜索抓取文章摘要与正文缓存"""
|
|
|
+import re, subprocess
|
|
|
+
|
|
|
+url = "https://weixin.sogou.com/weixin?type=2&query=Harness+%E5%B7%A5%E7%A8%8B+%E8%87%AA%E6%88%91%E6%94%B9%E8%BF%9B%E4%B8%8D%E4%BB%8E%E6%A8%A1%E5%9E%8B%E5%BC%80%E5%A7%8B"
|
|
|
+r = subprocess.run(["curl", "-sL", url,
|
|
|
+ "-H", "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
|
|
+ "--max-time", "25", "-o", "/tmp/sogou.html"], capture_output=True, timeout=40)
|
|
|
+raw = open("/tmp/sogou.html", encoding="utf-8", errors="ignore").read()
|
|
|
+print("len:", len(raw))
|
|
|
+items = re.findall(r"<h3>(.*?)</h3>", raw, re.S)
|
|
|
+for it in items[:3]:
|
|
|
+ txt = re.sub(r"<[^>]+>", "", it).strip()
|
|
|
+ print("·", txt[:80])
|
|
|
+# 摘要
|
|
|
+sps = re.findall(r'class="txt-info"[^>]*>(.*?)</p>', raw, re.S)
|
|
|
+for s in sps[:3]:
|
|
|
+ t = re.sub(r"<[^>]+>", "", s).strip()
|
|
|
+ print("摘要:", t[:220])
|
|
|
+# 找目标文章的跳转链接(搜狗转链)
|
|
|
+links = re.findall(r'href="(/link\?url=[^"]+)"', raw)
|
|
|
+print("links:", len(links))
|
|
|
+if links:
|
|
|
+ # 抓第一个link(通常就是最新相关文章) 跟踪跳转拿正文
|
|
|
+ follow = "https://weixin.sogou.com" + links[0].replace("&", "&")
|
|
|
+ r2 = subprocess.run(["curl", "-sL", "-v", follow,
|
|
|
+ "-H", "User-Agent: Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) MicroMessenger/8.0.49",
|
|
|
+ "-H", "Referer: https://weixin.sogou.com/",
|
|
|
+ "--max-time", "25", "-o", "/tmp/sogou-art.html"], capture_output=True, timeout=40, text=True)
|
|
|
+ import os
|
|
|
+ if not os.path.exists("/tmp/sogou-art.html"):
|
|
|
+ print("抓取失败, curl stderr尾部:", (r2.stderr or "")[-300:])
|
|
|
+ else:
|
|
|
+ art = open("/tmp/sogou-art.html", encoding="utf-8", errors="ignore").read()
|
|
|
+ print("art len:", len(art))
|
|
|
+ print("含js_content:", "js_content" in art, "| 含环境异常:", "环境异常" in art)
|
|
|
+ m = re.search(r'id="js_content"[^>]*>(.*)$', art, re.S)
|
|
|
+ if m:
|
|
|
+ seg = m.group(1)
|
|
|
+ text = re.sub(r"<style[^>]*>.*?</style>", "", seg, flags=re.S)
|
|
|
+ text = re.sub(r"<[^>]+>", "\n", text)
|
|
|
+ text = re.sub(r"\n\s*\n+", "\n", text).strip()
|
|
|
+ print("正文长度:", len(text))
|
|
|
+ open("/tmp/wx-article.txt", "w", encoding="utf-8").write(text)
|
|
|
+ print(text[:1000])
|