kangshifu-board/docs/workshop-report-design-20261109/inspect_reports.py

"""Keep public reference reports and readable structure for local design review."""
from html.parser import HTMLParser
from pathlib import Path
from urllib.request import Request, urlopen
import argparse
import hashlib
import json

OUT = Path(__file__).resolve().parent / 'reference'


class ReportParser(HTMLParser):
    def __init__(self):
        super().__init__()
        self.skip = 0
        self.lines = []
        self.headings = []
        self.heading = None
        self.links = []
        self.anchor = None
        self.tables = 0
        self.images = 0

    def handle_starttag(self, tag, attrs):
        attrs = dict(attrs)
        if tag in ('script', 'style'):
            self.skip += 1
        if tag in ('h1', 'h2', 'h3', 'h4'):
            self.heading = {'level': tag, 'id': attrs.get('id'), 'text': ''}
            self.lines.append('\n')
        if tag == 'a':
            self.anchor = {'href': attrs.get('href', ''), 'text': ''}
        if tag == 'table':
            self.tables += 1
        if tag == 'img':
            self.images += 1

    def handle_endtag(self, tag):
        if tag in ('script', 'style'):
            self.skip -= 1
        if tag in ('h1', 'h2', 'h3', 'h4') and self.heading:
            self.headings.append(self.heading)
            self.heading = None
        if tag == 'a' and self.anchor:
            self.links.append(self.anchor)
            self.anchor = None
        if tag in ('h1', 'h2', 'h3', 'h4', 'p', 'li', 'tr', 'section', 'article', 'blockquote'):
            self.lines.append('\n')

    def handle_data(self, data):
        if not self.skip and data.strip():
            self.lines.append(data.strip() + ' ')
            if self.heading:
                self.heading['text'] += data.strip()
            if self.anchor:
                self.anchor['text'] += data.strip()


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('name')
    parser.add_argument('url')
    args = parser.parse_args()
    OUT.mkdir(exist_ok=True)
    with urlopen(Request(args.url, headers={'User-Agent': 'Mozilla/5.0'}), timeout=40) as response:
        content = response.read()
        status = response.status
        final_url = response.url
    target = OUT / (args.name + '.html')
    if target.exists():
        assert target.read_bytes() == content, 'Reference changed; retain the earlier version before refreshing'
    else:
        target.write_bytes(content)
    parsed = ReportParser()
    parsed.feed(content.decode('utf-8', 'replace'))
    readable = '\n'.join(line.strip() for line in ''.join(parsed.lines).splitlines() if line.strip())
    (OUT / (args.name + '.txt')).write_text(readable + '\n', encoding='utf-8')
    result = {'url': args.url, 'finalUrl': final_url, 'httpStatus': status,
              'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest(),
              'headings': parsed.headings, 'tables': parsed.tables, 'images': parsed.images,
              'links': parsed.links}
    (OUT / (args.name + '.json')).write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
    print(json.dumps({k: v for k, v in result.items() if k != 'links'}, ensure_ascii=False, indent=2))


if __name__ == '__main__':
    main()