"""Keep public reference reports and readable structure for local design review.""" from html.parser import HTMLParser from pathlib import Path from urllib.request import Request, urlopen import argparse import hashlib import json OUT = Path(__file__).resolve().parent / 'reference' class ReportParser(HTMLParser): def __init__(self): super().__init__() self.skip = 0 self.lines = [] self.headings = [] self.heading = None self.links = [] self.anchor = None self.tables = 0 self.images = 0 def handle_starttag(self, tag, attrs): attrs = dict(attrs) if tag in ('script', 'style'): self.skip += 1 if tag in ('h1', 'h2', 'h3', 'h4'): self.heading = {'level': tag, 'id': attrs.get('id'), 'text': ''} self.lines.append('\n') if tag == 'a': self.anchor = {'href': attrs.get('href', ''), 'text': ''} if tag == 'table': self.tables += 1 if tag == 'img': self.images += 1 def handle_endtag(self, tag): if tag in ('script', 'style'): self.skip -= 1 if tag in ('h1', 'h2', 'h3', 'h4') and self.heading: self.headings.append(self.heading) self.heading = None if tag == 'a' and self.anchor: self.links.append(self.anchor) self.anchor = None if tag in ('h1', 'h2', 'h3', 'h4', 'p', 'li', 'tr', 'section', 'article', 'blockquote'): self.lines.append('\n') def handle_data(self, data): if not self.skip and data.strip(): self.lines.append(data.strip() + ' ') if self.heading: self.heading['text'] += data.strip() if self.anchor: self.anchor['text'] += data.strip() def main(): parser = argparse.ArgumentParser() parser.add_argument('name') parser.add_argument('url') args = parser.parse_args() OUT.mkdir(exist_ok=True) with urlopen(Request(args.url, headers={'User-Agent': 'Mozilla/5.0'}), timeout=40) as response: content = response.read() status = response.status final_url = response.url target = OUT / (args.name + '.html') if target.exists(): assert target.read_bytes() == content, 'Reference changed; retain the earlier version before refreshing' else: target.write_bytes(content) parsed = ReportParser() parsed.feed(content.decode('utf-8', 'replace')) readable = '\n'.join(line.strip() for line in ''.join(parsed.lines).splitlines() if line.strip()) (OUT / (args.name + '.txt')).write_text(readable + '\n', encoding='utf-8') result = {'url': args.url, 'finalUrl': final_url, 'httpStatus': status, 'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest(), 'headings': parsed.headings, 'tables': parsed.tables, 'images': parsed.images, 'links': parsed.links} (OUT / (args.name + '.json')).write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') print(json.dumps({k: v for k, v in result.items() if k != 'links'}, ensure_ascii=False, indent=2)) if __name__ == '__main__': main()