probe-industry-public-page.py 3.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869
  1. """Observe a normal public page; no login, cookie injection or challenge handling."""
  2. import argparse
  3. import base64
  4. import hashlib
  5. import json
  6. from pathlib import Path
  7. import subprocess
  8. import tempfile
  9. import time
  10. from urllib.parse import urlsplit,parse_qs
  11. from urllib.request import urlopen
  12. import websocket
  13. def main():
  14. p=argparse.ArgumentParser(description=__doc__);p.add_argument('--url',required=True);p.add_argument('--out',type=Path,required=True)
  15. a=p.parse_args();a.out.mkdir(parents=True,exist_ok=False)
  16. profile=Path(tempfile.mkdtemp(prefix='public-source-',dir=a.out))
  17. browser=subprocess.Popen(['C:/Program Files (x86)/Microsoft/Edge/Application/msedge.exe','--headless=new','--disable-gpu',
  18. '--disable-extensions','--no-first-run','--remote-debugging-port=0','--user-data-dir='+str(profile),'about:blank'],
  19. stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,creationflags=subprocess.CREATE_NO_WINDOW)
  20. ws=None;events=[];serial=0;report={'url':a.url,'paidCalls':0,'loginAttempted':False,'challengeHandling':False}
  21. try:
  22. end=time.monotonic()+15
  23. while not (profile/'DevToolsActivePort').exists():
  24. if time.monotonic()>end:raise TimeoutError('BROWSER_START_TIMEOUT')
  25. time.sleep(.1)
  26. port=(profile/'DevToolsActivePort').read_text().splitlines()[0]
  27. target=next(x for x in json.load(urlopen('http://127.0.0.1:'+port+'/json/list')) if x['type']=='page')
  28. ws=websocket.create_connection(target['webSocketDebuggerUrl'],timeout=20,suppress_origin=True)
  29. def call(method,params=None):
  30. nonlocal serial
  31. serial+=1;ws.send(json.dumps({'id':serial,'method':method,'params':params or {}}))
  32. while True:
  33. r=json.loads(ws.recv())
  34. if r.get('id')==serial:return r.get('result',{})
  35. events.append(r)
  36. call('Page.enable');call('Network.enable');call('Runtime.enable')
  37. call('Page.navigate',{'url':a.url})
  38. # Bounded normal page load. Never interact with sign-in or access challenges.
  39. time.sleep(12)
  40. result=call('Runtime.evaluate',{'expression':'({title:document.title,url:location.href,text:document.body?.innerText?.slice(0,5000)})','returnByValue':True})
  41. report['page']=result.get('result',{}).get('value')
  42. (a.out/'page.png').write_bytes(base64.b64decode(call('Page.captureScreenshot',{'format':'png'})['data']))
  43. responses=[e['params'] for e in events if e.get('method')=='Network.responseReceived']
  44. bodies=[];requests=[]
  45. for response in responses:
  46. r=response['response'];url=urlsplit(r['url']);query=parse_qs(url.query)
  47. info={'host':url.hostname,'path':url.path,'status':r['status'],'mimeType':r.get('mimeType'),
  48. 'functionId':query.get('functionId')}
  49. requests.append(info)
  50. if response.get('type') in ('XHR','Fetch') and 'json' in r.get('mimeType',''):
  51. try:
  52. body=call('Network.getResponseBody',{'requestId':response['requestId']})
  53. raw=base64.b64decode(body['body']) if body.get('base64Encoded') else body.get('body','').encode()
  54. if len(raw)<=5_000_000:
  55. digest=hashlib.sha256(raw).hexdigest();(a.out/(digest+'.json')).write_bytes(raw);bodies.append({**info,'sha256':digest,'bytes':len(raw)})
  56. except Exception:pass
  57. report.update(responses=requests,jsonResponses=bodies)
  58. finally:
  59. if ws:ws.close()
  60. browser.terminate()
  61. try:browser.wait(timeout=8)
  62. except subprocess.TimeoutExpired:browser.kill();browser.wait()
  63. (a.out/'receipt.json').write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n','utf-8')
  64. print(json.dumps({'page':report.get('page'),'jsonResponses':report.get('jsonResponses'),'responses':len(report.get('responses',[]))},ensure_ascii=False))
  65. if __name__=='__main__':main()