| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869 |
- """Observe a normal public page; no login, cookie injection or challenge handling."""
- import argparse
- import base64
- import hashlib
- import json
- from pathlib import Path
- import subprocess
- import tempfile
- import time
- from urllib.parse import urlsplit,parse_qs
- from urllib.request import urlopen
- import websocket
- def main():
- p=argparse.ArgumentParser(description=__doc__);p.add_argument('--url',required=True);p.add_argument('--out',type=Path,required=True)
- a=p.parse_args();a.out.mkdir(parents=True,exist_ok=False)
- profile=Path(tempfile.mkdtemp(prefix='public-source-',dir=a.out))
- browser=subprocess.Popen(['C:/Program Files (x86)/Microsoft/Edge/Application/msedge.exe','--headless=new','--disable-gpu',
- '--disable-extensions','--no-first-run','--remote-debugging-port=0','--user-data-dir='+str(profile),'about:blank'],
- stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,creationflags=subprocess.CREATE_NO_WINDOW)
- ws=None;events=[];serial=0;report={'url':a.url,'paidCalls':0,'loginAttempted':False,'challengeHandling':False}
- try:
- end=time.monotonic()+15
- while not (profile/'DevToolsActivePort').exists():
- if time.monotonic()>end:raise TimeoutError('BROWSER_START_TIMEOUT')
- time.sleep(.1)
- port=(profile/'DevToolsActivePort').read_text().splitlines()[0]
- target=next(x for x in json.load(urlopen('http://127.0.0.1:'+port+'/json/list')) if x['type']=='page')
- ws=websocket.create_connection(target['webSocketDebuggerUrl'],timeout=20,suppress_origin=True)
- def call(method,params=None):
- nonlocal serial
- serial+=1;ws.send(json.dumps({'id':serial,'method':method,'params':params or {}}))
- while True:
- r=json.loads(ws.recv())
- if r.get('id')==serial:return r.get('result',{})
- events.append(r)
- call('Page.enable');call('Network.enable');call('Runtime.enable')
- call('Page.navigate',{'url':a.url})
- # Bounded normal page load. Never interact with sign-in or access challenges.
- time.sleep(12)
- result=call('Runtime.evaluate',{'expression':'({title:document.title,url:location.href,text:document.body?.innerText?.slice(0,5000)})','returnByValue':True})
- report['page']=result.get('result',{}).get('value')
- (a.out/'page.png').write_bytes(base64.b64decode(call('Page.captureScreenshot',{'format':'png'})['data']))
- responses=[e['params'] for e in events if e.get('method')=='Network.responseReceived']
- bodies=[];requests=[]
- for response in responses:
- r=response['response'];url=urlsplit(r['url']);query=parse_qs(url.query)
- info={'host':url.hostname,'path':url.path,'status':r['status'],'mimeType':r.get('mimeType'),
- 'functionId':query.get('functionId')}
- requests.append(info)
- if response.get('type') in ('XHR','Fetch') and 'json' in r.get('mimeType',''):
- try:
- body=call('Network.getResponseBody',{'requestId':response['requestId']})
- raw=base64.b64decode(body['body']) if body.get('base64Encoded') else body.get('body','').encode()
- if len(raw)<=5_000_000:
- digest=hashlib.sha256(raw).hexdigest();(a.out/(digest+'.json')).write_bytes(raw);bodies.append({**info,'sha256':digest,'bytes':len(raw)})
- except Exception:pass
- report.update(responses=requests,jsonResponses=bodies)
- finally:
- if ws:ws.close()
- browser.terminate()
- try:browser.wait(timeout=8)
- except subprocess.TimeoutExpired:browser.kill();browser.wait()
- (a.out/'receipt.json').write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n','utf-8')
- print(json.dumps({'page':report.get('page'),'jsonResponses':report.get('jsonResponses'),'responses':len(report.get('responses',[]))},ensure_ascii=False))
- if __name__=='__main__':main()
|