test_library_channel.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176
  1. import base64
  2. import contextlib
  3. import hashlib
  4. import io
  5. import json
  6. import os
  7. from pathlib import Path
  8. import sqlite3
  9. import subprocess
  10. import tempfile
  11. import unittest
  12. from unittest.mock import patch
  13. import agent_batch
  14. import library_channel as client
  15. import library_receiver as receiver
  16. import update_library
  17. import voc_db as v
  18. class LibraryChannelTests(unittest.TestCase):
  19. def setUp(self):
  20. self.temp=tempfile.TemporaryDirectory();self.addCleanup(self.temp.cleanup)
  21. self.root=Path(self.temp.name);self.registry=self.root/'registry';self.intake=self.root/'intake';self.intake.mkdir(mode=0o700)
  22. self.rawroot=self.root/'input';(self.rawroot/'raw').mkdir(parents=True)
  23. self.raw=self.rawroot/'raw/response.json'
  24. self.raw.write_text(json.dumps({'product':{'id':'123','title':'康师傅香辣拌面'},'review':{'item':'123','id':'r1','text':'味道很好,一包吃不饱','default':False,'sku':'香辣拌面'}}),encoding='utf-8')
  25. self.batch={'schema':'ksf.voc-agent-batch.v2','batchId':'first','producer':'test-agent',
  26. 'sources':[{'id':'s','path':'raw/response.json','sha256':agent_batch.digest(self.raw),'platform':'jd','marketplace':'jd','requestItemId':'123'}],
  27. 'products':[{'sourceId':'s','pointer':'/product','fields':{'itemId':'/id','title':'/title'}}],
  28. 'reviews':[{'sourceId':'s','pointer':'/review','fields':{'itemId':'/item','reviewId':'/id','content':'/text','isDefault':'/default','purchasedSku':'/sku'}}]}
  29. self.path=self.rawroot/'batch.json';self.save()
  30. stage=self.root/'initial.sqlite';agent_batch.build_batch(self.path,stage,True)
  31. self.initial=update_library.publish(stage,self.registry,True)['generation']
  32. self.config={'schema':client.VERSION,'registry':str(self.registry),'intake':str(self.intake),'canPublish':True}
  33. def save(self):self.path.write_text(json.dumps(self.batch),encoding='utf-8')
  34. def request(self,apply=False):
  35. raw,h=client.pack(self.path)
  36. return {'schema':client.VERSION,'action':'intake','archiveBase64':base64.b64encode(raw).decode(),
  37. 'params':{'archiveSha256':h,'expectedGeneration':self.initial,'label':'真实批次测试','apply':apply}}
  38. def test_remote_reads_build_publish_and_same_generation_projection(self):
  39. self.batch['batchId']='second';self.save()
  40. request=self.request();before=(self.registry/'current.json').read_bytes()
  41. plan=receiver.dispatch(self.config,request)
  42. self.assertTrue(plan['readOnly']);self.assertFalse(plan['persistentFilesWritten']);self.assertEqual(list(self.intake.iterdir()),[])
  43. self.assertEqual(before,(self.registry/'current.json').read_bytes())
  44. request['params']['apply']=True;result=receiver.dispatch(self.config,request)
  45. self.assertTrue(result['remotePublished']);self.assertTrue(result['sourceArchivePreserved']);self.assertNotEqual(result['generation'],self.initial)
  46. contract=receiver.dispatch(self.config,{'schema':client.VERSION,'action':'read','command':'contract','params':{}})
  47. self.assertEqual(contract['generation'],result['generation']);self.assertEqual(contract['publicationBinding']['mode'],'ssh-channel')
  48. self.assertTrue(contract['publicationBinding']['remotePublisherEnabled'])
  49. schema=receiver.dispatch(self.config,{'schema':client.VERSION,'action':'read','command':'schema','params':{'table':'reviews','generation':result['generation']}})
  50. self.assertEqual(schema['generation'],result['generation']);self.assertTrue(schema['complete'])
  51. review=receiver.dispatch(self.config,{'schema':client.VERSION,'action':'read','command':'get','params':{'entity':'reviews','id':'jd:123:r1','generation':result['generation']}})
  52. self.assertEqual(review['record']['content'],'味道很好,一包吃不饱')
  53. source=receiver.dispatch(self.config,{'schema':client.VERSION,'action':'read','command':'source','params':{'id':review['record']['source_record_id'],'generation':result['generation']}})
  54. self.assertTrue(source['ok']);self.assertEqual(source['sourcePayload']['source_locator'],'/review')
  55. status=receiver.dispatch(self.config,{'schema':client.VERSION,'action':'status','params':{'archiveSha256':request['params']['archiveSha256']}})
  56. self.assertTrue(status['publicationFound']);self.assertEqual(status['publishedGeneration'],result['generation'])
  57. with self.assertRaisesRegex(v.LibraryError,'LIBRARY_GENERATION_CHANGED'):receiver.dispatch(self.config,request)
  58. request['params']['expectedGeneration']=result['generation']
  59. self.assertFalse(receiver.dispatch(self.config,request)['applied'])
  60. def test_receiver_rejects_arbitrary_paths_commands_permissions_and_changed_bytes(self):
  61. invalid=[{'schema':client.VERSION,'action':'read','command':'build','params':{}},
  62. {'schema':client.VERSION,'action':'read','command':'get','params':{'db':'/etc/shadow'}},
  63. {'schema':client.VERSION,'action':'shell','params':{}},
  64. {'schema':client.VERSION,'action':'read','command':'schema','params':{'table':'--db'}}]
  65. for request in invalid:
  66. with self.assertRaises(v.LibraryError):receiver.dispatch(self.config,request)
  67. request=self.request();request['params']['archiveSha256']='0'*64
  68. with self.assertRaisesRegex(v.LibraryError,'ARCHIVE_HASH_MISMATCH'):receiver.dispatch(self.config,request)
  69. with self.assertRaisesRegex(v.LibraryError,'PUBLICATION_DISABLED'):receiver.dispatch({**self.config,'canPublish':False},self.request())
  70. self.assertEqual(list(self.intake.iterdir()),[])
  71. def test_archive_paths_symlinks_extra_members_and_tampering_rejected(self):
  72. import zipfile
  73. for name in ('../escape','/absolute','raw/../../escape','raw/unreferenced.txt'):
  74. output=io.BytesIO()
  75. with zipfile.ZipFile(output,'w') as z:
  76. z.writestr('batch.json',self.path.read_bytes());z.writestr('raw/response.json',self.raw.read_bytes());z.writestr(name,'bad')
  77. with tempfile.TemporaryDirectory(dir=self.root) as td:
  78. with self.assertRaises(v.LibraryError):receiver.unpack(output.getvalue(),Path(td))
  79. self.assertFalse((self.root/'escape').exists())
  80. def test_transport_is_fixed_command_strict_host_check_and_never_retries_uncertain_publish(self):
  81. key=self.root/'key';hosts=self.root/'known_hosts';key.write_text('test-only-key');hosts.write_text('test-only-host')
  82. os.chmod(key,0o600);os.chmod(hosts,0o600)
  83. conf=self.root/'channel.json';conf.write_text(v.canonical({'schema':client.VERSION,'host':'127.0.0.1','port':22,'user':'publisher','identityFile':str(key),'knownHostsFile':str(hosts)}));os.chmod(conf,0o600)
  84. calls=[]
  85. def run(command,**kwargs):
  86. calls.append(command);self.assertEqual(command[-1],'ksf-library-v1');self.assertIn('StrictHostKeyChecking=yes',command)
  87. self.assertNotIn('shell',kwargs);return subprocess.CompletedProcess(command,0,b'{"ok":true}',b'')
  88. self.assertTrue(client.send(conf,{'action':'read','command':'contract','params':{}},run)['ok'])
  89. def timeout(command,**kwargs):calls.append(command);raise subprocess.TimeoutExpired(command,300)
  90. with self.assertRaisesRegex(v.LibraryError,'CHANNEL_RESULT_UNCERTAIN'):client.send(conf,{'action':'intake','params':{'apply':True}},timeout)
  91. self.assertEqual(len(calls),2)
  92. def test_hashing_and_publish_do_not_depend_on_file_digest(self):
  93. with patch.object(hashlib,'file_digest',None,create=True):
  94. self.assertEqual(agent_batch.digest(self.raw),hashlib.sha256(self.raw.read_bytes()).hexdigest())
  95. self.assertEqual(update_library.digest(self.raw),agent_batch.digest(self.raw))
  96. self.batch['batchId']='compat';self.save()
  97. new=self.root/'compat.sqlite';agent_batch.build_batch(self.path,new,True)
  98. self.assertTrue(update_library.publish(new,self.registry,True,expected_generation=self.initial)['applied'])
  99. def test_search_crosses_real_receiver_subprocess_without_changing_scope(self):
  100. query={'entities':['reviews'],'filters':{'platform':'jd','brand':'ksf','category':'banmian'},
  101. 'text':{'any':['吃不饱','味道']},'quality':'analysis','limit':20}
  102. before=(self.registry/'current.json').read_bytes()
  103. result=receiver.dispatch(self.config,{'schema':client.VERSION,'action':'read','command':'search',
  104. 'params':{'generation':self.initial,'input':query}})
  105. self.assertTrue(result['ok']);self.assertGreater(result['totalMatched'],0)
  106. self.assertEqual(result['generation'],self.initial)
  107. self.assertEqual(result['effectiveQuery']['filters'],{k:{'eq':v} for k,v in query['filters'].items()})
  108. self.assertEqual(result['effectiveQuery']['text']['groupsAll'],[['吃不饱','味道']])
  109. self.assertEqual(before,(self.registry/'current.json').read_bytes())
  110. self.assertEqual(result['databaseWrites'],0)
  111. def test_input_bom_and_local_errors_are_not_reported_as_ssh_failure(self):
  112. query={'entities':['reviews'],'text':{'any':['份量']}}
  113. with patch('sys.stdin',io.StringIO('\ufeff'+json.dumps(query))):
  114. self.assertEqual(client.read_input('-'),query)
  115. file=self.root/'query.json';file.write_bytes(b'\xef\xbb\xbf'+json.dumps(query).encode())
  116. self.assertEqual(client.read_input(str(file)),query)
  117. for value,code in [('{"entities":[]}','READ_INPUT_PATH_EXPECTED'),
  118. (str(self.root/'missing.json'),'READ_INPUT_FILE_NOT_FOUND')]:
  119. with self.assertRaisesRegex(v.LibraryError,code):client.read_input(value)
  120. file.write_text('{broken',encoding='utf-8')
  121. with self.assertRaisesRegex(v.LibraryError,'READ_INPUT_INVALID_JSON'):client.read_input(str(file))
  122. file.write_text('[]',encoding='utf-8')
  123. with self.assertRaisesRegex(v.LibraryError,'READ_INPUT_OBJECT_REQUIRED'):client.read_input(str(file))
  124. with patch.object(client,'send') as send,patch('sys.stderr',io.StringIO()) as err:
  125. exitcode=client.read_main(['search','--channel','unused','--input',str(self.root/'absent')])
  126. self.assertEqual(exitcode,1);send.assert_not_called()
  127. self.assertEqual(json.loads(err.getvalue())['error']['code'],'READ_INPUT_FILE_NOT_FOUND')
  128. def test_channel_failure_diagnostics_never_echo_stderr(self):
  129. secret=b'Authorization: Bearer sentinel-do-not-print'
  130. with patch.object(client,'config',return_value=dict(host='localhost',user='publisher',port=22,
  131. identityFile='private-key',knownHostsFile='known-hosts')):
  132. def failed(command,**kwargs):return subprocess.CompletedProcess(command,255,b'',secret)
  133. with self.assertRaises(client.ChannelError) as got:
  134. client.send('unused',{'action':'read','command':'search','params':{}},failed)
  135. self.assertEqual(str(got.exception),'CHANNEL_READ_FAILED')
  136. self.assertEqual(got.exception.diagnostics['exitCode'],255)
  137. self.assertEqual(got.exception.diagnostics['stderrSha256'],v.sha(secret))
  138. self.assertNotIn('sentinel',json.dumps(got.exception.diagnostics))
  139. with self.assertRaisesRegex(client.ChannelError,'CHANNEL_RESULT_UNCERTAIN'):
  140. client.send('unused',{'action':'intake','params':{'apply':True}},failed)
  141. def test_receiver_failure_result_keeps_safe_stage_and_structured_errors(self):
  142. request={'schema':client.VERSION,'action':'read','command':'search','params':{'input':{}}}
  143. with patch.object(receiver.subprocess,'run',return_value=subprocess.CompletedProcess([],1,b' \n',b'{"ok":false,"error":{"code":"INVALID_SEARCH_REQUEST"}}')):
  144. self.assertEqual(receiver.read(self.config,request)['error']['code'],'INVALID_SEARCH_REQUEST')
  145. with patch.object(receiver.subprocess,'run',return_value=subprocess.CompletedProcess([],1,b'',b'Traceback: secret-path-and-token')):
  146. with self.assertRaises(client.ChannelError) as got:receiver.read(self.config,request)
  147. self.assertEqual(str(got.exception),'INVALID_READ_RESULT')
  148. self.assertEqual(got.exception.diagnostics['stage'],'receiver-query')
  149. self.assertNotIn('secret',json.dumps(got.exception.diagnostics))
  150. def test_review_get_uses_physical_fields_and_preserves_unknowns(self):
  151. self.raw.write_text(json.dumps({'product':{'id':'124','title':'康师傅香辣拌面'},
  152. 'review':{'item':'124','id':'r2','text':'测试字段回读','date':'2026-11-06 10:49:00','rating':'5'}}),encoding='utf-8')
  153. self.batch['batchId']='review-fields'
  154. self.batch['sources'][0]['sha256']=agent_batch.digest(self.raw)
  155. self.batch['sources'][0]['requestItemId']='124'
  156. self.batch['reviews'][0]['fields']={'itemId':'/item','reviewId':'/id','content':'/text','reviewDate':'/date','rating':'/rating'}
  157. self.save()
  158. request=self.request(True);published=receiver.dispatch(self.config,request)
  159. params={'entity':'reviews','id':'jd:124:r2','generation':published['generation']}
  160. record=receiver.read(self.config,{'schema':client.VERSION,'action':'read','command':'get','params':params})['record']
  161. self.assertEqual(record['id'],'jd:124:r2')
  162. self.assertEqual(record['review_date_raw'],'2026-11-06 10:49:00')
  163. self.assertEqual(record['review_calendar_date'],'2026-11-06')
  164. self.assertEqual(record['rating_raw'],'5')
  165. self.assertIsNone(record['rating_scale']);self.assertIsNone(record['is_default'])
  166. self.assertIsNone(record['review_date_utc'])
  167. for wrong in ('recordid','platform','reviewday','ratingvalue','isdefault'):
  168. self.assertNotIn(wrong,record)
  169. if __name__=='__main__':unittest.main()