test_voc_db.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197
  1. import contextlib
  2. import importlib.util
  3. import io
  4. import json
  5. from pathlib import Path
  6. import sqlite3
  7. import tempfile
  8. import unittest
  9. from unittest.mock import patch
  10. import zipfile
  11. spec = importlib.util.spec_from_file_location('voc_db', Path(__file__).with_name('voc_db.py'))
  12. v = importlib.util.module_from_spec(spec)
  13. spec.loader.exec_module(v)
  14. class VocLibraryTests(unittest.TestCase):
  15. @classmethod
  16. def setUpClass(cls):
  17. cls.temp = tempfile.TemporaryDirectory()
  18. cls.base = Path(cls.temp.name)
  19. cls.archive = cls.base / 'fixture.zip'
  20. cls.db = cls.base / 'library.sqlite'
  21. product = lambda iid, title, platform='jd', market='jd': dict(platform=platform, item_id=iid, title=title, marketplace=market, source_file='raw/items.json', source_time='2026-11-05T10:00:00', collected_at='2026-11-05T02:00:00Z', price='19.90', price_type='search_listing_snapshot', shop_id='shop-1')
  22. products = [product('same','康师傅干拌面110g*5袋'), product('other-brand','白象干拌面110g*5袋'),
  23. product('same','康师傅干拌面110g*5袋','taobao','tmall'), product('soup','康师傅汤面'),
  24. product('sauce','仲景葱油拌面酱'), product('unknown','这是一款干拌面'),
  25. product('mixed','康师傅统一拌面组合'), product('mixed-category','康师傅拌面汤面组合'),
  26. product('unverified-channel','康师傅拌面','taobao','taobao_unverified')]
  27. def review(rid, default=False, match=True, sku='香辣干拌面'):
  28. return dict(platform='jd',marketplace='jd',item_id='same',review_id=rid,content='面量有点少',sku=sku,is_default_text=default,
  29. item_identity_match=match,observed_item_id='same',rating='5',rating_scale='jd原始星级',review_date='2026-11-04 10:30:00',source_file='raw/reviews.json')
  30. reviews = [review('106582280238337296'),review('default',True),review('bad-identity',match=False),review('wrong-category',sku='红烧汤面'), review('unclear-sku',sku='红烧牛肉5袋')]
  31. rows = dict(products=products,reviews=reviews,social=[dict(platform='xiaohongshu',id='post-1',title='康师傅与白象拌面对比',content='',content_scope='packaged_noodles',comments=1000)],
  32. documents=[dict(document_id='standard',title='方便面标准',status_on_20261104='current',source_file='raw/policy.txt',sha256=v.sha(b'policy'))],
  33. clauses=[dict(document_id='standard',clause='2.1',source_excerpt='摘要内容',application_note='应用建议',excerpt_method='人工摘要,非逐字')])
  34. with zipfile.ZipFile(cls.archive,'w') as z:
  35. for kind,name in v.PRIMARY.items():
  36. z.writestr('test/01_最新底库/'+name,'\n'.join(v.canonical(r) for r in rows[kind]))
  37. for f in ['items.json','reviews.json']:
  38. z.writestr('test/02_采集档案/raw/'+f,'{}')
  39. z.writestr('test/02_采集档案/raw/policy.txt','policy')
  40. cls.built = v.build(cls.archive,cls.db)
  41. @classmethod
  42. def tearDownClass(cls):
  43. cls.temp.cleanup()
  44. def q(self,**overrides):
  45. request = dict(entity='products',marketplace='京东',brand='康师傅',category='拌面',limit=10)
  46. request.update(overrides)
  47. with v.readonly(self.db) as c:
  48. return v.query(c,request,explain=True)
  49. def test_exact_three_dimensions_and_cross_platform_id(self):
  50. self.assertEqual([r['id'] for r in self.q()['records']],['jd:same'])
  51. self.assertEqual([r['id'] for r in self.q(marketplace='天猫')['records']],['taobao:same'])
  52. self.assertEqual([r['id'] for r in self.q(brand='白象')['records']],['jd:other-brand'])
  53. def test_cli_generation_guard_and_source_use_same_active_library(self):
  54. before=v.sha(self.db.read_bytes())
  55. registry=self.base/'cli-registry';registry.mkdir(exist_ok=True)
  56. (registry/'current.json').write_text(v.canonical({'file':'../library.sqlite'}),encoding='utf-8')
  57. with self.assertRaises(v.LibraryError):v.registry_database(registry)
  58. pointer={'file':'library.sqlite'}
  59. import shutil
  60. shutil.copyfile(self.db,registry/'library.sqlite')
  61. (registry/'current.json').write_text(v.canonical(pointer),encoding='utf-8')
  62. generation=self.built['generation']
  63. with v.readonly(self.db) as c:
  64. sid=c.execute('SELECT source_record_id FROM reviews LIMIT 1').fetchone()[0]
  65. requests=[['contract'],['aggregate','--input','-'],['get','--entity','products','--id','jd:same'],['source','--id',sid]]
  66. for command in requests:
  67. with self.subTest(command=command[0]):
  68. stdout,stderr=io.StringIO(),io.StringIO()
  69. argv=['query_voc.py',*command,'--registry',str(registry),'--generation',generation]
  70. with patch.object(v.sys,'argv',argv),patch.object(v.sys,'stdin',io.StringIO('{"entities":["products"],"groupBy":["platform"]}')),contextlib.redirect_stdout(stdout),contextlib.redirect_stderr(stderr):
  71. self.assertEqual(v.main(),0,stderr.getvalue())
  72. result=json.loads(stdout.getvalue())
  73. self.assertEqual(result['generation'],generation);self.assertTrue(result['readOnly']);self.assertEqual(result['databaseWrites'],0)
  74. stdout,stderr=io.StringIO(),io.StringIO()
  75. with patch.object(v.sys,'argv',['query_voc.py','get','--registry',str(registry),'--generation','stale','--entity','products','--id','jd:same']),contextlib.redirect_stdout(stdout),contextlib.redirect_stderr(stderr):
  76. self.assertEqual(v.main(),1)
  77. self.assertEqual(stdout.getvalue(),'');self.assertEqual(json.loads(stderr.getvalue())['error']['code'],'LIBRARY_GENERATION_CHANGED')
  78. self.assertEqual(v.sha(self.db.read_bytes()),before)
  79. def test_sauce_soup_mixed_unknown_do_not_enter_banmian(self):
  80. allbrands = self.q(brand='all')
  81. self.assertEqual({r['id'] for r in allbrands['records']},{'jd:same','jd:other-brand'})
  82. self.assertEqual(self.q(category='汤面')['records'][0]['id'],'jd:soup')
  83. self.assertEqual(self.q(brand='仲景',category='酱料调料')['records'][0]['id'],'jd:sauce')
  84. def test_brand_never_comes_from_shop_or_search(self):
  85. tax=v.taxonomy()
  86. self.assertIsNone(v.classify_brand('干拌面促销',tax)[0])
  87. self.assertIsNone(v.classify_brand('康师傅统一拌面',tax)[0])
  88. self.assertEqual(v.classify_brand('拌面',tax,'康师傅')[0],'ksf')
  89. self.assertIsNone(v.classify_brand('白象拌面',tax,'康师傅')[0])
  90. def test_tmall_is_not_unverified_taobao(self):
  91. self.assertEqual(self.q(marketplace='tmall')['totalMatched'],1)
  92. self.assertEqual(self.q(marketplace='taobao_unverified')['records'][0]['id'],'taobao:unverified-channel')
  93. def test_review_default_identity_and_sku_filters(self):
  94. r=self.q(entity='reviews')
  95. self.assertEqual([x['review_id'] for x in r['records']],['106582280238337296'])
  96. self.assertIsInstance(r['records'][0]['review_id'],str)
  97. self.assertEqual(self.q(entity='reviews',includeDefault=True)['totalMatched'],2)
  98. self.assertEqual(self.q(entity='reviews',reviewScope='listing')['totalMatched'],2)
  99. self.assertEqual(self.q(entity='reviews',marketplace='tmall')['totalMatched'],0)
  100. def test_zero_is_not_permission_or_query_error(self):
  101. self.assertEqual(self.q(brand='白象',category='汤面')['totalMatched'],0)
  102. with self.assertRaisesRegex(v.LibraryError,'UNKNOWN_OR_AMBIGUOUS'):
  103. self.q(brand='不存在的品牌')
  104. def test_missing_scope_and_unsupported_filters_rejected(self):
  105. with self.assertRaises(v.LibraryError):self.q(brand=None)
  106. with self.assertRaisesRegex(v.LibraryError,'UNKNOWN_QUERY_FIELD'):self.q(filters={'brand':'ksf'})
  107. with self.assertRaises(v.LibraryError):self.q(**{'from':'2026-11-01','timeBasis':'review_calendar_date'})
  108. with self.assertRaises(v.LibraryError):self.q(limit=True)
  109. def test_cursor_bound_to_scope_and_generation(self):
  110. first=self.q(brand='all',limit=1)
  111. self.assertIsNotNone(first['nextCursor'])
  112. second=self.q(brand='all',limit=1,cursor=first['nextCursor'])
  113. self.assertNotEqual(first['records'][0]['id'],second['records'][0]['id'])
  114. self.assertIsNone(second['nextCursor'])
  115. with self.assertRaisesRegex(v.LibraryError,'CURSOR_SCOPE_MISMATCH'):
  116. self.q(brand='ksf',cursor=first['nextCursor'])
  117. def test_parameterized_sql_injection(self):
  118. self.assertEqual(self.q(keyword="' OR 1=1 --")['totalMatched'],0)
  119. self.assertEqual(self.q(itemId="same' OR 1=1 --")['totalMatched'],0)
  120. with self.assertRaises(v.LibraryError):self.q(entity='products;DROP TABLE products')
  121. def test_time_not_fabricated_and_date_boundaries(self):
  122. r=self.q(entity='reviews')['records'][0]
  123. self.assertIsNone(r['review_date_utc'])
  124. self.assertEqual(r['review_calendar_date'],'2026-11-04')
  125. self.assertEqual(self.q(entity='reviews',timeBasis='review_calendar_date',**{'from':'2026-11-04','to':'2026-11-05'})['totalMatched'],1)
  126. self.assertEqual(self.q(entity='reviews',timeBasis='review_calendar_date',**{'from':'2026-11-03','to':'2026-11-04'})['totalMatched'],0)
  127. self.assertEqual(self.q(entity='reviews',timeBasis='review_utc',**{'from':'2026-11-01T00:00:00Z'})['totalMatched'],0)
  128. with self.assertRaises(v.LibraryError):self.q(entity='reviews',timeBasis='review_utc',**{'from':'2026-11-01T00:00:00'})
  129. def test_spec_candidates_do_not_create_false_unit_prices(self):
  130. s=self.q()['records'][0]['specs'][0]
  131. self.assertEqual(s['verification'],'title_candidate')
  132. self.assertEqual(s['total_weight_g'],550)
  133. p=self.q(entity='prices')['records'][0]
  134. self.assertEqual(p['amount_minor'],1990)
  135. self.assertIsNone(p['price_per_100g'])
  136. self.assertIsNone(p['source_time_utc'])
  137. self.assertIsNone(v.money('1?'))
  138. self.assertIsNone(v.money('19-29'))
  139. def test_source_citation_is_resolvable(self):
  140. s=self.q()['records'][0]['citation']
  141. self.assertEqual(s['locatorType'],'jsonl_line_1_based')
  142. self.assertTrue(all(r['resolution']=='matched' for r in s['upstreamSources']))
  143. with zipfile.ZipFile(self.archive) as z:
  144. self.assertEqual(v.sha(z.read(s['asset_path'])),s['assetSha256'])
  145. self.assertEqual(v.sha(z.read(s['asset_path']).decode().splitlines()[s['line_number']-1]),s['payload_sha256'])
  146. def test_social_mentions_and_counters_are_separate(self):
  147. with v.readonly(self.db) as c:
  148. r=v.query(c,dict(entity='social',marketplace='xiaohongshu',brand='ksf'))
  149. self.assertEqual(r['records'][0]['comment_count'],1000)
  150. self.assertEqual(r['records'][0]['fulltext_available'],0)
  151. self.assertEqual(c.execute('select count(*) from reviews').fetchone()[0],5)
  152. self.assertEqual(c.execute('select count(*) from social_brand_mentions').fetchone()[0],2)
  153. def test_policy_paraphrase_not_a_quote_and_status_is_dated(self):
  154. with v.readonly(self.db) as c:
  155. row=v.query(c,dict(entity='clauses',documentId='standard'))['records'][0]
  156. self.assertEqual(row['excerpt_type'],'paraphrase_or_transcription')
  157. self.assertEqual(row['status_as_of'],'2026-11-04')
  158. def test_readonly_and_idempotent_build(self):
  159. before=v.sha(self.db.read_bytes())
  160. with v.readonly(self.db) as c:
  161. with self.assertRaises(sqlite3.OperationalError):c.execute('delete from products')
  162. self.assertFalse(c.execute('pragma foreign_key_check').fetchall())
  163. receipt=v.build(self.archive,self.db)
  164. self.assertEqual(receipt['operation'],'unchanged')
  165. self.assertEqual(v.sha(self.db.read_bytes()),before)
  166. def test_foreign_keys_and_index_plan(self):
  167. self.assertTrue(any('idx_products_scope' in x for x in self.q()['queryPlan']))
  168. # With five fixture reviews SQLite may reasonably prefer a tiny scan.
  169. # The full-corpus acceptance separately checks the actual join plan.
  170. with v.readonly(self.db) as c:
  171. self.assertIsNotNone(c.execute("select 1 from sqlite_master where type='index' and name='idx_reviews_product'").fetchone())
  172. if __name__=='__main__':
  173. unittest.main()