| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197 |
- import contextlib
- import importlib.util
- import io
- import json
- from pathlib import Path
- import sqlite3
- import tempfile
- import unittest
- from unittest.mock import patch
- import zipfile
- spec = importlib.util.spec_from_file_location('voc_db', Path(__file__).with_name('voc_db.py'))
- v = importlib.util.module_from_spec(spec)
- spec.loader.exec_module(v)
- class VocLibraryTests(unittest.TestCase):
- @classmethod
- def setUpClass(cls):
- cls.temp = tempfile.TemporaryDirectory()
- cls.base = Path(cls.temp.name)
- cls.archive = cls.base / 'fixture.zip'
- cls.db = cls.base / 'library.sqlite'
- product = lambda iid, title, platform='jd', market='jd': dict(platform=platform, item_id=iid, title=title, marketplace=market, source_file='raw/items.json', source_time='2026-11-05T10:00:00', collected_at='2026-11-05T02:00:00Z', price='19.90', price_type='search_listing_snapshot', shop_id='shop-1')
- products = [product('same','康师傅干拌面110g*5袋'), product('other-brand','白象干拌面110g*5袋'),
- product('same','康师傅干拌面110g*5袋','taobao','tmall'), product('soup','康师傅汤面'),
- product('sauce','仲景葱油拌面酱'), product('unknown','这是一款干拌面'),
- product('mixed','康师傅统一拌面组合'), product('mixed-category','康师傅拌面汤面组合'),
- product('unverified-channel','康师傅拌面','taobao','taobao_unverified')]
- def review(rid, default=False, match=True, sku='香辣干拌面'):
- return dict(platform='jd',marketplace='jd',item_id='same',review_id=rid,content='面量有点少',sku=sku,is_default_text=default,
- item_identity_match=match,observed_item_id='same',rating='5',rating_scale='jd原始星级',review_date='2026-11-04 10:30:00',source_file='raw/reviews.json')
- reviews = [review('106582280238337296'),review('default',True),review('bad-identity',match=False),review('wrong-category',sku='红烧汤面'), review('unclear-sku',sku='红烧牛肉5袋')]
- rows = dict(products=products,reviews=reviews,social=[dict(platform='xiaohongshu',id='post-1',title='康师傅与白象拌面对比',content='',content_scope='packaged_noodles',comments=1000)],
- documents=[dict(document_id='standard',title='方便面标准',status_on_20261104='current',source_file='raw/policy.txt',sha256=v.sha(b'policy'))],
- clauses=[dict(document_id='standard',clause='2.1',source_excerpt='摘要内容',application_note='应用建议',excerpt_method='人工摘要,非逐字')])
- with zipfile.ZipFile(cls.archive,'w') as z:
- for kind,name in v.PRIMARY.items():
- z.writestr('test/01_最新底库/'+name,'\n'.join(v.canonical(r) for r in rows[kind]))
- for f in ['items.json','reviews.json']:
- z.writestr('test/02_采集档案/raw/'+f,'{}')
- z.writestr('test/02_采集档案/raw/policy.txt','policy')
- cls.built = v.build(cls.archive,cls.db)
- @classmethod
- def tearDownClass(cls):
- cls.temp.cleanup()
- def q(self,**overrides):
- request = dict(entity='products',marketplace='京东',brand='康师傅',category='拌面',limit=10)
- request.update(overrides)
- with v.readonly(self.db) as c:
- return v.query(c,request,explain=True)
- def test_exact_three_dimensions_and_cross_platform_id(self):
- self.assertEqual([r['id'] for r in self.q()['records']],['jd:same'])
- self.assertEqual([r['id'] for r in self.q(marketplace='天猫')['records']],['taobao:same'])
- self.assertEqual([r['id'] for r in self.q(brand='白象')['records']],['jd:other-brand'])
- def test_cli_generation_guard_and_source_use_same_active_library(self):
- before=v.sha(self.db.read_bytes())
- registry=self.base/'cli-registry';registry.mkdir(exist_ok=True)
- (registry/'current.json').write_text(v.canonical({'file':'../library.sqlite'}),encoding='utf-8')
- with self.assertRaises(v.LibraryError):v.registry_database(registry)
- pointer={'file':'library.sqlite'}
- import shutil
- shutil.copyfile(self.db,registry/'library.sqlite')
- (registry/'current.json').write_text(v.canonical(pointer),encoding='utf-8')
- generation=self.built['generation']
- with v.readonly(self.db) as c:
- sid=c.execute('SELECT source_record_id FROM reviews LIMIT 1').fetchone()[0]
- requests=[['contract'],['aggregate','--input','-'],['get','--entity','products','--id','jd:same'],['source','--id',sid]]
- for command in requests:
- with self.subTest(command=command[0]):
- stdout,stderr=io.StringIO(),io.StringIO()
- argv=['query_voc.py',*command,'--registry',str(registry),'--generation',generation]
- with patch.object(v.sys,'argv',argv),patch.object(v.sys,'stdin',io.StringIO('{"entities":["products"],"groupBy":["platform"]}')),contextlib.redirect_stdout(stdout),contextlib.redirect_stderr(stderr):
- self.assertEqual(v.main(),0,stderr.getvalue())
- result=json.loads(stdout.getvalue())
- self.assertEqual(result['generation'],generation);self.assertTrue(result['readOnly']);self.assertEqual(result['databaseWrites'],0)
- stdout,stderr=io.StringIO(),io.StringIO()
- with patch.object(v.sys,'argv',['query_voc.py','get','--registry',str(registry),'--generation','stale','--entity','products','--id','jd:same']),contextlib.redirect_stdout(stdout),contextlib.redirect_stderr(stderr):
- self.assertEqual(v.main(),1)
- self.assertEqual(stdout.getvalue(),'');self.assertEqual(json.loads(stderr.getvalue())['error']['code'],'LIBRARY_GENERATION_CHANGED')
- self.assertEqual(v.sha(self.db.read_bytes()),before)
- def test_sauce_soup_mixed_unknown_do_not_enter_banmian(self):
- allbrands = self.q(brand='all')
- self.assertEqual({r['id'] for r in allbrands['records']},{'jd:same','jd:other-brand'})
- self.assertEqual(self.q(category='汤面')['records'][0]['id'],'jd:soup')
- self.assertEqual(self.q(brand='仲景',category='酱料调料')['records'][0]['id'],'jd:sauce')
- def test_brand_never_comes_from_shop_or_search(self):
- tax=v.taxonomy()
- self.assertIsNone(v.classify_brand('干拌面促销',tax)[0])
- self.assertIsNone(v.classify_brand('康师傅统一拌面',tax)[0])
- self.assertEqual(v.classify_brand('拌面',tax,'康师傅')[0],'ksf')
- self.assertIsNone(v.classify_brand('白象拌面',tax,'康师傅')[0])
- def test_tmall_is_not_unverified_taobao(self):
- self.assertEqual(self.q(marketplace='tmall')['totalMatched'],1)
- self.assertEqual(self.q(marketplace='taobao_unverified')['records'][0]['id'],'taobao:unverified-channel')
- def test_review_default_identity_and_sku_filters(self):
- r=self.q(entity='reviews')
- self.assertEqual([x['review_id'] for x in r['records']],['106582280238337296'])
- self.assertIsInstance(r['records'][0]['review_id'],str)
- self.assertEqual(self.q(entity='reviews',includeDefault=True)['totalMatched'],2)
- self.assertEqual(self.q(entity='reviews',reviewScope='listing')['totalMatched'],2)
- self.assertEqual(self.q(entity='reviews',marketplace='tmall')['totalMatched'],0)
- def test_zero_is_not_permission_or_query_error(self):
- self.assertEqual(self.q(brand='白象',category='汤面')['totalMatched'],0)
- with self.assertRaisesRegex(v.LibraryError,'UNKNOWN_OR_AMBIGUOUS'):
- self.q(brand='不存在的品牌')
- def test_missing_scope_and_unsupported_filters_rejected(self):
- with self.assertRaises(v.LibraryError):self.q(brand=None)
- with self.assertRaisesRegex(v.LibraryError,'UNKNOWN_QUERY_FIELD'):self.q(filters={'brand':'ksf'})
- with self.assertRaises(v.LibraryError):self.q(**{'from':'2026-11-01','timeBasis':'review_calendar_date'})
- with self.assertRaises(v.LibraryError):self.q(limit=True)
- def test_cursor_bound_to_scope_and_generation(self):
- first=self.q(brand='all',limit=1)
- self.assertIsNotNone(first['nextCursor'])
- second=self.q(brand='all',limit=1,cursor=first['nextCursor'])
- self.assertNotEqual(first['records'][0]['id'],second['records'][0]['id'])
- self.assertIsNone(second['nextCursor'])
- with self.assertRaisesRegex(v.LibraryError,'CURSOR_SCOPE_MISMATCH'):
- self.q(brand='ksf',cursor=first['nextCursor'])
- def test_parameterized_sql_injection(self):
- self.assertEqual(self.q(keyword="' OR 1=1 --")['totalMatched'],0)
- self.assertEqual(self.q(itemId="same' OR 1=1 --")['totalMatched'],0)
- with self.assertRaises(v.LibraryError):self.q(entity='products;DROP TABLE products')
- def test_time_not_fabricated_and_date_boundaries(self):
- r=self.q(entity='reviews')['records'][0]
- self.assertIsNone(r['review_date_utc'])
- self.assertEqual(r['review_calendar_date'],'2026-11-04')
- self.assertEqual(self.q(entity='reviews',timeBasis='review_calendar_date',**{'from':'2026-11-04','to':'2026-11-05'})['totalMatched'],1)
- self.assertEqual(self.q(entity='reviews',timeBasis='review_calendar_date',**{'from':'2026-11-03','to':'2026-11-04'})['totalMatched'],0)
- self.assertEqual(self.q(entity='reviews',timeBasis='review_utc',**{'from':'2026-11-01T00:00:00Z'})['totalMatched'],0)
- with self.assertRaises(v.LibraryError):self.q(entity='reviews',timeBasis='review_utc',**{'from':'2026-11-01T00:00:00'})
- def test_spec_candidates_do_not_create_false_unit_prices(self):
- s=self.q()['records'][0]['specs'][0]
- self.assertEqual(s['verification'],'title_candidate')
- self.assertEqual(s['total_weight_g'],550)
- p=self.q(entity='prices')['records'][0]
- self.assertEqual(p['amount_minor'],1990)
- self.assertIsNone(p['price_per_100g'])
- self.assertIsNone(p['source_time_utc'])
- self.assertIsNone(v.money('1?'))
- self.assertIsNone(v.money('19-29'))
- def test_source_citation_is_resolvable(self):
- s=self.q()['records'][0]['citation']
- self.assertEqual(s['locatorType'],'jsonl_line_1_based')
- self.assertTrue(all(r['resolution']=='matched' for r in s['upstreamSources']))
- with zipfile.ZipFile(self.archive) as z:
- self.assertEqual(v.sha(z.read(s['asset_path'])),s['assetSha256'])
- self.assertEqual(v.sha(z.read(s['asset_path']).decode().splitlines()[s['line_number']-1]),s['payload_sha256'])
- def test_social_mentions_and_counters_are_separate(self):
- with v.readonly(self.db) as c:
- r=v.query(c,dict(entity='social',marketplace='xiaohongshu',brand='ksf'))
- self.assertEqual(r['records'][0]['comment_count'],1000)
- self.assertEqual(r['records'][0]['fulltext_available'],0)
- self.assertEqual(c.execute('select count(*) from reviews').fetchone()[0],5)
- self.assertEqual(c.execute('select count(*) from social_brand_mentions').fetchone()[0],2)
- def test_policy_paraphrase_not_a_quote_and_status_is_dated(self):
- with v.readonly(self.db) as c:
- row=v.query(c,dict(entity='clauses',documentId='standard'))['records'][0]
- self.assertEqual(row['excerpt_type'],'paraphrase_or_transcription')
- self.assertEqual(row['status_as_of'],'2026-11-04')
- def test_readonly_and_idempotent_build(self):
- before=v.sha(self.db.read_bytes())
- with v.readonly(self.db) as c:
- with self.assertRaises(sqlite3.OperationalError):c.execute('delete from products')
- self.assertFalse(c.execute('pragma foreign_key_check').fetchall())
- receipt=v.build(self.archive,self.db)
- self.assertEqual(receipt['operation'],'unchanged')
- self.assertEqual(v.sha(self.db.read_bytes()),before)
- def test_foreign_keys_and_index_plan(self):
- self.assertTrue(any('idx_products_scope' in x for x in self.q()['queryPlan']))
- # With five fixture reviews SQLite may reasonably prefer a tiny scan.
- # The full-corpus acceptance separately checks the actual join plan.
- with v.readonly(self.db) as c:
- self.assertIsNotNone(c.execute("select 1 from sqlite_master where type='index' and name='idx_reviews_product'").fetchone())
- if __name__=='__main__':
- unittest.main()
|