prepare-voc-handoff.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351
  1. """Review the 2026-11-06 VOC originals and prepare an isolated v2 batch.
  2. No networking, credential loading, collection, registry merge or publication.
  3. Normalized sources are explicitly derived: every copied value has an original
  4. JSON Pointer, and every adapter assertion is labelled as a mapping policy.
  5. """
  6. import argparse
  7. import collections
  8. import hashlib
  9. import json
  10. from pathlib import Path, PurePosixPath
  11. import re
  12. import sys
  13. import zipfile
  14. ROOT = Path(__file__).resolve().parents[1]
  15. sys.path.insert(0, str(ROOT / 'server/voc-library'))
  16. import agent_batch
  17. import structured_batch
  18. import voc_db
  19. NAMES = ('h1', 'p2', 'd1', 'd2', 'pr1', 'pr1b', 'c1', 'c2', 's1b', 't1', 'td1', 'tc1')
  20. ADAPTER = 'ksf.reviewed-handoff-adapter.20261106.v1'
  21. def require(condition, message):
  22. if not condition:
  23. raise ValueError(message)
  24. def sha(raw):
  25. return hashlib.sha256(raw).hexdigest()
  26. def write_json(path, value):
  27. path.parent.mkdir(parents=True, exist_ok=True)
  28. with path.open('x', encoding='utf-8', newline='\n') as f:
  29. json.dump(value, f, ensure_ascii=False, indent=2, allow_nan=False)
  30. f.write('\n')
  31. def decimal_id(value):
  32. # Never pass IDs through float or JavaScript Number.
  33. require(type(value) is int and value > 0 or isinstance(value, str) and value.isdecimal() and int(value) > 0,
  34. 'ID is not a positive decimal integer or string')
  35. return str(value)
  36. def valid_price(value):
  37. # Reject masked prices, ranges, currency prose, zero sentinels and negatives.
  38. return type(value) in (str, int, float) and bool(re.fullmatch(r'\d+(?:\.\d{1,2})?', str(value))) and float(value) > 0
  39. def baseline_review(path, staged):
  40. before = agent_batch.digest(path)
  41. with voc_db.readonly(path) as old, voc_db.readonly(staged) as new:
  42. old_products = {r['id']: dict(r) for r in old.execute('SELECT * FROM products')}
  43. old_reviews = {r['id']: dict(r) for r in old.execute('SELECT * FROM reviews')}
  44. product_overlaps, review_overlaps, downgrades = [], [], []
  45. for r in new.execute('SELECT * FROM products'):
  46. prior = old_products.get(r['id'])
  47. if not prior:
  48. continue
  49. product_overlaps.append(r['id'])
  50. for field in ('shop_id', 'shop_name', 'url', 'brand_code', 'category_code'):
  51. if prior[field] and not r[field]:
  52. downgrades.append({'entity': 'products', 'id': r['id'], 'field': field,
  53. 'reason': 'incoming value is missing; preserve existing evidence'})
  54. if prior['marketplace'] == 'tmall' and r['marketplace'] == 'taobao_unverified':
  55. downgrades.append({'entity': 'products', 'id': r['id'], 'field': 'marketplace',
  56. 'reason': 'new source does not disprove existing Tmall evidence'})
  57. for r in new.execute('SELECT * FROM reviews'):
  58. prior = old_reviews.get(r['id'])
  59. if not prior:
  60. continue
  61. review_overlaps.append(r['id'])
  62. for field in ('is_default', 'rating_scale'):
  63. if prior[field] is not None and r[field] is None:
  64. downgrades.append({'entity': 'reviews', 'id': r['id'], 'field': field,
  65. 'reason': 'incoming unknown must not erase a previously recorded value; verify original evidence'})
  66. result = {'scope': 'local-preview-only-not-live-production', 'generation': dict(old.execute('SELECT key,value FROM meta'))['generation'],
  67. 'sha256': before, 'productOverlaps': len(product_overlaps), 'reviewOverlaps': len(review_overlaps),
  68. 'newProducts': new.execute('SELECT count(*) FROM products').fetchone()[0] - len(product_overlaps),
  69. 'newReviews': new.execute('SELECT count(*) FROM reviews').fetchone()[0] - len(review_overlaps),
  70. 'potentialFieldDowngrades': downgrades, 'mergePerformed': False}
  71. require(agent_batch.digest(path) == before, 'Baseline changed during readonly comparison')
  72. result['sha256Unchanged'] = True
  73. return result
  74. def read_package(original, archive):
  75. require(original.is_dir() and not any(p.is_symlink() for p in (original, *original.parents)), 'Original directory missing or redirected')
  76. files = {}
  77. for line in (original / 'FILES.sha256').read_text(encoding='utf-8-sig').splitlines():
  78. digest, name = line.split(' ', 1)
  79. rel = PurePosixPath(name)
  80. require(re.fullmatch('[0-9a-f]{64}', digest) and not rel.is_absolute() and '..' not in rel.parts
  81. and '\\' not in name and ':' not in name and name not in files, 'Invalid manifest path/hash')
  82. path = original / name
  83. require(path.is_file() and not any(p.is_symlink() for p in (path, *path.parents)), 'Original missing or redirected')
  84. raw = path.read_bytes()
  85. require(sha(raw) == digest, 'Original hash mismatch: ' + name)
  86. files[name] = raw
  87. files['FILES.sha256'] = (original / 'FILES.sha256').read_bytes()
  88. require(len(files) == 18, 'Expected this handoff with 18 files')
  89. with zipfile.ZipFile(archive) as z:
  90. require(len(z.namelist()) == len(set(z.namelist())) and set(z.namelist()) == set(files), 'Archive inventory differs')
  91. for name, raw in files.items():
  92. require(z.read(name) == raw, 'Archive differs from original: ' + name)
  93. return files
  94. class Mapping:
  95. def __init__(self, files):
  96. self.files = files
  97. self.payload = {n: json.loads(files['raw/' + n + '.body.json']) for n in NAMES}
  98. self.docs = {}
  99. self.candidates = collections.defaultdict(list)
  100. def source(self, name):
  101. if name not in self.docs:
  102. rel = 'raw/' + name + '.body.json'
  103. self.docs[name] = {
  104. 'schema': 'ksf.voc-response-normalization.v1', 'adapter': ADAPTER,
  105. 'upstreamProducerVerified': False,
  106. 'original': {'path': 'raw/original/' + rel, 'sha256': sha(self.files[rel])},
  107. 'sourceTimePointer': '/data/recordTime',
  108. 'sourceTimeRaw': self.payload[name]['data']['recordTime'],
  109. 'records': collections.defaultdict(list)}
  110. return self.docs[name]
  111. def row(self, name, fields, policies=None):
  112. values, origins = {}, {}
  113. for field, loc in fields.items():
  114. value = agent_batch.pointer(self.payload[name], loc)
  115. transform = 'copy'
  116. if field in ('itemId', 'reviewId', 'shopId', 'skuId', 'rankId'):
  117. value = decimal_id(value)
  118. transform = 'lossless-decimal-string'
  119. values[field] = value
  120. origins[field] = {'pointer': loc, 'transform': transform}
  121. for field, (value, reason) in (policies or {}).items():
  122. values[field] = value
  123. origins[field] = {'kind': 'adapter-policy-not-upstream-field', 'reason': reason}
  124. return {'values': values, 'fieldOrigins': origins}
  125. def add(self, name, entity, fields, policies=None):
  126. row = self.row(name, fields, policies)
  127. self.source(name)['records'][entity].append(row)
  128. return row
  129. def product(self, name, priority, fields):
  130. row = self.row(name, fields)
  131. family = 'taobao' if name in ('t1', 'td1') else 'jd'
  132. self.candidates[(family, row['values']['itemId'])].append((priority, name, row))
  133. def price(self, name, iid, loc, basis):
  134. raw = agent_batch.pointer(self.payload[name], loc)
  135. require(valid_price(raw), 'Rejected nonnumeric/zero/masked price: ' + name + loc)
  136. return self.add(name, 'prices', {'itemId': iid, 'amount': loc}, {
  137. 'priceBasis': (basis, 'Field-specific display-price classification; not a transaction or coupon price'),
  138. 'currency': ('CNY', 'JD/Taobao domestic price mapping under the research CNY-yuan contract')})
  139. def normalize(self):
  140. for n in NAMES:
  141. d = self.payload[n]
  142. if n not in ('d2', 'pr1', 'pr1b'):
  143. require(d['code'] == 200 and d['data']['code'] == 0, 'Expected successful data envelope: ' + n)
  144. for n in ('h1', 'p2'):
  145. for i, _ in enumerate(self.payload[n]['data']['data']['ItemList']):
  146. loc = f'/data/data/ItemList/{i}'
  147. self.product(n, 1, {'itemId': loc + '/itemId', 'title': loc + '/title', 'url': loc + '/link',
  148. 'shopId': '/data/data/shopId', 'shopName': '/data/data/shopName'})
  149. seen_search = {}
  150. for i, item in enumerate(self.payload['s1b']['data']['data']['products']):
  151. loc = f'/data/data/products/{i}'
  152. self.product('s1b', 2, {'itemId': loc + '/id', 'title': loc + '/title', 'shopId': loc + '/shopId',
  153. 'shopName': loc + '/shopName', 'url': loc + '/click'})
  154. key = (item['id'], item['price'])
  155. if key not in seen_search:
  156. seen_search[key] = (item['shopId'], item['shopName'])
  157. self.price('s1b', loc + '/id', loc + '/price', 'search_listing_snapshot')
  158. require(seen_search[key] == (item['shopId'], item['shopName']), 'Search duplicate shop conflict')
  159. p = '/data/data/product'
  160. self.product('d1', 3, {'itemId': p + '/productId', 'title': p + '/skuName', 'shopId': p + '/shopId',
  161. 'shopName': '/data/data/stock/D/shopName', 'brandName': p + '/brandName'})
  162. self.add('d1', 'specs', {'itemId': p + '/productId', 'skuId': p + '/skuId', 'specText': p + '/color'}, {
  163. 'verification': ('source_attribute', 'Preserve the SKU attribute text; no verified food weight or pack count'),
  164. 'basis': ('product.color; product.weight is not verified food net weight', 'Mapping rule; shipping weight must not become netWeightG')})
  165. for i, _ in enumerate(self.payload['d1']['data']['data'].get('rankInfo', [])):
  166. loc = f'/data/data/rankInfo/{i}'
  167. self.add('d1', 'rankings', {'itemId': loc + '/id', 'rankId': loc + '/rankId',
  168. 'rankNumber': loc + '/rankNum', 'title': loc + '/name'}, {
  169. 'scope': ('detail_page_badge_not_complete_ranking', 'rankInfo is a detail badge; no full ranking coverage proved')})
  170. for n in ('c1', 'c2'):
  171. requested = self.payload[n]['data']['requestParams']['query']['itemId']
  172. for i, r in enumerate(self.payload[n]['data']['data']['data']):
  173. require(r['productId'] == requested, 'JD review item mismatch')
  174. loc = f'/data/data/data/{i}'
  175. self.add(n, 'reviews', {'itemId': loc + '/productId', 'reviewId': loc + '/commentId',
  176. 'content': loc + '/commentData', 'reviewDate': loc + '/commentDate',
  177. 'purchasedSku': loc + '/productColor', 'rating': loc + '/commentScore', 'appendContent': loc + '/afterComment'})
  178. for i, _ in enumerate(self.payload['t1']['data']['data']['result']):
  179. loc = f'/data/data/result/{i}/blockContent'
  180. # sellerId is request context, not a shopId and not proof of Tmall.
  181. self.product('t1', 1, {'itemId': loc + '/itemId', 'title': loc + '/title', 'url': loc + '/itemUrl'})
  182. self.price('t1', loc + '/itemId', loc + '/price', 'shop_listing_snapshot')
  183. self.price('t1', loc + '/itemId', loc + '/reservePrice', 'shop_reserve_price_not_transaction')
  184. self.product('td1', 3, {'itemId': '/data/data/item/itemId', 'title': '/data/data/item/title',
  185. 'shopId': '/data/data/seller/shopId', 'shopName': '/data/data/seller/shopName'})
  186. self.price('td1', '/data/data/item/itemId', '/data/data/item/price', 'detail_listing_snapshot')
  187. self.price('td1', '/data/data/item/itemId', '/data/data/item/originalPrice', 'detail_original_price_not_transaction')
  188. base = self.payload['td1']['data']['data']['skuBase']
  189. require(len(base['props']) == 1, 'SKU mapping requires one confirmed attribute dimension')
  190. for i, sku in enumerate(base['skus']):
  191. pid, vid = sku['propPath'].split(':')
  192. prop = base['props'][0]
  193. require(prop['pid'] == pid, 'SKU property ID mismatch')
  194. matches = [j for j, value in enumerate(prop['values']) if value['vid'] == vid]
  195. require(len(matches) == 1, 'SKU property value missing or ambiguous')
  196. self.add('td1', 'specs', {'itemId': '/data/data/item/itemId', 'skuId': f'/data/data/skuBase/skus/{i}/skuId',
  197. 'specText': f'/data/data/skuBase/props/0/values/{matches[0]}/name'}, {
  198. 'verification': ('source_attribute', 'Text selected by the exact propPath, pid and vid; quantities not independently verified'),
  199. 'basis': ('skuBase.skus.propPath -> skuBase.props.values', 'Exact property join, not positional guessing')})
  200. requested = self.payload['tc1']['data']['requestParams']['query']['itemId']
  201. for i, r in enumerate(self.payload['tc1']['data']['data']['comments']):
  202. require(r['auction']['aucNumId'] == requested, 'Taobao review item mismatch')
  203. loc = f'/data/data/comments/{i}'
  204. self.add('tc1', 'reviews', {'itemId': loc + '/auction/aucNumId', 'reviewId': loc + '/rateId',
  205. 'content': loc + '/content', 'reviewDate': loc + '/date', 'purchasedSku': loc + '/auction/sku',
  206. 'rating': loc + '/rate', 'appendContent': loc + '/append'})
  207. selection = []
  208. for (family, iid), candidates in sorted(self.candidates.items()):
  209. chosen = max(candidates, key=lambda r: (r[0], self.source(r[1])['sourceTimeRaw']))
  210. _, name, row = chosen
  211. self.source(name)['records']['products'].append(row)
  212. selection.append({'id': family + ':' + iid, 'chosenSource': name,
  213. 'allSourceCandidates': [n for _, n, _ in candidates]})
  214. for n, count, iid, total in [('c2', 19, '/data/requestParams/query/itemId', '/data/data/pageInfo/comment_count'),
  215. ('tc1', 20, '/data/requestParams/query/itemId', '/data/data/totalFuzzy')]:
  216. self.add(n, 'coverage', {'itemId': iid, 'reportedTotal': total}, {
  217. 'status': ('partial_sample', 'Only the handed-over pages were observed; cached JD maxPage values disagree'),
  218. 'availableReviews': (count, 'Number of distinct review IDs in the supplied sample, not platform availability'),
  219. 'historicalTotalComplete': (False, 'No complete historical review collection proved')})
  220. return selection
  221. def prepare(original, archive, out, baseline=None):
  222. files = read_package(original, archive)
  223. require(not out.exists() and not any(p.is_symlink() for p in (out, *out.parents)), 'A new isolated output directory is required')
  224. mapping = Mapping(files)
  225. selection = mapping.normalize() # Validate all pointers before creating output.
  226. out.mkdir(parents=True, mode=0o700)
  227. batch = {'schema': 'ksf.voc-agent-batch.v2', 'batchId': 'ksf-handoff-20261106-reviewed-r1',
  228. 'producer': ADAPTER, 'sources': [], 'attachments': []}
  229. for name, raw in files.items():
  230. rel = 'raw/original/' + name
  231. path = out / rel
  232. path.parent.mkdir(parents=True, exist_ok=True)
  233. with path.open('xb') as f:
  234. f.write(raw)
  235. batch['attachments'].append({'path': rel, 'sha256': sha(raw)})
  236. for name, doc in mapping.docs.items():
  237. rel = 'raw/normalized/' + name + '.json'
  238. write_json(out / rel, doc)
  239. family = 'taobao' if name in ('t1', 'td1', 'tc1') else 'jd'
  240. source = {'id': name, 'path': rel, 'sha256': sha((out / rel).read_bytes()), 'platform': family,
  241. 'marketplace': 'jd' if family == 'jd' else 'taobao_unverified', 'observedAt': doc['sourceTimeRaw']}
  242. query = mapping.payload[name]['data'].get('requestParams', {}).get('query', {})
  243. if query.get('itemId'):
  244. source['requestItemId'] = decimal_id(query['itemId'])
  245. batch['sources'].append(source)
  246. for entity, rows in doc['records'].items():
  247. for i, row in enumerate(rows):
  248. batch.setdefault(entity, []).append({'sourceId': name, 'pointer': f'/records/{entity}/{i}/values',
  249. 'fields': {field: '/' + field for field in row['values']}})
  250. write_json(out / 'batch.json', batch)
  251. dry = agent_batch.build_batch(out / 'batch.json', out / 'staging.sqlite')
  252. built = agent_batch.build_batch(out / 'batch.json', out / 'staging.sqlite', True)
  253. with voc_db.readonly(out / 'staging.sqlite') as c:
  254. counts = voc_db.table_counts(c)
  255. checks = {
  256. 'foreignKeysValid': not bool(c.execute('PRAGMA foreign_key_check').fetchall()),
  257. 'reviewDefaultsRemainUnknown': c.execute('SELECT count(*) FROM reviews WHERE is_default IS NOT NULL').fetchone()[0] == 0,
  258. 'allReviewsHaveExactProductMatch': c.execute("SELECT count(*) FROM reviews WHERE identity_state<>'matched'").fetchone()[0] == 0,
  259. 'taobaoNotPromotedToTmall': c.execute("SELECT count(*) FROM products WHERE platform_family='taobao' AND marketplace<>'taobao_unverified'").fetchone()[0] == 0,
  260. 't1SellerIdNotUsedAsShopId': c.execute("SELECT count(*) FROM products WHERE shop_id='2932889983'").fetchone()[0] == 0,
  261. 'noInventedUnitPrices': c.execute('SELECT count(*) FROM prices WHERE price_per_100g IS NOT NULL').fetchone()[0] == 0,
  262. 'maskedJDDetailPriceNotImported': c.execute("SELECT count(*) FROM prices p JOIN source_records s ON s.id=p.source_record_id WHERE s.payload_json LIKE '%d1.json%' AND p.product_id='jd:100005702440'").fetchone()[0] == 0,
  263. }
  264. quality = [dict(r) for r in c.execute('SELECT entity_type,code,count(*) AS count FROM quality_issues GROUP BY entity_type,code ORDER BY entity_type,code')]
  265. import retrieval
  266. analysis = retrieval.search(c, {'entities': ['reviews'], 'quality': 'analysis', 'limit': 100})
  267. audit_reviews = retrieval.search(c, {'entities': ['reviews'], 'quality': 'audit', 'limit': 100})
  268. checks['auditReviewsAllReadable'] = audit_reviews['totalMatched'] == 39
  269. checks['unknownDefaultsExcludedFromAnalysis'] = analysis['totalMatched'] == 0
  270. r = c.execute("SELECT source_record_id FROM reviews WHERE platform_family='taobao' LIMIT 1").fetchone()
  271. checks['sourceReadbackWorks'] = bool(voc_db.get_source(c, r[0])['ok'])
  272. require(all(checks.values()), 'Isolated readback failed: ' + str(checks))
  273. statuses = [{'name': n, 'outerCode': p.get('code'), 'innerCode': (p.get('data') or {}).get('code'),
  274. 'recordTime': (p.get('data') or {}).get('recordTime'), 'httpStatus': None}
  275. for n, p in mapping.payload.items()]
  276. h1, p2, search = (mapping.payload[n]['data']['data'] for n in ('h1', 'p2', 's1b'))
  277. ids = [{r['itemId'] for r in d['ItemList']} for d in (h1, p2)]
  278. report = {'schema': 'ksf.voc-handoff-review.v1', 'adapter': ADAPTER,
  279. 'archive': {'bytes': archive.stat().st_size, 'sha256': sha(archive.read_bytes()), 'files': len(files), 'manifestMatched': len(files) - 1},
  280. 'originalsUnchanged': all((original / name).read_bytes() == raw for name, raw in files.items()),
  281. 'upstreamProducerVerified': False, 'networkRequests': 0, 'collectionCostMinor': 0,
  282. 'productionImported': False, 'activeLibraryModified': False,
  283. 'representedAttempts': {'directResponseArtifacts': 12, 'dataSuccess': 9, 'failure': 3,
  284. 'mcpTextOnly': 1, 'totalArtifacts': 13, 'completeNetworkAttemptLedger': False},
  285. 'statuses': statuses,
  286. 'jdShopPages': {'rows': [24, 24], 'distinctItems': len(ids[0] | ids[1]), 'overlap': len(ids[0] & ids[1]),
  287. 'reportedTotal': h1['totalResults'], 'reportedPages': h1['TotalPage'],
  288. 'reportedTotalPointer': '/data/data/totalResults', 'reportedPagesPointer': '/data/data/TotalPage', 'complete': False},
  289. 'jdSearch': {'rows': len(search['products']), 'distinctItems': len({r['id'] for r in search['products']}),
  290. 'distinctShops': len({r['shopId'] for r in search['products']}), 'platform': 'jd'},
  291. 'jdMaskedDetailPrice': {'raw': mapping.payload['d1']['data']['data']['priceFloor']['price'], 'usable': False},
  292. 'shopIdentity': {'t1RequestedSellerId': '2932889983', 'td1ShopId': '250707370',
  293. 'sameShopProved': False, 'differentShopProved': False, 'tmallProved': False},
  294. 'billing': {'actualCostMinor': None, 'independentlyVerifiedCharges': 0,
  295. 'reportedTraceChargeMinor': 30, 'directCallsReserveMinor': 360, 'otherUnsettledReserveMinor': 600,
  296. 'provisionalBudgetOccupationMinor': 990, 'remainingAfterProvisionalReservesMinor': 4010,
  297. 'rawUsageRowsPresent': False, 'aggregatesMayOnlyBeEstimates': True},
  298. 'missingEvidence': ['per-request collector metadata and headers', 'raw usage/charge rows',
  299. 'original batch.json', 'original staging database and 49 quality issue rows',
  300. 'MCP structuredContent or files already saved under its outputDir'],
  301. 'productSelection': selection, 'dry': dry, 'build': built, 'databaseCounts': counts,
  302. 'qualityIssuesByCode': quality, 'readbackChecks': checks,
  303. 'publicationHold': 'Review before merge: older/sparse observations and unknown review defaults must not overwrite richer current records.'}
  304. if baseline:
  305. report['baselineComparison'] = baseline_review(baseline, out / 'staging.sqlite')
  306. require(report['originalsUnchanged'], 'Original files changed during preparation')
  307. write_json(out / 'review.json', report)
  308. return report
  309. def main():
  310. p = argparse.ArgumentParser(description=__doc__)
  311. p.add_argument('--original', required=True, type=Path)
  312. p.add_argument('--archive', required=True, type=Path)
  313. p.add_argument('--out', required=True, type=Path)
  314. p.add_argument('--baseline-db', type=Path, help='Optional local preview: readonly overlap/downgrade review only')
  315. args = p.parse_args()
  316. report = prepare(args.original.absolute(), args.archive.absolute(), args.out.absolute(), args.baseline_db)
  317. print(json.dumps({k: report[k] for k in ('archive', 'originalsUnchanged', 'networkRequests', 'productionImported',
  318. 'databaseCounts', 'readbackChecks')}, ensure_ascii=False, indent=2))
  319. if __name__ == '__main__':
  320. main()