| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351 |
- """Review the 2026-11-06 VOC originals and prepare an isolated v2 batch.
- No networking, credential loading, collection, registry merge or publication.
- Normalized sources are explicitly derived: every copied value has an original
- JSON Pointer, and every adapter assertion is labelled as a mapping policy.
- """
- import argparse
- import collections
- import hashlib
- import json
- from pathlib import Path, PurePosixPath
- import re
- import sys
- import zipfile
- ROOT = Path(__file__).resolve().parents[1]
- sys.path.insert(0, str(ROOT / 'server/voc-library'))
- import agent_batch
- import structured_batch
- import voc_db
- NAMES = ('h1', 'p2', 'd1', 'd2', 'pr1', 'pr1b', 'c1', 'c2', 's1b', 't1', 'td1', 'tc1')
- ADAPTER = 'ksf.reviewed-handoff-adapter.20261106.v1'
- def require(condition, message):
- if not condition:
- raise ValueError(message)
- def sha(raw):
- return hashlib.sha256(raw).hexdigest()
- def write_json(path, value):
- path.parent.mkdir(parents=True, exist_ok=True)
- with path.open('x', encoding='utf-8', newline='\n') as f:
- json.dump(value, f, ensure_ascii=False, indent=2, allow_nan=False)
- f.write('\n')
- def decimal_id(value):
- # Never pass IDs through float or JavaScript Number.
- require(type(value) is int and value > 0 or isinstance(value, str) and value.isdecimal() and int(value) > 0,
- 'ID is not a positive decimal integer or string')
- return str(value)
- def valid_price(value):
- # Reject masked prices, ranges, currency prose, zero sentinels and negatives.
- return type(value) in (str, int, float) and bool(re.fullmatch(r'\d+(?:\.\d{1,2})?', str(value))) and float(value) > 0
- def baseline_review(path, staged):
- before = agent_batch.digest(path)
- with voc_db.readonly(path) as old, voc_db.readonly(staged) as new:
- old_products = {r['id']: dict(r) for r in old.execute('SELECT * FROM products')}
- old_reviews = {r['id']: dict(r) for r in old.execute('SELECT * FROM reviews')}
- product_overlaps, review_overlaps, downgrades = [], [], []
- for r in new.execute('SELECT * FROM products'):
- prior = old_products.get(r['id'])
- if not prior:
- continue
- product_overlaps.append(r['id'])
- for field in ('shop_id', 'shop_name', 'url', 'brand_code', 'category_code'):
- if prior[field] and not r[field]:
- downgrades.append({'entity': 'products', 'id': r['id'], 'field': field,
- 'reason': 'incoming value is missing; preserve existing evidence'})
- if prior['marketplace'] == 'tmall' and r['marketplace'] == 'taobao_unverified':
- downgrades.append({'entity': 'products', 'id': r['id'], 'field': 'marketplace',
- 'reason': 'new source does not disprove existing Tmall evidence'})
- for r in new.execute('SELECT * FROM reviews'):
- prior = old_reviews.get(r['id'])
- if not prior:
- continue
- review_overlaps.append(r['id'])
- for field in ('is_default', 'rating_scale'):
- if prior[field] is not None and r[field] is None:
- downgrades.append({'entity': 'reviews', 'id': r['id'], 'field': field,
- 'reason': 'incoming unknown must not erase a previously recorded value; verify original evidence'})
- result = {'scope': 'local-preview-only-not-live-production', 'generation': dict(old.execute('SELECT key,value FROM meta'))['generation'],
- 'sha256': before, 'productOverlaps': len(product_overlaps), 'reviewOverlaps': len(review_overlaps),
- 'newProducts': new.execute('SELECT count(*) FROM products').fetchone()[0] - len(product_overlaps),
- 'newReviews': new.execute('SELECT count(*) FROM reviews').fetchone()[0] - len(review_overlaps),
- 'potentialFieldDowngrades': downgrades, 'mergePerformed': False}
- require(agent_batch.digest(path) == before, 'Baseline changed during readonly comparison')
- result['sha256Unchanged'] = True
- return result
- def read_package(original, archive):
- require(original.is_dir() and not any(p.is_symlink() for p in (original, *original.parents)), 'Original directory missing or redirected')
- files = {}
- for line in (original / 'FILES.sha256').read_text(encoding='utf-8-sig').splitlines():
- digest, name = line.split(' ', 1)
- rel = PurePosixPath(name)
- require(re.fullmatch('[0-9a-f]{64}', digest) and not rel.is_absolute() and '..' not in rel.parts
- and '\\' not in name and ':' not in name and name not in files, 'Invalid manifest path/hash')
- path = original / name
- require(path.is_file() and not any(p.is_symlink() for p in (path, *path.parents)), 'Original missing or redirected')
- raw = path.read_bytes()
- require(sha(raw) == digest, 'Original hash mismatch: ' + name)
- files[name] = raw
- files['FILES.sha256'] = (original / 'FILES.sha256').read_bytes()
- require(len(files) == 18, 'Expected this handoff with 18 files')
- with zipfile.ZipFile(archive) as z:
- require(len(z.namelist()) == len(set(z.namelist())) and set(z.namelist()) == set(files), 'Archive inventory differs')
- for name, raw in files.items():
- require(z.read(name) == raw, 'Archive differs from original: ' + name)
- return files
- class Mapping:
- def __init__(self, files):
- self.files = files
- self.payload = {n: json.loads(files['raw/' + n + '.body.json']) for n in NAMES}
- self.docs = {}
- self.candidates = collections.defaultdict(list)
- def source(self, name):
- if name not in self.docs:
- rel = 'raw/' + name + '.body.json'
- self.docs[name] = {
- 'schema': 'ksf.voc-response-normalization.v1', 'adapter': ADAPTER,
- 'upstreamProducerVerified': False,
- 'original': {'path': 'raw/original/' + rel, 'sha256': sha(self.files[rel])},
- 'sourceTimePointer': '/data/recordTime',
- 'sourceTimeRaw': self.payload[name]['data']['recordTime'],
- 'records': collections.defaultdict(list)}
- return self.docs[name]
- def row(self, name, fields, policies=None):
- values, origins = {}, {}
- for field, loc in fields.items():
- value = agent_batch.pointer(self.payload[name], loc)
- transform = 'copy'
- if field in ('itemId', 'reviewId', 'shopId', 'skuId', 'rankId'):
- value = decimal_id(value)
- transform = 'lossless-decimal-string'
- values[field] = value
- origins[field] = {'pointer': loc, 'transform': transform}
- for field, (value, reason) in (policies or {}).items():
- values[field] = value
- origins[field] = {'kind': 'adapter-policy-not-upstream-field', 'reason': reason}
- return {'values': values, 'fieldOrigins': origins}
- def add(self, name, entity, fields, policies=None):
- row = self.row(name, fields, policies)
- self.source(name)['records'][entity].append(row)
- return row
- def product(self, name, priority, fields):
- row = self.row(name, fields)
- family = 'taobao' if name in ('t1', 'td1') else 'jd'
- self.candidates[(family, row['values']['itemId'])].append((priority, name, row))
- def price(self, name, iid, loc, basis):
- raw = agent_batch.pointer(self.payload[name], loc)
- require(valid_price(raw), 'Rejected nonnumeric/zero/masked price: ' + name + loc)
- return self.add(name, 'prices', {'itemId': iid, 'amount': loc}, {
- 'priceBasis': (basis, 'Field-specific display-price classification; not a transaction or coupon price'),
- 'currency': ('CNY', 'JD/Taobao domestic price mapping under the research CNY-yuan contract')})
- def normalize(self):
- for n in NAMES:
- d = self.payload[n]
- if n not in ('d2', 'pr1', 'pr1b'):
- require(d['code'] == 200 and d['data']['code'] == 0, 'Expected successful data envelope: ' + n)
- for n in ('h1', 'p2'):
- for i, _ in enumerate(self.payload[n]['data']['data']['ItemList']):
- loc = f'/data/data/ItemList/{i}'
- self.product(n, 1, {'itemId': loc + '/itemId', 'title': loc + '/title', 'url': loc + '/link',
- 'shopId': '/data/data/shopId', 'shopName': '/data/data/shopName'})
- seen_search = {}
- for i, item in enumerate(self.payload['s1b']['data']['data']['products']):
- loc = f'/data/data/products/{i}'
- self.product('s1b', 2, {'itemId': loc + '/id', 'title': loc + '/title', 'shopId': loc + '/shopId',
- 'shopName': loc + '/shopName', 'url': loc + '/click'})
- key = (item['id'], item['price'])
- if key not in seen_search:
- seen_search[key] = (item['shopId'], item['shopName'])
- self.price('s1b', loc + '/id', loc + '/price', 'search_listing_snapshot')
- require(seen_search[key] == (item['shopId'], item['shopName']), 'Search duplicate shop conflict')
- p = '/data/data/product'
- self.product('d1', 3, {'itemId': p + '/productId', 'title': p + '/skuName', 'shopId': p + '/shopId',
- 'shopName': '/data/data/stock/D/shopName', 'brandName': p + '/brandName'})
- self.add('d1', 'specs', {'itemId': p + '/productId', 'skuId': p + '/skuId', 'specText': p + '/color'}, {
- 'verification': ('source_attribute', 'Preserve the SKU attribute text; no verified food weight or pack count'),
- 'basis': ('product.color; product.weight is not verified food net weight', 'Mapping rule; shipping weight must not become netWeightG')})
- for i, _ in enumerate(self.payload['d1']['data']['data'].get('rankInfo', [])):
- loc = f'/data/data/rankInfo/{i}'
- self.add('d1', 'rankings', {'itemId': loc + '/id', 'rankId': loc + '/rankId',
- 'rankNumber': loc + '/rankNum', 'title': loc + '/name'}, {
- 'scope': ('detail_page_badge_not_complete_ranking', 'rankInfo is a detail badge; no full ranking coverage proved')})
- for n in ('c1', 'c2'):
- requested = self.payload[n]['data']['requestParams']['query']['itemId']
- for i, r in enumerate(self.payload[n]['data']['data']['data']):
- require(r['productId'] == requested, 'JD review item mismatch')
- loc = f'/data/data/data/{i}'
- self.add(n, 'reviews', {'itemId': loc + '/productId', 'reviewId': loc + '/commentId',
- 'content': loc + '/commentData', 'reviewDate': loc + '/commentDate',
- 'purchasedSku': loc + '/productColor', 'rating': loc + '/commentScore', 'appendContent': loc + '/afterComment'})
- for i, _ in enumerate(self.payload['t1']['data']['data']['result']):
- loc = f'/data/data/result/{i}/blockContent'
- # sellerId is request context, not a shopId and not proof of Tmall.
- self.product('t1', 1, {'itemId': loc + '/itemId', 'title': loc + '/title', 'url': loc + '/itemUrl'})
- self.price('t1', loc + '/itemId', loc + '/price', 'shop_listing_snapshot')
- self.price('t1', loc + '/itemId', loc + '/reservePrice', 'shop_reserve_price_not_transaction')
- self.product('td1', 3, {'itemId': '/data/data/item/itemId', 'title': '/data/data/item/title',
- 'shopId': '/data/data/seller/shopId', 'shopName': '/data/data/seller/shopName'})
- self.price('td1', '/data/data/item/itemId', '/data/data/item/price', 'detail_listing_snapshot')
- self.price('td1', '/data/data/item/itemId', '/data/data/item/originalPrice', 'detail_original_price_not_transaction')
- base = self.payload['td1']['data']['data']['skuBase']
- require(len(base['props']) == 1, 'SKU mapping requires one confirmed attribute dimension')
- for i, sku in enumerate(base['skus']):
- pid, vid = sku['propPath'].split(':')
- prop = base['props'][0]
- require(prop['pid'] == pid, 'SKU property ID mismatch')
- matches = [j for j, value in enumerate(prop['values']) if value['vid'] == vid]
- require(len(matches) == 1, 'SKU property value missing or ambiguous')
- self.add('td1', 'specs', {'itemId': '/data/data/item/itemId', 'skuId': f'/data/data/skuBase/skus/{i}/skuId',
- 'specText': f'/data/data/skuBase/props/0/values/{matches[0]}/name'}, {
- 'verification': ('source_attribute', 'Text selected by the exact propPath, pid and vid; quantities not independently verified'),
- 'basis': ('skuBase.skus.propPath -> skuBase.props.values', 'Exact property join, not positional guessing')})
- requested = self.payload['tc1']['data']['requestParams']['query']['itemId']
- for i, r in enumerate(self.payload['tc1']['data']['data']['comments']):
- require(r['auction']['aucNumId'] == requested, 'Taobao review item mismatch')
- loc = f'/data/data/comments/{i}'
- self.add('tc1', 'reviews', {'itemId': loc + '/auction/aucNumId', 'reviewId': loc + '/rateId',
- 'content': loc + '/content', 'reviewDate': loc + '/date', 'purchasedSku': loc + '/auction/sku',
- 'rating': loc + '/rate', 'appendContent': loc + '/append'})
- selection = []
- for (family, iid), candidates in sorted(self.candidates.items()):
- chosen = max(candidates, key=lambda r: (r[0], self.source(r[1])['sourceTimeRaw']))
- _, name, row = chosen
- self.source(name)['records']['products'].append(row)
- selection.append({'id': family + ':' + iid, 'chosenSource': name,
- 'allSourceCandidates': [n for _, n, _ in candidates]})
- for n, count, iid, total in [('c2', 19, '/data/requestParams/query/itemId', '/data/data/pageInfo/comment_count'),
- ('tc1', 20, '/data/requestParams/query/itemId', '/data/data/totalFuzzy')]:
- self.add(n, 'coverage', {'itemId': iid, 'reportedTotal': total}, {
- 'status': ('partial_sample', 'Only the handed-over pages were observed; cached JD maxPage values disagree'),
- 'availableReviews': (count, 'Number of distinct review IDs in the supplied sample, not platform availability'),
- 'historicalTotalComplete': (False, 'No complete historical review collection proved')})
- return selection
- def prepare(original, archive, out, baseline=None):
- files = read_package(original, archive)
- require(not out.exists() and not any(p.is_symlink() for p in (out, *out.parents)), 'A new isolated output directory is required')
- mapping = Mapping(files)
- selection = mapping.normalize() # Validate all pointers before creating output.
- out.mkdir(parents=True, mode=0o700)
- batch = {'schema': 'ksf.voc-agent-batch.v2', 'batchId': 'ksf-handoff-20261106-reviewed-r1',
- 'producer': ADAPTER, 'sources': [], 'attachments': []}
- for name, raw in files.items():
- rel = 'raw/original/' + name
- path = out / rel
- path.parent.mkdir(parents=True, exist_ok=True)
- with path.open('xb') as f:
- f.write(raw)
- batch['attachments'].append({'path': rel, 'sha256': sha(raw)})
- for name, doc in mapping.docs.items():
- rel = 'raw/normalized/' + name + '.json'
- write_json(out / rel, doc)
- family = 'taobao' if name in ('t1', 'td1', 'tc1') else 'jd'
- source = {'id': name, 'path': rel, 'sha256': sha((out / rel).read_bytes()), 'platform': family,
- 'marketplace': 'jd' if family == 'jd' else 'taobao_unverified', 'observedAt': doc['sourceTimeRaw']}
- query = mapping.payload[name]['data'].get('requestParams', {}).get('query', {})
- if query.get('itemId'):
- source['requestItemId'] = decimal_id(query['itemId'])
- batch['sources'].append(source)
- for entity, rows in doc['records'].items():
- for i, row in enumerate(rows):
- batch.setdefault(entity, []).append({'sourceId': name, 'pointer': f'/records/{entity}/{i}/values',
- 'fields': {field: '/' + field for field in row['values']}})
- write_json(out / 'batch.json', batch)
- dry = agent_batch.build_batch(out / 'batch.json', out / 'staging.sqlite')
- built = agent_batch.build_batch(out / 'batch.json', out / 'staging.sqlite', True)
- with voc_db.readonly(out / 'staging.sqlite') as c:
- counts = voc_db.table_counts(c)
- checks = {
- 'foreignKeysValid': not bool(c.execute('PRAGMA foreign_key_check').fetchall()),
- 'reviewDefaultsRemainUnknown': c.execute('SELECT count(*) FROM reviews WHERE is_default IS NOT NULL').fetchone()[0] == 0,
- 'allReviewsHaveExactProductMatch': c.execute("SELECT count(*) FROM reviews WHERE identity_state<>'matched'").fetchone()[0] == 0,
- 'taobaoNotPromotedToTmall': c.execute("SELECT count(*) FROM products WHERE platform_family='taobao' AND marketplace<>'taobao_unverified'").fetchone()[0] == 0,
- 't1SellerIdNotUsedAsShopId': c.execute("SELECT count(*) FROM products WHERE shop_id='2932889983'").fetchone()[0] == 0,
- 'noInventedUnitPrices': c.execute('SELECT count(*) FROM prices WHERE price_per_100g IS NOT NULL').fetchone()[0] == 0,
- 'maskedJDDetailPriceNotImported': c.execute("SELECT count(*) FROM prices p JOIN source_records s ON s.id=p.source_record_id WHERE s.payload_json LIKE '%d1.json%' AND p.product_id='jd:100005702440'").fetchone()[0] == 0,
- }
- quality = [dict(r) for r in c.execute('SELECT entity_type,code,count(*) AS count FROM quality_issues GROUP BY entity_type,code ORDER BY entity_type,code')]
- import retrieval
- analysis = retrieval.search(c, {'entities': ['reviews'], 'quality': 'analysis', 'limit': 100})
- audit_reviews = retrieval.search(c, {'entities': ['reviews'], 'quality': 'audit', 'limit': 100})
- checks['auditReviewsAllReadable'] = audit_reviews['totalMatched'] == 39
- checks['unknownDefaultsExcludedFromAnalysis'] = analysis['totalMatched'] == 0
- r = c.execute("SELECT source_record_id FROM reviews WHERE platform_family='taobao' LIMIT 1").fetchone()
- checks['sourceReadbackWorks'] = bool(voc_db.get_source(c, r[0])['ok'])
- require(all(checks.values()), 'Isolated readback failed: ' + str(checks))
- statuses = [{'name': n, 'outerCode': p.get('code'), 'innerCode': (p.get('data') or {}).get('code'),
- 'recordTime': (p.get('data') or {}).get('recordTime'), 'httpStatus': None}
- for n, p in mapping.payload.items()]
- h1, p2, search = (mapping.payload[n]['data']['data'] for n in ('h1', 'p2', 's1b'))
- ids = [{r['itemId'] for r in d['ItemList']} for d in (h1, p2)]
- report = {'schema': 'ksf.voc-handoff-review.v1', 'adapter': ADAPTER,
- 'archive': {'bytes': archive.stat().st_size, 'sha256': sha(archive.read_bytes()), 'files': len(files), 'manifestMatched': len(files) - 1},
- 'originalsUnchanged': all((original / name).read_bytes() == raw for name, raw in files.items()),
- 'upstreamProducerVerified': False, 'networkRequests': 0, 'collectionCostMinor': 0,
- 'productionImported': False, 'activeLibraryModified': False,
- 'representedAttempts': {'directResponseArtifacts': 12, 'dataSuccess': 9, 'failure': 3,
- 'mcpTextOnly': 1, 'totalArtifacts': 13, 'completeNetworkAttemptLedger': False},
- 'statuses': statuses,
- 'jdShopPages': {'rows': [24, 24], 'distinctItems': len(ids[0] | ids[1]), 'overlap': len(ids[0] & ids[1]),
- 'reportedTotal': h1['totalResults'], 'reportedPages': h1['TotalPage'],
- 'reportedTotalPointer': '/data/data/totalResults', 'reportedPagesPointer': '/data/data/TotalPage', 'complete': False},
- 'jdSearch': {'rows': len(search['products']), 'distinctItems': len({r['id'] for r in search['products']}),
- 'distinctShops': len({r['shopId'] for r in search['products']}), 'platform': 'jd'},
- 'jdMaskedDetailPrice': {'raw': mapping.payload['d1']['data']['data']['priceFloor']['price'], 'usable': False},
- 'shopIdentity': {'t1RequestedSellerId': '2932889983', 'td1ShopId': '250707370',
- 'sameShopProved': False, 'differentShopProved': False, 'tmallProved': False},
- 'billing': {'actualCostMinor': None, 'independentlyVerifiedCharges': 0,
- 'reportedTraceChargeMinor': 30, 'directCallsReserveMinor': 360, 'otherUnsettledReserveMinor': 600,
- 'provisionalBudgetOccupationMinor': 990, 'remainingAfterProvisionalReservesMinor': 4010,
- 'rawUsageRowsPresent': False, 'aggregatesMayOnlyBeEstimates': True},
- 'missingEvidence': ['per-request collector metadata and headers', 'raw usage/charge rows',
- 'original batch.json', 'original staging database and 49 quality issue rows',
- 'MCP structuredContent or files already saved under its outputDir'],
- 'productSelection': selection, 'dry': dry, 'build': built, 'databaseCounts': counts,
- 'qualityIssuesByCode': quality, 'readbackChecks': checks,
- 'publicationHold': 'Review before merge: older/sparse observations and unknown review defaults must not overwrite richer current records.'}
- if baseline:
- report['baselineComparison'] = baseline_review(baseline, out / 'staging.sqlite')
- require(report['originalsUnchanged'], 'Original files changed during preparation')
- write_json(out / 'review.json', report)
- return report
- def main():
- p = argparse.ArgumentParser(description=__doc__)
- p.add_argument('--original', required=True, type=Path)
- p.add_argument('--archive', required=True, type=Path)
- p.add_argument('--out', required=True, type=Path)
- p.add_argument('--baseline-db', type=Path, help='Optional local preview: readonly overlap/downgrade review only')
- args = p.parse_args()
- report = prepare(args.original.absolute(), args.archive.absolute(), args.out.absolute(), args.baseline_db)
- print(json.dumps({k: report[k] for k in ('archive', 'originalsUnchanged', 'networkRequests', 'productionImported',
- 'databaseCounts', 'readbackChecks')}, ensure_ascii=False, indent=2))
- if __name__ == '__main__':
- main()
|