prepare-review-append-restoration.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223
  1. """Prepare source-proven append texts omitted from the public review pages.
  2. Build only a new local staging database using the installed batch contract.
  3. All review fields map directly to original response pointers. No collection or
  4. production write; the existing publisher must check its actual merge dry plan.
  5. """
  6. import argparse
  7. import hashlib
  8. import json
  9. from pathlib import Path
  10. import shutil
  11. import sys
  12. import tempfile
  13. import zipfile
  14. ROOT = Path(__file__).resolve().parents[1]
  15. sys.path.insert(0, str(ROOT / 'server/voc-library'))
  16. import agent_batch
  17. import library_channel
  18. import structured_batch
  19. import update_library
  20. import voc_db
  21. def save(path, value):
  22. path.parent.mkdir(parents=True, exist_ok=True)
  23. with path.open('x', encoding='utf-8', newline='\n') as stream:
  24. json.dump(value, stream, ensure_ascii=False, indent=2)
  25. stream.write('\n')
  26. def prepare(args):
  27. source = Path(args.evidence).resolve()
  28. out = Path(args.out).resolve()
  29. if out.exists():
  30. raise ValueError('NEW_OUTPUT_DIRECTORY_REQUIRED')
  31. report = json.loads((source / 'verification.json').read_text(encoding='utf-8'))
  32. failures = json.loads((source / 'failures.json').read_text(encoding='utf-8'))
  33. original = json.loads((source / 'selected-original-reviews.json').read_text(encoding='utf-8'))
  34. selected = {r['record_id']: r for r in original['records']}
  35. issues = failures['selected']
  36. if len(issues) != 2 or any(r['fields'] != ['append_content'] for r in issues):
  37. raise ValueError('EXPECTED_EXACTLY_TWO_APPEND_ONLY_FAILURES')
  38. rows = [selected[r['reviewId']] for r in issues]
  39. append_only_id = None
  40. if args.include_append_only:
  41. pid = 'jd:100010259226'
  42. name = 'originals/100010259226.json'
  43. path = source / 'source' / name
  44. original_body = json.loads(path.read_text(encoding='utf-8'))
  45. original_row = agent_batch.pointer(original_body, '/data/data/data/6')
  46. if (original_row['commentId'] != '106713640233346432' or
  47. original_row['productId'] != '100010259226' or
  48. original_row['afterComment']['content'] != '非常好吃,下回还要回购。'):
  49. raise ValueError('APPEND_ONLY_ORIGINAL_MISMATCH')
  50. append_only_id = pid + ':' + original_row['commentId']
  51. if append_only_id in selected:
  52. raise ValueError('APPEND_ONLY_ALREADY_SELECTED')
  53. actual_ids = set()
  54. for page in (source / 'board').glob('reviews-audit-p*.json'):
  55. if not page.name.endswith('-request.json'):
  56. actual_ids.update(r['record_id'] for r in json.loads(page.read_text(encoding='utf-8'))['records'])
  57. if append_only_id in actual_ids:
  58. raise ValueError('APPEND_ONLY_ALREADY_IN_PRODUCTION_BASELINE')
  59. rows.append({
  60. 'record_id': append_only_id, 'product_id': pid, 'review_id': original_row['commentId'],
  61. 'content': original_row['commentData'],
  62. 'append_content': original_row['afterComment']['content'],
  63. 'origin': {'file': name, 'pointer': '/data/data/data/6', 'sha256': agent_batch.digest(path)},
  64. })
  65. out.mkdir(parents=True)
  66. batch = {
  67. 'schema': 'ksf.voc-agent-batch.v2',
  68. 'batchId': ('review-three-append-restoration-20261107-r1' if args.include_append_only
  69. else 'review-append-restoration-20261107-r1'),
  70. 'producer': 'Codex: preserve source-proven original append texts without changing default eligibility',
  71. 'sources': [], 'attachments': [], 'products': [], 'reviews': [],
  72. }
  73. def copy(path, name):
  74. target = out / name
  75. target.parent.mkdir(parents=True, exist_ok=True)
  76. shutil.copyfile(path, target)
  77. return agent_batch.digest(target)
  78. archive = source / 'ksf-review-originals-60-20261107-r2.zip'
  79. archive_name = 'raw/evidence/delivered-originals-60.zip'
  80. archive_sha = copy(archive, archive_name)
  81. if archive_sha != report['sourceArchiveSha256']:
  82. raise ValueError('ORIGINAL_ARCHIVE_CHANGED')
  83. batch['attachments'].append({'path': archive_name, 'sha256': archive_sha})
  84. for name in ['manifest.csv', 'FILES.sha256']:
  85. relative = 'raw/evidence/delivered-' + name
  86. digest = copy(source / 'source' / name, relative)
  87. batch['attachments'].append({'path': relative, 'sha256': digest})
  88. checks = []
  89. for index, row in enumerate(rows):
  90. pid = row['product_id']
  91. iid = pid.split(':', 1)[1]
  92. context_path = source / 'append-correction-context' / (pid.replace(':', '-') + '.json')
  93. context = json.loads(context_path.read_text(encoding='utf-8'))
  94. if context['generation'] != report['generation'] or context['record']['id'] != pid:
  95. raise ValueError('PRODUCT_CONTEXT_GENERATION_MISMATCH')
  96. name = 'raw/context/' + context_path.name
  97. digest = copy(context_path, name)
  98. sid = 'product-' + iid
  99. current = context['record']
  100. batch['sources'].append({
  101. 'id': sid, 'path': name, 'sha256': digest, 'platform': 'jd', 'marketplace': 'jd',
  102. 'requestItemId': iid, 'observedAt': current.get('source_time_raw'),
  103. 'url': current.get('url'),
  104. })
  105. batch['products'].append({'sourceId': sid, 'pointer': '/record', 'fields': {
  106. 'itemId': '/item_id', 'title': '/title', 'url': '/url',
  107. 'shopId': '/shop_id', 'shopName': '/shop_name', 'sourceTime': '/source_time_raw',
  108. }})
  109. origin = row['origin']
  110. original_path = source / 'source' / origin['file']
  111. payload = json.loads(original_path.read_text(encoding='utf-8'))
  112. upstream = agent_batch.pointer(payload, origin['pointer'])
  113. if upstream['commentId'] != row['review_id'] or upstream['productId'] != iid:
  114. raise ValueError('RAW_REVIEW_POINTER_IDENTITY_MISMATCH')
  115. if not upstream['afterComment'].get('content'):
  116. raise ValueError('ORIGINAL_APPEND_TEXT_MISSING')
  117. name = 'raw/originals/' + original_path.name
  118. digest = copy(original_path, name)
  119. if digest != origin['sha256']:
  120. raise ValueError('ORIGINAL_REVIEW_RESPONSE_CHANGED')
  121. sid = 'review-' + str(index)
  122. batch['sources'].append({
  123. 'id': sid, 'path': name, 'sha256': digest, 'platform': 'jd', 'marketplace': 'jd',
  124. 'requestItemId': iid, 'observedAt': payload['data']['recordTime'],
  125. })
  126. batch['reviews'].append({'sourceId': sid, 'pointer': origin['pointer'], 'fields': {
  127. 'itemId': '/productId', 'reviewId': '/commentId', 'content': '/commentData',
  128. 'appendContent': '/afterComment', 'rating': '/commentScore',
  129. 'reviewDate': '/commentDate', 'purchasedSku': '/productColor',
  130. }})
  131. detail = source / ('append-loss-detail-' + row['record_id'].replace(':', '-') + '.json')
  132. detail_name = None
  133. if row['record_id'] != append_only_id:
  134. detail_name = 'raw/context/' + detail.name
  135. digest = copy(detail, detail_name)
  136. batch['attachments'].append({'path': detail_name, 'sha256': digest})
  137. checks.append({'reviewId': row['record_id'], 'productId': pid,
  138. 'appendText': upstream['afterComment']['content'],
  139. 'sourceFile': detail_name, 'originalFile': 'raw/originals/' + original_path.name,
  140. 'originalPointer': origin['pointer'], 'originalSha256': origin['sha256']})
  141. save(out / 'batch.json', batch)
  142. dry = structured_batch.build(out / 'batch.json', out / 'staging.sqlite')
  143. structured_batch.build(out / 'batch.json', out / 'staging.sqlite', True)
  144. update_library.verified(out / 'staging.sqlite')
  145. with voc_db.readonly(out / 'staging.sqlite') as connection:
  146. staged = {r['id']: dict(r) for r in connection.execute('SELECT * FROM reviews')}
  147. if staged.keys() != {r['record_id'] for r in rows}:
  148. raise ValueError('STAGING_REVIEW_IDS_MISMATCH')
  149. decisions = []
  150. for row in rows:
  151. current = staged[row['record_id']]
  152. if (current['content'] != row['content'] or current['is_default'] is not None or
  153. current['rating_scale'] is not None or current['identity_state'] != 'matched'):
  154. raise ValueError('STAGING_CORE_FIELD_MISMATCH')
  155. if json.loads(current['append_content_json'])['content'] != row['append_content']:
  156. raise ValueError('STAGING_APPEND_TEXT_MISMATCH')
  157. if row['record_id'] == append_only_id:
  158. # Keep the placeholder main text verbatim and default state unknown.
  159. decisions.append({'reviewId': row['record_id'], 'decision': 'new_review_candidate',
  160. 'scope': 'absent_from_complete_public_readback_requires_actual_production_dry'})
  161. continue
  162. detail = json.loads((source / ('append-loss-detail-' + row['record_id'].replace(':', '-') + '.json')).read_text(encoding='utf-8'))['record']
  163. previous = {key: detail[key] for key in current}
  164. decision, reason = update_library.choose_record('reviews', previous, current)
  165. if decision != 'used_incoming' or reason['codes'] != ['NONCONFLICTING_ENRICHMENT']:
  166. raise ValueError('EXPECTED_NONCONFLICTING_APPEND_RESTORATION')
  167. decisions.append({'reviewId': row['record_id'], 'decision': decision, 'reason': reason,
  168. 'scope': 'public_detail_field_comparison_not_actual_production_merge_dry'})
  169. wire, digest = library_channel.pack(out / 'batch.json')
  170. (out / 'channel-source.zip').write_bytes(wire)
  171. # Rebuild from exactly the transferable source archive, compare complete rows.
  172. with tempfile.TemporaryDirectory(prefix='append-transfer-check-', dir=out) as temp:
  173. target = Path(temp)
  174. with zipfile.ZipFile(io_bytes(wire)) as z:
  175. for info in z.infolist():
  176. path = target / info.filename
  177. if not path.resolve().is_relative_to(target):
  178. raise ValueError('UNSAFE_TRANSFER_MEMBER')
  179. path.parent.mkdir(parents=True, exist_ok=True)
  180. path.write_bytes(z.read(info))
  181. structured_batch.build(target / 'batch.json', target / 'rebuilt.sqlite', True)
  182. update_library.verified(target / 'rebuilt.sqlite')
  183. with voc_db.readonly(target / 'rebuilt.sqlite') as connection:
  184. rebuilt = {r['id']: dict(r) for r in connection.execute('SELECT * FROM reviews')}
  185. if rebuilt != staged:
  186. raise ValueError('TRANSFER_REBUILD_FIELD_MISMATCH')
  187. result = {
  188. 'schema': 'ksf.review-append-restoration.prepared.v1', 'ok': True,
  189. 'expectedProductionGeneration': report['generation'],
  190. 'products': len(rows), 'reviews': len(rows), 'newReviewIdsExpected': int(args.include_append_only),
  191. 'existingReviewIdsToCheck': 2, 'appendOnlyOriginalsRetained': int(args.include_append_only),
  192. 'appendTextsRestoredInStaging': len(rows), 'defaultStatesRemainUnknown': True,
  193. 'ratingScalesRemainUnknown': True, 'analysisEligibilityNotRelaxed': True,
  194. 'originalArchiveSha256': archive_sha, 'sourceFilesVerified': dry['sourceFilesVerified'],
  195. 'sourceArchiveSha256': digest, 'sourceArchiveBytes': len(wire),
  196. 'transferRebuiltAndAllReviewFieldsMatched': True,
  197. 'publicDetailNonconflictingEnrichmentChecks': decisions,
  198. 'appendEvidence': checks, 'productionImported': False,
  199. 'actualProductionMergeDryExecuted': False, 'productionWrites': 0, 'paidCalls': 0,
  200. }
  201. save(out / 'verification.json', result)
  202. print(json.dumps(result, ensure_ascii=False, indent=2))
  203. def io_bytes(raw):
  204. import io
  205. return io.BytesIO(raw)
  206. if __name__ == '__main__':
  207. parser = argparse.ArgumentParser(description=__doc__)
  208. parser.add_argument('--evidence', required=True)
  209. parser.add_argument('--out', required=True)
  210. parser.add_argument('--include-append-only', action='store_true')
  211. prepare(parser.parse_args())