| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223 |
- """Prepare source-proven append texts omitted from the public review pages.
- Build only a new local staging database using the installed batch contract.
- All review fields map directly to original response pointers. No collection or
- production write; the existing publisher must check its actual merge dry plan.
- """
- import argparse
- import hashlib
- import json
- from pathlib import Path
- import shutil
- import sys
- import tempfile
- import zipfile
- ROOT = Path(__file__).resolve().parents[1]
- sys.path.insert(0, str(ROOT / 'server/voc-library'))
- import agent_batch
- import library_channel
- import structured_batch
- import update_library
- import voc_db
- def save(path, value):
- path.parent.mkdir(parents=True, exist_ok=True)
- with path.open('x', encoding='utf-8', newline='\n') as stream:
- json.dump(value, stream, ensure_ascii=False, indent=2)
- stream.write('\n')
- def prepare(args):
- source = Path(args.evidence).resolve()
- out = Path(args.out).resolve()
- if out.exists():
- raise ValueError('NEW_OUTPUT_DIRECTORY_REQUIRED')
- report = json.loads((source / 'verification.json').read_text(encoding='utf-8'))
- failures = json.loads((source / 'failures.json').read_text(encoding='utf-8'))
- original = json.loads((source / 'selected-original-reviews.json').read_text(encoding='utf-8'))
- selected = {r['record_id']: r for r in original['records']}
- issues = failures['selected']
- if len(issues) != 2 or any(r['fields'] != ['append_content'] for r in issues):
- raise ValueError('EXPECTED_EXACTLY_TWO_APPEND_ONLY_FAILURES')
- rows = [selected[r['reviewId']] for r in issues]
- append_only_id = None
- if args.include_append_only:
- pid = 'jd:100010259226'
- name = 'originals/100010259226.json'
- path = source / 'source' / name
- original_body = json.loads(path.read_text(encoding='utf-8'))
- original_row = agent_batch.pointer(original_body, '/data/data/data/6')
- if (original_row['commentId'] != '106713640233346432' or
- original_row['productId'] != '100010259226' or
- original_row['afterComment']['content'] != '非常好吃,下回还要回购。'):
- raise ValueError('APPEND_ONLY_ORIGINAL_MISMATCH')
- append_only_id = pid + ':' + original_row['commentId']
- if append_only_id in selected:
- raise ValueError('APPEND_ONLY_ALREADY_SELECTED')
- actual_ids = set()
- for page in (source / 'board').glob('reviews-audit-p*.json'):
- if not page.name.endswith('-request.json'):
- actual_ids.update(r['record_id'] for r in json.loads(page.read_text(encoding='utf-8'))['records'])
- if append_only_id in actual_ids:
- raise ValueError('APPEND_ONLY_ALREADY_IN_PRODUCTION_BASELINE')
- rows.append({
- 'record_id': append_only_id, 'product_id': pid, 'review_id': original_row['commentId'],
- 'content': original_row['commentData'],
- 'append_content': original_row['afterComment']['content'],
- 'origin': {'file': name, 'pointer': '/data/data/data/6', 'sha256': agent_batch.digest(path)},
- })
- out.mkdir(parents=True)
- batch = {
- 'schema': 'ksf.voc-agent-batch.v2',
- 'batchId': ('review-three-append-restoration-20261107-r1' if args.include_append_only
- else 'review-append-restoration-20261107-r1'),
- 'producer': 'Codex: preserve source-proven original append texts without changing default eligibility',
- 'sources': [], 'attachments': [], 'products': [], 'reviews': [],
- }
- def copy(path, name):
- target = out / name
- target.parent.mkdir(parents=True, exist_ok=True)
- shutil.copyfile(path, target)
- return agent_batch.digest(target)
- archive = source / 'ksf-review-originals-60-20261107-r2.zip'
- archive_name = 'raw/evidence/delivered-originals-60.zip'
- archive_sha = copy(archive, archive_name)
- if archive_sha != report['sourceArchiveSha256']:
- raise ValueError('ORIGINAL_ARCHIVE_CHANGED')
- batch['attachments'].append({'path': archive_name, 'sha256': archive_sha})
- for name in ['manifest.csv', 'FILES.sha256']:
- relative = 'raw/evidence/delivered-' + name
- digest = copy(source / 'source' / name, relative)
- batch['attachments'].append({'path': relative, 'sha256': digest})
- checks = []
- for index, row in enumerate(rows):
- pid = row['product_id']
- iid = pid.split(':', 1)[1]
- context_path = source / 'append-correction-context' / (pid.replace(':', '-') + '.json')
- context = json.loads(context_path.read_text(encoding='utf-8'))
- if context['generation'] != report['generation'] or context['record']['id'] != pid:
- raise ValueError('PRODUCT_CONTEXT_GENERATION_MISMATCH')
- name = 'raw/context/' + context_path.name
- digest = copy(context_path, name)
- sid = 'product-' + iid
- current = context['record']
- batch['sources'].append({
- 'id': sid, 'path': name, 'sha256': digest, 'platform': 'jd', 'marketplace': 'jd',
- 'requestItemId': iid, 'observedAt': current.get('source_time_raw'),
- 'url': current.get('url'),
- })
- batch['products'].append({'sourceId': sid, 'pointer': '/record', 'fields': {
- 'itemId': '/item_id', 'title': '/title', 'url': '/url',
- 'shopId': '/shop_id', 'shopName': '/shop_name', 'sourceTime': '/source_time_raw',
- }})
- origin = row['origin']
- original_path = source / 'source' / origin['file']
- payload = json.loads(original_path.read_text(encoding='utf-8'))
- upstream = agent_batch.pointer(payload, origin['pointer'])
- if upstream['commentId'] != row['review_id'] or upstream['productId'] != iid:
- raise ValueError('RAW_REVIEW_POINTER_IDENTITY_MISMATCH')
- if not upstream['afterComment'].get('content'):
- raise ValueError('ORIGINAL_APPEND_TEXT_MISSING')
- name = 'raw/originals/' + original_path.name
- digest = copy(original_path, name)
- if digest != origin['sha256']:
- raise ValueError('ORIGINAL_REVIEW_RESPONSE_CHANGED')
- sid = 'review-' + str(index)
- batch['sources'].append({
- 'id': sid, 'path': name, 'sha256': digest, 'platform': 'jd', 'marketplace': 'jd',
- 'requestItemId': iid, 'observedAt': payload['data']['recordTime'],
- })
- batch['reviews'].append({'sourceId': sid, 'pointer': origin['pointer'], 'fields': {
- 'itemId': '/productId', 'reviewId': '/commentId', 'content': '/commentData',
- 'appendContent': '/afterComment', 'rating': '/commentScore',
- 'reviewDate': '/commentDate', 'purchasedSku': '/productColor',
- }})
- detail = source / ('append-loss-detail-' + row['record_id'].replace(':', '-') + '.json')
- detail_name = None
- if row['record_id'] != append_only_id:
- detail_name = 'raw/context/' + detail.name
- digest = copy(detail, detail_name)
- batch['attachments'].append({'path': detail_name, 'sha256': digest})
- checks.append({'reviewId': row['record_id'], 'productId': pid,
- 'appendText': upstream['afterComment']['content'],
- 'sourceFile': detail_name, 'originalFile': 'raw/originals/' + original_path.name,
- 'originalPointer': origin['pointer'], 'originalSha256': origin['sha256']})
- save(out / 'batch.json', batch)
- dry = structured_batch.build(out / 'batch.json', out / 'staging.sqlite')
- structured_batch.build(out / 'batch.json', out / 'staging.sqlite', True)
- update_library.verified(out / 'staging.sqlite')
- with voc_db.readonly(out / 'staging.sqlite') as connection:
- staged = {r['id']: dict(r) for r in connection.execute('SELECT * FROM reviews')}
- if staged.keys() != {r['record_id'] for r in rows}:
- raise ValueError('STAGING_REVIEW_IDS_MISMATCH')
- decisions = []
- for row in rows:
- current = staged[row['record_id']]
- if (current['content'] != row['content'] or current['is_default'] is not None or
- current['rating_scale'] is not None or current['identity_state'] != 'matched'):
- raise ValueError('STAGING_CORE_FIELD_MISMATCH')
- if json.loads(current['append_content_json'])['content'] != row['append_content']:
- raise ValueError('STAGING_APPEND_TEXT_MISMATCH')
- if row['record_id'] == append_only_id:
- # Keep the placeholder main text verbatim and default state unknown.
- decisions.append({'reviewId': row['record_id'], 'decision': 'new_review_candidate',
- 'scope': 'absent_from_complete_public_readback_requires_actual_production_dry'})
- continue
- detail = json.loads((source / ('append-loss-detail-' + row['record_id'].replace(':', '-') + '.json')).read_text(encoding='utf-8'))['record']
- previous = {key: detail[key] for key in current}
- decision, reason = update_library.choose_record('reviews', previous, current)
- if decision != 'used_incoming' or reason['codes'] != ['NONCONFLICTING_ENRICHMENT']:
- raise ValueError('EXPECTED_NONCONFLICTING_APPEND_RESTORATION')
- decisions.append({'reviewId': row['record_id'], 'decision': decision, 'reason': reason,
- 'scope': 'public_detail_field_comparison_not_actual_production_merge_dry'})
- wire, digest = library_channel.pack(out / 'batch.json')
- (out / 'channel-source.zip').write_bytes(wire)
- # Rebuild from exactly the transferable source archive, compare complete rows.
- with tempfile.TemporaryDirectory(prefix='append-transfer-check-', dir=out) as temp:
- target = Path(temp)
- with zipfile.ZipFile(io_bytes(wire)) as z:
- for info in z.infolist():
- path = target / info.filename
- if not path.resolve().is_relative_to(target):
- raise ValueError('UNSAFE_TRANSFER_MEMBER')
- path.parent.mkdir(parents=True, exist_ok=True)
- path.write_bytes(z.read(info))
- structured_batch.build(target / 'batch.json', target / 'rebuilt.sqlite', True)
- update_library.verified(target / 'rebuilt.sqlite')
- with voc_db.readonly(target / 'rebuilt.sqlite') as connection:
- rebuilt = {r['id']: dict(r) for r in connection.execute('SELECT * FROM reviews')}
- if rebuilt != staged:
- raise ValueError('TRANSFER_REBUILD_FIELD_MISMATCH')
- result = {
- 'schema': 'ksf.review-append-restoration.prepared.v1', 'ok': True,
- 'expectedProductionGeneration': report['generation'],
- 'products': len(rows), 'reviews': len(rows), 'newReviewIdsExpected': int(args.include_append_only),
- 'existingReviewIdsToCheck': 2, 'appendOnlyOriginalsRetained': int(args.include_append_only),
- 'appendTextsRestoredInStaging': len(rows), 'defaultStatesRemainUnknown': True,
- 'ratingScalesRemainUnknown': True, 'analysisEligibilityNotRelaxed': True,
- 'originalArchiveSha256': archive_sha, 'sourceFilesVerified': dry['sourceFilesVerified'],
- 'sourceArchiveSha256': digest, 'sourceArchiveBytes': len(wire),
- 'transferRebuiltAndAllReviewFieldsMatched': True,
- 'publicDetailNonconflictingEnrichmentChecks': decisions,
- 'appendEvidence': checks, 'productionImported': False,
- 'actualProductionMergeDryExecuted': False, 'productionWrites': 0, 'paidCalls': 0,
- }
- save(out / 'verification.json', result)
- print(json.dumps(result, ensure_ascii=False, indent=2))
- def io_bytes(raw):
- import io
- return io.BytesIO(raw)
- if __name__ == '__main__':
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument('--evidence', required=True)
- parser.add_argument('--out', required=True)
- parser.add_argument('--include-append-only', action='store_true')
- prepare(parser.parse_args())
|