"""Seeded uniform random sample (without replacement) of records from an Adelante library file.
Read-only. Joins each sampled record with the rule-based findings of corpusAudit.ts.
Usage: python3 -I draw_sample.py <library.json> <corpus-audit.json> <n> <seed> <out.json>"""
import json, random, re, sys
lib, audit, n, seed, out = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4]), sys.argv[5]
records = sorted(json.load(open(lib, encoding='utf-8')), key=lambda x: x['id'])
report = json.load(open(audit, encoding='utf-8'))
findings = {f['id']: f['issues'] for f in report['findings']}
template_ids = set(i for p in report['repeatedPhrases'] for i in p['ids'])
near_ids = set(i for p in report['nearDuplicates'] for i in p['ids'])
idx = sorted(random.Random(seed).sample(range(len(records)), n))
NUM = re.compile(r'\d|\b(?:one|two|three|four|five|six|seven|eight|nine|ten|hundred|thousand|million|percent|twice|half)\b', re.I)
rows = []
for k, i in enumerate(idx):
    x = records[i]
    rows.append({
        'k': k, 'index': i, 'id': x['id'], 'categories': x['categories'], 'religion': x['religion'],
        'author': x['author'], 'contentType': x['contentType'], 'licenseStatus': x['licenseStatus'], 'text': x['text'],
        'rule': {
            'generated': x['id'].startswith('gen-'),
            'review_date_missing': 'review_date_missing' in findings.get(x['id'], []),
            'flags_true_without_date': bool(x.get('reviewed') and x.get('approved') and x.get('verified')) and not x.get('lastReviewedAt'),
            'quality_score_saturated': 'quality_score_saturated' in findings.get(x['id'], []),
            'template_phrase_ge4': x['id'] in template_ids,
            'near_duplicate': x['id'] in near_ids,
            'numeric_token': bool(NUM.search(x['text'])),
            'external_attribution': x['author'] != 'Adelante',
            'rights_basis_ambiguous': 'rights_basis_ambiguous' in findings.get(x['id'], []),
        },
    })
json.dump({'population': len(records), 'n': n, 'seed': seed, 'sortedBy': 'id', 'rows': rows}, open(out, 'w', encoding='utf-8'), indent=1, ensure_ascii=False)
print(len(records), n, seed)