"""Rates with Wilson 95% intervals for the rule-based and annotated classes in the 200-record sample,
plus census rates over the whole asset for the rule-based classes. Usage:
python3 -I summarise_sample.py results/sample-200.json annotations-200.json results/corpus-audit-af1aea2.json data/library-af1aea2.json"""
import json, math, re, sys
sample = json.load(open(sys.argv[1], encoding='utf-8')); ann = json.load(open(sys.argv[2], encoding='utf-8'))
audit = json.load(open(sys.argv[3], encoding='utf-8')); lib = json.load(open(sys.argv[4], encoding='utf-8'))
def wilson(k, n, z=1.959963984540054):
    p = k / n; d = 1 + z*z/n; c = (p + z*z/(2*n)) / d; h = z*math.sqrt(p*(1-p)/n + z*z/(4*n*n)) / d
    return [round(max(0, c-h), 4), round(min(1, c+h), 4)]
rows = sample['rows']; n = len(rows); N = len(lib)
out = {'n': n, 'population': N, 'rule_sample': {}, 'rule_census': {}, 'annotated': {}}
for key in rows[0]['rule']:
    k = sum(r['rule'][key] for r in rows); out['rule_sample'][key] = {'k': k, 'rate': round(k/n, 4), 'wilson95': wilson(k, n)}
findings = {f['id']: f['issues'] for f in audit['findings']}
template_ids = set(i for p in audit['repeatedPhrases'] for i in p['ids'])
NUM = re.compile(r'\d|\b(?:one|two|three|four|five|six|seven|eight|nine|ten|hundred|thousand|million|percent|twice|half)\b', re.I)
census = {
  'generated': sum(x['id'].startswith('gen-') for x in lib),
  'review_date_missing': sum('review_date_missing' in findings.get(x['id'], []) for x in lib),
  'flags_true_without_date': sum(bool(x.get('reviewed') and x.get('approved') and x.get('verified')) and not x.get('lastReviewedAt') for x in lib),
  'quality_score_saturated': sum('quality_score_saturated' in findings.get(x['id'], []) for x in lib),
  'template_phrase_ge4': sum(x['id'] in template_ids for x in lib),
  'numeric_token': sum(bool(NUM.search(x['text'])) for x in lib),
  'external_attribution': sum(x['author'] != 'Adelante' for x in lib),
}
out['rule_census'] = {k: {'k': v, 'rate': round(v/N, 4)} for k, v in census.items()}
for cls, ks in ann['positives'].items():
    k = len(set(ks)); out['annotated'][cls] = {'k': k, 'rate': round(k/n, 4), 'wilson95': wilson(k, n), 'projected_records_point': round(k/n*N)}
print(json.dumps(out, indent=2))