# Read-only audit (Adelante docs/research/content_quality_audit.py at af1aea2), changed ONLY so the
# input library and the output file are arguments instead of the repo asset and /tmp.
# Usage: python3 -I content_quality_audit.py <library.json> <metrics-out.json>
import json,re,unicodedata,collections,hashlib,random,itertools,math,sys
from pathlib import Path
data=Path(sys.argv[1]).read_bytes();r=json.loads(data)
def tokens(s):return re.findall(r'\w+',unicodedata.normalize('NFKC',s).casefold())
norm=lambda s:' '.join(tokens(s))
sets=[set(tokens(x['text'])) for x in r]
# Deterministic 32-permutation MinHash: candidate generation only; verify every pair.
rng=random.Random(917);prime=(1<<61)-1;perms=[(rng.randrange(1,prime),rng.randrange(prime)) for _ in range(32)]
cache={}
for ts in sets:
 for t in ts:
  if t not in cache:
   h=int.from_bytes(hashlib.sha256(t.encode()).digest()[:8],'big')%prime
   cache[t]=[(a*h+b)%prime for a,b in perms]
bands=collections.defaultdict(list)
for i,ts in enumerate(sets):
 sig=[min(cache[t][k] for t in ts) for k in range(32)]
 for b in range(8):bands[(b,tuple(sig[b*4:b*4+4]))].append(i)
candidates=set()
for group in bands.values():
 for a,b in itertools.combinations(group,2):candidates.add((a,b))
pairs=[]
for a,b in sorted(candidates):
 if min(len(sets[a]),len(sets[b]))<6:continue
 score=len(sets[a]&sets[b])/len(sets[a]|sets[b])
 if score>=.8:pairs.append((a,b,score))
parent=list(range(len(r)))
def find(i):
 while parent[i]!=i:
  parent[i]=parent[parent[i]];i=parent[i]
 return i
for a,b,s in pairs:parent[find(a)]=find(b)
groups=collections.defaultdict(list)
for i in set(j for a,b,s in pairs for j in [a,b]):groups[find(i)].append(i)
cats=sorted(set(c for x in r for c in x['categories']))
religions=sorted(set(x['religion'] for x in r if x['religion']))
def countrows(rows):return {'total':len(rows),'generated':sum(x['id'].startswith('gen-') for x in rows),'scripture':sum(x['contentType'] in ['ayah','verse'] for x in rows),'external_author':sum(x['author']!='Adelante' for x in rows),'with_url':sum(bool(x['sourceUrl']) for x in rows)}
examples=[]
for cat in cats:
 rows=[x for x in r if cat in x['categories']]
 # Deterministic distributed sample: 3 ID-sorted generated entries per category.
 generated=sorted([x for x in rows if x['id'].startswith('gen-')],key=lambda x:x['id']) or sorted(rows,key=lambda x:x['id'])
 picks=[generated[int((len(generated)-1)*p)] for p in [0,.5,1]]
 examples.append({'category':cat,'counts':countrows(rows),'samples':[{'id':x['id'],'text':x['text'],'shortText':x['shortText'],'religion':x['religion']} for x in picks]})
faith=[]
for rel in religions:
 rows=sorted([x for x in r if x['religion']==rel],key=lambda x:x['id'])
 faith.append({'religion':rel,'counts':countrows(rows),'samples':[{'id':rows[int((len(rows)-1)*p)]['id'],'text':rows[int((len(rows)-1)*p)]['text']} for p in [0,.5,1]]})
phrases=collections.defaultdict(set)
for i,x in enumerate(r):
 words=tokens(x['text'])
 for j in range(max(0,len(words)-6+1)):phrases[' '.join(words[j:j+6])].add(i)
repeated=sorted(((p,idx) for p,idx in phrases.items() if len(idx)>=15),key=lambda z:(-len(z[1]),z[0]))
metric={'sha256':hashlib.sha256(data).hexdigest(),'count':len(r),'generated':sum(x['id'].startswith('gen-') for x in r),'categories':examples,'faith':faith,'contentTypes':dict(collections.Counter(x['contentType'] for x in r)),'missing_url_external_ids':[x['id'] for x in r if x['author']!='Adelante' and not x['sourceUrl']],'missing_translation_external_ids':[x['id'] for x in r if x['author']!='Adelante' and not x['translation']],'short_differs':sum(norm(x['shortText'])!=norm(x['text']) for x in r),'short_not_substring':[x['id'] for x in r if norm(x['shortText']) not in norm(x['text'])],'near':{'candidate_pairs':len(candidates),'verified_pairs':len(pairs),'affected_records':sum(len(g) for g in groups.values()),'connected_groups':len(groups),'cross_primary_category_pairs':sum(r[a]['categories'][0]!=r[b]['categories'][0] for a,b,s in pairs),'pairs':[{'score':round(s,4),'a':r[a]['id'],'b':r[b]['id'],'text_a':r[a]['text'],'text_b':r[b]['text']} for a,b,s in sorted(pairs,key=lambda v:-v[2])[:50]],'largest_groups':[[r[i]['id'] for i in sorted(g)] for g in sorted(groups.values(),key=len,reverse=True)[:10]]},'repeated_six_word_phrases':[{'phrase':p,'records':len(idx),'categories':sorted(set(c for i in idx for c in r[i]['categories']))} for p,idx in repeated[:20]]}
Path(sys.argv[2]).write_text(json.dumps(metric,indent=2))
print(json.dumps({k:v for k,v in metric.items() if k not in ['categories','faith','missing_url_external_ids','missing_translation_external_ids','short_not_substring','near']},indent=2))
print('NEAR',json.dumps({k:v for k,v in metric['near'].items() if k not in ['pairs','largest_groups']},indent=2))
for p in metric['near']['pairs'][:20]: print(p)