#!/usr/bin/env python3
"""Compute every number in the paper from the public dataset and draw the
figures as plain SVG (no plotting library needed).

Reads  data/coding_pass1.csv, data/interventions.csv, data/consistency_check.csv,
       data/chains.csv, data/memory_rules.csv, data/messages_index.csv
Writes data/stats.json, data/tables.md and four SVG figures into --figdir.
"""
import argparse, collections, csv, json, os
from xml.sax.saxutils import escape

CATS = collections.OrderedDict([
    ("FUNC", "Broken behaviour found in use"),
    ("REG", "Regression or still-unfixed bug"),
    ("DEVL", "Layout and rendering breakage"),
    ("FOLLOW", "Unfinished or unverified work"),
    ("INST", "Earlier instruction ignored"),
    ("INTENT", "Misread intent or wrong diagnosis"),
    ("FACT", "Wrong fact, path or instruction"),
    ("STALE", "Stale build, asset or public page"),
    ("PROC", "Process and release hygiene"),
    ("SCOPE", "Wrong scope"),
    ("BLOCK", "False blocker or premature stop"),
    ("OVER", "Overclaim of status"),
    ("SEC", "Missing security or abuse control"),
    ("COST", "Cost surprise"),
])
HOW = collections.OrderedDict([
    ("DEVICE", "Using the app on a phone, simulator or browser"),
    ("VISUAL", "Looking at a rendered page, screenshot or mock-up"),
    ("READ", "Reading the agent's report, diff or repo"),
    ("MEMORY", "Remembering an earlier instruction"),
    ("DOMAIN", "Product or platform knowledge"),
    ("EXTERNAL", "An outside system (console, store, bill)"),
])
HOW_SHORT = collections.OrderedDict([("DEVICE", "Device use"), ("VISUAL", "Viewing output"),
                                    ("READ", "Reading"), ("MEMORY", "Memory"), ("DOMAIN", "Know-how"),
                                    ("EXTERNAL", "Outside system")])
HOW_COL = {"DEVICE": "#1f5f8b", "VISUAL": "#5b9bd0", "READ": "#d9822b", "MEMORY": "#8c6bb1",
           "DOMAIN": "#4d9a6a", "EXTERNAL": "#9a9a9a"}


def rd(p):
    return list(csv.DictReader(open(p)))


def pct(a, b):
    return round(100.0 * a / b, 1) if b else 0.0


def wilson(k, n, z=1.96):
    if not n:
        return [0, 0]
    p = k / n
    d = 1 + z * z / n
    c = (p + z * z / (2 * n)) / d
    h = z * ((p * (1 - p) / n + z * z / (4 * n * n)) ** 0.5) / d
    return [round(100 * (c - h), 1), round(100 * (c + h), 1)]


def kappa(pairs):
    n = len(pairs)
    po = sum(1 for a, b in pairs if a == b) / n
    ca = collections.Counter(a for a, _ in pairs)
    cb = collections.Counter(b for _, b in pairs)
    pe = sum(ca[k] * cb[k] for k in set(ca) | set(cb)) / (n * n)
    return round((po - pe) / (1 - pe), 3) if pe < 1 else 1.0


# ---------------- tiny SVG helpers ----------------
FONT = "font-family='Helvetica, Arial, sans-serif'"


def svg_open(w, h):
    return [f"<svg xmlns='http://www.w3.org/2000/svg' width='{w}' height='{h}' viewBox='0 0 {w} {h}' role='img'>",
            f"<rect x='0' y='0' width='{w}' height='{h}' fill='#ffffff'/>"]


def text(x, y, s, size=13, anchor="start", weight="normal", fill="#222222"):
    return (f"<text x='{x:.1f}' y='{y:.1f}' {FONT} font-size='{size}' text-anchor='{anchor}' "
            f"font-weight='{weight}' fill='{fill}'>{escape(str(s))}</text>")


def rect(x, y, w, h, fill):
    return f"<rect x='{x:.1f}' y='{y:.1f}' width='{max(w,0):.1f}' height='{h:.1f}' fill='{fill}'/>"


def legend(out, items, x, y, size=12, gap=16):
    cx = x
    for label, col in items:
        out.append(rect(cx, y - 10, 12, 12, col))
        out.append(text(cx + 17, y, label, size))
        cx += 17 + len(label) * size * 0.56 + gap
    return out


def fig_taxonomy(inter, path):
    nv = [r for r in inter if r["category"] != "VIS"]
    cnt = collections.Counter(r["category"] for r in nv)
    order = [c for c in sorted(CATS, key=lambda c: -cnt[c]) if cnt[c]]
    W, left, top, rowh = 600, 222, 64, 26
    H = top + rowh * len(order) + 60
    scale = (W - left - 60) / max(cnt.values())
    out = svg_open(W, H)
    out.append(text(16, 26, f"What the reviewer caught ({len(nv)} non-visual interventions)", 15, weight="bold"))
    legend(out, [(HOW_SHORT[k], HOW_COL[k]) for k in HOW], 16, 48, 13, 8)
    for i, c in enumerate(order):
        y = top + i * rowh
        out.append(text(left - 8, y + 16, CATS[c], 13, "end"))
        x = left
        byhow = collections.Counter(r["how_noticed"] for r in nv if r["category"] == c)
        for h in HOW:
            w = byhow[h] * scale
            out.append(rect(x, y + 3, w, rowh - 8, HOW_COL[h]))
            x += w
        out.append(text(x + 6, y + 16, cnt[c], 13))
    out.append(text(16, H - 14, "Colour = how the problem was noticed. Visual-taste corrections excluded.", 13, fill="#555555"))
    out.append("</svg>")
    open(path, "w").write("\n".join(out))


def fig_claims(inter, path):
    nv = [r for r in inter if r["category"] != "VIS"]
    groups = list(HOW_SHORT.items())
    W, left, top, rowh = 600, 190, 92, 30
    H = top + rowh * len(groups) + 56
    out = svg_open(W, H)
    out.append(text(16, 26, "Had the agent just reported success?", 15, weight="bold"))
    cols = [("Y", "#c0392b", "Yes: done, passing or live"), ("N", "#bdbdbd", "No"),
            ("U", "#e6e6e6", "Unclear")]
    legend(out, [(c[2], c[1]) for c in cols], 16, 52, 13, 10)
    barw = W - left - 80
    for i, (h, lab) in enumerate(groups):
        rows = [r for r in nv if r["how_noticed"] == h]
        if not rows:
            continue
        y = top + i * rowh
        out.append(text(left - 8, y + 18, f"{lab} (n={len(rows)})", 13, "end"))
        x = left
        c = collections.Counter(r["agent_reported_success"] for r in rows)
        for k, col, _ in cols:
            w = barw * c[k] / len(rows)
            out.append(rect(x, y + 4, w, rowh - 10, col))
            x += w
        out.append(text(left + barw + 6, y + 18, f"{pct(c['Y'], len(rows)):.0f}%", 13))
    out.append(text(16, H - 14, "Rows grouped by how the problem was noticed. Small groups: read with care.", 13, fill="#555555"))
    out.append("</svg>")
    open(path, "w").write("\n".join(out))


def fig_chains(chains, path):
    import datetime as dt
    d0, d1 = dt.date(2026, 6, 13), dt.date(2026, 11, 2)
    span = (d1 - d0).days
    W, left, top, rowh = 600, 212, 60, 30
    H = top + rowh * len(chains) + 70
    pw = W - left - 30

    def X(d):
        d = dt.date.fromisoformat(d)
        return left + pw * (d - d0).days / span
    out = svg_open(W, H)
    out.append(text(16, 26, "Repeat reports, and when a written rule appeared", 15, weight="bold"))
    legend(out, [("Report by the reviewer", "#1f5f8b"), ("Written rule", "#c0392b")], 16, 48, 13, 14)
    for m, lab in [("2026-06-13", "13 Jun"), ("2026-07-01", "Jul"), ("2026-08-01", "Aug"), ("2026-09-01", "Sep"), ("2026-10-01", "Oct"), ("2026-11-01", "Nov")]:
        x = X(m)
        out.append(f"<line x1='{x:.1f}' y1='{top-4}' x2='{x:.1f}' y2='{H-46}' stroke='#e3e3e3'/>")
        out.append(text(x, H - 30, lab, 12, "middle", fill="#555555"))
    for i, c in enumerate(chains):
        y = top + i * rowh + 14
        lab = c["short"]
        out.append(text(left - 8, y + 5, f"{lab} ({c['n_reports']})", 13, "end"))
        out.append(f"<line x1='{left}' y1='{y}' x2='{left+pw}' y2='{y}' stroke='#f0f0f0'/>")
        dates = collections.Counter(c["dates"])
        for d, k in dates.items():
            r = 4 + 1.6 * (k - 1)
            out.append(f"<circle cx='{X(d):.1f}' cy='{y}' r='{r:.1f}' fill='#1f5f8b' fill-opacity='0.85'/>")
        if c["rule_date"]:
            x = X(c["rule_date"])
            out.append(f"<line x1='{x:.1f}' y1='{y-10}' x2='{x:.1f}' y2='{y+10}' stroke='#c0392b' stroke-width='3'/>")
    out.append(text(16, H - 10, "Bigger dot = more reports that day. No red mark = no rule. Purposive chains.", 13, fill="#555555"))
    out.append("</svg>")
    open(path, "w").write("\n".join(out))


def fig_rules(rules, chains, path):
    W, H = 600, 360
    out = svg_open(W, H)
    out.append(text(16, 26, "The 32 written feedback rules", 15, weight="bold"))
    groups = [("owner", "Owner's correction"), ("mixed", "Owner's report, agent's lesson"),
              ("agent-review", "Agent's adversarial review"), ("agent-other", "Agent's own debugging")]
    left, top, rowh, barw = 210, 60, 34, 330
    mx = max(sum(1 for r in rules if r["origin"] == g) for g, _ in groups)
    for i, (g, lab) in enumerate(groups):
        y = top + i * rowh
        rs = [r for r in rules if r["origin"] == g]
        vis = sum(1 for r in rs if r["visual"] == "1")
        out.append(text(left - 8, y + 19, lab, 13, "end"))
        out.append(rect(left, y + 5, barw * (len(rs) - vis) / mx, rowh - 12, "#1f5f8b"))
        out.append(rect(left + barw * (len(rs) - vis) / mx, y + 5, barw * vis / mx, rowh - 12, "#9ec5e8"))
        out.append(text(left + barw * len(rs) / mx + 6, y + 19, len(rs), 13))
    legend(out, [("Non-visual", "#1f5f8b"), ("Visual taste", "#9ec5e8")], left, top + 4 * rowh + 18, 13)
    parts = []
    for k, lab in [("exact", "Rule names the exact failure"), ("adjacent", "Rule covers a neighbouring path"), ("habit", "Rule asks for a habit")]:
        cs = [c for c in chains if c["rule_match"] == k]
        parts.append((lab, len(cs), sum(int(c["reports_after_rule_day"] or 0) for c in cs)))
    out.append(text(16, H - 70, "Reports after the rule day (9 purposive chains with a rule):", 13, weight="bold"))
    for j, (lab, n, s_) in enumerate(parts):
        out.append(text(28, H - 48 + 19 * j, f"{lab}: {n} chains, {s_} later reports", 13))
    out.append("</svg>")
    open(path, "w").write("\n".join(out))


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--data", required=True)
    ap.add_argument("--figdir", required=True)
    a = ap.parse_args()
    D = a.data
    p1 = rd(os.path.join(D, "coding_pass1.csv"))
    inter = rd(os.path.join(D, "interventions.csv"))
    cons = rd(os.path.join(D, "consistency_check.csv"))
    chains = rd(os.path.join(D, "chains.csv"))
    rules = rd(os.path.join(D, "memory_rules.csv"))
    idx = rd(os.path.join(D, "messages_index.csv"))
    S = {}

    # corpus
    S["corpus_messages"] = len(idx)
    S["corpus_prefilter"] = sum(1 for r in idx if r["prefilter"] == "1")
    S["corpus_date_range"] = [min(r["date"] for r in idx), max(r["date"] for r in idx)]
    S["corpus_by_month"] = dict(sorted(collections.Counter(r["date"][:7] for r in idx).items()))

    # sample status
    st = {}
    for s in ("A_prefilter", "B_nonprefilter"):
        rows = [r for r in p1 if r["stratum"] == s]
        c = collections.Counter(r["status"] for r in rows)
        nv = sum(1 for r in rows if r["status"] == "intervention" and any(x != "VIS" for x in r["categories"].split(";")))
        vis_only = sum(1 for r in rows if r["status"] == "intervention" and set(r["categories"].split(";")) == {"VIS"})
        st[s] = {"n": len(rows), "intervention": c["intervention"], "none": c["none"], "excluded": c["excluded"],
                 "with_nonvisual": nv, "visual_only": vis_only}
    S["sample"] = st
    nA, nB = S["corpus_prefilter"], S["corpus_messages"] - S["corpus_prefilter"]
    rA = st["A_prefilter"]["intervention"] / st["A_prefilter"]["n"]
    rB = st["B_nonprefilter"]["intervention"] / st["B_nonprefilter"]["n"]
    S["est_intervention_messages_corpus"] = round(rA * nA + rB * nB)
    S["est_prefilter_recall_pct"] = pct(rA * nA, rA * nA + rB * nB)
    rAn = st["A_prefilter"]["with_nonvisual"] / st["A_prefilter"]["n"]
    rBn = st["B_nonprefilter"]["with_nonvisual"] / st["B_nonprefilter"]["n"]
    S["est_nonvisual_intervention_messages_corpus"] = round(rAn * nA + rBn * nB)
    S["est_prefilter_recall_nonvisual_pct"] = pct(rAn * nA, rAn * nA + rBn * nB)
    S["est_intervention_share_corpus_pct"] = pct(rA * nA + rB * nB, nA + nB)
    # rough range for non-visual pre-filter recall: Wilson bounds on each stratum's rate
    lA, hA = [x / 100 for x in wilson(st["A_prefilter"]["with_nonvisual"], st["A_prefilter"]["n"])]
    lB, hB = [x / 100 for x in wilson(st["B_nonprefilter"]["with_nonvisual"], st["B_nonprefilter"]["n"])]
    S["est_prefilter_recall_nonvisual_range_pct"] = [pct(lA * nA, lA * nA + hB * nB), pct(hA * nA, hA * nA + lB * nB)]
    S["est_intervention_share_corpus_range_pct"] = [pct(lA2 * nA + lB2 * nB, nA + nB) for lA2, lB2 in [
        [x / 100 for x in (wilson(st["A_prefilter"]["intervention"], st["A_prefilter"]["n"])[0], wilson(st["B_nonprefilter"]["intervention"], st["B_nonprefilter"]["n"])[0])],
        [x / 100 for x in (wilson(st["A_prefilter"]["intervention"], st["A_prefilter"]["n"])[1], wilson(st["B_nonprefilter"]["intervention"], st["B_nonprefilter"]["n"])[1])]]]
    S["sample_coded_nonexcluded"] = sum(1 for r in p1 if r["status"] != "excluded")
    S["sample_intervention_msgs"] = sum(1 for r in p1 if r["status"] == "intervention")

    # interventions
    S["rows_total"] = len(inter)
    S["rows_visual"] = sum(1 for r in inter if r["category"] == "VIS")
    nv = [r for r in inter if r["category"] != "VIS"]
    S["rows_nonvisual"] = len(nv)
    S["msgs_with_nonvisual"] = len({r["msg_id"] for r in nv})
    S["by_category"] = {c: sum(1 for r in nv if r["category"] == c) for c in CATS}
    S["by_category_pct"] = {c: pct(v, len(nv)) for c, v in S["by_category"].items()}
    S["by_how"] = {h: sum(1 for r in nv if r["how_noticed"] == h) for h in HOW}
    S["by_how_pct"] = {h: pct(v, len(nv)) for h, v in S["by_how"].items()}
    dv = [r for r in nv if r["how_noticed"] in ("DEVICE", "VISUAL")]
    S["device_or_visual"] = len(dv)
    S["device_or_visual_pct"] = pct(len(dv), len(nv))
    S["agent_success_Y"] = sum(1 for r in nv if r["agent_reported_success"] == "Y")
    S["agent_success_Y_pct"] = pct(S["agent_success_Y"], len(nv))
    S["agent_success_by_how"] = {h: [sum(1 for r in nv if r["how_noticed"] == h and r["agent_reported_success"] == "Y"),
                                     sum(1 for r in nv if r["how_noticed"] == h)] for h in HOW}
    S["checks_cited"] = sum(1 for r in nv if r["checks_cited"] == "1")
    S["checks_cited_pct"] = pct(S["checks_cited"], len(nv))
    S["success_Y_and_device_visual"] = sum(1 for r in dv if r["agent_reported_success"] == "Y")
    S["claim_or_checks"] = sum(1 for r in nv if r["agent_reported_success"] == "Y" or r["checks_cited"] == "1")
    S["claim_or_checks_pct"] = pct(S["claim_or_checks"], len(nv))
    S["wilson95"] = {"device_or_visual": wilson(len(dv), len(nv)),
                     "agent_success_Y": wilson(S["agent_success_Y"], len(nv)),
                     "claim_or_checks": wilson(S["claim_or_checks"], len(nv))}
    S["recurred_Y"] = sum(1 for r in nv if r["recurred"] == "Y")
    S["recurred_Y_pct"] = pct(S["recurred_Y"], len(nv))
    S["success_Y_by_category_of_n"] = {c: [sum(1 for r in nv if r["category"] == c and r["agent_reported_success"] == "Y"), S["by_category"][c]] for c in CATS}
    S["recurred_by_category"] = {c: sum(1 for r in nv if r["category"] == c and r["recurred"] == "Y") for c in CATS}
    S["by_project"] = dict(collections.Counter(r["project"] for r in nv).most_common())
    S["projects_nonvisual"] = sorted(S["by_project"])
    S["cat_x_how"] = {c: {h: sum(1 for r in nv if r["category"] == c and r["how_noticed"] == h) for h in HOW} for c in CATS}
    S["cat_success_Y"] = {c: sum(1 for r in nv if r["category"] == c and r["agent_reported_success"] == "Y") for c in CATS}
    groups = {"bugs_in_use": ["FUNC", "REG", "DEVL"], "follow_through_and_instructions": ["FOLLOW", "INST", "SCOPE", "BLOCK"],
              "understanding": ["INTENT", "FACT"], "truth_of_status": ["OVER", "STALE", "PROC"], "risk": ["SEC", "COST"]}
    S["groups"] = {g: sum(S["by_category"][c] for c in cs) for g, cs in groups.items()}
    S["groups_x_how"] = {g: {h: sum(S["cat_x_how"][c][h] for c in cs) for h in HOW} for g, cs in groups.items()}
    S["bugs_in_use_device_visual"] = sum(1 for r in nv if r["category"] in groups["bugs_in_use"] and r["how_noticed"] in ("DEVICE", "VISUAL"))
    S["non_bug_device_visual"] = sum(1 for r in nv if r["category"] not in groups["bugs_in_use"] and r["how_noticed"] in ("DEVICE", "VISUAL"))
    S["non_bug_total"] = sum(1 for r in nv if r["category"] not in groups["bugs_in_use"])

    # further Wilson intervals for every proportion quoted in the paper
    def w(k, n):
        return {"k": k, "n": n, "pct": pct(k, n), "ci95": wilson(k, n)}
    reg = [r for r in nv if r["category"] == "REG"]
    S["wilson95_more"] = {
        "device_only": w(S["by_how"]["DEVICE"], len(nv)),
        "read": w(S["by_how"]["READ"], len(nv)),
        "checks_cited": w(S["checks_cited"], len(nv)),
        "recurred": w(S["recurred_Y"], len(nv)),
        "bugs_in_use_share": w(S["groups"]["bugs_in_use"], len(nv)),
        "bugs_device_visual": w(S["bugs_in_use_device_visual"], S["groups"]["bugs_in_use"]),
        "non_bug_device_visual": w(S["non_bug_device_visual"], S["non_bug_total"]),
        "regression_after_claim": w(sum(1 for r in reg if r["agent_reported_success"] == "Y"), len(reg)),
        "bupples_share": w(S["by_project"].get("bupples", 0), len(nv)),
        "visual_share_all_rows": w(S["rows_visual"], len(inter)),
    }
    # sensitivity 1: weight rows by inverse sampling fraction of their stratum
    wt = {"A_prefilter": nA / st["A_prefilter"]["n"], "B_nonprefilter": nB / st["B_nonprefilter"]["n"]}
    def wpct(pred):
        den = sum(wt[r["stratum"]] for r in nv)
        return pct(sum(wt[r["stratum"]] for r in nv if pred(r)), den)
    S["stratum_weighted_pct"] = {
        "device_or_visual": wpct(lambda r: r["how_noticed"] in ("DEVICE", "VISUAL")),
        "agent_success_Y": wpct(lambda r: r["agent_reported_success"] == "Y"),
        "recurred": wpct(lambda r: r["recurred"] == "Y"),
        "read": wpct(lambda r: r["how_noticed"] == "READ"),
    }
    S["rows_nonvisual_by_stratum"] = dict(collections.Counter(r["stratum"] for r in nv))
    # sensitivity 2: message as the unit (any non-visual row in the message)
    by_msg = collections.defaultdict(list)
    for r in nv:
        by_msg[r["msg_id"]].append(r)
    nm = len(by_msg)
    S["message_level"] = {
        "device_or_visual": w(sum(1 for rs in by_msg.values() if any(r["how_noticed"] in ("DEVICE", "VISUAL") for r in rs)), nm),
        "agent_success_Y": w(sum(1 for rs in by_msg.values() if any(r["agent_reported_success"] == "Y" for r in rs)), nm),
        "recurred": w(sum(1 for rs in by_msg.values() if any(r["recurred"] == "Y" for r in rs)), nm),
    }

    # consistency
    def status(s):
        return "EXCL" if s == "EXCL" else ("NONE" if s == "NONE" else "INT")
    def nvset(s):
        return set(x for x in s.split(";") if x not in ("VIS", "NONE", "EXCL"))
    pairs_status = [(status(r["pass1"]), status(r["pass2"])) for r in cons]
    S["cons_n"] = len(cons)
    S["cons_status_agree"] = sum(1 for a_, b_ in pairs_status if a_ == b_)
    S["cons_status_kappa"] = kappa(pairs_status)
    both = [r for r in cons if status(r["pass1"]) == "INT" and status(r["pass2"]) == "INT"]
    S["cons_both_intervention"] = len(both)
    S["cons_category_exact"] = sum(1 for r in both if set(r["pass1"].split(";")) == set(r["pass2"].split(";")))
    S["cons_nonvisual_set_exact"] = sum(1 for r in both if nvset(r["pass1"]) == nvset(r["pass2"]))
    S["cons_disagreements"] = [r["msg_id"] + ": " + r["pass1"] + " -> " + r["pass2"] for r in cons if r["pass1"] != r["pass2"]]

    # chains
    S["chains"] = []
    kinds = {r["rule_id"]: r["rule_kind"] for r in rules}
    short = {"C1": "Small no-photo bubbles", "C2": "Profile scrolls into blank", "C3": "Currencies summed together",
             "C4": "Settle up lost direct view", "C5": "Web Google sign-in dead", "C6": "Native glass bar ignored",
             "C7": "Is the work running?", "C8": "Did you do all I asked?", "C9": "Product, not a demo",
             "C10": "Private repo + showcase", "C11": "Database cost surprise"}
    idxd = {r["msg_id"]: r["date"] for r in idx}
    fig_chain_rows = []
    for c in chains:
        k = kinds.get(c["rule_id"], "")
        c["rule_kind"] = k
        c["rule_match"] = c.get("rule_match", "")
        S["chains"].append({kk: c[kk] for kk in ("chain_id", "description", "category", "n_reports", "rule_id", "rule_date",
                                                 "reports_on_or_before_rule", "reports_after_rule", "reports_after_rule_day",
                                                 "success_claim_before_report", "rule_match")} | {"rule_kind": k})
        fig_chain_rows.append({"short": short[c["chain_id"]], "n_reports": c["n_reports"], "rule_date": c["rule_date"],
                               "dates": [idxd[m] for m in c["msg_ids"].split(";")]})
    S["chain_reports_total"] = sum(int(c["n_reports"]) for c in chains)
    S["chain_success_claims"] = sum(int(c["success_claim_before_report"] or 0) for c in chains)
    S["chains_by_match"] = {}
    for k in ("exact", "adjacent", "habit"):
        cs = [c for c in chains if c["rule_match"] == k]
        S["chains_by_match"][k] = {"n": len(cs), "ids": [c["chain_id"] for c in cs],
                                   "reports_before": sum(int(c["reports_on_or_before_rule"] or 0) for c in cs),
                                   "after_rule_day": sum(int(c["reports_after_rule_day"] or 0) for c in cs)}
    S["chains_no_rule"] = [c["chain_id"] for c in chains if not c["rule_id"]]

    # memory rules
    S["rules_n"] = len(rules)
    S["rules_by_origin"] = dict(collections.Counter(r["origin"] for r in rules))
    S["rules_visual"] = sum(1 for r in rules if r["visual"] == "1")
    S["rules_owner_nonvisual"] = sum(1 for r in rules if r["origin"] in ("owner", "mixed") and r["visual"] == "0")
    S["rules_with_why"] = sum(1 for r in rules if r["has_why"] == "1")
    S["rules_with_how_to_apply"] = sum(1 for r in rules if r["has_how_to_apply"] == "1")
    S["rules_by_kind"] = dict(collections.Counter(r["rule_kind"] for r in rules))

    json.dump(S, open(os.path.join(D, "stats.json"), "w"), indent=1)

    # tables.md
    L = ["# Tables (generated by scripts/analyse.py)", "", "## Non-visual interventions by category", "",
         "| Code | Category | n | % | caught by device/visual | agent had claimed success | recurred |", "|---|---|---|---|---|---|---|"]
    for c in CATS:
        n = S["by_category"][c]
        dvc = S["cat_x_how"][c]["DEVICE"] + S["cat_x_how"][c]["VISUAL"]
        L.append(f"| {c} | {CATS[c]} | {n} | {S['by_category_pct'][c]} | {dvc} | {S['cat_success_Y'][c]} | {S['recurred_by_category'][c]} |")
    L += ["", "## How noticed", "", "| How | n | % | agent had claimed success |", "|---|---|---|---|"]
    for h in HOW:
        y, n = S["agent_success_by_how"][h]
        L.append(f"| {HOW[h]} | {S['by_how'][h]} | {S['by_how_pct'][h]} | {y} of {n} |")
    L += ["", "## Chains", "", "| Chain | Reports | Rule | Rule kind | Reports after rule day |", "|---|---|---|---|---|"]
    for c in S["chains"]:
        L.append(f"| {c['chain_id']} {c['description']} | {c['n_reports']} | {c['rule_id'] or '-'} | {c['rule_kind'] or '-'} | {c['reports_after_rule_day'] or '-'} |")
    open(os.path.join(D, "tables.md"), "w").write("\n".join(L) + "\n")

    os.makedirs(a.figdir, exist_ok=True)
    fig_taxonomy(inter, os.path.join(a.figdir, "fig1-taxonomy.svg"))
    fig_claims(inter, os.path.join(a.figdir, "fig2-claimed-success.svg"))
    fig_chains(fig_chain_rows, os.path.join(a.figdir, "fig3-recurrence.svg"))
    fig_rules(rules, chains, os.path.join(a.figdir, "fig4-rules.svg"))
    print(json.dumps({k: S[k] for k in ("rows_nonvisual", "rows_visual", "device_or_visual_pct", "agent_success_Y_pct",
                                         "recurred_Y_pct", "cons_status_kappa")}))


if __name__ == "__main__":
    main()