#!/usr/bin/env python3 """Tables, figures and the second-pass consistency check for "Green but Wrong". Reads data/defects.csv (from build_dataset.py) and data/census.csv (from mine_commits.py). Writes data/tables.md, data/stats.json, data/consistency_check.csv and SVG figures to --figdir. python3 analyse.py --figdir ../../../../images/research/green-but-wrong """ import argparse import collections import csv import json import os import random import re import matplotlib matplotlib.use("svg") import matplotlib.pyplot as plt # noqa: E402 plt.rcParams.update({"font.family": "sans-serif", "font.sans-serif": ["Helvetica", "Arial", "DejaVu Sans"], "font.size": 12, "svg.fonttype": "none", "axes.spines.top": False, "axes.spines.right": False, "axes.edgecolor": "#555", "figure.facecolor": "white", "axes.facecolor": "white"}) CAT = {"AUTHZ": "Authorisation and tenant isolation", "PRIV": "Privacy and data exposure", "LOGIC": "Business-rule and workflow logic", "CONC": "Concurrency, locking, idempotency", "STALE": "Stale state and caching", "UI": "UI state, visibility, layout", "ABUSE": "Abuse, injection, resource limits", "PERF": "Performance and resource lifecycle", "CONTRACT": "Cross-component contract mismatch", "TEST": "Test adequacy", "MONEY": "Money arithmetic and ledger", "PLAT": "Platform-specific runtime", "ERR": "Error handling and honest states", "UNITS": "Units, frames, precision", "BUILD": "Build, configuration, operations", "DOCS": "Overclaiming or stale docs"} DET = {"REVIEW": "Adversarial or audit review", "SELF": "Implementer's own verification", "OWNER": "Owner use or device testing", "GATE": "Automated gate (CI, test, axe)", "UNK": "Not recorded"} RED = {"RV": "Revert/mutation verified RED", "RF": "Observed failing before fix", "RM": "Reproduced manually, no committed test", "RO": "Test committed first (inferred)", "T": "Test added, RED not recorded", "U": "No test recorded", "N": "No test (stated or evident)", "X": "Not fixed (open)"} RED_ORDER = ["RV", "RF", "RO", "RM", "T", "U", "N", "X"] RED_COL = {"RV": "#1b5e20", "RF": "#43a047", "RO": "#a5d6a7", "RM": "#81d4fa", "T": "#ffcc80", "U": "#e0e0e0", "N": "#bdbdbd", "X": "#757575"} SEV_COL = {"H": "#b71c1c", "M": "#ef8a62", "L": "#fddbc7"} PRODUCTS = ["Bupples", "CareerOS", "WayClub", "Photoshoot", "CodeAtlas", "Adelante", "Planket", "HACKFAM"] # Second-pass coder: deterministic keyword rules over the short title only, # written after the first coding and applied blind to the first codes. RULES = [ ("TEST", r"\btest|suite|proof|mutation|verifier walked|coverage set"), ("DOCS", r"readme|roadmap|docs? (overstated|listed)|overstated"), ("MONEY", r"money|payment|balance|receipt total|ledger|price|currency|formula ignoring"), ("PRIV", r"leak|readable by|exposed?|disclos|email address|pii|listable|export omitted|retract|orphan|allowbackup|left stored|stop queued"), ("AUTHZ", r"forg|authoris|takeover|hijack|cross-tenant|could move a claim|evict|rls|publicly callable|stolen session|public execute|tenant per request|verification not gating"), ("ABUSE", r"limit|redos|backtracking|injection|denial|ssrf|passwords|validation|pre-plant|bodies read"), ("CONC", r"concurren|race|lock|twice|double|fork|interleav|stranded|inode|duplicate retries|cancel flash|two achievements"), ("UNITS", r"iso|y-down|mirror|seconds|whole seconds|date\.now|idempotent|bleach"), ("PLAT", r"windows|ios|android|safari|webgl|d3d11|requestvideoframe|gpu|native|scenedelegate"), ("PERF", r"decode|listeners?|per-vsync|re-ran|sorted|eager|139 s|three times|rebuild"), ("STALE", r"stale|cached|never refreshed|drift|re-scope|keyed per account|re-armed"), ("CONTRACT", r"allowlist|disjoint|sent no body|never collected|request constructor|bridged|subselect"), ("ERR", r"silent|claimed success|empty state|honest|unclaimed frame|lost input|quit waited|edge cases|raw postgres|redelivery|zero count|scan mid-flight"), ("BUILD", r"build|fingerprint|cron|typecheck|kdebugmode|nul bytes|seed"), ("UI", r"visib|unstyled|layout|dialog|tick|breadcrumb|hydration|ellips|colour|contrast|drag|swipe|screen|toolbar|blur|ink|img|eyes|paint"), ("LOGIC", r".*"), ] def recode(title): t = title.lower() for cat, pat in RULES: if re.search(pat, t): return cat return "LOGIC" def kappa(a, b): n = len(a) po = sum(x == y for x, y in zip(a, b)) / n ca, cb = collections.Counter(a), collections.Counter(b) pe = sum(ca[k] * cb[k] for k in set(a) | set(b)) / (n * n) return po, (po - pe) / (1 - pe) if pe < 1 else 1.0 def md_table(header, rows): out = ["| " + " | ".join(header) + " |", "|" + "---|" * len(header)] out += ["| " + " | ".join(str(c) for c in r) + " |" for r in rows] return "\n".join(out) def save(fig, figdir, name): path = os.path.join(figdir, name) fig.savefig(path, format="svg", bbox_inches="tight") plt.close(fig) # strip matplotlib's metadata date so output is reproducible s = open(path).read() s = re.sub(r".*?", "", s) open(path, "w").write(s) def main(): ap = argparse.ArgumentParser() here = os.path.dirname(os.path.abspath(__file__)) ap.add_argument("--data", default=os.path.join(here, "..", "data")) ap.add_argument("--figdir", required=True) a = ap.parse_args() os.makedirs(a.figdir, exist_ok=True) D = list(csv.DictReader(open(os.path.join(a.data, "defects.csv")))) census = list(csv.DictReader(open(os.path.join(a.data, "census.csv")))) n = len(D) C = collections.Counter st = {"n": n} st["by_product"] = C(d["product"] for d in D) st["by_path"] = C(d["path"] for d in D) st["by_category"] = C(d["category"] for d in D) st["by_severity"] = C(d["severity"] for d in D) st["by_detection"] = C(d["detection"] for d in D) st["by_red"] = C(d["red"] for d in D) st["by_author"] = C(d["author"] for d in D) st["by_tier"] = C(d["attr_tier"] for d in D) st["green_over"] = C(d["green_over"] for d in D) st["green_over_cats"] = C(d["category"] for d in D if d["green_over"] == "Y") st["green_over_detection"] = C(d["detection"] for d in D if d["green_over"] == "Y") fixed = [d for d in D if d["red"] != "X"] st["fixed"] = len(fixed) strong = [d for d in fixed if d["red"] in ("RV", "RF")] st["red_strong"] = len(strong) st["red_any_evidence"] = len([d for d in fixed if d["red"] in ("RV", "RF", "RO", "RM")]) st["high"] = [d["id"] for d in D if d["severity"] == "H"] hi = [d for d in fixed if d["severity"] == "H"] st["high_fixed"] = len(hi) st["high_red_strong"] = len([d for d in hi if d["red"] in ("RV", "RF")]) st["red_by_detection"] = {k: dict(C(d["red"] for d in D if d["detection"] == k)) for k in DET} st["red_by_product"] = {p: dict(C(d["red"] for d in D if d["product"] == p)) for p in PRODUCTS} st["sev_by_detection"] = {k: dict(C(d["severity"] for d in D if d["detection"] == k)) for k in DET} st["cat_by_detection"] = {c: dict(C(d["detection"] for d in D if d["category"] == c)) for c in CAT} st["cat_by_sev"] = {c: dict(C(d["severity"] for d in D if d["category"] == c)) for c in CAT} st["author_by_tier"] = {t: dict(C(d["author"] for d in D if d["attr_tier"] == t)) for t in "ABCD"} st["security"] = len([d for d in D if d["category"] in ("AUTHZ", "PRIV", "ABUSE")]) st["security_high"] = len([d for d in D if d["category"] in ("AUTHZ", "PRIV", "ABUSE") and d["severity"] == "H"]) rev = [d for d in D if d["detection"] == "REVIEW"] st["review_security_share"] = round(sum(d["category"] in ("AUTHZ", "PRIV", "ABUSE") for d in rev) / len(rev), 3) own = [d for d in D if d["detection"] == "OWNER"] st["owner_ui_plat_share"] = round(sum(d["category"] in ("UI", "PLAT", "STALE", "ERR") for d in own) / len(own), 3) st["census"] = census st["census_total_commits"] = sum(int(c["commits"]) for c in census) st["census_claude"] = sum(int(c["kind_claude"]) + int(c["kind_codex+claude"]) for c in census) st["census_codex"] = sum(int(c["kind_codex"]) for c in census) st["census_unknown"] = sum(int(c["kind_unknown"]) for c in census) st["census_fix_candidates"] = sum(int(c["fix_candidates"]) for c in census) # consistency check: seeded sample of 40, second-pass coder vs first coding rng = random.Random(20261102) sample = rng.sample(D, 40) rows = [] for d in sample: rc = recode(d["title"]) rows.append({"id": d["id"], "title": d["title"], "first": d["category"], "second": rc, "agree": int(rc == d["category"])}) with open(os.path.join(a.data, "consistency_check.csv"), "w", newline="") as f: w = csv.DictWriter(f, fieldnames=list(rows[0])) w.writeheader() w.writerows(rows) po, k = kappa([r["first"] for r in rows], [r["second"] for r in rows]) st["consistency"] = {"n": 40, "agreement": round(po, 3), "kappa": round(k, 3), "disagreements": [(r["id"], r["first"], r["second"]) for r in rows if not r["agree"]]} # full-dataset agreement as a robustness figure pa, ka = kappa([d["category"] for d in D], [recode(d["title"]) for d in D]) st["consistency_full"] = {"agreement": round(pa, 3), "kappa": round(ka, 3)} json.dump(st, open(os.path.join(a.data, "stats.json"), "w"), indent=1, default=dict) # ---------- tables.md ---------- T = [] T.append("## Defects by product and inclusion path\n") T.append(md_table(["Product", "D (docs)", "C (commit screen)", "S (sample)", "Total"], [[p] + [len([d for d in D if d["product"] == p and d["path"] == x]) for x in "DCS"] + [st["by_product"][p]] for p in PRODUCTS])) T.append("\n## Category by severity\n") T.append(md_table(["Category", "H", "M", "L", "Total"], [[CAT[c]] + [st["cat_by_sev"][c].get(s, 0) for s in "HML"] + [st["by_category"][c]] for c in sorted(CAT, key=lambda c: -st["by_category"][c])])) T.append("\n## Detection by severity\n") T.append(md_table(["Detection", "H", "M", "L", "Total"], [[DET[k]] + [st["sev_by_detection"][k].get(s, 0) for s in "HML"] + [st["by_detection"][k]] for k in DET])) T.append("\n## Red-proof level by detection\n") T.append(md_table(["Detection"] + RED_ORDER, [[DET[k]] + [st["red_by_detection"][k].get(r, 0) for r in RED_ORDER] for k in DET])) T.append("\n## Red-proof level by product\n") T.append(md_table(["Product"] + RED_ORDER, [[p] + [st["red_by_product"][p].get(r, 0) for r in RED_ORDER] for p in PRODUCTS])) T.append("\n## Category by detection\n") T.append(md_table(["Category"] + list(DET), [[c] + [st["cat_by_detection"][c].get(k, 0) for k in DET] for c in CAT])) T.append("\n## Attributed author by tier\n") T.append(md_table(["Tier", "claude", "codex", "agent", "unknown"], [[t] + [st["author_by_tier"][t].get(x, 0) for x in ("claude", "codex", "agent", "unknown")] for t in "ABCD"])) open(os.path.join(a.data, "tables.md"), "w").write("\n".join(T) + "\n") # ---------- Figure 1 (hero): categories, stacked by severity ---------- cats = sorted(CAT, key=lambda c: st["by_category"][c]) fig, ax = plt.subplots(figsize=(7.2, 5.6)) left = [0] * len(cats) for s, lab in (("H", "High"), ("M", "Medium"), ("L", "Low")): vals = [st["cat_by_sev"][c].get(s, 0) for c in cats] ax.barh([CAT[c] for c in cats], vals, left=left, color=SEV_COL[s], label=lab, edgecolor="white", linewidth=0.6) left = [l + v for l, v in zip(left, vals)] for i, c in enumerate(cats): ax.text(left[i] + 0.4, i, str(st["by_category"][c]), va="center", fontsize=11, color="#222") ax.set_xlabel(f"Defects (n = {n}, eight products)") ax.legend(title="Severity", frameon=False, loc="lower right") ax.set_title("What the agents got wrong: 183 defects by kind and severity", loc="left", fontsize=12) save(fig, a.figdir, "fig1-categories.svg") # ---------- Figure 2: proof ladder by detection ---------- dets = [k for k in DET] fig, ax = plt.subplots(figsize=(7.2, 4.6)) left = [0] * len(dets) for r in RED_ORDER: vals = [st["red_by_detection"][k].get(r, 0) for k in dets] ax.barh([DET[k] for k in dets], vals, left=left, color=RED_COL[r], label=RED[r], edgecolor="white", linewidth=0.6) left = [l + v for l, v in zip(left, vals)] ax.invert_yaxis() ax.set_xlabel("Defects") ax.legend(frameon=False, fontsize=11, loc="upper center", bbox_to_anchor=(0.35, -0.2), ncol=2) ax.set_title("How each defect was found, and how strongly its fix was proved", loc="left", fontsize=12) save(fig, a.figdir, "fig2-proof-by-detection.svg") # ---------- Figure 3: detection x category dot matrix ---------- cats3 = sorted(CAT, key=lambda c: -st["by_category"][c]) fig, ax = plt.subplots(figsize=(7.2, 5.4)) for yi, c in enumerate(cats3): for xi, k in enumerate(dets): v = st["cat_by_detection"][c].get(k, 0) if v: ax.scatter(xi, yi, s=60 + v * 34, color="#2b6cb0" if v >= 3 else "#bee3f8") ax.text(xi, yi, str(v), ha="center", va="center", fontsize=10, color="white" if v >= 3 else "#1a365d") ax.set_xticks(range(len(dets))) ax.set_xticklabels(["Review", "Self-\nverification", "Owner /\ndevice", "Automated\ngate", "Not\nrecorded"]) ax.set_yticks(range(len(cats3))) ax.set_yticklabels([CAT[c] for c in cats3]) ax.invert_yaxis() ax.set_xlim(-0.6, len(dets) - 0.4) ax.grid(False) ax.set_title("Which practice caught which kind of defect", loc="left", fontsize=12) save(fig, a.figdir, "fig3-detection-matrix.svg") # ---------- Figure 4: green over a defect ---------- gc = st["green_over_cats"] cats4 = sorted(gc, key=lambda c: gc[c]) fig, ax = plt.subplots(figsize=(7.2, 3.4)) ax.barh([CAT[c] for c in cats4], [gc[c] for c in cats4], color="#2e7d32") for i, c in enumerate(cats4): ax.text(gc[c] + 0.1, i, str(gc[c]), va="center") ax.set_xlabel(f"Defects with a documented passing test or claim over them ({sum(gc.values())} of {n})") ax.set_title("Green but wrong: where a passing check covered the defect", loc="left", fontsize=12) save(fig, a.figdir, "fig4-green-over.svg") for k in ("n", "fixed", "red_strong", "red_any_evidence", "high_fixed", "high_red_strong", "security", "security_high", "review_security_share", "owner_ui_plat_share", "consistency", "consistency_full", "census_total_commits", "census_claude", "census_codex", "census_unknown", "census_fix_candidates"): print(k, st[k]) for k in ("by_product", "by_category", "by_severity", "by_detection", "by_red", "by_author", "by_tier", "green_over", "green_over_cats", "green_over_detection", "by_path"): print(k, dict(st[k])) print("red_by_detection", st["red_by_detection"]) print("sev_by_detection", st["sev_by_detection"]) print("author_by_tier", st["author_by_tier"]) if __name__ == "__main__": main()