#!/usr/bin/env python3
"""Extract the owner's own typed messages from Claude Code session transcripts,
each paired with the immediately preceding assistant text (read-only).

Inclusion rule (applied in this order):
  1. Only top-level session files (<projects>/<project>/<session>.jsonl).
     Subagent transcripts are skipped: their "user" turns are prompts written
     by another agent, not by the owner.
  2. Only records with type == "user", isSidechain false, isMeta not true, and
     origin.kind absent or "human" (drops task notifications, peer messages and
     subagent hand-backs).
  3. Only text blocks (tool results are never read). Text starting with a
     harness marker (<command-, <local-command, <system-reminder>,
     <task-notification>, [Image, [Request interrupted, "This session is being
     continued", "Another Claude session", <bash-, Caveat:) is dropped.
  4. Sessions passed with --exclude (the session that commissioned this study)
     are skipped.
  5. Messages longer than 4,000 characters are dropped (almost always pasted
     documents, logs or prompt packs, not a reaction to the agent).
  6. Only messages with a timestamp at or before --until (default
     2026-11-01T17:01:00Z, the snapshot used in the paper) are kept, so the
     corpus does not grow while other sessions are still running.
  7. Exact duplicates (same normalised text) are kept once, earliest first;
     resumed and forked sessions replay the same turns.

For context only, the last assistant *text* block seen before each kept
message in the same file is stored (truncated to its final 900 characters).

Outputs:
  --out-private  JSONL with message text and preceding-assistant text. This
                 stays OUTSIDE the public artifact: it can contain names, links
                 and other people's data.
  --out-public   CSV with: msg_id, date, project_hint, chars, has_image,
                 prefilter (0/1), prefilter_hits, text_sha256_12. No text.
"""
import argparse, csv, hashlib, json, os, re

SKIP_PREFIX = ("<command-", "<local-command", "<system-reminder", "<task-notification",
               "[Image", "[Request interrupted", "This session is being continued",
               "Another Claude session", "<bash-", "Caveat:", "<user-prompt-submit-hook")

# Intervention pre-filter: words that mark a correction, rejection, redirect or
# catch. Deliberately broad (recall over precision); manual coding decides.
PREFILTER = re.compile(
    r"\b(still|again|not working|doesn'?t work|does not work|didn'?t work|isn'?t working|broken|broke|"
    r"bug|crash(es|ed|ing)?|error|wrong|why (is|are|does|did|do|isn'?t|aren'?t|u|you)|"
    r"you said|u said|u told|you told|i told|i said|i asked|i already|already (told|said|asked)|"
    r"remember|forgot|forget|reminder|make sure|all devices|every device|other devices|on my (phone|iphone|device|android)|"
    r"on (the )?device|ipad|android|simulator|screenshot|stale|old|outdated|"
    r"live|demo|fake|lie|lying|honest|actually|real(ly)?|invented|made up|"
    r"cost|bill|charged|pay(ing)?|price|account|repo|push(ed)?|commit(ted)?|private|public|"
    r"secure|security|leak|privacy|exposed|revert|undo|came back|back again|regress\w*|"
    r"missing|not (done|finished|complete)|didn'?t (do|finish|fix|change|update)|haven'?t|hasn'?t|"
    r"instead|not what|that'?s not|i (don'?t|do not) want|i meant|i mean|no[, ]|stop|"
    r"fix|issue|problem|glitch|lag|slow|stuck|freez\w*|blank|overlap\w*|cut off|clipp\w*|"
    r"test(s|ed)?|verify|check)\b", re.I)


def texts(rec):
    c = rec.get("message", {}).get("content")
    if isinstance(c, str):
        return [c]
    if isinstance(c, list):
        return [b.get("text", "") for b in c if isinstance(b, dict) and b.get("type") == "text"]
    return []


def load_private_aliases():
    """Optional extra (keyword, project) aliases from a private, unshipped file
    named by the ALIAS_FILE environment variable: one 'keyword<TAB>project' per
    line. Used for projects whose keyword is a person's name, which must never
    appear in published files."""
    path = os.environ.get("ALIAS_FILE")
    out = []
    if path and os.path.exists(path):
        for line in open(path):
            if "\t" in line:
                k, v = line.rstrip("\n").split("\t", 1)
                out.append((k.lower(), v))
    return out


PRIVATE_ALIASES = load_private_aliases()


def project_hint(cwd, text, prev):
    s = ((cwd or "") + " " + text + " " + (prev or "")[-400:]).lower()
    for key, name in [("bupples", "bupples"), ("adelante", "adelante"), ("wayclub", "wayclub"),
                      ("joinlet", "hackfam"), ("hackfam", "hackfam"), ("hackathon", "hackfam"),
                      ("planket", "planket"), ("photoshoot", "photoshoot"), ("photo booth", "photoshoot"),
                      ("careerapp", "careeros"), ("careeros", "careeros"), ("nutri", "nutrigo"),
                      ("himalaya", "himalaya"), ("tajweed", "tajweed"), ("onplan", "onplan"),
                      ("gitspark", "codeatlas"), ("codeatlas", "codeatlas"),
                      ("spell", "spellquest"), ("yeegz.github.io", "portfolio"), ("portfolio", "portfolio"),
                      ("experts", "experts-marine"), ("marine", "experts-marine")] + PRIVATE_ALIASES:
        if key in s:
            return name
    return "unknown"


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--root", default=os.path.expanduser("~/.claude/projects"))
    ap.add_argument("--exclude", action="append", default=[])
    ap.add_argument("--out-private", required=True)
    ap.add_argument("--out-public", required=True)
    ap.add_argument("--until", default="2026-11-01T17:01:00Z")
    a = ap.parse_args()
    rows, seen = [], set()
    st = {"sessions": 0, "human_msgs": 0, "too_long": 0, "dupes": 0, "kept": 0, "prefilter": 0}
    for proj in sorted(os.listdir(a.root)):
        pdir = os.path.join(a.root, proj)
        if not os.path.isdir(pdir):
            continue
        for fn in sorted(os.listdir(pdir)):
            if not fn.endswith(".jsonl"):
                continue
            sid = fn[:-6]
            if any(sid.startswith(x) for x in a.exclude):
                continue
            st["sessions"] += 1
            last_asst = ""
            with open(os.path.join(pdir, fn), errors="replace") as fh:
                for line in fh:
                    if '"assistant"' not in line and '"user"' not in line:
                        continue
                    try:
                        r = json.loads(line)
                    except Exception:
                        continue
                    if r.get("isSidechain"):
                        continue
                    if r.get("type") == "assistant":
                        tt = [t for t in texts(r) if t.strip()]
                        if tt:
                            last_asst = tt[-1][-900:]
                        continue
                    if r.get("type") != "user" or r.get("isMeta"):
                        continue
                    if (r.get("timestamp") or "") > a.until:
                        continue
                    o = r.get("origin")
                    if isinstance(o, dict) and o.get("kind") not in (None, "human"):
                        continue
                    c = r.get("message", {}).get("content")
                    has_img = isinstance(c, list) and any(isinstance(b, dict) and b.get("type") == "image" for b in c)
                    for t in texts(r):
                        t = t.strip()
                        if not t or t.startswith(SKIP_PREFIX):
                            continue
                        st["human_msgs"] += 1
                        if len(t) > 4000:
                            st["too_long"] += 1
                            continue
                        norm = re.sub(r"\s+", " ", t.lower())
                        if norm in seen:
                            st["dupes"] += 1
                            continue
                        seen.add(norm)
                        hits = sorted({m.group(0).lower() for m in PREFILTER.finditer(t)})
                        rows.append({"session": sid, "ts": r.get("timestamp", ""), "cwd": r.get("cwd", ""),
                                     "project_hint": project_hint(r.get("cwd", ""), t, last_asst),
                                     "text": t, "prev": last_asst, "hits": hits, "prefilter": int(bool(hits)),
                                     "has_image": bool(has_img or re.search(r"\.(png|jpe?g|heic)\b|/uploads/", t, re.I))})
    rows.sort(key=lambda x: x["ts"])
    for i, x in enumerate(rows, 1):
        x["msg_id"] = f"M{i:04d}"
    st["kept"] = len(rows)
    st["prefilter"] = sum(x["prefilter"] for x in rows)
    with open(a.out_private, "w") as f:
        for x in rows:
            f.write(json.dumps(x, ensure_ascii=False) + "\n")
    with open(a.out_public, "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["msg_id", "date", "project_hint", "chars", "has_image", "prefilter", "prefilter_hits", "text_sha256_12"])
        for x in rows:
            w.writerow([x["msg_id"], x["ts"][:10], x["project_hint"], len(x["text"]), int(x["has_image"]),
                        x["prefilter"], ";".join(x["hits"][:12]), hashlib.sha256(x["text"].encode()).hexdigest()[:12]])
    print(json.dumps(st))


if __name__ == "__main__":
    main()