"""Gamma-wall history backfill study — pre-reg studies/2026-09-09-gamma-backfill-prereg.md (frozen before pull)."""
import gzip, json, glob, os, numpy as np, pandas as pd, warnings
from scipy import stats
warnings.filterwarnings("ignore")
src=open("crown_policies.py").read().split("\ndf=run()")[0]; cp={}; exec(src, cp)
gp={}; exec(open("gamma_policy.py").read().split("\nsnaps={}")[0], gp)   # reuse compute_gamma + helpers verbatim
FP=cp["FP"]; d=cp["load_nq"](); QUAD=cp["QUAD"]; compute_gamma=gp["compute_gamma"]; f=gp["f"]
HIST=f"{FP}/gex/history"
# ---------- 1. load history (EOD book of date D) ----------
hist={}
for p in sorted(glob.glob(f"{HIST}/*.json.gz")):
    j=json.load(gzip.open(p)); dt=pd.Timestamp(j["date"]).date()
    q=compute_gamma(j["qqq"], j["qqq_close"]); n=None
    if j.get("ndx"):
        n=compute_gamma(j["ndx"], j.get("ndx_close") or j["qqq_close"]*41.0)   # NDX/QQQ ratio 40.9–41.3 over 2023–26; agreement stat only
    if q: hist[dt]=dict(q=q,n=n,close=j["qqq_close"],raw_q=j["qqq"],raw_n=j.get("ndx") or [])
hd=sorted(hist); print("history days:",len(hd),hd[0],"→",hd[-1])
sessions=sorted(d["sday"].unique()); prev_of={sessions[i]:sessions[i-1] for i in range(1,len(sessions))}
nq_0910={}; nq_close={}
for sday,g in d.groupby("sday"):
    pre=g[g["min"]<=550]; rth=g[(g["min"]>=570)&(g["min"]<960)]
    if len(pre): nq_0910[sday]=float(pre.iloc[-1]["close"])
    if len(rth): nq_close[sday]=float(rth.iloc[-1]["close"])
def proxy_for(sday):
    """Amendment 2: S−1 close book evaluated at the pre-open spot (S−1 QQQ close × NQ 09:10 / NQ prior 16:00)."""
    p=prev_of.get(sday); h=hist.get(p) if p else None
    if not h or sday not in nq_0910 or p not in nq_close: return None
    spot=h["close"]*nq_0910[sday]/nq_close[p]
    q=compute_gamma(h["raw_q"], spot); n=compute_gamma(h["raw_n"], spot*41.0) if h["raw_n"] else None
    return dict(q=q,n=n,close=h["close"],spot=spot) if q else None
# ---------- 2. proxy validation on the live 09:10 overlap ----------
val=[]
for path in sorted(glob.glob(f"{FP}/gex/2026/QQQ_*T0910.json.gz")):
    j=json.load(gzip.open(path)); dd=j["data"]; st=dd.get("stock_state") or {}; spot=f(st.get("close")) or f(st.get("prev_close"))
    live=compute_gamma(dd.get("greek_exposure_strike") or [], spot) if spot else None
    sday=pd.Timestamp(j["et_date"]).date(); px=proxy_for(sday)
    if not (live and px): continue
    q=px["q"]; val.append(dict(date=sday, cw_ok=abs(q["call"]-live["call"])/live["call"]<=0.005, pw_ok=abs(q["put"]-live["put"])/live["put"]<=0.005, reg_ok=q["regime"]==live["regime"], live_cw=live["call"],px_cw=q["call"],live_pw=live["put"],px_pw=q["put"],live_reg=live["regime"],px_reg=q["regime"]))
V=pd.DataFrame(val)
if len(V):
    wall_agree=(V.cw_ok.mean()+V.pw_ok.mean())/2
    print(f"PROXY VALIDATION n={len(V)}: call wall within 0.5% {V.cw_ok.mean()*100:.0f}%, put wall {V.pw_ok.mean()*100:.0f}%, regime {V.reg_ok.mean()*100:.0f}%")
else: wall_agree=float("nan")
FIT = len(V)>=10 and wall_agree>=0.70
# ---------- 3. per-session walk ----------
rows=[]; fade=[]; hist5=[]   # hist5: rolling list of median 5m TR per session (for the Part D range proxy)
for sday,g in d.groupby("sday"):
    rth=g[(g["min"]>=570)&(g["min"]<960)].reset_index(drop=True)
    if len(rth)<300 or sday in QUAD: continue
    f5=cp["five_min"](rth)
    tr5=np.median([max(f5.h[k]-f5.l[k], abs(f5.h[k]-f5.c[k-1]), abs(f5.l[k]-f5.c[k-1])) for k in range(1,len(f5))])
    px=proxy_for(sday)
    if px and len(hist5)>=5:
        gm=px["q"]; o=rth.iloc[0]["open"]; k=o/gm["spot"]; cw=gm["call"]*k; pw=gm["put"]*k; flip=gm["flip"]*k
        c=f5.c.values
        def fd(i):
            if i==0: return 0
            if c[i-1]>cw and c[i]<=cw: return -1
            if c[i-1]<pw and c[i]>=pw: return 1
            return 0
        brk=lambda i: (1 if c[i]>cw else (-1 if c[i]<pw else 0))
        sigs={"P8_longg_fade":cp["first_signal"](f5,fd) if gm["regime"]=="long" else (None,0),
              "P9_shortg_break":cp["first_signal"](f5,brk) if gm["regime"]=="short" else (None,0)}
        close=rth.iloc[-1]["close"]; caged=pw<=close<=cw
        ndx_agree = (px["n"] is not None) and (px["n"]["regime"]==gm["regime"])
        for pol,(i5,side) in sigs.items():
            elig=(pol=="P8_longg_fade" and gm["regime"]=="long") or (pol=="P9_shortg_break" and gm["regime"]=="short")
            w=cp["walk"](rth,int(f5.endmin[i5]),side,f5,i5) if (i5 is not None and side) else None
            rows.append(dict(date=sday,pol=pol,elig=elig,trig=w is not None,side=side,regime=gm["regime"],tilt=round(gm["tilt"],3),cw=round(cw),pw=round(pw),caged=caged,ndx_agree=ndx_agree,**(w or {})))
        rows.append(dict(date=sday,pol="P10_cage",elig=True,trig=False,side=0,regime=gm["regime"],tilt=round(gm["tilt"],3),cw=round(cw),pw=round(pw),caged=caged,ndx_agree=ndx_agree))
        # ---- Part D first-touch limit fade ----
        exp5=float(np.median(hist5[-5:])); sD=max(12.0, 0.5*exp5)
        for wall,side,variant in ((cw,-1,"D"),(pw,1,"D"),(cw,-1,"D2"),(pw,1,"D2")):
            if (side<0 and wall<=o) or (side>0 and wall>=o): continue   # wrong side of the open
            touched=None
            for idx,b in rth.iterrows():
                hit = b["high"]>=wall if side<0 else b["low"]<=wall
                if hit: touched=idx; break
            if touched is not None and variant=="D2":
                i5=int(rth.iloc[touched]["min"]//5*5); i5=int(np.searchsorted(f5.m.values, i5)); a=cp["atr3"](f5,i5)
                s=max(12.0, 2.0*a) if not np.isnan(a) else max(12.0, 2.0*exp5/1.0)
            else: s=sD
            rec=dict(date=sday,variant=variant,side="call" if side<0 else "put",regime=gm["regime"],tilt=round(gm["tilt"],3),wall=round(wall,2),stop_pts=round(s,2),touched=touched is not None)
            if touched is not None:
                b=rth.iloc[touched]; fill = wall   # limit at the wall fills at the wall (gap-through fills at the open would only be better; keep conservative)
                stop = wall+s if side<0 else wall-s; tgt = wall-s if side<0 else wall+s
                res="close"; r=None; ex=None
                for idx2 in range(touched,len(rth)):
                    bb=rth.iloc[idx2]
                    hs = bb["high"]>=stop if side<0 else bb["low"]<=stop
                    ht = (bb["low"]<=tgt-0.25 if side<0 else bb["high"]>=tgt+0.25) and idx2>touched   # touch bar: the extreme may predate the fill → no target credit (look-ahead guard)
                    if hs: res="loss"; pts=side*(stop-fill); ex=int(bb["min"]); break
                    if ht: res="win"; pts=side*(tgt-fill); ex=int(bb["min"]); break
                else: pts=side*(rth.iloc[-1]["close"]-fill); ex=959
                rec.update(res=res,pts=round(pts,2),r=round(pts/s,3),touch_min=int(b["min"]),exit=ex,first_half_hour=int(b["min"])<600)
            fade.append(rec)
    hist5.append(tr5)
df=pd.DataFrame(rows); F=pd.DataFrame(fade); os.makedirs("data",exist_ok=True); df.to_parquet("data/gamma_backfill.parquet"); F.to_parquet("data/gamma_backfill_fade.parquet")
# ---------- 4. report ----------
def pval(x): 
    x=np.asarray(x,float); return stats.ttest_1samp(x,0,alternative="greater").pvalue if len(x)>2 else float("nan")
mid=pd.Timestamp("2025-03-15").date()
L=[f"# Gamma-wall history backfill — results · {pd.Timestamp.now():%Y-%m-%d %H:%M}\n",
   f"Pre-registration: `2026-09-09-gamma-backfill-prereg.md` (frozen before the pull). History days {len(hd)} ({hd[0]} → {hd[-1]}); sessions evaluated {df.date.nunique()} (quad-witching excluded). Regimes: "+", ".join(f"{k} {v}" for k,v in df[df.pol=='P10_cage'].regime.value_counts().items())+f". NDX regime agreement with QQQ: {df[df.pol=='P10_cage'].ndx_agree.mean()*100:.0f}%.\n",
   "## Proxy validation (S−1 close book vs live 09:10 book)"]
if len(V): L.append(f"n={len(V)} overlap sessions: call wall within 0.5% **{V.cw_ok.mean()*100:.0f}%**, put wall **{V.pw_ok.mean()*100:.0f}%**, regime agreement **{V.reg_ok.mean()*100:.0f}%** → proxy **{'FIT' if FIT else 'UNFIT (threshold 70%)'}**.\n")
else: L.append("no overlap sessions found → proxy UNFIT.\n")
if not FIT: L.append("**No policy verdict is issued (pre-registered stop).** Figures below are descriptive only.\n")
L.append("## Part C — wall policies")
for pol,g in df.groupby("pol"):
    if pol=="P10_cage":
        L.append(f"- **P10 cage held**: {g.caged.mean()*100:.0f}% of {len(g)} sessions; by regime "+", ".join(f"{k} {v*100:.0f}%" for k,v in g.groupby('regime').caged.mean().items())+"."); continue
    e=g[g.elig]; t=g[g.trig]
    if len(t)<20: L.append(f"- **{pol}**: eligible {len(e)}, triggers {len(t)} — n too small."); continue
    h1=t[t.date<mid]; h2=t[t.date>=mid]; p=pval(t.r)
    ok = (t.r.mean()>0) and (p<0.025) and (h1.r.mean()>0) and (h2.r.mean()>0)
    L.append(f"- **{pol}**: eligible {len(e)}, triggers {len(t)}, W/L/T {(t.res=='win').sum()}/{(t.res=='loss').sum()}/{(t.res=='timeout').sum()}, R/trigger **{t.r.mean():+.3f}** (p={p:.3f}), halves {h1.r.mean():+.3f} (n={len(h1)}) / {h2.r.mean():+.3f} (n={len(h2)}), $ at 1 NQ {t.pts.sum()*20:+,.0f} → **{('VALIDATED' if ok else 'REFUTED') if FIT else 'NO VERDICT (proxy unfit)'}**.")
for variant,label in (("D","Part D — first-touch wall fade (limit at the wall, stop max(12, 0.5×exp 5m range), 1R)"),("D2","Part D2 — same fade, doctrine stop max(12, 2×ATR3 5m), 1R (single pre-declared variant)")):
  L.append(f"\n## {label}")
  FV=F[F.variant==variant] if len(F) else F
  if len(FV):
    T=FV[FV.touched]
    L.append(f"Walls on the correct side of the open: {len(FV)}; touched {len(T)} ({len(T)/max(1,len(FV))*100:.0f}%).")
    if len(T)>=20:
        hit=(T.res=='win').mean(); h1=T[T.date<mid]; h2=T[T.date>=mid]
        byreg=T.groupby('regime').apply(lambda x:(x.res=='win').mean()); lg=byreg.get('long',np.nan); sg=byreg.get('short',np.nan)
        ok = hit>=0.57 and (h1.res=='win').mean()>=0.57 and (h2.res=='win').mean()>=0.57 and (np.isnan(lg) or np.isnan(sg) or lg>=sg)
        L.append(f"- Hit rate at 1R **{hit*100:.1f}%** (W/L/close {(T.res=='win').sum()}/{(T.res=='loss').sum()}/{(T.res=='close').sum()}), mean R {T.r.mean():+.3f}, $ at 1 NQ {T.pts.sum()*20:+,.0f}; halves {(h1.res=='win').mean()*100:.1f}% (n={len(h1)}) / {(h2.res=='win').mean()*100:.1f}% (n={len(h2)}).")
        L.append("- By side: "+", ".join(f"{k} {(x.res=='win').mean()*100:.1f}% (n={len(x)}, R {x.r.mean():+.3f})" for k,x in T.groupby('side'))+".")
        L.append("- By regime: "+", ".join(f"{k} {(x.res=='win').mean()*100:.1f}% (n={len(x)}, R {x.r.mean():+.3f})" for k,x in T.groupby('regime'))+".")
        L.append("- By touch time: "+", ".join(f"{'09:30–10:00' if k else 'after 10:00'} {(x.res=='win').mean()*100:.1f}% (n={len(x)}, R {x.r.mean():+.3f})" for k,x in T.groupby('first_half_hour'))+".")
        L.append(f"- Per-year hit rate: "+", ".join(f"{y} {(x.res=='win').mean()*100:.0f}% (n={len(x)})" for y,x in T.groupby(pd.to_datetime(T.date).dt.year))+".")
        L.append(f"→ **{('VALIDATED' if ok else 'REFUTED') if FIT else 'NO VERDICT (proxy unfit)'}** (rule: ≥57% in both halves, long-γ ≥ short-γ).")
    else: L.append("- n too small.")
res="\n".join(L); open(f"{FP}/studies/2026-09-09-gamma-backfill-results.md","w").write(res); print(res)