// Backfill end-of-day strike GEX (QQQ, NDX) + QQQ regular close per date from Unusual Whales.
// usage: UW_KEY_FILE=<envfile> node backfill_history.mjs 2023-10-10 2026-09-08
import fs from "fs"; import zlib from "zlib";
const env=Object.fromEntries(fs.readFileSync(process.env.UW_KEY_FILE,"utf8").split("\n").filter(l=>/^[A-Z_]+=/.test(l)).map(l=>{const i=l.indexOf("=");return [l.slice(0,i),l.slice(i+1).replace(/^"|"$/g,"")]}));
const key=env.UW_API_KEY||env.UNUSUAL_WHALES_API_KEY||env.UW_TOKEN; const H={Authorization:`Bearer ${key}`,Accept:"application/json"}; const B="https://api.unusualwhales.com";
const OUT="<data-root>/fp-data/gex/history"; const [start,end]=process.argv.slice(2);
async function get(u){ for(let a=0;a<4;a++){ try{ const r=await fetch(B+u,{headers:H}); if(r.status===429){await new Promise(s=>setTimeout(s,3000)); continue;} const j=await r.json().catch(()=>({})); return {s:r.status,j}; }catch(e){ await new Promise(s=>setTimeout(s,2000)); } } return {s:0,j:{}}; }
const days=[]; for(let d=new Date(start+"T12:00:00Z"); d<=new Date(end+"T12:00:00Z"); d.setUTCDate(d.getUTCDate()+1)){ const dow=d.getUTCDay(); if(dow>0&&dow<6) days.push(d.toISOString().slice(0,10)); }
let ok=0,skip=0,miss=0;
async function one(day){
  const f=`${OUT}/${day}.json.gz`; if(fs.existsSync(f)){skip++;return;}
  const [q,n,o]=await Promise.all([get(`/api/stock/QQQ/greek-exposure/strike?date=${day}`),get(`/api/stock/NDX/greek-exposure/strike?date=${day}`),get(`/api/stock/QQQ/ohlc/1d?date=${day}`)]);
  const qd=q.j.data||[], nd=n.j.data||[], od=o.j.data||[]; const reg=od.find(r=>r.market_time==="r")||od[od.length-1];
  if(!qd.length||!reg){ miss++; fs.writeFileSync(`${OUT}/${day}.missing`, JSON.stringify({q:q.s,n:n.s,o:o.s,msg:(q.j.message||"").slice(0,120)})); return; }
  const rec={date:day, qqq_close:+reg.close, qqq_ohlc:od, qqq:qd, ndx:nd, pulled:new Date().toISOString()};
  fs.writeFileSync(f, zlib.gzipSync(JSON.stringify(rec))); ok++;
}
const CONC=4; let i=0; async function worker(){ while(i<days.length){ const d=days[i++]; await one(d); if((ok+miss)%50===0&&(ok+miss)>0) console.log(new Date().toISOString(),"ok",ok,"miss",miss,"skip",skip,"at",d); } }
await Promise.all(Array.from({length:CONC},worker));
console.log("DONE ok",ok,"miss",miss,"skip",skip,"days",days.length);