Chan/research/step2_walkforward.py

"""Step 2：Walk-forward 重放，检验缠论买卖点在实时环境下是否真实可交易。

与 Step 1 的区别：入场时刻不再取引擎自报的 sure_time，而是取信号在滑动窗口重放中
「第一次真正可被观测到」的时刻。这才是没有未来函数的口径。
"""
from __future__ import annotations

import argparse
import sys
import time
from pathlib import Path

import numpy as np
import pandas as pd

sys.path.insert(0, str(Path(__file__).resolve().parent))

from lib.bsp_eval import DEFAULT_HORIZONS, baseline_stats, run_pipeline
from lib.data import fetch_ohlcv
from lib.walkforward import analyze_stability, replay

pd.set_option("display.width", 220)

from chanlun.core.ChanEnum import Chan_BSP_DIR


def realtime_event_study(life: pd.DataFrame, df: pd.DataFrame, horizons=DEFAULT_HORIZONS) -> pd.DataFrame:
    """以 first_seen 为入场点做事件研究（无未来函数）。"""
    closes = df["close"].to_numpy(dtype=float)
    n = len(df)
    rows = []
    for _, r in life.iterrows():
        i = int(r["first_seen_idx"])
        d = int(r["direction"])
        entry = closes[i]
        row = {"bsp_type": r["bsp_type"], "direction": d,
               "side": "LONG" if d == 1 else "SHORT", "entry_idx": i}
        for h in horizons:
            j = i + h
            row[f"ret_{h}"] = d * (closes[j] - entry) / entry if j < n else np.nan
        rows.append(row)
    return pd.DataFrame(rows)


def summarize_rt(fwd: pd.DataFrame, df: pd.DataFrame, horizons=DEFAULT_HORIZONS) -> pd.DataFrame:
    base = baseline_stats(df, horizons).set_index("horizon")
    groups = [("ALL", fwd), ("ALL_LONG", fwd[fwd.direction == 1]), ("ALL_SHORT", fwd[fwd.direction == -1])]
    groups += [(t, g) for t, g in fwd.groupby("bsp_type")]
    rows = []
    for name, g in groups:
        if g.empty:
            continue
        for h in horizons:
            r = g[f"ret_{h}"].dropna().to_numpy()
            if len(r) == 0:
                continue
            dirs = g.loc[g[f"ret_{h}"].notna(), "direction"].to_numpy()
            base_mean = float(np.mean(dirs) * base.loc[h, "base_mean_long"])
            sd = r.std(ddof=1) if len(r) > 1 else np.nan
            rows.append({
                "group": name, "horizon": h, "n": len(r),
                "mean": r.mean(), "winrate": (r > 0).mean(),
                "excess": r.mean() - base_mean,
                "tstat": r.mean() / (sd / np.sqrt(len(r))) if sd else np.nan,
            })
    return pd.DataFrame(rows)


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--symbol", default="BTC/USDT:USDT")
    ap.add_argument("--tf", default="1h")
    ap.add_argument("--limit", type=int, default=50000)
    ap.add_argument("--window", type=int, default=3000)
    ap.add_argument("--step", type=int, default=6)
    ap.add_argument("--tail", type=int, default=0, help="只用最后N根做快速验证，0=全量")
    args = ap.parse_args()

    df = fetch_ohlcv(args.symbol, args.tf, args.limit)
    if args.tail:
        df = df.tail(args.tail).reset_index(drop=True)
    print(f"[data] {args.symbol} {args.tf} rows={len(df)} {df['date'].iloc[0]} -> {df['date'].iloc[-1]}")

    _, final_bsp = run_pipeline(df, args.tf)
    final_keys = {
        (str(b.type).replace("Chan_BSP_TYPE.", ""),
         1 if b.dir == Chan_BSP_DIR.BUY else -1,
         str(b.end_time))
        for b in final_bsp if b.is_sure and b.sure_time is not None
    }
    print(f"[final] 全量口径买卖点 = {len(final_keys)}")

    n_windows = len(range(args.window, len(df) + 1, args.step))
    print(f"[replay] window={args.window} step={args.step} 共 {n_windows} 个窗口")
    t0 = time.time()
    result = replay(
        df, args.tf, window=args.window, step=args.step,
        cache_key=f"{args.symbol.replace('/', '_').replace(':', '-')}_{args.tf}",
    )
    print(f"[replay] 完成，用时 {time.time() - t0:.0f}s")

    life = analyze_stability(result, final_keys, df, window=args.window)
    out_dir = Path(__file__).parent / "out"
    out_dir.mkdir(exist_ok=True)
    tag = f"{args.symbol.split('/')[0]}_{args.tf}" + (f"_tail{args.tail}" if args.tail else "")
    life.to_csv(out_dir / f"step2_life_{tag}.csv", index=False)

    valid = life[~life["truncated"]].copy()
    mature = valid[~valid["immature"]].copy()
    print(f"\n[信号总数] 重放中出现 = {len(life)}；剔除左截断后 = {len(valid)}；其中已成熟 = {len(mature)}")
    if valid.empty:
        return

    print("\n=== 信号稳定性 ===")
    print(f"[幻影率] 实时报出但完整历史上不存在: {(~mature['in_final']).mean() * 100:.1f}%"
          f"  ({(~mature['in_final']).sum()}/{len(mature)})")
    print(f"[撤销率] 窗口内报出后又被抹掉      : {(~mature['alive_at_scope_end']).mean() * 100:.1f}%"
          f"  ({(~mature['alive_at_scope_end']).sum()}/{len(mature)})")
    print(f"[存活度] persist_ratio 均值        : {mature['persist_ratio'].mean():.3f}")
    print(
        mature.groupby("bsp_type")
        .agg(n=("in_final", "size"),
             phantom=("in_final", lambda s: 1 - s.mean()),
             revoked=("alive_at_scope_end", lambda s: 1 - s.mean()),
             persist=("persist_ratio", "mean"))
        .assign(phantom=lambda d: (d["phantom"] * 100).map(lambda v: f"{v:.1f}%"),
                revoked=lambda d: (d["revoked"] * 100).map(lambda v: f"{v:.1f}%"),
                persist=lambda d: d["persist"].map(lambda v: f"{v:.3f}"))
        .to_string()
    )

    print("\n=== 真实滞后 ===")
    idx_of = {t: i for i, t in enumerate(df["date"].dt.strftime("%Y-%m-%d %H:%M:%S"))}
    sure_lag = {}
    for b in final_bsp:
        if b.is_sure and b.sure_time is not None and str(b.sure_time) in idx_of:
            k = str(b.end_time)
            if k in idx_of:
                sure_lag[k] = idx_of[str(b.sure_time)] - idx_of[k]
    valid["sure_lag"] = valid["fx_time"].map(sure_lag)
    print(f"[实时首见滞后] 中位数 = {valid['observed_lag'].median():.0f} 根"
          f"  均值 = {valid['observed_lag'].mean():.1f} 根")
    both = valid.dropna(subset=["sure_lag"])
    if not both.empty:
        print(f"[引擎自报滞后] 中位数 = {both['sure_lag'].median():.0f} 根"
              f"  均值 = {both['sure_lag'].mean():.1f} 根")
        print(f"[被低估的滞后] 中位数 = {(both['observed_lag'] - both['sure_lag']).median():.0f} 根"
              f"  均值 = {(both['observed_lag'] - both['sure_lag']).mean():.1f} 根")

    print("\n=== 无未来函数事件研究（以实时首见时刻入场）===")
    fwd = realtime_event_study(valid, df)
    s = summarize_rt(fwd, df)
    out = s.copy()
    for c in ("mean", "excess"):
        out[c] = out[c].map(lambda v: f"{v * 100:+.2f}%")
    out["winrate"] = out["winrate"].map(lambda v: f"{v * 100:.0f}%")
    out["tstat"] = out["tstat"].map(lambda v: f"{v:+.2f}")
    print(out.to_string(index=False))

    fwd.to_csv(out_dir / f"step2_rt_events_{tag}.csv", index=False)
    s.to_csv(out_dir / f"step2_rt_summary_{tag}.csv", index=False)
    print(f"\n明细已写入 {out_dir}/step2_*_{tag}.csv")


if __name__ == "__main__":
    main()