Chan/research/step24_signal_clustering.py

"""Step 24：同一大级别分型下的多信号问题。

疑问：15m 分型出现后、趋势反转之前，小级别可能连出好几个三买。
这些信号此前被无差别全部计入，带来两个隐患：
  1. 同源信号高度相关，独立同分布假设被破坏，t 值会虚高
  2. 时间上重叠的仓位在实盘要占多份保证金，回测的收益无法照搬

本步回答：一个分型平均带出几个信号、第 N 个信号还值不值得做、
以及去掉同源重复后统计量掉多少。
"""
from __future__ import annotations

import argparse
import os
import sys
import warnings
from concurrent.futures import ProcessPoolExecutor, as_completed
from pathlib import Path

import numpy as np
import pandas as pd

warnings.filterwarnings("ignore")
for v in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS"):
    os.environ.setdefault(v, "1")

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
sys.path.insert(0, str(HERE.parent))
pd.set_option("display.width", 300)

SL, TP, MAXB = 1.5, 3.0, 48
FEE, SLIP = 0.0004, 0.0001


def run_one(task: tuple) -> dict | None:
    import warnings as _w
    _w.filterwarnings("ignore")
    sys.path.insert(0, str(HERE))
    sys.path.insert(0, str(HERE.parent))

    from chanlun import TF_DF
    from lib.breakout import run_trades
    from lib.data import fetch_ohlcv
    from lib.fast_bsp3 import find_fast_bsp3
    from lib.fx_signal import extract_fx_signals, signals_to_frame
    from lib.nested_bsp import attach_htf_context, htf_fx_timeline
    from lib.nested_level import build_htf_zones

    sym, ltf, h1, h2 = task
    try:
        df_l = fetch_ohlcv(f"{sym}/USDT:USDT", ltf, 10**9)
        if df_l is None or len(df_l) < 3000:
            return None
        chan_l = TF_DF(df_l, 1, ltf)
        cdf = chan_l.dataframe
        sig = find_fast_bsp3(cdf, build_htf_zones(cdf, ltf, chan=chan_l))
        if sig.empty or len(sig) < 10:
            return None
        for tf, pref in ((h1, "h1"), (h2, "h2")):
            df_h = fetch_ohlcv(f"{sym}/USDT:USDT", tf, 10**9)
            if df_h is None or len(df_h) < 300:
                continue
            chan_h = TF_DF(df_h, 1, tf)
            s = signals_to_frame(extract_fx_signals(chan_h, chan_h.dataframe))
            sig = attach_htf_context(sig, cdf, htf_fx_timeline(s, chan_h.dataframe), pref)

        entries = list(zip(sig["entry_idx"].astype(int), sig["direction"].astype(int)))
        tr = run_trades(cdf, entries, SL, TP, MAXB, fee=0.0, entry_delay=1)
        if tr.empty:
            return None
        m = sig.set_index("entry_idx")
        tr["symbol"], tr["ltf"] = sym, ltf
        tr["date"] = cdf["date"].to_numpy()[tr["entry_idx"].to_numpy()]
        for c in ("h1_agree", "h2_agree", "h1_fx_ts", "h1_age_bars"):
            tr[c] = tr["entry_idx"].map(m[c]) if c in m.columns else np.nan
        return {"task": f"{sym} {ltf}", "trades": tr}
    except Exception as e:
        return {"task": f"{sym} {ltf}", "error": repr(e)[:200]}


def desc(r: np.ndarray, label: str) -> dict:
    if len(r) < 15:
        return {}
    w, o = r[r > 0], r[r <= 0]
    sd = r.std(ddof=1)
    return {
        "分组": label, "笔数": len(r),
        "胜率": f"{(r > 0).mean() * 100:.1f}%",
        "均收益": f"{r.mean() * 100:+.3f}%",
        "中位": f"{np.median(r) * 100:+.3f}%",
        "PF": f"{w.sum() / abs(o.sum()):.2f}" if len(o) else "inf",
        "t值": f"{r.mean() / (sd / np.sqrt(len(r))):+.2f}",
    }


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--pairs", default="5m:15m:1h,15m:1h:4h,30m:2h:4h")
    ap.add_argument("--symbols", default="BTC,ETH,SOL")
    ap.add_argument("--workers", type=int, default=6)
    ap.add_argument("--reuse", action="store_true")
    args = ap.parse_args()

    cache = HERE / "out" / "step24_clustered.csv"
    if args.reuse and cache.exists():
        allt = pd.read_csv(cache, parse_dates=["date"])
        print(f"[复用] {len(allt)} 笔\n")
    else:
        tasks = [(s, *tuple(p.split(":")))
                 for p in args.pairs.split(",") if p
                 for s in args.symbols.split(",")]
        print(f"[聚集分析] {len(tasks)} 个任务\n", flush=True)
        res = []
        with ProcessPoolExecutor(max_workers=args.workers) as ex:
            futs = {ex.submit(run_one, t): t for t in tasks}
            for i, f in enumerate(as_completed(futs), 1):
                r = f.result()
                if r is None or "error" in (r or {}):
                    print(f"  [{i}] 跳过 {(r or {}).get('error','')}", flush=True)
                    continue
                res.append(r)
                print(f"  [{i}/{len(tasks)}] {r['task']} — {len(r['trades'])} 笔", flush=True)
        if not res:
            return
        allt = pd.concat([r["trades"] for r in res], ignore_index=True)
        allt.to_csv(cache, index=False)

    allt["date"] = pd.to_datetime(allt["date"])
    allt = allt[allt["h1_agree"] == 1].copy()
    allt["ret_net"] = allt["gross"] - FEE - SLIP
    # 同一 (品种,级别,分型) 下按时间排序，标出这是该分型的第几个信号
    allt = allt.sort_values(["symbol", "ltf", "h1_fx_ts", "entry_idx"])
    allt["seq"] = allt.groupby(["symbol", "ltf", "h1_fx_ts"]).cumcount() + 1
    grp = allt.groupby(["symbol", "ltf", "h1_fx_ts"])
    allt["n_in_fx"] = grp["seq"].transform("max")

    print("=" * 110)
    print("########## 1. 一个大级别分型平均带出几个小级别三买 ##########")
    rows = []
    for tf, g in allt.groupby("ltf"):
        k = g.groupby(["symbol", "h1_fx_ts"]).size()
        rows.append({
            "级别": tf, "分型数": len(k), "信号数": len(g),
            "均信号/分型": f"{k.mean():.2f}",
            "只有1个": f"{(k == 1).mean() * 100:.0f}%",
            "2个": f"{(k == 2).mean() * 100:.0f}%",
            "3个及以上": f"{(k >= 3).mean() * 100:.0f}%",
            "最多": int(k.max()),
        })
    print(pd.DataFrame(rows).to_string(index=False))

    print("\n########## 2. 第 N 个信号的质量（这是你问的核心）##########")
    for tf, g in allt.groupby("ltf"):
        rows = [desc(g[g.seq == 1]["ret_net"].to_numpy(), f"{tf} 第1个"),
                desc(g[g.seq == 2]["ret_net"].to_numpy(), f"{tf} 第2个"),
                desc(g[g.seq >= 3]["ret_net"].to_numpy(), f"{tf} 第3个+")]
        rows = [r for r in rows if r]
        if rows:
            print(pd.DataFrame(rows).to_string(index=False))
    print("  若第2个不比第1个差，说明重复入场是有效加仓而非噪声。")

    print("\n########## 3. 只保留每个分型的第一个信号 vs 全要 ##########")
    rows = []
    for tf, g in allt.groupby("ltf"):
        rows.append(desc(g["ret_net"].to_numpy(), f"{tf} 全要"))
        rows.append(desc(g[g.seq == 1]["ret_net"].to_numpy(), f"{tf} 仅第1个"))
    print(pd.DataFrame([r for r in rows if r]).to_string(index=False))
    print("  t 值下降主要来自样本变少；关键看均收益/PF 是否维持。")

    print("\n########## 4. 持仓重叠程度（实盘保证金约束）##########")
    rows = []
    for (sym, tf), g in allt.groupby(["symbol", "ltf"]):
        g = g.sort_values("entry_idx")
        e = g["entry_idx"].to_numpy()
        x = g["exit_idx"].to_numpy()
        # 每笔开仓时，有多少笔尚未平仓
        overlap = [(x[:i] > e[i]).sum() for i in range(len(e))]
        rows.append({
            "品种": sym, "级别": tf, "笔数": len(g),
            "开仓时已有持仓均值": f"{np.mean(overlap):.2f}",
            "无重叠占比": f"{(np.array(overlap) == 0).mean() * 100:.0f}%",
            "最大同时持仓": int(np.max(overlap)) + 1,
        })
    print(pd.DataFrame(rows).to_string(index=False))

    print("\n########## 5. 去重后的组合表现（每分型仅第1个，5m需过滤）##########")
    parts = []
    for tf in ("5m", "15m", "30m"):
        g = allt[allt.ltf == tf]
        if g.empty:
            continue
        if tf == "5m":
            q = g["risk_pct"].quantile(0.67)
            g = g[(g["h2_agree"] == 1) & (g["risk_pct"] > q)]
        parts.append((tf, g))
    for name, pick in (("全要", lambda g: g), ("仅第1个", lambda g: g[g.seq == 1])):
        sub = pd.concat([pick(g) for _, g in parts]).sort_values("date")
        r = sub["ret_net"].to_numpy()
        lev = np.clip(0.01 / np.clip(sub["risk_pct"].to_numpy(), 0.002, None), 0, 20)
        pnl = r * lev
        eq = np.cumprod(1 + pnl)
        yrs = (sub["date"].max() - sub["date"].min()).days / 365.25
        dd = (1 - eq / np.maximum.accumulate(eq)).max()
        print(f"  {name:>6}: n={len(r):>4}  年化 {(eq[-1] ** (1 / yrs) - 1) * 100:+6.1f}%  "
              f"回撤 {dd * 100:4.1f}%  Sharpe "
              f"{pnl.mean() / pnl.std(ddof=1) * np.sqrt(len(pnl) / yrs):4.2f}  "
              f"中位 {np.median(r) * 100:+.3f}%  年 {len(r) / yrs:.0f} 笔")


if __name__ == "__main__":
    main()