Chan/scripts/wyckoff_gate_robustness_slices.py

#!/usr/bin/env python3
"""
Gate v1.1 冻结前小范围稳健性确认（不改 Spring / 不调 soft-score）

在 Baseline SPRING_LONG 全集上：
  - 按年份、era 切片
  - 看 blocked 是否仍主要来自 distribution
  - kept vs blocked 的 PF 关系是否稳定

range 只作观察桶，不改交易规则。
"""
from __future__ import annotations

import json
import logging
import sys
from collections import defaultdict
from pathlib import Path
from typing import Any

import numpy as np
import pandas as pd

ROOT = Path(__file__).resolve().parents[3]
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / "user_data/Chan"))

from engine.market_state import compute_market_state_8h  # noqa: E402
from user_data.Chan.scripts.wyckoff_tf_grid import install_offline_markets  # noqa: E402

AUDIT = ROOT / "user_data/Chan/scripts/wyckoff_negative_domain_audit_result.json"
OUT = ROOT / "user_data/Chan/scripts/wyckoff_gate_robustness_slices_result.json"
PAIR = "BTC/USDT:USDT"
CFG = ROOT / "user_data/Chan/config/Wyckoff_BTC_V1_BASELINE.json"


def _pf(ps: list[float]) -> float:
	wins = [p for p in ps if p > 0]
	losses = [-p for p in ps if p <= 0]
	gw, gl = sum(wins), sum(losses)
	if gl <= 0:
		return 999.0 if gw > 0 else 0.0
	return gw / gl


def _stats(ps: list[float]) -> dict[str, Any]:
	if not ps:
		return {"n": 0, "pf": 0.0, "sum_pct": 0.0, "winrate": 0.0}
	return {
		"n": len(ps),
		"pf": round(_pf(ps), 3),
		"sum_pct": round(100.0 * float(np.sum(ps)), 2),
		"winrate": round(100.0 * sum(1 for p in ps if p > 0) / len(ps), 1),
	}


def load_annotated_springs() -> list[dict[str, Any]]:
	"""复用 audit 逻辑，产出逐笔 annotated SPRING。"""
	from freqtrade.configuration import Configuration
	from freqtrade.enums import RunMode
	from freqtrade.optimize.backtesting import Backtesting
	from freqtrade.persistence import LocalTrade
	import freqtrade.optimize.optimize_reports.bt_output as bt_output

	bt_output.show_backtest_results = lambda *a, **k: None  # type: ignore
	for mod in list(sys.modules):
		if "Wyckoff_BTC" in mod:
			del sys.modules[mod]

	cfg = Configuration.from_files([str(CFG)])
	cfg.update(
		{
			"strategy": "Wyckoff_BTC_V1_BASELINE",
			"strategy_path": str(ROOT / "user_data/Chan/strategies"),
			"timerange": "20190901-",
			"timeframe": "1h",
			"export": "none",
			"runmode": RunMode.BACKTEST,
			"datadir": ROOT / "user_data/data/binance",
			"user_data_dir": ROOT / "user_data",
			"enable_protections": False,
			"fee": 0.0010,
			"exchange": {
				**cfg.get("exchange", {}),
				"name": "binance",
				"pair_whitelist": [PAIR],
			},
		}
	)
	bt = Backtesting(cfg)
	bt.start()

	h8 = pd.read_feather(ROOT / "user_data/data/binance/futures/BTC_USDT_USDT-8h-futures.feather")
	h8["date"] = pd.to_datetime(h8["date"], utc=True)
	h8 = compute_market_state_8h(h8).set_index("date").sort_index()

	rows = []
	for t in LocalTrade.bt_trades:
		if "SPRING" not in (t.enter_tag or ""):
			continue
		ed = pd.Timestamp(t.open_date_utc)
		if ed.tzinfo is None:
			ed = ed.tz_localize("UTC")
		idx = h8.index.get_indexer([ed], method="ffill")[0]
		if idx < 0:
			continue
		st = h8.iloc[idx]
		state = str(st["market_state"])
		rows.append(
			{
				"entry_date": ed.isoformat(),
				"year": str(ed.year),
				"era": "2023plus" if ed >= pd.Timestamp("2023-01-01", tz="UTC") else "pre_2023",
				"market_state": state,
				"allow_spring": bool(st["allow_spring"]),
				"profit_ratio": float(t.close_profit or 0.0),
			}
		)
	return rows


def slice_report(rows: list[dict], key: str) -> dict[str, Any]:
	out: dict[str, Any] = {}
	groups: dict[str, list[dict]] = defaultdict(list)
	for r in rows:
		groups[str(r[key])].append(r)
	for k, rs in sorted(groups.items()):
		kept = [x for x in rs if x["allow_spring"]]
		blocked = [x for x in rs if not x["allow_spring"]]
		b_by_state: dict[str, list[float]] = defaultdict(list)
		for x in blocked:
			b_by_state[x["market_state"]].append(x["profit_ratio"])
		blocked_states = {s: _stats(ps) for s, ps in b_by_state.items()}
		dist_n = blocked_states.get("distribution", {}).get("n", 0)
		blocked_n = len(blocked)
		out[k] = {
			"n_total": len(rs),
			"kept": _stats([x["profit_ratio"] for x in kept]),
			"blocked": _stats([x["profit_ratio"] for x in blocked]),
			"blocked_by_state": blocked_states,
			"blocked_distribution_share": round(dist_n / blocked_n, 3) if blocked_n else None,
			"blocked_all_bad": (
				all(s in ("distribution", "markdown", "range") for s in blocked_states)
				if blocked_n
				else True
			),
		}
	return out


def main() -> None:
	logging.getLogger("freqtrade").setLevel(logging.ERROR)
	install_offline_markets([PAIR])

	print("===== Annotate SPRING_LONG =====", flush=True)
	rows = load_annotated_springs()
	print(f"  n={len(rows)}", flush=True)

	by_year = slice_report(rows, "year")
	by_era = slice_report(rows, "era")

	# 稳定性：有 blocked 的切片里，distribution 是否为第一大来源
	dist_primary = []
	for label, block in {**{f"year:{k}": v for k, v in by_year.items()}, **{f"era:{k}": v for k, v in by_era.items()}}.items():
		bn = block["blocked"]["n"]
		if bn < 2:
			continue
		states = block["blocked_by_state"]
		top = max(states.items(), key=lambda x: x[1]["n"])[0] if states else None
		dist_primary.append(
			{
				"slice": label,
				"blocked_n": bn,
				"top_blocked_state": top,
				"distribution_share": block["blocked_distribution_share"],
				"blocked_pf": block["blocked"]["pf"],
				"kept_pf": block["kept"]["pf"],
			}
		)

	n_slices = len(dist_primary)
	n_dist_top = sum(1 for x in dist_primary if x["top_blocked_state"] == "distribution")
	n_dist_ge_50 = sum(
		1 for x in dist_primary if (x["distribution_share"] or 0) >= 0.5
	)

	result = {
		"n_spring": len(rows),
		"by_year": by_year,
		"by_era": by_era,
		"slice_summaries": dist_primary,
		"range_observation_only": {
			"note": "range 不作交易规则；仅观察 blocked 中的占比与 PF",
			"blocked_range_global": _stats(
				[r["profit_ratio"] for r in rows if (not r["allow_spring"] and r["market_state"] == "range")]
			),
		},
		"verdict": {
			"slices_with_blocked_ge_2": n_slices,
			"distribution_is_top_blocked_state": n_dist_top,
			"distribution_share_ge_50pct_slices": n_dist_ge_50,
			"distribution_attribution_stable": (
				n_slices > 0 and (n_dist_top / n_slices) >= 0.6
			),
			"status": (
				"PASS"
				if n_slices > 0 and (n_dist_top / n_slices) >= 0.6
				else "PARTIAL"
				if n_dist_ge_50 >= max(1, n_slices // 2)
				else "FAIL"
			),
			"note": "PASS = across year/era slices, blocked mass still led by distribution.",
		},
	}

	print("\n===== By year (blocked focus) =====", flush=True)
	for y, b in by_year.items():
		print(
			f"  {y}: total={b['n_total']} kept_pf={b['kept']['pf']} "
			f"blocked_n={b['blocked']['n']} blocked_pf={b['blocked']['pf']} "
			f"dist_share={b['blocked_distribution_share']} states={list(b['blocked_by_state'])}",
			flush=True,
		)
	print("\n===== By era =====", flush=True)
	for e, b in by_era.items():
		print(
			f"  {e}: total={b['n_total']} kept_pf={b['kept']['pf']} "
			f"blocked_n={b['blocked']['n']} blocked_pf={b['blocked']['pf']} "
			f"dist_share={b['blocked_distribution_share']} states={list(b['blocked_by_state'])}",
			flush=True,
		)
	print("\n===== Verdict =====", flush=True)
	print(json.dumps(result["verdict"], indent=2, ensure_ascii=False))

	OUT.write_text(json.dumps(result, indent=2, ensure_ascii=False))
	print(f"\nSaved {OUT}")


if __name__ == "__main__":
	main()