#!/usr/bin/env python3
"""
Regime Attribution Study — 策略完全冻结

问题：为什么 Spring 在 2023+ BTC 有效，全历史 / 多品种不稳健？
方法：逐笔交易打市场状态标签，按桶看 net PF（不改任何入场逻辑）

输出:
  - scripts/wyckoff_regime_attribution_trades.jsonl  逐笔
  - scripts/wyckoff_regime_attribution_result.json   汇总
  - research/VALIDITY_BOUNDARY.md                    适用域草案
"""
from __future__ import annotations

import json
import logging
import sys
from collections import defaultdict
from pathlib import Path
from typing import Any, Optional

import numpy as np
import pandas as pd
import talib.abstract as ta

ROOT = Path(__file__).resolve().parents[3]
sys.path.insert(0, str(ROOT))

from user_data.Chan.scripts.wyckoff_tf_grid import install_offline_markets  # noqa: E402

STRAT = "Wyckoff_BTC_V1_BASELINE"
CONFIG = ROOT / "user_data/Chan/config/Wyckoff_BTC_V1_BASELINE.json"
DATADIR = ROOT / "user_data/data/binance/futures"
OUT_JSON = ROOT / "user_data/Chan/scripts/wyckoff_regime_attribution_result.json"
OUT_TRADES = ROOT / "user_data/Chan/scripts/wyckoff_regime_attribution_trades.jsonl"
OUT_BOUNDARY = ROOT / "user_data/Chan/research/VALIDITY_BOUNDARY.md"
STATUS = ROOT / "user_data/Chan/research/SYSTEM_STATUS.md"

PAIR = "BTC/USDT:USDT"
TIMERANGE = "20190901-"
FEE = 0.0005
SLIP = 0.0005  # 评价用 net


def _pf(profits: list[float]) -> float:
	wins = [p for p in profits if p > 0]
	losses = [-p for p in profits if p <= 0]
	gw, gl = sum(wins), sum(losses)
	if gl <= 0:
		return 999.0 if gw > 0 else 0.0
	return gw / gl


def _bucket_stats(rows: list[dict], key: str) -> dict[str, Any]:
	groups: dict[str, list[float]] = defaultdict(list)
	for r in rows:
		groups[str(r.get(key, "na"))].append(float(r["profit_ratio"]))
	out = {}
	for k, ps in sorted(groups.items(), key=lambda x: -len(x[1])):
		out[k] = {
			"n": len(ps),
			"winrate": 100.0 * sum(1 for p in ps if p > 0) / len(ps),
			"avg_pct": 100.0 * float(np.mean(ps)),
			"sum_pct": 100.0 * float(np.sum(ps)),
			"pf": round(_pf(ps), 3),
		}
	return out


def build_feature_frames(pair_file: str = "BTC_USDT_USDT") -> tuple[pd.DataFrame, pd.DataFrame]:
	"""1h ATR percentile + 8h structure features（与策略无关的分析层）。"""
	h1 = pd.read_feather(DATADIR / f"{pair_file}-1h-futures.feather")
	h1["date"] = pd.to_datetime(h1["date"], utc=True)
	h1 = h1.sort_values("date").reset_index(drop=True)
	h1["atr"] = ta.ATR(h1, timeperiod=14)
	# 滚动 90 天 ≈ 2160 根 1h 的 ATR 分位
	win = 2160
	h1["atr_percentile"] = h1["atr"].rolling(win, min_periods=200).apply(
		lambda x: pd.Series(x).rank(pct=True).iloc[-1], raw=False
	)

	h8 = pd.read_feather(DATADIR / f"{pair_file}-8h-futures.feather")
	h8["date"] = pd.to_datetime(h8["date"], utc=True)
	h8 = h8.sort_values("date").reset_index(drop=True)
	h8["ema50"] = ta.EMA(h8, timeperiod=50)
	h8["ema200"] = ta.EMA(h8, timeperiod=200)
	h8["adx"] = ta.ADX(h8, timeperiod=14)
	h8["ema_slope"] = (h8["ema50"] - h8["ema50"].shift(6)) / h8["ema50"].shift(6)
	h8["dist_ema200"] = (h8["close"] - h8["ema200"]) / h8["ema200"]
	h8["bull"] = (h8["close"] > h8["ema200"]) & (h8["ema50"] > h8["ema200"])
	h8["bear"] = (h8["close"] < h8["ema200"]) & (h8["ema50"] < h8["ema200"])

	# Cycle（粗粒度威科夫语境，非策略信号）
	slope = h8["ema_slope"]
	cycle = np.full(len(h8), "transition", dtype=object)
	cycle[(h8["bear"]) & (slope < -0.01)] = "markdown"
	cycle[(h8["bear"]) & (slope >= -0.01)] = "accumulation_like"
	cycle[(h8["bull"]) & (slope > 0.005)] = "markup"
	cycle[(h8["bull"]) & (slope <= 0.005)] = "distribution_like"
	h8["btc_cycle"] = cycle

	# trend strength
	ts = np.full(len(h8), "weak", dtype=object)
	ts[(h8["adx"] >= 25) & (h8["adx"] < 35)] = "moderate"
	ts[h8["adx"] >= 35] = "strong"
	h8["trend_strength"] = ts

	regime = np.full(len(h8), "range", dtype=object)
	regime[h8["bull"].fillna(False)] = "bull"
	regime[h8["bear"].fillna(False)] = "bear"
	h8["market_regime"] = regime
	return h1, h8


def atr_bucket(p: float) -> str:
	if pd.isna(p):
		return "atr_unknown"
	if p < 0.33:
		return "atr_low"
	if p < 0.66:
		return "atr_mid"
	return "atr_high"


def slope_bucket(s: float) -> str:
	if pd.isna(s):
		return "slope_unknown"
	if s > 0.01:
		return "slope_up_strong"
	if s > 0:
		return "slope_up_mild"
	if s > -0.01:
		return "slope_flat_down"
	return "slope_down_strong"


def run_backtest_trades() -> list[dict[str, Any]]:
	from freqtrade.configuration import Configuration
	from freqtrade.enums import RunMode
	from freqtrade.optimize.backtesting import Backtesting
	from freqtrade.persistence import LocalTrade
	import freqtrade.optimize.optimize_reports.bt_output as bt_output

	bt_output.show_backtest_results = lambda *a, **k: None  # type: ignore
	for mod in list(sys.modules):
		if "Wyckoff_BTC" in mod:
			del sys.modules[mod]

	config = Configuration.from_files([str(CONFIG)])
	config.update(
		{
			"strategy": STRAT,
			"strategy_path": str(ROOT / "user_data/Chan/strategies"),
			"timerange": TIMERANGE,
			"timeframe": "1h",
			"export": "none",
			"runmode": RunMode.BACKTEST,
			"datadir": ROOT / "user_data/data/binance",
			"user_data_dir": ROOT / "user_data",
			"enable_protections": False,
			"fee": FEE + SLIP,
			"exchange": {
				**config.get("exchange", {}),
				"name": "binance",
				"pair_whitelist": [PAIR],
			},
		}
	)
	bt = Backtesting(config)
	bt.start()

	rows = []
	for t in LocalTrade.bt_trades:
		rows.append(
			{
				"pair": t.pair,
				"enter_tag": t.enter_tag or "",
				"is_short": bool(t.is_short),
				"entry_date": t.open_date_utc.isoformat(),
				"exit_date": t.close_date_utc.isoformat() if t.close_date_utc else None,
				"profit_ratio": float(t.close_profit or 0.0),
				"exit_reason": t.exit_reason or "",
			}
		)
	return rows


def attribute(trades: list[dict], h1: pd.DataFrame, h8: pd.DataFrame) -> list[dict]:
	h1 = h1.set_index("date").sort_index()
	h8 = h8.set_index("date").sort_index()
	out = []
	for t in trades:
		ed = pd.Timestamp(t["entry_date"])
		if ed.tzinfo is None:
			ed = ed.tz_localize("UTC")
		# asof merge：入场前最后一根已收盘特征
		i1 = h1.index.get_indexer([ed], method="ffill")[0]
		i8 = h8.index.get_indexer([ed], method="ffill")[0]
		if i1 < 0 or i8 < 0:
			continue
		r1 = h1.iloc[i1]
		r8 = h8.iloc[i8]
		ap = float(r1["atr_percentile"]) if pd.notna(r1["atr_percentile"]) else float("nan")
		slope = float(r8["ema_slope"]) if pd.notna(r8["ema_slope"]) else float("nan")
		adx = float(r8["adx"]) if pd.notna(r8["adx"]) else float("nan")
		era = "2023plus" if ed >= pd.Timestamp("2023-01-01", tz="UTC") else "pre_2023"
		rec = {
			**t,
			"market_regime": str(r8["market_regime"]),
			"8h_adx": round(adx, 2) if not np.isnan(adx) else None,
			"8h_ema_slope": round(slope, 5) if not np.isnan(slope) else None,
			"atr_percentile": round(ap, 3) if not np.isnan(ap) else None,
			"btc_cycle": str(r8["btc_cycle"]),
			"trend_strength": str(r8["trend_strength"]),
			"dist_ema200": round(float(r8["dist_ema200"]), 4) if pd.notna(r8["dist_ema200"]) else None,
			"atr_bucket": atr_bucket(ap),
			"slope_bucket": slope_bucket(slope),
			"era": era,
			"setup": t["enter_tag"] or ("UTAD_SHORT" if t["is_short"] else "SPRING_LONG"),
			"result": "win" if t["profit_ratio"] > 0 else "loss",
		}
		out.append(rec)
	return out


def write_boundary(summary: dict[str, Any]) -> None:
	# 从桶结果提炼适用域草案（描述性，非自动交易规则）
	atr = summary["by_atr_bucket"]
	cycle = summary["by_btc_cycle"]
	era = summary["by_era"]
	ts = summary["by_trend_strength"]

	def best_worst(d: dict) -> tuple[str, str]:
		items = [(k, v) for k, v in d.items() if v["n"] >= 5]
		if not items:
			return "n/a", "n/a"
		best = max(items, key=lambda x: x[1]["pf"])
		worst = min(items, key=lambda x: x[1]["pf"])
		return f"{best[0]} (PF {best[1]['pf']}, n={best[1]['n']})", f"{worst[0]} (PF {worst[1]['pf']}, n={worst[1]['n']})"

	ab, aw = best_worst(atr)
	cb, cw = best_worst(cycle)
	tb, tw = best_worst(ts)

	text = f"""# Validity Boundary — Spring Baseline (draft)

> 策略规则冻结。本文仅来自 Regime Attribution，**不是**新入场条件。

## Evidence snapshot

| Era | n | PF (net) | sum%% |
|-----|---|----------|-------|
| pre_2023 | {era.get('pre_2023', {}).get('n', 0)} | {era.get('pre_2023', {}).get('pf', 0)} | {era.get('pre_2023', {}).get('sum_pct', 0):.1f} |
| 2023plus | {era.get('2023plus', {}).get('n', 0)} | {era.get('2023plus', {}).get('pf', 0)} | {era.get('2023plus', {}).get('sum_pct', 0):.1f} |

## Observed favorable (descriptive)

- ATR bucket best: **{ab}**
- Cycle best: **{cb}**
- Trend strength best: **{tb}**

## Observed unfavorable (descriptive)

- ATR bucket worst: **{aw}**
- Cycle worst: **{cw}**
- Trend strength worst: **{tw}**

## Draft Validity Boundary

```
Spring Strategy (BTC)
适用（研究假设，待 Decision Engine 验证）:
  ✓ BTC（非默认跨资产）
  ✓ 2023+ 类「明确资金方向 / Markup 启动」环境
  ✓ 高/中波动（ATR rising / mid-high percentile）若数据支持
  ✓ Accumulation_like → Markup 过渡语境

不适用（当前证据）:
  ✗ 默认全历史无条件交易
  ✗ 横盘 / range regime
  ✗ 跨资产默认开启（ETH/SOL Phase3 未过）
  ✗ 熊市 Markdown 快速崩跌阶段（若桶显示 PF 差）
```

## Next for Decision Engine

Market State 先判定「是否落在适用域」→ 再允许 SPRING_LONG / UTAD_SHORT 信号。  
**禁止**把本文件桶标签直接写回 Baseline 参数扫参。
"""
	OUT_BOUNDARY.write_text(text)


def main() -> None:
	logging.getLogger("freqtrade").setLevel(logging.ERROR)
	install_offline_markets([PAIR])

	print("===== 1) Frozen baseline backtest (BTC, net cost) =====", flush=True)
	raw = run_backtest_trades()
	print(f"  trades={len(raw)}", flush=True)

	print("===== 2) Build regime features =====", flush=True)
	h1, h8 = build_feature_frames()
	rows = attribute(raw, h1, h8)
	print(f"  attributed={len(rows)}", flush=True)

	with OUT_TRADES.open("w") as f:
		for r in rows:
			f.write(json.dumps(r, ensure_ascii=False) + "\n")

	summary: dict[str, Any] = {
		"pair": PAIR,
		"timerange": TIMERANGE,
		"fee_model": f"fee {FEE}+slip {SLIP}",
		"n": len(rows),
		"overall_pf": round(_pf([r["profit_ratio"] for r in rows]), 3),
		"by_era": _bucket_stats(rows, "era"),
		"by_setup": _bucket_stats(rows, "setup"),
		"by_market_regime": _bucket_stats(rows, "market_regime"),
		"by_atr_bucket": _bucket_stats(rows, "atr_bucket"),
		"by_trend_strength": _bucket_stats(rows, "trend_strength"),
		"by_slope_bucket": _bucket_stats(rows, "slope_bucket"),
		"by_btc_cycle": _bucket_stats(rows, "btc_cycle"),
		"by_era_x_cycle": {},
		"by_era_x_atr": {},
		"interpretation": [],
	}

	# 交叉：era × cycle / atr
	for era in ("pre_2023", "2023plus"):
		sub = [r for r in rows if r["era"] == era]
		summary["by_era_x_cycle"][era] = _bucket_stats(sub, "btc_cycle")
		summary["by_era_x_atr"][era] = _bucket_stats(sub, "atr_bucket")

	# 自动写几条解释线索（非交易规则）
	era = summary["by_era"]
	if era.get("2023plus", {}).get("pf", 0) > era.get("pre_2023", {}).get("pf", 0):
		summary["interpretation"].append(
			"2023plus PF 显著高于 pre_2023 → 存在 regime/cycle 依赖，非随机噪声单一窗口。"
		)
	cyc = summary["by_btc_cycle"]
	if cyc:
		best_c = max(cyc.items(), key=lambda x: (x[1]["n"] >= 5, x[1]["pf"]))
		summary["interpretation"].append(
			f"全样本 cycle 最优桶（n≥5 优先）: {best_c[0]} PF={best_c[1]['pf']} n={best_c[1]['n']}"
		)

	print("\n===== 3) Attribution tables =====", flush=True)
	for name in (
		"by_era", "by_setup", "by_market_regime", "by_atr_bucket",
		"by_trend_strength", "by_slope_bucket", "by_btc_cycle",
	):
		print(f"\n-- {name} --")
		for k, v in summary[name].items():
			print(f"  {k:<22} n={v['n']:<3} pf={v['pf']:<6} wr={v['winrate']:.0f}% sum={v['sum_pct']:.1f}%")

	print("\n-- by_era_x_cycle --")
	print(json.dumps(summary["by_era_x_cycle"], indent=2, ensure_ascii=False))

	write_boundary(summary)
	OUT_JSON.write_text(json.dumps(summary, indent=2, ensure_ascii=False))

	# 更新 SYSTEM_STATUS
	if STATUS.exists():
		st = STATUS.read_text()
		marker = "## Frozen Baseline"
		block = (
			"**Status update (Regime Attribution):**\n"
			"Evidence: PASS (2023+ BTC) · Robustness: FAILED (multi-cycle) · "
			"Confidence: LOW-MEDIUM · Next: Decision Engine validity gate "
			f"(see `VALIDITY_BOUNDARY.md`, trades=`{OUT_TRADES.name}`).\n\n"
		)
		if "Status update (Regime Attribution)" not in st:
			st = st.replace(marker, block + marker)
			STATUS.write_text(st)

	print(f"\nSaved {OUT_JSON}")
	print(f"Saved {OUT_TRADES}")
	print(f"Saved {OUT_BOUNDARY}")


if __name__ == "__main__":
	main()