chore: 移除不再使用的 ChanMacro、system、tests。
这些目录已废弃，从仓库中清理。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
jackyu66git
@@ -1,5 +0,0 @@
|
||||
"""Validation Framework — Phase 0: verify every factor before trusting it."""
|
||||
from .factor_validator import FactorValidator
|
||||
from .regime_validator import RegimeValidator
|
||||
from .transition_validator import TransitionValidator
|
||||
from .reporter import ValidationReporter
|
||||
@@ -1,174 +0,0 @@
|
||||
"""
|
||||
validation/factor_validator.py — Validates a factor's predictive power.
|
||||
|
||||
Tests: IC, ICIR, Hit Ratio, Quantile Spread, Lead-Lag analysis.
|
||||
Answers: "Does this factor predict future returns?"
|
||||
"""
|
||||
|
||||
from datetime import date as Date
|
||||
from typing import Optional
|
||||
import sqlite3
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from config import config
|
||||
from .metrics import (
|
||||
information_coefficient, icir, hit_ratio,
|
||||
quantile_spread, lead_lag_ic,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FactorReport:
|
||||
"""Structured report for a single factor's validation results."""
|
||||
|
||||
def __init__(self, factor_name: str):
|
||||
self.factor_name = factor_name
|
||||
self.ic_mean: float = 0.0
|
||||
self.ic_std: float = 0.0
|
||||
self.icir: float = 0.0
|
||||
self.hit_ratio: float = 0.0
|
||||
self.quantile_spread: float = 0.0
|
||||
self.is_leading: bool = False
|
||||
self.lead_days: int = 0
|
||||
self.lead_ic: float = 0.0
|
||||
self.n_observations: int = 0
|
||||
self.conclusion: str = ""
|
||||
|
||||
def summary(self) -> str:
|
||||
lines = [
|
||||
f"Factor: {self.factor_name}",
|
||||
f" N={self.n_observations}",
|
||||
f" IC mean={self.ic_mean:.4f} std={self.ic_std:.4f} ICIR={self.icir:.2f}",
|
||||
f" Hit Ratio={self.hit_ratio:.1%} Top-Bot Spread={self.quantile_spread:.4f}",
|
||||
f" Best Lead: {self.lead_days}d (IC={self.lead_ic:.4f})" if self.is_leading else " Leading: No (synchronous/lagging)",
|
||||
f" → {self.conclusion}",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class FactorValidator:
|
||||
"""
|
||||
Validates a factor's predictive power using standard quant metrics.
|
||||
|
||||
For each forward horizon (1d, 3d, 5d, 7d, 14d), computes:
|
||||
- IC (Spearman rank correlation)
|
||||
- ICIR (IC stability)
|
||||
- Hit Ratio (direction accuracy)
|
||||
- Quantile spread (top vs bottom bucket)
|
||||
- Lead-lag profile
|
||||
|
||||
A factor is valid if IC > 0.03 and ICIR > 0.5.
|
||||
For regime factors, also check regime_validator.
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: Optional[str] = None):
|
||||
self.db_path = db_path or config.db_path
|
||||
|
||||
def validate(self, factor_name: str, factor_scores: pd.Series,
|
||||
forward_returns: dict[str, pd.Series]) -> FactorReport:
|
||||
"""
|
||||
Args:
|
||||
factor_name: Human-readable name
|
||||
factor_scores: Series indexed by date, values 0-100
|
||||
forward_returns: Dict of horizon → Series indexed by date (e.g. "1d" → returns)
|
||||
"""
|
||||
report = FactorReport(factor_name)
|
||||
|
||||
# Align series to common dates
|
||||
common_idx = factor_scores.index
|
||||
for ret in forward_returns.values():
|
||||
common_idx = common_idx.intersection(ret.index)
|
||||
|
||||
if len(common_idx) < 30:
|
||||
report.conclusion = "INSUFFICIENT DATA (< 30 observations)"
|
||||
return report
|
||||
|
||||
f = factor_scores[common_idx]
|
||||
report.n_observations = len(common_idx)
|
||||
|
||||
# Test against 7d forward returns (primary horizon)
|
||||
primary_ret = forward_returns.get("7d")
|
||||
if primary_ret is None:
|
||||
# Use first available
|
||||
primary_ret = list(forward_returns.values())[0]
|
||||
|
||||
r = primary_ret[common_idx]
|
||||
|
||||
# IC
|
||||
ic = information_coefficient(f, r)
|
||||
report.ic_mean = round(ic, 4)
|
||||
|
||||
# Rolling IC for ICIR
|
||||
rolling_ics = []
|
||||
for i in range(30, len(f)):
|
||||
ic_i = information_coefficient(f.iloc[:i], r.iloc[:i])
|
||||
rolling_ics.append(ic_i)
|
||||
ic_series = pd.Series(rolling_ics)
|
||||
report.ic_std = round(ic_series.std(), 4)
|
||||
report.icir = round(icir(ic_series), 2)
|
||||
|
||||
# Hit ratio
|
||||
report.hit_ratio = round(hit_ratio(f, r), 4)
|
||||
|
||||
# Quantile spread
|
||||
report.quantile_spread = round(quantile_spread(f, r), 4)
|
||||
|
||||
# Lead-lag
|
||||
lead = lead_lag_ic(f, r, max_lag=14)
|
||||
report.is_leading = lead["is_leading"]
|
||||
report.lead_days = lead["lead_days"]
|
||||
report.lead_ic = round(lead["best_ic"], 4)
|
||||
|
||||
# Conclusion
|
||||
if abs(report.ic_mean) > 0.05 and report.icir > 1.0:
|
||||
report.conclusion = "STRONG: significant predictive power"
|
||||
elif abs(report.ic_mean) > 0.03 and report.icir > 0.5:
|
||||
report.conclusion = "VALID: moderate predictive power"
|
||||
elif abs(report.ic_mean) < 0.02:
|
||||
report.conclusion = "CONFIRMING: describes current state, not predictive"
|
||||
else:
|
||||
report.conclusion = "WEAK: borderline, monitor or downweight"
|
||||
|
||||
return report
|
||||
|
||||
def validate_from_db(self, factor_name: str,
|
||||
score_query: str,
|
||||
horizon_days: int = 7) -> FactorReport:
|
||||
"""
|
||||
Convenience: load scores from DB and OHLCV returns, then validate.
|
||||
|
||||
score_query: SQL that returns (date, score) pairs.
|
||||
"""
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
|
||||
scores_df = pd.read_sql_query(score_query, conn)
|
||||
if scores_df.empty:
|
||||
conn.close()
|
||||
r = FactorReport(factor_name)
|
||||
r.conclusion = "NO DATA"
|
||||
return r
|
||||
|
||||
scores_df["date"] = pd.to_datetime(scores_df["date"])
|
||||
scores = scores_df.set_index("date")["score"]
|
||||
|
||||
# Load forward returns from OHLCV
|
||||
ohlcv = pd.read_sql_query(
|
||||
"SELECT date, close FROM ohlcv_daily WHERE symbol='BTC/USDT:USDT' ORDER BY date",
|
||||
conn
|
||||
)
|
||||
conn.close()
|
||||
|
||||
ohlcv["date"] = pd.to_datetime(ohlcv["date"])
|
||||
ohlcv = ohlcv.set_index("date")
|
||||
ohlcv["ret"] = ohlcv["close"].pct_change().shift(-1) # forward 1d
|
||||
|
||||
# Build forward returns for multiple horizons
|
||||
forward = {}
|
||||
for h in [1, 3, 5, 7, 14]:
|
||||
forward[str(h) + "d"] = ohlcv["close"].pct_change(periods=h).shift(-h)
|
||||
|
||||
return self.validate(factor_name, scores, forward)
|
||||
@@ -1,192 +0,0 @@
|
||||
"""
|
||||
validation/metrics.py — Shared statistical metrics for factor and regime validation.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from scipy import stats
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def information_coefficient(factor: pd.Series, forward_returns: pd.Series) -> float:
|
||||
"""Spearman rank IC between factor values and forward returns."""
|
||||
mask = factor.notna() & forward_returns.notna()
|
||||
if mask.sum() < 10:
|
||||
return 0.0
|
||||
ic, _ = stats.spearmanr(factor[mask], forward_returns[mask])
|
||||
return float(ic) if not np.isnan(ic) else 0.0
|
||||
|
||||
|
||||
def icir(ic_series: pd.Series) -> float:
|
||||
"""Information Coefficient IR = mean(IC) / std(IC)."""
|
||||
if len(ic_series) < 5 or ic_series.std() == 0:
|
||||
return 0.0
|
||||
return float(ic_series.mean() / ic_series.std())
|
||||
|
||||
|
||||
def hit_ratio(factor: pd.Series, forward_returns: pd.Series) -> float:
|
||||
"""Fraction of times factor direction matches return direction."""
|
||||
mask = factor.notna() & forward_returns.notna()
|
||||
if mask.sum() < 10:
|
||||
return 0.5
|
||||
# Compare sign of factor deviation from median vs sign of returns
|
||||
factor_median = factor[mask].median()
|
||||
factor_sign = np.sign(factor[mask] - factor_median)
|
||||
return_sign = np.sign(forward_returns[mask])
|
||||
return float((factor_sign == return_sign).mean())
|
||||
|
||||
|
||||
def quantile_spread(factor: pd.Series, forward_returns: pd.Series,
|
||||
n_quantiles: int = 5) -> float:
|
||||
"""Top vs bottom quantile return spread (分层回测)."""
|
||||
mask = factor.notna() & forward_returns.notna()
|
||||
if mask.sum() < n_quantiles * 3:
|
||||
return 0.0
|
||||
f = factor[mask]
|
||||
r = forward_returns[mask]
|
||||
labels = pd.qcut(f, n_quantiles, labels=False, duplicates="drop")
|
||||
top_ret = r[labels == labels.max()].mean()
|
||||
bot_ret = r[labels == labels.min()].mean()
|
||||
return float(top_ret - bot_ret)
|
||||
|
||||
|
||||
def lead_lag_ic(factor: pd.Series, returns: pd.Series,
|
||||
max_lag: int = 14) -> dict:
|
||||
"""Find the best leading/trailing relationship by computing IC at each lag."""
|
||||
results = {}
|
||||
for lag in range(-max_lag, max_lag + 1):
|
||||
if lag < 0:
|
||||
shifted = factor.shift(abs(lag))
|
||||
ic = information_coefficient(shifted, returns)
|
||||
results[f"lead_{abs(lag)}d"] = ic
|
||||
elif lag > 0:
|
||||
shifted = returns.shift(lag)
|
||||
ic = information_coefficient(factor, shifted)
|
||||
results[f"lag_{lag}d"] = ic
|
||||
else:
|
||||
ic = information_coefficient(factor, returns)
|
||||
results["sync"] = ic
|
||||
|
||||
# Find best lead period
|
||||
lead_ics = {k: v for k, v in results.items() if k.startswith("lead_")}
|
||||
best_lead = max(lead_ics, key=lead_ics.get) if lead_ics else "sync"
|
||||
best_ic = lead_ics.get(best_lead, results.get("sync", 0))
|
||||
|
||||
return {
|
||||
"best_lead": best_lead,
|
||||
"best_ic": best_ic,
|
||||
"ic_curve": results,
|
||||
"is_leading": best_lead.startswith("lead_") and abs(best_ic) > 0.03,
|
||||
"lead_days": int(best_lead.split("_")[1].rstrip("d")) if best_lead.startswith("lead_") else 0,
|
||||
}
|
||||
|
||||
|
||||
def mutual_information(factor: pd.Series, labels: pd.Series,
|
||||
n_bins: int = 10) -> float:
|
||||
"""Mutual information between factor (binned) and discrete regime labels."""
|
||||
mask = factor.notna() & labels.notna()
|
||||
if mask.sum() < 20:
|
||||
return 0.0
|
||||
f = factor[mask]
|
||||
l = labels[mask]
|
||||
try:
|
||||
f_binned = pd.qcut(f, n_bins, labels=False, duplicates="drop")
|
||||
except ValueError:
|
||||
f_binned = pd.cut(f, n_bins, labels=False)
|
||||
mi = 0.0
|
||||
for fi in range(n_bins):
|
||||
p_f = (f_binned == fi).mean()
|
||||
if p_f == 0:
|
||||
continue
|
||||
for li in l.unique():
|
||||
p_l = (l == li).mean()
|
||||
p_joint = ((f_binned == fi) & (l == li)).mean()
|
||||
if p_joint > 0:
|
||||
mi += p_joint * np.log(p_joint / (p_f * p_l))
|
||||
return float(mi)
|
||||
|
||||
|
||||
def kl_divergence(factor: pd.Series, labels: pd.Series,
|
||||
regime_a: str, regime_b: str, n_bins: int = 10) -> float:
|
||||
"""KL divergence between factor distributions in two regimes."""
|
||||
mask_a = (labels == regime_a) & factor.notna()
|
||||
mask_b = (labels == regime_b) & factor.notna()
|
||||
if mask_a.sum() < 10 or mask_b.sum() < 10:
|
||||
return 0.0
|
||||
try:
|
||||
hist_a, edges = np.histogram(factor[mask_a], bins=n_bins, density=True)
|
||||
hist_b, _ = np.histogram(factor[mask_b], bins=edges, density=True)
|
||||
except ValueError:
|
||||
return 0.0
|
||||
hist_a = np.clip(hist_a, 1e-10, None)
|
||||
hist_b = np.clip(hist_b, 1e-10, None)
|
||||
return float((hist_a * np.log(hist_a / hist_b)).sum())
|
||||
|
||||
|
||||
def anova_f_score(factor: pd.Series, labels: pd.Series) -> float:
|
||||
"""ANOVA F-statistic: how well factor separates different regimes."""
|
||||
mask = factor.notna() & labels.notna()
|
||||
if mask.sum() < 20:
|
||||
return 0.0
|
||||
groups = [factor[mask][labels[mask] == lbl] for lbl in labels[mask].unique()]
|
||||
groups = [g for g in groups if len(g) > 1]
|
||||
if len(groups) < 2:
|
||||
return 0.0
|
||||
f_stat, _ = stats.f_oneway(*groups)
|
||||
return float(f_stat) if not np.isnan(f_stat) else 0.0
|
||||
|
||||
|
||||
def transition_matrix(labels: pd.Series) -> pd.DataFrame:
|
||||
"""Compute Markov transition matrix from regime sequence."""
|
||||
unique = sorted(labels.dropna().unique())
|
||||
n = len(unique)
|
||||
matrix = np.zeros((n, n))
|
||||
seq = labels.dropna().values
|
||||
for i in range(len(seq) - 1):
|
||||
from_idx = unique.index(seq[i])
|
||||
to_idx = unique.index(seq[i + 1])
|
||||
matrix[from_idx][to_idx] += 1
|
||||
|
||||
# Row-normalize
|
||||
row_sums = matrix.sum(axis=1, keepdims=True)
|
||||
row_sums[row_sums == 0] = 1
|
||||
matrix = matrix / row_sums
|
||||
|
||||
return pd.DataFrame(matrix, index=unique, columns=unique)
|
||||
|
||||
|
||||
def regime_duration_stats(labels: pd.Series) -> dict:
|
||||
"""Compute average duration, flip rate, state entropy for regime sequence."""
|
||||
seq = labels.dropna().values
|
||||
if len(seq) < 2:
|
||||
return {"avg_duration": 0, "flip_rate": 0, "state_entropy": 0, "n_days": len(seq)}
|
||||
|
||||
# Count durations
|
||||
durations = []
|
||||
current = seq[0]
|
||||
count = 1
|
||||
flips = 0
|
||||
for i in range(1, len(seq)):
|
||||
if seq[i] == current:
|
||||
count += 1
|
||||
else:
|
||||
durations.append(count)
|
||||
current = seq[i]
|
||||
count = 1
|
||||
flips += 1
|
||||
durations.append(count)
|
||||
|
||||
avg_dur = float(np.mean(durations)) if durations else 0
|
||||
flip_rate = flips / len(seq)
|
||||
|
||||
# State entropy
|
||||
_, counts = np.unique(seq, return_counts=True)
|
||||
probs = counts / counts.sum()
|
||||
entropy = float(-(probs * np.log2(probs + 1e-10)).sum())
|
||||
|
||||
return {
|
||||
"avg_duration": round(avg_dur, 1),
|
||||
"flip_rate": round(flip_rate, 3),
|
||||
"state_entropy": round(entropy, 3),
|
||||
"n_days": len(seq),
|
||||
}
|
||||
@@ -1,144 +0,0 @@
|
||||
"""
|
||||
validation/regime_validator.py — Validates factors as regime separators.
|
||||
|
||||
Tests: Mutual Information, KL Divergence, ANOVA F-score.
|
||||
Answers: "Does this factor distinguish different market regimes?"
|
||||
|
||||
Key insight: a factor may have low IC (poor return predictor) but high
|
||||
regime separation (good regime classifier). Breadth is the prime example.
|
||||
"""
|
||||
|
||||
from datetime import date as Date
|
||||
from typing import Optional
|
||||
import sqlite3
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from config import config
|
||||
from .metrics import (
|
||||
mutual_information, kl_divergence, anova_f_score,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RegimeReport:
|
||||
"""Structured report for regime separation validation."""
|
||||
|
||||
def __init__(self, factor_name: str):
|
||||
self.factor_name = factor_name
|
||||
self.mutual_info: float = 0.0
|
||||
self.anova_f: float = 0.0
|
||||
self.kl_pairs: dict = {} # (regime_a, regime_b) → KL divergence
|
||||
self.best_separates: list[str] = []
|
||||
self.separation_score: float = 0.0
|
||||
self.is_regime_factor: bool = False
|
||||
self.conclusion: str = ""
|
||||
|
||||
def summary(self) -> str:
|
||||
lines = [
|
||||
f"Factor: {self.factor_name}",
|
||||
f" Mutual Information: {self.mutual_info:.4f}",
|
||||
f" ANOVA F: {self.anova_f:.1f}",
|
||||
f" Best separates: {', '.join(self.best_separates) if self.best_separates else 'none'}",
|
||||
f" Regime Factor: {'YES' if self.is_regime_factor else 'No'}",
|
||||
f" → {self.conclusion}",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class RegimeValidator:
|
||||
"""
|
||||
Validates a factor's ability to separate different market regimes.
|
||||
|
||||
A good regime factor has:
|
||||
- Mutual Information > 0.1
|
||||
- KL Divergence between regimes > 0.5
|
||||
- ANOVA F-score high
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: Optional[str] = None):
|
||||
self.db_path = db_path or config.db_path
|
||||
|
||||
def validate(self, factor_name: str, factor_scores: pd.Series,
|
||||
regime_labels: pd.Series) -> RegimeReport:
|
||||
"""
|
||||
Args:
|
||||
factor_name: Human-readable name
|
||||
factor_scores: Series indexed by date, values 0-100
|
||||
regime_labels: Series indexed by date, values = 'TREND'/'RANGE'/'PANIC'
|
||||
"""
|
||||
report = RegimeReport(factor_name)
|
||||
|
||||
# Align
|
||||
common_idx = factor_scores.index.intersection(regime_labels.index)
|
||||
if len(common_idx) < 30:
|
||||
report.conclusion = "INSUFFICIENT DATA"
|
||||
return report
|
||||
|
||||
f = factor_scores[common_idx]
|
||||
labels = regime_labels[common_idx]
|
||||
|
||||
# Mutual Information
|
||||
report.mutual_info = round(mutual_information(f, labels), 4)
|
||||
|
||||
# ANOVA
|
||||
report.anova_f = round(anova_f_score(f, labels), 1)
|
||||
|
||||
# KL Divergence between each pair of regimes
|
||||
unique_regimes = sorted(labels.unique())
|
||||
for i, ra in enumerate(unique_regimes):
|
||||
for rb in unique_regimes[i + 1:]:
|
||||
kl = kl_divergence(f, labels, ra, rb)
|
||||
report.kl_pairs[f"{ra}↔{rb}"] = round(kl, 4)
|
||||
|
||||
# Best separation
|
||||
if report.kl_pairs:
|
||||
sorted_pairs = sorted(report.kl_pairs, key=report.kl_pairs.get, reverse=True)
|
||||
report.best_separates = sorted_pairs[:2]
|
||||
|
||||
# Separation score (0-1 composite)
|
||||
mi_norm = min(report.mutual_info / 0.5, 1.0)
|
||||
kl_avg = np.mean(list(report.kl_pairs.values())) if report.kl_pairs else 0
|
||||
kl_norm = min(kl_avg / 1.0, 1.0)
|
||||
report.separation_score = round(0.5 * mi_norm + 0.5 * kl_norm, 2)
|
||||
|
||||
# Is this a good regime factor?
|
||||
report.is_regime_factor = (
|
||||
report.mutual_info > 0.1 and
|
||||
kl_avg > 0.5
|
||||
)
|
||||
|
||||
if report.separation_score > 0.8:
|
||||
report.conclusion = "EXCELLENT regime separator"
|
||||
elif report.separation_score > 0.5:
|
||||
report.conclusion = "GOOD regime separator"
|
||||
elif report.separation_score > 0.3:
|
||||
report.conclusion = "MODERATE — some regime separation"
|
||||
else:
|
||||
report.conclusion = "WEAK regime separator"
|
||||
|
||||
return report
|
||||
|
||||
def validate_from_db(self, factor_name: str,
|
||||
score_query: str) -> RegimeReport:
|
||||
"""Load scores and regime labels from DB, then validate."""
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
|
||||
scores_df = pd.read_sql_query(score_query, conn)
|
||||
regimes_df = pd.read_sql_query(
|
||||
"SELECT date, regime FROM regime_history", conn
|
||||
)
|
||||
conn.close()
|
||||
|
||||
if scores_df.empty or regimes_df.empty:
|
||||
r = RegimeReport(factor_name)
|
||||
r.conclusion = "NO DATA"
|
||||
return r
|
||||
|
||||
scores = scores_df.set_index("date")["score"]
|
||||
regimes = regimes_df.set_index("date")["regime"]
|
||||
|
||||
return self.validate(factor_name, scores, regimes)
|
||||
@@ -1,120 +0,0 @@
|
||||
"""
|
||||
validation/reporter.py — Aggregates all validation reports into a unified summary.
|
||||
|
||||
Used by: python main.py validate
|
||||
"""
|
||||
|
||||
from datetime import date as Date
|
||||
from typing import Optional
|
||||
import logging
|
||||
|
||||
from .factor_validator import FactorValidator, FactorReport
|
||||
from .regime_validator import RegimeValidator, RegimeReport
|
||||
from .transition_validator import TransitionValidator, TransitionReport
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ValidationReporter:
|
||||
"""
|
||||
Orchestrates full validation pipeline:
|
||||
|
||||
1. Factor validation (IC, ICIR, Hit Ratio) for each factor
|
||||
2. Regime validation (MI, KL, ANOVA) for each factor
|
||||
3. Transition validation (stability, flip rate)
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: Optional[str] = None):
|
||||
from config import config
|
||||
self.db_path = db_path or config.db_path
|
||||
self.factor_validator = FactorValidator(self.db_path)
|
||||
self.regime_validator = RegimeValidator(self.db_path)
|
||||
self.transition_validator = TransitionValidator(self.db_path)
|
||||
|
||||
def run_all(self) -> str:
|
||||
"""Run all validations and return a formatted report string."""
|
||||
lines = []
|
||||
lines.append("=" * 70)
|
||||
lines.append(f" ChanMacro Validation Report — {Date.today()}")
|
||||
lines.append("=" * 70)
|
||||
|
||||
# ── Factor Validation ──────────────────────────
|
||||
lines.append("")
|
||||
lines.append("─" * 50)
|
||||
lines.append(" FACTOR VALIDATION (Predictive Power)")
|
||||
lines.append("─" * 50)
|
||||
|
||||
factor_queries = {
|
||||
"Price Structure": "SELECT date, score FROM ohlcv_daily WHERE ema20 IS NOT NULL",
|
||||
"Breadth": """
|
||||
SELECT bd.date,
|
||||
(bd.advance_top50*1.0/(bd.advance_top50+bd.decline_top50+1)*100*0.30
|
||||
+ bd.above_ema20_top50*1.0/50*100*0.35
|
||||
+ bd.new_highs_20d_top50*1.0/50*100*0.20
|
||||
+ 50*0.15) as score
|
||||
FROM breadth_daily bd
|
||||
""",
|
||||
}
|
||||
|
||||
factor_reports: list[FactorReport] = []
|
||||
for name, query in factor_queries.items():
|
||||
try:
|
||||
report = self.factor_validator.validate_from_db(name, query)
|
||||
factor_reports.append(report)
|
||||
lines.append(report.summary())
|
||||
lines.append("")
|
||||
except Exception as e:
|
||||
logger.warning(f"Factor validation failed for {name}: {e}")
|
||||
|
||||
# ── Regime Validation ──────────────────────────
|
||||
lines.append("─" * 50)
|
||||
lines.append(" REGIME VALIDATION (Regime Separation)")
|
||||
lines.append("─" * 50)
|
||||
|
||||
regime_reports: list[RegimeReport] = []
|
||||
for name, query in factor_queries.items():
|
||||
try:
|
||||
report = self.regime_validator.validate_from_db(name, query)
|
||||
regime_reports.append(report)
|
||||
lines.append(report.summary())
|
||||
lines.append("")
|
||||
except Exception as e:
|
||||
logger.warning(f"Regime validation failed for {name}: {e}")
|
||||
|
||||
# ── Transition Validation ──────────────────────
|
||||
lines.append("─" * 50)
|
||||
lines.append(" TRANSITION VALIDATION (Regime Stability)")
|
||||
lines.append("─" * 50)
|
||||
|
||||
try:
|
||||
t_report = self.transition_validator.validate_from_db()
|
||||
lines.append(t_report.summary())
|
||||
except Exception as e:
|
||||
logger.warning(f"Transition validation failed: {e}")
|
||||
|
||||
# ── Summary ────────────────────────────────────
|
||||
lines.append("")
|
||||
lines.append("=" * 70)
|
||||
lines.append(" SUMMARY")
|
||||
lines.append("=" * 70)
|
||||
|
||||
# Factor ranking by IC
|
||||
if factor_reports:
|
||||
ranked = sorted(factor_reports, key=lambda r: abs(r.ic_mean), reverse=True)
|
||||
lines.append(" Factor Ranking (by |IC|):")
|
||||
for i, r in enumerate(ranked):
|
||||
tag = "★★★" if abs(r.ic_mean) > 0.05 else "★★" if abs(r.ic_mean) > 0.03 else "★"
|
||||
lines.append(f" {i+1}. {r.factor_name:20s} IC={r.ic_mean:+.4f} {tag} {r.conclusion}")
|
||||
|
||||
# Regime factor ranking
|
||||
if regime_reports:
|
||||
ranked_r = sorted(regime_reports, key=lambda r: r.separation_score, reverse=True)
|
||||
lines.append("")
|
||||
lines.append(" Regime Factor Ranking (by Separation Score):")
|
||||
for i, r in enumerate(ranked_r):
|
||||
lines.append(f" {i+1}. {r.factor_name:20s} Score={r.separation_score:.2f} {r.conclusion}")
|
||||
|
||||
lines.append("")
|
||||
lines.append("=" * 70)
|
||||
|
||||
return "\n".join(lines)
|
||||
@@ -1,131 +0,0 @@
|
||||
"""
|
||||
validation/transition_validator.py — Validates regime stability.
|
||||
|
||||
Tests: Transition matrix, average duration, flip rate, state entropy.
|
||||
Answers: "Does the regime design produce stable, persistent states?"
|
||||
|
||||
Hard requirements:
|
||||
- avg_duration > 5 days
|
||||
- flip_rate < 15%
|
||||
- Fails → regime definition needs redesign.
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
import sqlite3
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from config import config
|
||||
from .metrics import transition_matrix, regime_duration_stats
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TransitionReport:
|
||||
"""Structured report for regime stability validation."""
|
||||
|
||||
def __init__(self):
|
||||
self.avg_duration: float = 0.0
|
||||
self.flip_rate: float = 0.0
|
||||
self.state_entropy: float = 0.0
|
||||
self.n_days: int = 0
|
||||
self.transition_matrix: Optional[pd.DataFrame] = None
|
||||
self.persistence_score: float = 0.0
|
||||
self.is_stable: bool = False
|
||||
self.conclusion: str = ""
|
||||
self.warnings: list[str] = []
|
||||
|
||||
def summary(self) -> str:
|
||||
lines = [
|
||||
f"Regime Stability (N={self.n_days} days)",
|
||||
f" Avg Duration: {self.avg_duration:.1f} days (need > {config.regime_min_avg_duration})",
|
||||
f" Flip Rate: {self.flip_rate:.1%} (need < {config.regime_max_flip_rate:.0%})",
|
||||
f" State Entropy: {self.state_entropy:.3f}",
|
||||
f" Persistence Score: {self.persistence_score:.2f}",
|
||||
f" Stable: {'YES' if self.is_stable else 'NO — redesign needed'}",
|
||||
]
|
||||
if self.warnings:
|
||||
lines.append(f" Warnings: {'; '.join(self.warnings)}")
|
||||
if self.transition_matrix is not None:
|
||||
lines.append(f" Transition Matrix:\n{self.transition_matrix.to_string()}")
|
||||
lines.append(f" → {self.conclusion}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class TransitionValidator:
|
||||
"""
|
||||
Validates regime temporal stability.
|
||||
|
||||
Regime must persist — not flip daily.
|
||||
If flip_rate > 20% or avg_duration < 3 days → regime definition failed.
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: Optional[str] = None):
|
||||
self.db_path = db_path or config.db_path
|
||||
|
||||
def validate(self, regime_labels: pd.Series) -> TransitionReport:
|
||||
"""Validate a regime sequence for stability."""
|
||||
report = TransitionReport()
|
||||
report.n_days = len(regime_labels)
|
||||
|
||||
if len(regime_labels) < 30:
|
||||
report.conclusion = "INSUFFICIENT DATA (< 30 days)"
|
||||
return report
|
||||
|
||||
# Duration stats
|
||||
stats = regime_duration_stats(regime_labels)
|
||||
report.avg_duration = stats["avg_duration"]
|
||||
report.flip_rate = stats["flip_rate"]
|
||||
report.state_entropy = stats["state_entropy"]
|
||||
|
||||
# Transition matrix
|
||||
report.transition_matrix = transition_matrix(regime_labels)
|
||||
|
||||
# Persistence: how often does regime stay the same?
|
||||
diag = np.diag(report.transition_matrix.values)
|
||||
report.persistence_score = round(float(np.mean(diag)), 2)
|
||||
|
||||
# Stability check
|
||||
report.is_stable = (
|
||||
report.avg_duration >= config.regime_min_avg_duration and
|
||||
report.flip_rate <= config.regime_max_flip_rate
|
||||
)
|
||||
|
||||
# Warnings
|
||||
if report.avg_duration < 3:
|
||||
report.warnings.append(f"CRITICAL: avg duration={report.avg_duration:.1f}d — regime flips too fast")
|
||||
elif report.avg_duration < config.regime_min_avg_duration:
|
||||
report.warnings.append(f"WARNING: avg duration={report.avg_duration:.1f}d < {config.regime_min_avg_duration}")
|
||||
|
||||
if report.flip_rate > 0.20:
|
||||
report.warnings.append(f"CRITICAL: flip rate={report.flip_rate:.1%} — regime unstable")
|
||||
elif report.flip_rate > config.regime_max_flip_rate:
|
||||
report.warnings.append(f"WARNING: flip rate={report.flip_rate:.1%} > {config.regime_max_flip_rate:.0%}")
|
||||
|
||||
if report.state_entropy > 2.0:
|
||||
report.warnings.append(f"NOTE: high state entropy={report.state_entropy:.2f}, regimes may be too fine-grained")
|
||||
|
||||
if report.is_stable:
|
||||
report.conclusion = "PASS: regime design is stable"
|
||||
else:
|
||||
report.conclusion = "FAIL: regime definition needs adjustment"
|
||||
|
||||
return report
|
||||
|
||||
def validate_from_db(self) -> TransitionReport:
|
||||
"""Load regime history from DB and validate stability."""
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
df = pd.read_sql_query(
|
||||
"SELECT date, regime FROM regime_history ORDER BY date", conn
|
||||
)
|
||||
conn.close()
|
||||
|
||||
if df.empty:
|
||||
r = TransitionReport()
|
||||
r.conclusion = "NO DATA"
|
||||
return r
|
||||
|
||||
regimes = df.set_index("date")["regime"]
|
||||
return self.validate(regimes)
|
||||
Reference in New Issue
Block a user