feat: 新增 ChanMacro 宏观 regime 检测模块

ChanMacro/expectancy/__init__.py

@@ -0,0 +1,4 @@
+"""Expectancy Engine — Signal tracking, Bayesian inference, time decay."""
+from .tracker import SignalTracker
+from .decay import TimeDecay
+from .engine import BayesianExpectancyEngine, SufficiencyGuard

ChanMacro/expectancy/decay.py

@@ -0,0 +1,55 @@
+"""
+expectancy/decay.py — Time-weighted sample decay.
+
+2024 market structure ≠ 2026 market structure.
+Recent samples get higher weight via exponential decay.
+"""
+
+from datetime import date as Date
+from typing import Optional
+import numpy as np
+
+
+class TimeDecay:
+    """Exponential time decay for sample weighting."""
+
+    def __init__(self, half_life_days: int = 180):
+        self.half_life = half_life_days
+        self._decay_rate = np.log(2) / half_life_days
+
+    def weight(self, sample_date: Date, reference_date: Optional[Date] = None) -> float:
+        """
+        Compute decay weight for a sample.
+        weight = exp(-days_ago * decay_rate)
+        """
+        if reference_date is None:
+            reference_date = Date.today()
+        days = (reference_date - sample_date).days
+        return np.exp(-days * self._decay_rate)
+
+    def weights(self, dates: list[Date], reference_date: Optional[Date] = None) -> np.ndarray:
+        """Compute decay weights for a list of dates."""
+        return np.array([self.weight(d, reference_date) for d in dates])
+
+    def weighted_win_rate(self, wins: np.ndarray, weights: np.ndarray) -> float:
+        """Weighted win rate: sum(wins * weights) / sum(weights)."""
+        total_weight = weights.sum()
+        if total_weight == 0:
+            return 0.0
+        return float((wins * weights).sum() / total_weight)
+
+    def weighted_mean(self, values: np.ndarray, weights: np.ndarray) -> float:
+        """Weighted mean."""
+        total_weight = weights.sum()
+        if total_weight == 0:
+            return 0.0
+        return float((values * weights).sum() / total_weight)
+
+    def effective_samples(self, weights: np.ndarray) -> float:
+        """Effective number of samples after decay weighting."""
+        return float(weights.sum())
+
+    @staticmethod
+    def weight_at_age(days_ago: int, half_life_days: int = 180) -> float:
+        """Quick weight lookup for a given age in days."""
+        return np.exp(-days_ago * np.log(2) / half_life_days)

ChanMacro/expectancy/engine.py

@@ -0,0 +1,295 @@
+"""
+expectancy/engine.py — Bayesian Expectancy Engine.
+
+Core algorithm:
+  1. LeveledExpectancy: filter layer-by-layer, stop at highest valid level
+  2. Empirical Bayes prior: prior = signal's global historical winrate
+  3. Dynamic Beta strength: adaptive to sample size
+  4. Time decay: recent samples weighted higher (half_life=180d)
+  5. SufficiencyGuard: refuse output if effective_samples < 30
+  6. KNN Fallback: similarity search when strict filtering fails (Phase D)
+"""
+
+from datetime import date as Date
+from typing import Optional
+import sqlite3
+import logging
+
+import numpy as np
+import pandas as pd
+
+from models import (
+    MarketStateVector, ExpectancyReport, ExpectancyLayer,
+    SufficiencyLevel, MarketRegime,
+)
+from config import config
+from .decay import TimeDecay
+
+logger = logging.getLogger(__name__)
+
+
+class SufficiencyGuard:
+    """Prevents trading advice from insufficient samples."""
+
+    def __init__(self, min_effective: int = 30, low: int = 50, medium: int = 100):
+        self.MIN = min_effective
+        self.LOW = low
+        self.MEDIUM = medium
+
+    def evaluate(self, effective_samples: float) -> SufficiencyLevel:
+        if effective_samples < self.MIN:
+            return SufficiencyLevel.INSUFFICIENT
+        elif effective_samples < self.LOW:
+            return SufficiencyLevel.LOW
+        elif effective_samples < self.MEDIUM:
+            return SufficiencyLevel.MEDIUM
+        return SufficiencyLevel.HIGH
+
+
+class BayesianExpectancyEngine:
+    """
+    Leveled Bayesian Expectancy Engine.
+
+    Query layers from coarse to fine. Stop when effective_samples drops below threshold.
+    Uses Empirical Bayes prior (signal's global winrate, not fixed 50%).
+    """
+
+    # Expectancy query levels: name → WHERE clause template
+    LEVELS = [
+        ("Base",            "signal_type = '{signal}'"),
+        ("+ Regime",        "signal_type = '{signal}' AND regime = '{regime}'"),
+        ("+ Breadth",       "signal_type = '{signal}' AND regime = '{regime}' AND breadth_bucket = '{breadth}'"),
+        ("+ OI State",      "signal_type = '{signal}' AND regime = '{regime}' AND breadth_bucket = '{breadth}' AND oi_state = '{oi}'"),
+        ("+ Volatility",    "signal_type = '{signal}' AND regime = '{regime}' AND breadth_bucket = '{breadth}' AND oi_state = '{oi}' AND volatility_regime = '{vol}'"),
+    ]
+
+    def __init__(self, db_path: Optional[str] = None,
+                 half_life_days: int = 180,
+                 level_min_samples: int = 50):
+        self.db_path = db_path or config.db_path
+        self.decay = TimeDecay(half_life_days)
+        self.guard = SufficiencyGuard(
+            min_effective=config.sufficiency_min_effective,
+            low=config.sufficiency_low,
+            medium=config.sufficiency_medium,
+        )
+        self.level_min = level_min_samples
+
+    def estimate(self, state: MarketStateVector,
+                 signal_type: str = "B3",
+                 target_date: Optional[Date] = None) -> ExpectancyReport:
+        """
+        Compute layered Bayesian expectancy for a signal in current market state.
+
+        Returns the estimate at the deepest level with >= level_min effective samples.
+        """
+        if target_date is None:
+            target_date = Date.today()
+
+        conn = sqlite3.connect(self.db_path)
+
+        # Get global signal winrate for Empirical Bayes prior
+        global_rate = self._global_winrate(conn, signal_type)
+
+        layers = []
+        best_result = None
+
+        for level_name, template in self.LEVELS:
+            where = template.format(
+                signal=signal_type,
+                regime=state.regime.value,
+                breadth=state.breadth_bucket.value,
+                oi=state.oi_state.value,
+                vol=state.volatility_regime.value,
+            )
+            query = f"SELECT * FROM signal_features WHERE {where}"
+            df = pd.read_sql_query(query, conn)
+
+            if df.empty:
+                layers.append(ExpectancyLayer(
+                    name=level_name, posterior_winrate=0.0,
+                    samples=0, effective_samples=0.0,
+                ))
+                continue
+
+            # Time-weighted stats
+            dates_list = [Date.fromisoformat(d) for d in df["date"]]
+            weights = self.decay.weights(dates_list, target_date)
+            eff_n = self.decay.effective_samples(weights)
+
+            wins = pd.to_numeric(df["is_win_7d"].fillna(0), errors="coerce").fillna(0).values
+            returns = pd.to_numeric(df["result_7d"].fillna(0), errors="coerce").fillna(0).values
+
+            raw_wr = float(wins.mean()) if len(wins) > 0 else 0.0
+            weighted_wr = self.decay.weighted_win_rate(wins, weights)
+            weighted_ret = self.decay.weighted_mean(returns, weights)
+
+            # Empirical Bayes posterior
+            posterior = self._bayesian_posterior(
+                global_rate=global_rate,
+                wins=wins.sum(),
+                samples=len(df),
+            )
+
+            layer = ExpectancyLayer(
+                name=level_name,
+                posterior_winrate=round(posterior, 4),
+                raw_winrate=round(raw_wr, 4),
+                samples=len(df),
+                effective_samples=round(eff_n, 1),
+                avg_return=round(weighted_ret, 2),
+            )
+            layers.append(layer)
+
+            # Level-based fallback: keep going while samples sufficient
+            if eff_n >= self.level_min:
+                best_result = layer
+
+        conn.close()
+
+        if best_result is None and layers:
+            # Fallback to the deepest layer that had any samples
+            for layer in reversed(layers):
+                if layer.samples > 0:
+                    best_result = layer
+                    break
+
+        if best_result is None:
+            return ExpectancyReport(
+                signal_type=signal_type,
+                date=target_date,
+                layers=layers,
+                final_estimate=0.0,
+                sufficiency=SufficiencyLevel.INSUFFICIENT,
+                source="insufficient",
+            )
+
+        sufficiency = self.guard.evaluate(
+            best_result.effective_samples
+        )
+
+        # Compute profit factor and MAE from the SAME level as best_result
+        profit_factor = None
+        avg_mae = None
+        if best_result and best_result.samples > 0:
+            # Re-query the level that produced best_result
+            best_level_idx = next(
+                i for i, l in enumerate(layers) if l.name == best_result.name
+            )
+            where = self.LEVELS[best_level_idx][1].format(
+                signal=signal_type, regime=state.regime.value,
+                breadth=state.breadth_bucket.value, oi=state.oi_state.value,
+                vol=state.volatility_regime.value,
+            )
+            query = f"SELECT result_7d, max_adverse_excursion FROM signal_features WHERE {where}"
+            conn2 = sqlite3.connect(self.db_path)
+            df_detail = pd.read_sql_query(query, conn2)
+            conn2.close()
+            if not df_detail.empty:
+                returns_7d = df_detail["result_7d"].dropna()
+                if len(returns_7d) > 0:
+                    gains = returns_7d[returns_7d > 0].sum()
+                    losses = abs(returns_7d[returns_7d < 0].sum())
+                    profit_factor = round(gains / losses, 2) if losses > 0 else None
+                maes = df_detail["max_adverse_excursion"].dropna()
+                if len(maes) > 0:
+                    avg_mae = round(float(maes.mean()), 2)
+
+        return ExpectancyReport(
+            signal_type=signal_type,
+            date=target_date,
+            layers=layers,
+            final_estimate=round(best_result.posterior_winrate, 4),
+            sufficiency=sufficiency,
+            prior_strength=self._prior_strength(best_result.samples),
+            half_life_days=self.decay.half_life,
+            source="bayesian",
+            avg_return_7d=best_result.avg_return,
+            profit_factor=profit_factor,
+            max_adverse_excursion=avg_mae,
+        )
+
+    def _global_winrate(self, conn: sqlite3.Connection,
+                        signal_type: str) -> float:
+        """Get global historical winrate for a signal type (Empirical Bayes prior)."""
+        row = conn.execute(
+            "SELECT AVG(is_win_7d) as wr, COUNT(*) as cnt "
+            "FROM signal_features WHERE signal_type = ? AND is_win_7d IS NOT NULL",
+            (signal_type,)
+        ).fetchone()
+        if row and row[1] and row[1] > 0:
+            return float(row[0])
+        return 0.50  # default: neutral
+
+    def _prior_strength(self, samples: int) -> int:
+        """Dynamic prior strength based on sample count."""
+        if samples < 100:
+            return 20  # Beta(10,10)
+        elif samples < 500:
+            return 40  # Beta(20,20)
+        else:
+            return 100  # Beta(50,50) — data dominates
+
+    def _bayesian_posterior(self, global_rate: float, wins: float,
+                            samples: int) -> float:
+        """
+        Empirical Bayes posterior: prior = global signal winrate.
+
+        posterior = (alpha + wins) / (alpha + beta + samples)
+        where alpha/(alpha+beta) = global_rate
+        """
+        prior_strength = self._prior_strength(samples)
+        alpha = max(global_rate * prior_strength, 1.0)  # floor at 1 to ensure shrinkage
+        beta = max((1 - global_rate) * prior_strength, 1.0)
+        return (alpha + wins) / (alpha + beta + samples)
+
+    def precompute_cache(self):
+        """
+        Precompute expectancy for all state_hashes in signal_features.
+        Populates expectancy_cache table with raw weighted counts (not posteriors).
+        """
+        conn = sqlite3.connect(self.db_path)
+        conn.row_factory = sqlite3.Row
+
+        hashes = conn.execute(
+            "SELECT DISTINCT market_state_hash, signal_type FROM signal_features"
+        ).fetchall()
+
+        today = Date.today()
+        count = 0
+
+        for row in hashes:
+            h = row["market_state_hash"]
+            sig = row["signal_type"]
+
+            df = pd.read_sql_query(
+                "SELECT date, is_win_7d, result_7d "
+                "FROM signal_features WHERE market_state_hash = ? AND signal_type = ?",
+                conn, params=(h, sig)
+            )
+
+            if df.empty:
+                continue
+
+            dates_list = [Date.fromisoformat(d) for d in df["date"]]
+            weights = self.decay.weights(dates_list, today)
+            wins_w = (df["is_win_7d"].fillna(0).values * weights).sum()
+            losses_w = ((1 - df["is_win_7d"].fillna(0)).values * weights).sum()
+            ret_sum = (df["result_7d"].fillna(0).values * weights).sum()
+            ret_sq = ((df["result_7d"].fillna(0).values ** 2) * weights).sum()
+            eff_n = weights.sum()
+
+            sufficiency = self.guard.evaluate(eff_n).value
+
+            conn.execute("""
+                INSERT OR REPLACE INTO expectancy_cache
+                (state_hash, signal_type, wins_weighted, losses_weighted,
+                 sum_return_7d, sum_return_sq_7d, effective_samples, sufficiency)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            """, (h, sig, wins_w, losses_w, ret_sum, ret_sq, eff_n, sufficiency))
+            count += 1
+
+        conn.commit()
+        conn.close()
+        logger.info(f"Precomputed expectancy cache: {count} state×signal combos")
+        return count

ChanMacro/expectancy/tracker.py

@@ -0,0 +1,271 @@
+"""
+expectancy/tracker.py — SignalTracker: records signals with full market state
+and computes forward outcomes.
+
+This is the entry point for populating signal_features — THE moat table.
+"""
+
+from datetime import date as Date, timedelta
+from typing import Optional
+import sqlite3
+import json
+import logging
+
+import pandas as pd
+import numpy as np
+
+from models import (
+    MarketStateVector, SignalFeatureRecord, MarketRegime,
+    OIState, BreadthBucket, VolRegime, SignalGrade,
+)
+from config import config
+
+logger = logging.getLogger(__name__)
+
+
+class SignalTracker:
+    """
+    Records trading signals with full market state context.
+
+    Usage:
+        tracker = SignalTracker()
+        tracker.record(
+            date=Date(2026, 6, 24),
+            signal_type="B3",
+            entry_price=96500.0,
+            state=market_state_vector,  # from scoring pipeline
+            signal_grade="A",
+        )
+    """
+
+    def __init__(self, db_path: Optional[str] = None):
+        self.db_path = db_path or config.db_path
+
+    def record(self, date: Date, signal_type: str, entry_price: float,
+               state: MarketStateVector,
+               signal_version: str = "b3_v1",
+               signal_grade: Optional[str] = None,
+               signal_strength: Optional[float] = None) -> int:
+        """
+        Record a signal with market state snapshot and compute forward outcomes.
+
+        Returns the record ID in signal_features.
+        """
+        conn = sqlite3.connect(self.db_path)
+
+        # Compute forward outcomes
+        outcomes = self._compute_outcomes(conn, date, entry_price)
+
+        # Build embedding
+        embedding = json.dumps(state.state_embedding())
+
+        record_id = conn.execute("""
+            INSERT INTO signal_features
+            (date, signal_type, signal_version, symbol,
+             regime_version, signal_grade, signal_strength,
+             regime, regime_confidence, regime_maturity_score,
+             market_state_hash, state_embedding,
+             breadth_top20, breadth_top30, breadth_top50,
+             breadth_bucket, breadth_divergence,
+             oi_state, volatility_regime, price_structure_score,
+             entry_price,
+             result_1d, result_3d, result_5d, result_7d, result_14d,
+             max_favorable_excursion, max_adverse_excursion,
+             is_win_7d)
+            VALUES (?, ?, ?, ?, ?, ?, ?,
+                    ?, ?, ?,
+                    ?, ?,
+                    ?, ?, ?,
+                    ?, ?,
+                    ?, ?, ?,
+                    ?,
+                    ?, ?, ?, ?, ?,
+                    ?, ?,
+                    ?)
+        """, (
+            str(date), signal_type, signal_version, state.symbol,
+            state.regime_version, signal_grade, signal_strength,
+            state.regime.value, state.regime_confidence, state.regime_maturity_score,
+            state.market_state_hash, embedding,
+            state.breadth_top20, state.breadth_top30, state.breadth_top50,
+            state.breadth_bucket.value, state.breadth_divergence,
+            state.oi_state.value, state.volatility_regime.value,
+            state.price_structure_score.score,
+            entry_price,
+            outcomes.get("result_1d"), outcomes.get("result_3d"),
+            outcomes.get("result_5d"), outcomes.get("result_7d"),
+            outcomes.get("result_14d"),
+            outcomes.get("mfe"), outcomes.get("mae"),
+            outcomes.get("is_win_7d"),
+        )).lastrowid
+
+        conn.commit()
+        conn.close()
+
+        is_win = outcomes.get("is_win_7d", 0)
+        ret_7d = outcomes.get("result_7d", 0) or 0
+        logger.info(
+            f"Recorded {signal_type} on {date} @ {entry_price:.0f} "
+            f"(regime={state.regime.value}, breadth={state.breadth_bucket.value}, "
+            f"oi={state.oi_state.value}) → 7d={ret_7d:+.1f}%"
+        )
+        return record_id
+
+    def _compute_outcomes(self, conn: sqlite3.Connection, date: Date,
+                          entry_price: float) -> dict:
+        """
+        Compute forward returns, MFE, MAE from OHLCV data.
+
+        Queries future daily bars relative to the signal date.
+        """
+        # Get future OHLCV data
+        df = pd.read_sql_query(
+            "SELECT date, high, low, close FROM ohlcv_daily "
+            "WHERE date > ? AND symbol = 'BTC/USDT:USDT' "
+            "ORDER BY date ASC LIMIT 20",
+            conn, params=(str(date),)
+        )
+
+        if df.empty:
+            return {}
+
+        outcomes = {}
+        entry = entry_price
+
+        # Forward returns
+        for horizon_days, col in [(1, "result_1d"), (3, "result_3d"),
+                                   (5, "result_5d"), (7, "result_7d"),
+                                   (14, "result_14d")]:
+            if len(df) >= horizon_days:
+                exit_price = float(df.iloc[horizon_days - 1]["close"])
+                outcomes[col] = round((exit_price - entry) / entry * 100, 2)
+
+        # MFE / MAE
+        if len(df) > 0:
+            highs = df["high"].astype(float).values[:14]
+            lows = df["low"].astype(float).values[:14]
+            outcomes["mfe"] = round((max(highs) - entry) / entry * 100, 2)
+            outcomes["mae"] = round((min(lows) - entry) / entry * 100, 2)
+
+        # is_win_7d
+        outcomes["is_win_7d"] = 1 if outcomes.get("result_7d", 0) > 0 else 0
+
+        return outcomes
+
+    def backfill_signals(self, signals: list[dict]) -> int:
+        """
+        Backfill multiple signals from historical data.
+
+        Each signal dict:
+            {"date": Date, "signal_type": str, "entry_price": float,
+             "signal_grade": str (optional), "signal_strength": float (optional)}
+
+        This requires the scoring pipeline to have been run for those dates
+        (breadth_daily, ohlcv_daily, derivatives all populated).
+        """
+        from scoring.price_structure import PriceStructureScorer
+        from scoring.breadth_scorer import BreadthScorer
+        from scoring.oi_matrix import OIMatrixScorer
+        from scoring.volatility_regime import VolatilityRegimeScorer
+        from regime_detector import RegimeDetector
+
+        detector = RegimeDetector()
+        count = 0
+
+        for sig in signals:
+            target = sig["date"]
+            try:
+                # Compute market state for this date
+                ps = PriceStructureScorer(self.db_path).compute(target)
+                br = BreadthScorer(self.db_path).compute(target)
+                oi = OIMatrixScorer(self.db_path).compute(target)
+                vol = VolatilityRegimeScorer(self.db_path).compute(target)
+
+                regime_result = detector.detect(
+                    price_structure_score=ps.score,
+                    breadth_score=br.breadth_top50,
+                    volatility_regime=vol.vol_regime.value,
+                    date=target,
+                )
+
+                state = MarketStateVector(
+                    date=target,
+                    regime=regime_result.regime,
+                    regime_confidence=regime_result.confidence,
+                    regime_version=regime_result.regime_version,
+                    regime_maturity_score=regime_result.maturity_score,
+                    breadth_top20=br.breadth_top20,
+                    breadth_top30=br.breadth_top30,
+                    breadth_top50=br.breadth_top50,
+                    breadth_bucket=br.breadth_bucket,
+                    breadth_divergence=br.breadth_divergence,
+                    oi_state=oi.oi_state,
+                    volatility_regime=vol.vol_regime,
+                    price_structure_score=ps,
+                    breadth_score=br,
+                    oi_matrix_score=oi,
+                    volatility_regime_score=vol,
+                )
+                state.market_state_hash = state.compute_hash()
+
+                self.record(
+                    date=target,
+                    signal_type=sig["signal_type"],
+                    entry_price=sig["entry_price"],
+                    state=state,
+                    signal_grade=sig.get("signal_grade"),
+                    signal_strength=sig.get("signal_strength"),
+                )
+                count += 1
+            except Exception as e:
+                logger.warning(f"Failed to backfill {sig['signal_type']} on {target}: {e}")
+
+        return count
+
+    def get_samples(self, signal_type: Optional[str] = None,
+                    regime: Optional[str] = None,
+                    breadth_bucket: Optional[str] = None,
+                    oi_state: Optional[str] = None,
+                    volatility_regime: Optional[str] = None,
+                    limit: int = 5000) -> list[dict]:
+        """Query signal_features with optional filters."""
+        conn = sqlite3.connect(self.db_path)
+        conn.row_factory = sqlite3.Row
+
+        query = "SELECT * FROM signal_features WHERE 1=1"
+        params = []
+
+        if signal_type:
+            query += " AND signal_type = ?"
+            params.append(signal_type)
+        if regime:
+            query += " AND regime = ?"
+            params.append(regime)
+        if breadth_bucket:
+            query += " AND breadth_bucket = ?"
+            params.append(breadth_bucket)
+        if oi_state:
+            query += " AND oi_state = ?"
+            params.append(oi_state)
+        if volatility_regime:
+            query += " AND volatility_regime = ?"
+            params.append(volatility_regime)
+
+        query += " ORDER BY date DESC LIMIT ?"
+        params.append(limit)
+
+        rows = conn.execute(query, params).fetchall()
+        conn.close()
+        return [dict(r) for r in rows]
+
+    def count_samples(self) -> dict:
+        """Count signal_features by signal_type and regime."""
+        conn = sqlite3.connect(self.db_path)
+        rows = conn.execute("""
+            SELECT signal_type, regime, COUNT(*) as cnt
+            FROM signal_features
+            GROUP BY signal_type, regime
+            ORDER BY signal_type, regime
+        """).fetchall()
+        conn.close()
+        return {f"{r[0]}/{r[1]}": r[2] for r in rows}