From 71951019fb8af416ca46cb1394482e3803ad8adf Mon Sep 17 00:00:00 2001 From: jackyu66git Date: Wed, 24 Jun 2026 17:44:55 +0800 Subject: [PATCH] =?UTF-8?q?chanmacro:=20Signal=20Expectancy=20Engine=20V1?= =?UTF-8?q?=20=E2=80=94=20Market=20Memory=20System?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase A-C complete: 4 core factors, regime detection, signal tracking, Bayesian expectancy. chanmacro/ (32 files, ~4000 lines): - models: 12 enums + 15 Pydantic v2 models (DateAwareModel, MarketStateVector, etc.) - fetchers: OHLCV + Breadth (from data_provider) + Derivatives (new endpoint) - scoring: Price Structure / Breadth (quantile buckets) / OI Matrix (5 discrete states) / Volatility Regime - regime_detector: 3-state (TREND/RANGE/PANIC), factor-locked (Price+Breadth+Vol), versioned, 2-day confirmation - expectancy: SignalTracker (record+outcomes), TimeDecay (half-life=180d), BayesianExpectancyEngine (Empirical Bayes, Leveled, SufficiencyGuard) - validation: FactorValidator (IC/ICIR/Hit Ratio), RegimeValidator (MI/KL/ANOVA), TransitionValidator (stability) - CLI: fetch|score|regime|track|backfill|expectancy|validate|serve - tests: 52 passing (models, scoring, regime, expectancy) data_provider: - /api/derivatives endpoint: funding rate, OI, OI change, basis - _derivatives storage: same persist pattern as K-line (merge→lock→snapshot→atomic write) - background refresh every 60s Co-Authored-By: Claude --- ChanMacro/.env.example | 12 + ChanMacro/__init__.py | 9 + ChanMacro/cli.py | 364 ++++++++++++++++++ ChanMacro/config.json | 23 ++ ChanMacro/config.py | 114 ++++++ ChanMacro/database.py | 224 +++++++++++ ChanMacro/expectancy/__init__.py | 4 + ChanMacro/expectancy/decay.py | 55 +++ ChanMacro/expectancy/engine.py | 295 +++++++++++++++ ChanMacro/expectancy/tracker.py | 271 ++++++++++++++ ChanMacro/fetchers/__init__.py | 5 + ChanMacro/fetchers/base.py | 69 ++++ ChanMacro/fetchers/breadth.py | 189 ++++++++++ ChanMacro/fetchers/derivatives.py | 66 ++++ ChanMacro/fetchers/ohlcv.py | 157 ++++++++ ChanMacro/main.py | 17 + ChanMacro/models.py | 370 +++++++++++++++++++ ChanMacro/regime_detector.py | 213 +++++++++++ ChanMacro/report/__init__.py | 0 ChanMacro/requirements.txt | 7 + ChanMacro/run_tests.sh | 11 + ChanMacro/scoring/__init__.py | 6 + ChanMacro/scoring/base.py | 28 ++ ChanMacro/scoring/breadth_scorer.py | 218 +++++++++++ ChanMacro/scoring/constants.py | 98 +++++ ChanMacro/scoring/oi_matrix.py | 137 +++++++ ChanMacro/scoring/price_structure.py | 248 +++++++++++++ ChanMacro/scoring/volatility_regime.py | 143 +++++++ ChanMacro/tests/__init__.py | 0 ChanMacro/tests/conftest.py | 134 +++++++ ChanMacro/tests/test_expectancy.py | 173 +++++++++ ChanMacro/tests/test_models.py | 130 +++++++ ChanMacro/tests/test_regime.py | 109 ++++++ ChanMacro/tests/test_scoring.py | 121 ++++++ ChanMacro/trend_detector.py | 81 ++++ ChanMacro/validation/__init__.py | 5 + ChanMacro/validation/factor_validator.py | 174 +++++++++ ChanMacro/validation/metrics.py | 192 ++++++++++ ChanMacro/validation/regime_validator.py | 144 ++++++++ ChanMacro/validation/reporter.py | 120 ++++++ ChanMacro/validation/transition_validator.py | 131 +++++++ ChanMacro/web/__init__.py | 0 data_provider/main.py | 171 ++++++++- 43 files changed, 5036 insertions(+), 2 deletions(-) create mode 100644 ChanMacro/.env.example create mode 100644 ChanMacro/__init__.py create mode 100644 ChanMacro/cli.py create mode 100644 ChanMacro/config.json create mode 100644 ChanMacro/config.py create mode 100644 ChanMacro/database.py create mode 100644 ChanMacro/expectancy/__init__.py create mode 100644 ChanMacro/expectancy/decay.py create mode 100644 ChanMacro/expectancy/engine.py create mode 100644 ChanMacro/expectancy/tracker.py create mode 100644 ChanMacro/fetchers/__init__.py create mode 100644 ChanMacro/fetchers/base.py create mode 100644 ChanMacro/fetchers/breadth.py create mode 100644 ChanMacro/fetchers/derivatives.py create mode 100644 ChanMacro/fetchers/ohlcv.py create mode 100644 ChanMacro/main.py create mode 100644 ChanMacro/models.py create mode 100644 ChanMacro/regime_detector.py create mode 100644 ChanMacro/report/__init__.py create mode 100644 ChanMacro/requirements.txt create mode 100755 ChanMacro/run_tests.sh create mode 100644 ChanMacro/scoring/__init__.py create mode 100644 ChanMacro/scoring/base.py create mode 100644 ChanMacro/scoring/breadth_scorer.py create mode 100644 ChanMacro/scoring/constants.py create mode 100644 ChanMacro/scoring/oi_matrix.py create mode 100644 ChanMacro/scoring/price_structure.py create mode 100644 ChanMacro/scoring/volatility_regime.py create mode 100644 ChanMacro/tests/__init__.py create mode 100644 ChanMacro/tests/conftest.py create mode 100644 ChanMacro/tests/test_expectancy.py create mode 100644 ChanMacro/tests/test_models.py create mode 100644 ChanMacro/tests/test_regime.py create mode 100644 ChanMacro/tests/test_scoring.py create mode 100644 ChanMacro/trend_detector.py create mode 100644 ChanMacro/validation/__init__.py create mode 100644 ChanMacro/validation/factor_validator.py create mode 100644 ChanMacro/validation/metrics.py create mode 100644 ChanMacro/validation/regime_validator.py create mode 100644 ChanMacro/validation/reporter.py create mode 100644 ChanMacro/validation/transition_validator.py create mode 100644 ChanMacro/web/__init__.py diff --git a/ChanMacro/.env.example b/ChanMacro/.env.example new file mode 100644 index 0000000..857693d --- /dev/null +++ b/ChanMacro/.env.example @@ -0,0 +1,12 @@ +# Data Provider URL (existing chan data_provider service) +PROVIDER_URL=http://127.0.0.1:80 + +# Database path +DB_PATH=data/macro.db + +# Telegram (reuse bsp_monitor config) +# TELEGRAM_BOT_TOKEN=your_bot_token +# TELEGRAM_CHAT_ID=your_chat_id + +# AI API (for daily report, Phase 5+) +# ANTHROPIC_API_KEY=sk-ant-... diff --git a/ChanMacro/__init__.py b/ChanMacro/__init__.py new file mode 100644 index 0000000..729473e --- /dev/null +++ b/ChanMacro/__init__.py @@ -0,0 +1,9 @@ +""" +ChanMacro — Crypto Market Memory System (Signal Expectancy Engine). + +V1: 4 factors (Price Structure, Breadth, OI State, Volatility Regime) + 3 regimes (TREND / RANGE / PANIC) + Factor-locked: Regime = f(Price, Breadth, Vol) — forever. +""" + +__version__ = "1.0.0" diff --git a/ChanMacro/cli.py b/ChanMacro/cli.py new file mode 100644 index 0000000..862eec0 --- /dev/null +++ b/ChanMacro/cli.py @@ -0,0 +1,364 @@ +""" +cli.py — Command-line interface for ChanMacro. +""" + +import argparse +import json +import logging +from datetime import date as Date, datetime, timedelta + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", +) +logger = logging.getLogger("chanmacro") + + +def parse_date(date_str: str) -> Date: + """Parse YYYY-MM-DD string to Date.""" + return datetime.strptime(date_str, "%Y-%m-%d").date() + + +def _build_market_state(target: Date) -> tuple: + """Shared helper: compute all scores → (MarketStateVector, RegimeResult).""" + from config import config + from scoring.price_structure import PriceStructureScorer + from scoring.breadth_scorer import BreadthScorer + from scoring.oi_matrix import OIMatrixScorer + from scoring.volatility_regime import VolatilityRegimeScorer + from regime_detector import RegimeDetector + from models import MarketStateVector + + ps = PriceStructureScorer().compute(target) + br = BreadthScorer().compute(target) + oi = OIMatrixScorer().compute(target) + vol = VolatilityRegimeScorer().compute(target) + + detector = RegimeDetector() + detector.load_state(config.db_path) + r = detector.detect(ps.score, br.breadth_top50, vol.vol_regime.value, target) + + state = MarketStateVector( + date=target, regime=r.regime, regime_confidence=r.confidence, + regime_version=r.regime_version, regime_maturity_score=r.maturity_score, + breadth_top20=br.breadth_top20, breadth_top30=br.breadth_top30, + breadth_top50=br.breadth_top50, breadth_bucket=br.breadth_bucket, + breadth_divergence=br.breadth_divergence, + oi_state=oi.oi_state, volatility_regime=vol.vol_regime, + price_structure_score=ps, breadth_score=br, + oi_matrix_score=oi, volatility_regime_score=vol, + ) + state.market_state_hash = state.compute_hash() + return state, r + + +def cmd_fetch(args): + """Fetch raw data and store to DB.""" + from database import init_db + from fetchers.ohlcv import OHLCVFetcher + from fetchers.breadth import BreadthFetcher + + target = parse_date(args.date) if args.date else Date.today() + init_db() + + module = args.module or "all" + + if module in ("ohlcv", "all"): + logger.info(f"Fetching OHLCV for {target}...") + fetcher = OHLCVFetcher() + df = fetcher.fetch(target) + if not df.empty: + n = fetcher.store_df(df) + logger.info(f"OHLCV: stored {n} rows") + + if module in ("breadth", "all"): + logger.info(f"Fetching Breadth for {target}...") + fetcher = BreadthFetcher() + record = fetcher.fetch(target) + if record: + fetcher.store(record=record) + logger.info(f"Breadth: stored (adv={record.get('advance_top50')}, " + f"dec={record.get('decline_top50')}, " + f"ema20={record.get('above_ema20_top50')})") + + if module in ("derivatives", "all"): + logger.info(f"Fetching Derivatives for {target}...") + from fetchers.derivatives import DerivativesFetcher + fetcher = DerivativesFetcher() + records = fetcher.fetch(target) + if records: + n = fetcher.store(records=records) + logger.info(f"Derivatives: stored {n} records") + + +def cmd_score(args): + """Compute all factor scores and regime for a date.""" + from database import init_db, get_connection + + target = parse_date(args.date) if args.date else Date.today() + init_db() + logger.info(f"Computing scores for {target}...") + + state, regime_result = _build_market_state(target) + + # Output + ps = state.price_structure_score + br = state.breadth_score + oi = state.oi_matrix_score + vol = state.volatility_regime_score + + print(f"\n{'='*60}") + print(f" {target} Market State") + print(f"{'='*60}") + print(f" Regime: {state.regime.value} (conf={state.regime_confidence:.2f}, " + f"v={state.regime_version})") + print(f" Maturity: {state.regime_maturity_score:.0f}/100") + print(f" Breadth: {state.breadth_bucket.value} " + f"(T20={state.breadth_top20:.0f} T30={state.breadth_top30:.0f} " + f"T50={state.breadth_top50:.0f} div={state.breadth_divergence:+.0f})") + print(f" OI State: {state.oi_state.value}") + print(f" Volatility: {state.volatility_regime.value}") + print(f"{'='*60}") + print(f" Scores:") + print(f" Price Structure: {ps.score:.0f} {ps.label}") + print(f" Breadth: {br.score:.0f} {br.breadth_bucket.value}") + print(f" OI Matrix: {oi.score:.0f} {oi.oi_state.value}") + print(f" Volatility: {vol.score:.0f} {vol.vol_regime.value}") + print(f"{'='*60}") + print(f" Market State Hash: {state.market_state_hash}") + print() + + # Store regime to DB + conn = get_connection() + conn.execute(""" + INSERT OR REPLACE INTO regime_history + (date, regime, confidence, regime_version, maturity_score, all_scores_json, + prior_regime, confirmation_days) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, ( + str(target), + state.regime.value, + state.regime_confidence, + state.regime_version, + state.regime_maturity_score, + json.dumps({k.value: v for k, v in regime_result.all_scores.items()}), + regime_result.prior_regime.value if regime_result.prior_regime else None, + regime_result.confirmation_days, + )) + conn.commit() + conn.close() + + return state + + +def cmd_regime(args): + """Show regime history.""" + from database import get_connection + days = args.days or 30 + conn = get_connection() + rows = conn.execute( + "SELECT date, regime, confidence, maturity_score, confirmation_days " + "FROM regime_history ORDER BY date DESC LIMIT ?", + (days,) + ).fetchall() + conn.close() + + print(f"\n{'='*50}") + print(f" Regime History (last {days} days)") + print(f"{'='*50}") + for r in rows: + print(f" {r['date']} {r['regime']:7s} conf={r['confidence']:.2f} " + f"mat={r['maturity_score']:.0f} days={r['confirmation_days']}") + print() + + +def cmd_track(args): + """Record a trading signal with current market state.""" + from database import init_db + from expectancy.tracker import SignalTracker + + target = parse_date(args.date) if args.date else Date.today() + init_db() + + logger.info(f"Recording {args.signal} on {target} @ {args.price}") + + state, _ = _build_market_state(target) + + tracker = SignalTracker() + rid = tracker.record( + date=target, signal_type=args.signal, entry_price=args.price, + state=state, signal_grade=args.grade, signal_strength=args.strength, + ) + logger.info(f"Signal recorded: id={rid}") + + +def cmd_backfill(args): + """Backfill historical scores and/or signals.""" + from datetime import date as Date, timedelta + from database import init_db, get_connection + from fetchers.ohlcv import OHLCVFetcher + + start = parse_date(args.from_date) + end = parse_date(args.to_date) if args.to_date else Date.today() + init_db() + + # First, backfill OHLCV data + logger.info(f"Backfilling OHLCV from {start} to {end}...") + fetcher = OHLCVFetcher() + df = fetcher.fetch() + if not df.empty: + fetcher.store_df(df) + + # Then compute scores for each date + from scoring.price_structure import PriceStructureScorer + from scoring.breadth_scorer import BreadthScorer + from scoring.oi_matrix import OIMatrixScorer + from scoring.volatility_regime import VolatilityRegimeScorer + from regime_detector import RegimeDetector + + detector = RegimeDetector() + conn = get_connection() + + current = start + count = 0 + while current <= end: + try: + ps = PriceStructureScorer().compute(current) + br = BreadthScorer().compute(current) + if br.score == 50.0 and br.label == "No Data": + current += timedelta(days=1) + continue + + oi = OIMatrixScorer().compute(current) + vol = VolatilityRegimeScorer().compute(current) + r = detector.detect(ps.score, br.breadth_top50, + vol.vol_regime.value, current) + + conn.execute(""" + INSERT OR REPLACE INTO regime_history + (date, regime, confidence, regime_version, maturity_score, + all_scores_json, confirmation_days) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, ( + str(current), r.regime.value, r.confidence, + r.regime_version, r.maturity_score, + json.dumps(r.all_scores), r.confirmation_days, + )) + count += 1 + if count % 30 == 0: + conn.commit() + logger.info(f" Backfilled {count} days... ({current})") + except Exception as e: + logger.debug(f" Skip {current}: {e}") + current += timedelta(days=1) + + conn.commit() + conn.close() + logger.info(f"Backfill complete: {count} days scored") + + +def cmd_expectancy(args): + """Query signal expectancy for current market state.""" + from database import init_db + from expectancy.engine import BayesianExpectancyEngine + + target = parse_date(args.date) if args.date else Date.today() + init_db() + + state, _ = _build_market_state(target) + + engine = BayesianExpectancyEngine() + signal = args.signal or "B3" + report = engine.estimate(state, signal_type=signal, target_date=target) + + print(f"\n{'='*60}") + print(f" {target} Signal Expectancy: {signal}") + print(f"{'='*60}") + print(f" Regime: {state.regime.value} (conf={state.regime_confidence:.2f})") + print(f" Breadth: {state.breadth_bucket.value} (T50={state.breadth_top50:.0f})") + print(f" OI State: {state.oi_state.value}") + print(f" Volatility: {state.volatility_regime.value}") + print(f"{'='*60}") + + for layer in report.layers: + print(f" {layer.name:15s} N={layer.samples:4d} eff={layer.effective_samples:.0f} " + f"raw={layer.raw_winrate or 0:.1%} post={layer.posterior_winrate:.1%} " + f"ret={layer.avg_return or 0:+.1f}%") + + print(f"{'='*60}") + print(f" Final: {report.final_estimate:.1%} " + f"(sufficiency={report.sufficiency.value}, source={report.source})") + if report.profit_factor: + print(f" PF={report.profit_factor} MAE={report.max_adverse_excursion}%") + print() + + +def main(): + parser = argparse.ArgumentParser( + description="ChanMacro — Crypto Market Memory System" + ) + sub = parser.add_subparsers(dest="command", help="Commands") + + # fetch + p_fetch = sub.add_parser("fetch", help="Fetch raw data") + p_fetch.add_argument("--date", help="Target date (YYYY-MM-DD)") + p_fetch.add_argument("--module", choices=["ohlcv", "breadth", "derivatives", "all"]) + + # score + p_score = sub.add_parser("score", help="Compute scores and regime") + p_score.add_argument("--date", help="Target date (YYYY-MM-DD)") + + # regime + p_regime = sub.add_parser("regime", help="Show regime history") + p_regime.add_argument("--days", type=int, default=30) + + # track + p_track = sub.add_parser("track", help="Record a trading signal") + p_track.add_argument("--date", help="Signal date (YYYY-MM-DD)") + p_track.add_argument("--signal", required=True, help="Signal type (B1/B2/B3/S1/S2/S3)") + p_track.add_argument("--price", type=float, required=True, help="Entry price") + p_track.add_argument("--grade", choices=["A", "B", "C"], help="Signal quality grade") + p_track.add_argument("--strength", type=float, help="Signal strength 0-100") + + # backfill + p_backfill = sub.add_parser("backfill", help="Backfill historical scores") + p_backfill.add_argument("--from", dest="from_date", required=True) + p_backfill.add_argument("--to", dest="to_date") + + # expectancy + p_expectancy = sub.add_parser("expectancy", help="Query signal expectancy") + p_expectancy.add_argument("--date", help="Target date (YYYY-MM-DD)") + p_expectancy.add_argument("--signal", default="B3", help="Signal type") + + # validate + p_validate = sub.add_parser("validate", help="Run validation framework") + + # serve + p_serve = sub.add_parser("serve", help="Start web dashboard") + + args = parser.parse_args() + + if args.command == "fetch": + cmd_fetch(args) + elif args.command == "score": + cmd_score(args) + elif args.command == "regime": + cmd_regime(args) + elif args.command == "track": + cmd_track(args) + elif args.command == "backfill": + cmd_backfill(args) + elif args.command == "expectancy": + cmd_expectancy(args) + elif args.command == "validate": + from validation.reporter import ValidationReporter + report = ValidationReporter().run_all() + print(report) + elif args.command == "serve": + logger.info("Web dashboard not yet implemented (Phase 7)") + else: + parser.print_help() + + +if __name__ == "__main__": + main() diff --git a/ChanMacro/config.json b/ChanMacro/config.json new file mode 100644 index 0000000..0c30e2c --- /dev/null +++ b/ChanMacro/config.json @@ -0,0 +1,23 @@ +{ + "provider_url": "http://127.0.0.1:80", + "db_path": "data/macro.db", + "btc_symbol": "BTC/USDT:USDT", + "regime_version": "v1_price_breadth_vol", + "half_life_days": 180, + "sufficiency_min_effective": 30, + "sufficiency_low": 50, + "sufficiency_medium": 100, + "level_min_samples": 50, + "knn_max_distance": 0.35, + "knn_k": 200, + "oi_price_threshold_pct": 0.5, + "oi_oi_threshold_pct": 0.5, + "vol_low_threshold": 2.0, + "vol_high_threshold": 5.0, + "vol_explosive_threshold": 10.0, + "regime_w_price": 0.35, + "regime_w_breadth": 0.50, + "regime_w_vol": 0.15, + "trend_w_price": 0.30, + "trend_w_breadth": 0.70 +} diff --git a/ChanMacro/config.py b/ChanMacro/config.py new file mode 100644 index 0000000..ed8e108 --- /dev/null +++ b/ChanMacro/config.py @@ -0,0 +1,114 @@ +""" +config.py — Global configuration for ChanMacro. + +All weights, thresholds, and paths are configurable. +V1 weights are deliberately simple; they will be tuned via Phase 0 validation. +""" + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + + +@dataclass +class Config: + """Global configuration. Override via config.json or env vars.""" + + # ── Paths ────────────────────────────────────────────── + db_path: str = "data/macro.db" + data_dir: str = "data" + + # ── Data Provider ────────────────────────────────────── + provider_url: str = "http://127.0.0.1:80" + btc_symbol: str = "BTC/USDT:USDT" + top50_symbols: list[str] = field(default_factory=lambda: [ + "BTC/USDT:USDT", "ETH/USDT:USDT", "SOL/USDT:USDT", + "BNB/USDT:USDT", "XRP/USDT:USDT", "DOGE/USDT:USDT", + "ADA/USDT:USDT", "AVAX/USDT:USDT", "DOT/USDT:USDT", + "LINK/USDT:USDT", "MATIC/USDT:USDT", "UNI/USDT:USDT", + "ATOM/USDT:USDT", "LTC/USDT:USDT", "ETC/USDT:USDT", + "FIL/USDT:USDT", "APT/USDT:USDT", "ARB/USDT:USDT", + "OP/USDT:USDT", "NEAR/USDT:USDT", + "INJ/USDT:USDT", "TIA/USDT:USDT", "SUI/USDT:USDT", + "SEI/USDT:USDT", "RUNE/USDT:USDT", + ]) + + # ── Breadth ──────────────────────────────────────────── + breadth_top_n: list[int] = field(default_factory=lambda: [20, 30, 50]) + breadth_ema_period: int = 20 + breadth_new_high_window: int = 20 + + # ── Regime (factor-locked: Price + Breadth + Vol) ───── + regime_version: str = "v1_price_breadth_vol" + # Weights for trend_score within regime detection + regime_w_price: float = 0.35 + regime_w_breadth: float = 0.50 + regime_w_vol: float = 0.15 + # Weights for panic_score + regime_panic_w_anti_trend: float = 0.60 + regime_panic_w_vol_extreme: float = 0.40 + + # ── Price Structure ──────────────────────────────────── + ps_ema_fast: int = 20 + ps_ema_mid: int = 60 + ps_ema_slow: int = 120 + ps_adx_period: int = 14 + ps_adx_threshold: int = 25 + ps_atr_period: int = 14 + ps_bb_period: int = 20 + ps_roc_periods: list[int] = field(default_factory=lambda: [5, 10, 20]) + + # ── OI Matrix ────────────────────────────────────────── + oi_price_threshold_pct: float = 0.5 # min price change% to classify + oi_oi_threshold_pct: float = 0.5 # min OI change% to classify + + # ── Volatility Regime ────────────────────────────────── + vol_atr_period: int = 14 + vol_hv_short: int = 20 + vol_hv_long: int = 60 + # Thresholds (ATR/Close %) + vol_low_threshold: float = 2.0 + vol_high_threshold: float = 5.0 + vol_explosive_threshold: float = 10.0 + + # ── Trend (L2 aggregation) ───────────────────────────── + trend_w_price: float = 0.30 + trend_w_breadth: float = 0.70 + + # ── Maturity Score ───────────────────────────────────── + maturity_w_trend: float = 0.50 + maturity_w_breadth: float = 0.30 + maturity_w_vol: float = 0.20 + + # ── Expectancy ───────────────────────────────────────── + half_life_days: int = 180 + sufficiency_min_effective: int = 30 + sufficiency_low: int = 50 + sufficiency_medium: int = 100 + level_min_samples: int = 50 + knn_max_distance: float = 0.35 + knn_k: int = 200 + + # ── Validation ───────────────────────────────────────── + min_history_days: int = 365 + regime_min_avg_duration: int = 5 + regime_max_flip_rate: float = 0.15 + + @classmethod + def from_json(cls, path: str = "config.json") -> "Config": + """Load config from JSON file, overriding defaults.""" + import json + config = cls() + try: + with open(path) as f: + data = json.load(f) + for key, value in data.items(): + if hasattr(config, key): + setattr(config, key, value) + except FileNotFoundError: + pass + return config + + +# Global singleton +config = Config() diff --git a/ChanMacro/database.py b/ChanMacro/database.py new file mode 100644 index 0000000..6d387c4 --- /dev/null +++ b/ChanMacro/database.py @@ -0,0 +1,224 @@ +""" +database.py — SQLite schema initialization and connection management. +""" + +import sqlite3 +import os +from pathlib import Path + +SCHEMA = """ +-- ═══════════════════════════════════════════════ +-- L0: Raw data tables +-- ═══════════════════════════════════════════════ + +CREATE TABLE IF NOT EXISTS ohlcv_daily ( + date TEXT NOT NULL, + symbol TEXT NOT NULL DEFAULT 'BTC/USDT:USDT', + open REAL, + high REAL, + low REAL, + close REAL, + volume REAL, + ema20 REAL, + ema60 REAL, + ema120 REAL, + atr_14 REAL, + bb_width REAL, + adx_14 REAL, + PRIMARY KEY (date, symbol) +); + +CREATE TABLE IF NOT EXISTS breadth_daily ( + date TEXT PRIMARY KEY, + total_tracked INTEGER DEFAULT 50, + advance_top50 INTEGER DEFAULT 0, + decline_top50 INTEGER DEFAULT 0, + above_ema20_top50 INTEGER DEFAULT 0, + new_highs_20d_top50 INTEGER DEFAULT 0, + btc_dominance REAL, + advance_top20 INTEGER DEFAULT 0, + advance_top30 INTEGER DEFAULT 0, + above_ema20_top20 INTEGER DEFAULT 0, + above_ema20_top30 INTEGER DEFAULT 0, + new_highs_20d_top20 INTEGER DEFAULT 0, + new_highs_20d_top30 INTEGER DEFAULT 0, + fetched_at TEXT DEFAULT (datetime('now')) +); + +CREATE TABLE IF NOT EXISTS derivatives ( + date TEXT NOT NULL, + symbol TEXT NOT NULL DEFAULT 'BTC/USDT:USDT', + funding_rate REAL, + open_interest REAL, + oi_24h_change_pct REAL, + long_liquidations REAL, + short_liquidations REAL, + basis_annualised_pct REAL, + source TEXT DEFAULT 'binance', + fetched_at TEXT DEFAULT (datetime('now')), + PRIMARY KEY (date, symbol) +); + +CREATE TABLE IF NOT EXISTS etf_flow ( + date TEXT NOT NULL, + product TEXT NOT NULL, + net_flow_million REAL NOT NULL, + price REAL, + source TEXT DEFAULT 'farside', + fetched_at TEXT DEFAULT (datetime('now')), + PRIMARY KEY (date, product) +); + +CREATE TABLE IF NOT EXISTS stablecoin_supply ( + date TEXT NOT NULL, + token TEXT NOT NULL, + chain TEXT NOT NULL DEFAULT 'all', + supply REAL NOT NULL, + source TEXT DEFAULT 'defillama', + fetched_at TEXT DEFAULT (datetime('now')), + PRIMARY KEY (date, token, chain) +); + +-- ═══════════════════════════════════════════════ +-- L3: Regime history +-- ═══════════════════════════════════════════════ + +CREATE TABLE IF NOT EXISTS regime_history ( + date TEXT PRIMARY KEY, + regime TEXT NOT NULL, + confidence REAL, + regime_version TEXT NOT NULL DEFAULT 'v1_price_breadth_vol', + maturity_score REAL DEFAULT 50.0, + all_scores_json TEXT DEFAULT '{}', + prior_regime TEXT, + confirmation_days INTEGER DEFAULT 1, + created_at TEXT DEFAULT (datetime('now')) +); + +-- ═══════════════════════════════════════════════ +-- ★ signal_features — THE moat +-- ═══════════════════════════════════════════════ + +CREATE TABLE IF NOT EXISTS signal_features ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + date TEXT NOT NULL, + signal_type TEXT NOT NULL, + signal_version TEXT NOT NULL DEFAULT 'b3_v1', + symbol TEXT DEFAULT 'BTC/USDT:USDT', + + -- ★★ Version control (most important fields) + regime_version TEXT NOT NULL DEFAULT 'v1_price_breadth_vol', + signal_grade TEXT, + signal_strength REAL, + + -- Market State Vector snapshot + regime TEXT NOT NULL, + regime_confidence REAL, + regime_maturity_score REAL DEFAULT 50.0, + market_state_hash TEXT, + state_embedding TEXT DEFAULT '[]', + breadth_top20 REAL, + breadth_top30 REAL, + breadth_top50 REAL, + breadth_bucket TEXT, + breadth_divergence REAL, + oi_state TEXT, + volatility_regime TEXT, + price_structure_score REAL, + + -- Chan context (V5+) + chan_trend_direction TEXT, + chan_pivot_count INTEGER, + chan_divergence_type TEXT, + + -- Outcomes + entry_price REAL, + result_1d REAL, + result_3d REAL, + result_5d REAL, + result_7d REAL, + result_14d REAL, + max_favorable_excursion REAL, + max_adverse_excursion REAL, + is_win_7d INTEGER, + + created_at TEXT DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_sf_regime ON signal_features(regime); +CREATE INDEX IF NOT EXISTS idx_sf_signal ON signal_features(signal_type); +CREATE INDEX IF NOT EXISTS idx_sf_oi_state ON signal_features(oi_state); +CREATE INDEX IF NOT EXISTS idx_sf_date ON signal_features(date); +CREATE INDEX IF NOT EXISTS idx_sf_state_hash ON signal_features(market_state_hash); +CREATE INDEX IF NOT EXISTS idx_sf_regime_version ON signal_features(regime_version); +CREATE INDEX IF NOT EXISTS idx_sf_signal_version ON signal_features(signal_version); + +-- ═══════════════════════════════════════════════ +-- Expectancy cache (raw counts, NOT posteriors) +-- ═══════════════════════════════════════════════ + +CREATE TABLE IF NOT EXISTS expectancy_cache ( + state_hash TEXT NOT NULL, + signal_type TEXT NOT NULL, + wins_weighted REAL DEFAULT 0, + losses_weighted REAL DEFAULT 0, + sum_return_7d REAL DEFAULT 0, + sum_return_sq_7d REAL DEFAULT 0, + effective_samples REAL DEFAULT 0, + sufficiency TEXT DEFAULT 'INSUFFICIENT', + updated_at TEXT DEFAULT (datetime('now')), + PRIMARY KEY (state_hash, signal_type) +); + +-- ═══════════════════════════════════════════════ +-- Similarity outcome (KNN weight learning, Phase D) +-- ═══════════════════════════════════════════════ + +CREATE TABLE IF NOT EXISTS similarity_outcome ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + state_a_hash TEXT, + state_b_hash TEXT, + distance REAL, + actual_return_gap REAL, + dimension_weights_json TEXT DEFAULT '{}', + created_at TEXT DEFAULT (datetime('now')) +); + +-- ═══════════════════════════════════════════════ +-- chan_context — Chan theory integration (V1 empty) +-- ═══════════════════════════════════════════════ + +CREATE TABLE IF NOT EXISTS chan_context ( + date TEXT NOT NULL, + timeframe TEXT NOT NULL DEFAULT '1d', + trend_direction TEXT, + trend_strength REAL, + pivot_count INTEGER, + pivot_level TEXT, + signal_type TEXT, + signal_strength REAL, + divergence_type TEXT, + chan_structure_score REAL, + alignment_score REAL, + raw_context_json TEXT DEFAULT '{}', + PRIMARY KEY (date, timeframe) +); +""" + + +def init_db(db_path: str = "data/macro.db") -> sqlite3.Connection: + """Initialize database: create directory and all tables.""" + Path(db_path).parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(db_path) + conn.executescript(SCHEMA) + conn.commit() + return conn + + +def get_connection(db_path: str = "data/macro.db") -> sqlite3.Connection: + """Get a database connection. Creates tables if first run.""" + if not os.path.exists(db_path): + return init_db(db_path) + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + return conn diff --git a/ChanMacro/expectancy/__init__.py b/ChanMacro/expectancy/__init__.py new file mode 100644 index 0000000..baffc40 --- /dev/null +++ b/ChanMacro/expectancy/__init__.py @@ -0,0 +1,4 @@ +"""Expectancy Engine — Signal tracking, Bayesian inference, time decay.""" +from .tracker import SignalTracker +from .decay import TimeDecay +from .engine import BayesianExpectancyEngine, SufficiencyGuard diff --git a/ChanMacro/expectancy/decay.py b/ChanMacro/expectancy/decay.py new file mode 100644 index 0000000..c1fd42a --- /dev/null +++ b/ChanMacro/expectancy/decay.py @@ -0,0 +1,55 @@ +""" +expectancy/decay.py — Time-weighted sample decay. + +2024 market structure ≠ 2026 market structure. +Recent samples get higher weight via exponential decay. +""" + +from datetime import date as Date +from typing import Optional +import numpy as np + + +class TimeDecay: + """Exponential time decay for sample weighting.""" + + def __init__(self, half_life_days: int = 180): + self.half_life = half_life_days + self._decay_rate = np.log(2) / half_life_days + + def weight(self, sample_date: Date, reference_date: Optional[Date] = None) -> float: + """ + Compute decay weight for a sample. + weight = exp(-days_ago * decay_rate) + """ + if reference_date is None: + reference_date = Date.today() + days = (reference_date - sample_date).days + return np.exp(-days * self._decay_rate) + + def weights(self, dates: list[Date], reference_date: Optional[Date] = None) -> np.ndarray: + """Compute decay weights for a list of dates.""" + return np.array([self.weight(d, reference_date) for d in dates]) + + def weighted_win_rate(self, wins: np.ndarray, weights: np.ndarray) -> float: + """Weighted win rate: sum(wins * weights) / sum(weights).""" + total_weight = weights.sum() + if total_weight == 0: + return 0.0 + return float((wins * weights).sum() / total_weight) + + def weighted_mean(self, values: np.ndarray, weights: np.ndarray) -> float: + """Weighted mean.""" + total_weight = weights.sum() + if total_weight == 0: + return 0.0 + return float((values * weights).sum() / total_weight) + + def effective_samples(self, weights: np.ndarray) -> float: + """Effective number of samples after decay weighting.""" + return float(weights.sum()) + + @staticmethod + def weight_at_age(days_ago: int, half_life_days: int = 180) -> float: + """Quick weight lookup for a given age in days.""" + return np.exp(-days_ago * np.log(2) / half_life_days) diff --git a/ChanMacro/expectancy/engine.py b/ChanMacro/expectancy/engine.py new file mode 100644 index 0000000..ce4e198 --- /dev/null +++ b/ChanMacro/expectancy/engine.py @@ -0,0 +1,295 @@ +""" +expectancy/engine.py — Bayesian Expectancy Engine. + +Core algorithm: + 1. LeveledExpectancy: filter layer-by-layer, stop at highest valid level + 2. Empirical Bayes prior: prior = signal's global historical winrate + 3. Dynamic Beta strength: adaptive to sample size + 4. Time decay: recent samples weighted higher (half_life=180d) + 5. SufficiencyGuard: refuse output if effective_samples < 30 + 6. KNN Fallback: similarity search when strict filtering fails (Phase D) +""" + +from datetime import date as Date +from typing import Optional +import sqlite3 +import logging + +import numpy as np +import pandas as pd + +from models import ( + MarketStateVector, ExpectancyReport, ExpectancyLayer, + SufficiencyLevel, MarketRegime, +) +from config import config +from .decay import TimeDecay + +logger = logging.getLogger(__name__) + + +class SufficiencyGuard: + """Prevents trading advice from insufficient samples.""" + + def __init__(self, min_effective: int = 30, low: int = 50, medium: int = 100): + self.MIN = min_effective + self.LOW = low + self.MEDIUM = medium + + def evaluate(self, effective_samples: float) -> SufficiencyLevel: + if effective_samples < self.MIN: + return SufficiencyLevel.INSUFFICIENT + elif effective_samples < self.LOW: + return SufficiencyLevel.LOW + elif effective_samples < self.MEDIUM: + return SufficiencyLevel.MEDIUM + return SufficiencyLevel.HIGH + + +class BayesianExpectancyEngine: + """ + Leveled Bayesian Expectancy Engine. + + Query layers from coarse to fine. Stop when effective_samples drops below threshold. + Uses Empirical Bayes prior (signal's global winrate, not fixed 50%). + """ + + # Expectancy query levels: name → WHERE clause template + LEVELS = [ + ("Base", "signal_type = '{signal}'"), + ("+ Regime", "signal_type = '{signal}' AND regime = '{regime}'"), + ("+ Breadth", "signal_type = '{signal}' AND regime = '{regime}' AND breadth_bucket = '{breadth}'"), + ("+ OI State", "signal_type = '{signal}' AND regime = '{regime}' AND breadth_bucket = '{breadth}' AND oi_state = '{oi}'"), + ("+ Volatility", "signal_type = '{signal}' AND regime = '{regime}' AND breadth_bucket = '{breadth}' AND oi_state = '{oi}' AND volatility_regime = '{vol}'"), + ] + + def __init__(self, db_path: Optional[str] = None, + half_life_days: int = 180, + level_min_samples: int = 50): + self.db_path = db_path or config.db_path + self.decay = TimeDecay(half_life_days) + self.guard = SufficiencyGuard( + min_effective=config.sufficiency_min_effective, + low=config.sufficiency_low, + medium=config.sufficiency_medium, + ) + self.level_min = level_min_samples + + def estimate(self, state: MarketStateVector, + signal_type: str = "B3", + target_date: Optional[Date] = None) -> ExpectancyReport: + """ + Compute layered Bayesian expectancy for a signal in current market state. + + Returns the estimate at the deepest level with >= level_min effective samples. + """ + if target_date is None: + target_date = Date.today() + + conn = sqlite3.connect(self.db_path) + + # Get global signal winrate for Empirical Bayes prior + global_rate = self._global_winrate(conn, signal_type) + + layers = [] + best_result = None + + for level_name, template in self.LEVELS: + where = template.format( + signal=signal_type, + regime=state.regime.value, + breadth=state.breadth_bucket.value, + oi=state.oi_state.value, + vol=state.volatility_regime.value, + ) + query = f"SELECT * FROM signal_features WHERE {where}" + df = pd.read_sql_query(query, conn) + + if df.empty: + layers.append(ExpectancyLayer( + name=level_name, posterior_winrate=0.0, + samples=0, effective_samples=0.0, + )) + continue + + # Time-weighted stats + dates_list = [Date.fromisoformat(d) for d in df["date"]] + weights = self.decay.weights(dates_list, target_date) + eff_n = self.decay.effective_samples(weights) + + wins = pd.to_numeric(df["is_win_7d"].fillna(0), errors="coerce").fillna(0).values + returns = pd.to_numeric(df["result_7d"].fillna(0), errors="coerce").fillna(0).values + + raw_wr = float(wins.mean()) if len(wins) > 0 else 0.0 + weighted_wr = self.decay.weighted_win_rate(wins, weights) + weighted_ret = self.decay.weighted_mean(returns, weights) + + # Empirical Bayes posterior + posterior = self._bayesian_posterior( + global_rate=global_rate, + wins=wins.sum(), + samples=len(df), + ) + + layer = ExpectancyLayer( + name=level_name, + posterior_winrate=round(posterior, 4), + raw_winrate=round(raw_wr, 4), + samples=len(df), + effective_samples=round(eff_n, 1), + avg_return=round(weighted_ret, 2), + ) + layers.append(layer) + + # Level-based fallback: keep going while samples sufficient + if eff_n >= self.level_min: + best_result = layer + + conn.close() + + if best_result is None and layers: + # Fallback to the deepest layer that had any samples + for layer in reversed(layers): + if layer.samples > 0: + best_result = layer + break + + if best_result is None: + return ExpectancyReport( + signal_type=signal_type, + date=target_date, + layers=layers, + final_estimate=0.0, + sufficiency=SufficiencyLevel.INSUFFICIENT, + source="insufficient", + ) + + sufficiency = self.guard.evaluate( + best_result.effective_samples + ) + + # Compute profit factor and MAE from the SAME level as best_result + profit_factor = None + avg_mae = None + if best_result and best_result.samples > 0: + # Re-query the level that produced best_result + best_level_idx = next( + i for i, l in enumerate(layers) if l.name == best_result.name + ) + where = self.LEVELS[best_level_idx][1].format( + signal=signal_type, regime=state.regime.value, + breadth=state.breadth_bucket.value, oi=state.oi_state.value, + vol=state.volatility_regime.value, + ) + query = f"SELECT result_7d, max_adverse_excursion FROM signal_features WHERE {where}" + conn2 = sqlite3.connect(self.db_path) + df_detail = pd.read_sql_query(query, conn2) + conn2.close() + if not df_detail.empty: + returns_7d = df_detail["result_7d"].dropna() + if len(returns_7d) > 0: + gains = returns_7d[returns_7d > 0].sum() + losses = abs(returns_7d[returns_7d < 0].sum()) + profit_factor = round(gains / losses, 2) if losses > 0 else None + maes = df_detail["max_adverse_excursion"].dropna() + if len(maes) > 0: + avg_mae = round(float(maes.mean()), 2) + + return ExpectancyReport( + signal_type=signal_type, + date=target_date, + layers=layers, + final_estimate=round(best_result.posterior_winrate, 4), + sufficiency=sufficiency, + prior_strength=self._prior_strength(best_result.samples), + half_life_days=self.decay.half_life, + source="bayesian", + avg_return_7d=best_result.avg_return, + profit_factor=profit_factor, + max_adverse_excursion=avg_mae, + ) + + def _global_winrate(self, conn: sqlite3.Connection, + signal_type: str) -> float: + """Get global historical winrate for a signal type (Empirical Bayes prior).""" + row = conn.execute( + "SELECT AVG(is_win_7d) as wr, COUNT(*) as cnt " + "FROM signal_features WHERE signal_type = ? AND is_win_7d IS NOT NULL", + (signal_type,) + ).fetchone() + if row and row[1] and row[1] > 0: + return float(row[0]) + return 0.50 # default: neutral + + def _prior_strength(self, samples: int) -> int: + """Dynamic prior strength based on sample count.""" + if samples < 100: + return 20 # Beta(10,10) + elif samples < 500: + return 40 # Beta(20,20) + else: + return 100 # Beta(50,50) — data dominates + + def _bayesian_posterior(self, global_rate: float, wins: float, + samples: int) -> float: + """ + Empirical Bayes posterior: prior = global signal winrate. + + posterior = (alpha + wins) / (alpha + beta + samples) + where alpha/(alpha+beta) = global_rate + """ + prior_strength = self._prior_strength(samples) + alpha = max(global_rate * prior_strength, 1.0) # floor at 1 to ensure shrinkage + beta = max((1 - global_rate) * prior_strength, 1.0) + return (alpha + wins) / (alpha + beta + samples) + + def precompute_cache(self): + """ + Precompute expectancy for all state_hashes in signal_features. + Populates expectancy_cache table with raw weighted counts (not posteriors). + """ + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + + hashes = conn.execute( + "SELECT DISTINCT market_state_hash, signal_type FROM signal_features" + ).fetchall() + + today = Date.today() + count = 0 + + for row in hashes: + h = row["market_state_hash"] + sig = row["signal_type"] + + df = pd.read_sql_query( + "SELECT date, is_win_7d, result_7d " + "FROM signal_features WHERE market_state_hash = ? AND signal_type = ?", + conn, params=(h, sig) + ) + + if df.empty: + continue + + dates_list = [Date.fromisoformat(d) for d in df["date"]] + weights = self.decay.weights(dates_list, today) + wins_w = (df["is_win_7d"].fillna(0).values * weights).sum() + losses_w = ((1 - df["is_win_7d"].fillna(0)).values * weights).sum() + ret_sum = (df["result_7d"].fillna(0).values * weights).sum() + ret_sq = ((df["result_7d"].fillna(0).values ** 2) * weights).sum() + eff_n = weights.sum() + + sufficiency = self.guard.evaluate(eff_n).value + + conn.execute(""" + INSERT OR REPLACE INTO expectancy_cache + (state_hash, signal_type, wins_weighted, losses_weighted, + sum_return_7d, sum_return_sq_7d, effective_samples, sufficiency) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, (h, sig, wins_w, losses_w, ret_sum, ret_sq, eff_n, sufficiency)) + count += 1 + + conn.commit() + conn.close() + logger.info(f"Precomputed expectancy cache: {count} state×signal combos") + return count diff --git a/ChanMacro/expectancy/tracker.py b/ChanMacro/expectancy/tracker.py new file mode 100644 index 0000000..c5cc7ea --- /dev/null +++ b/ChanMacro/expectancy/tracker.py @@ -0,0 +1,271 @@ +""" +expectancy/tracker.py — SignalTracker: records signals with full market state +and computes forward outcomes. + +This is the entry point for populating signal_features — THE moat table. +""" + +from datetime import date as Date, timedelta +from typing import Optional +import sqlite3 +import json +import logging + +import pandas as pd +import numpy as np + +from models import ( + MarketStateVector, SignalFeatureRecord, MarketRegime, + OIState, BreadthBucket, VolRegime, SignalGrade, +) +from config import config + +logger = logging.getLogger(__name__) + + +class SignalTracker: + """ + Records trading signals with full market state context. + + Usage: + tracker = SignalTracker() + tracker.record( + date=Date(2026, 6, 24), + signal_type="B3", + entry_price=96500.0, + state=market_state_vector, # from scoring pipeline + signal_grade="A", + ) + """ + + def __init__(self, db_path: Optional[str] = None): + self.db_path = db_path or config.db_path + + def record(self, date: Date, signal_type: str, entry_price: float, + state: MarketStateVector, + signal_version: str = "b3_v1", + signal_grade: Optional[str] = None, + signal_strength: Optional[float] = None) -> int: + """ + Record a signal with market state snapshot and compute forward outcomes. + + Returns the record ID in signal_features. + """ + conn = sqlite3.connect(self.db_path) + + # Compute forward outcomes + outcomes = self._compute_outcomes(conn, date, entry_price) + + # Build embedding + embedding = json.dumps(state.state_embedding()) + + record_id = conn.execute(""" + INSERT INTO signal_features + (date, signal_type, signal_version, symbol, + regime_version, signal_grade, signal_strength, + regime, regime_confidence, regime_maturity_score, + market_state_hash, state_embedding, + breadth_top20, breadth_top30, breadth_top50, + breadth_bucket, breadth_divergence, + oi_state, volatility_regime, price_structure_score, + entry_price, + result_1d, result_3d, result_5d, result_7d, result_14d, + max_favorable_excursion, max_adverse_excursion, + is_win_7d) + VALUES (?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, + ?, ?, + ?, ?, ?, + ?, ?, + ?, ?, ?, + ?, + ?, ?, ?, ?, ?, + ?, ?, + ?) + """, ( + str(date), signal_type, signal_version, state.symbol, + state.regime_version, signal_grade, signal_strength, + state.regime.value, state.regime_confidence, state.regime_maturity_score, + state.market_state_hash, embedding, + state.breadth_top20, state.breadth_top30, state.breadth_top50, + state.breadth_bucket.value, state.breadth_divergence, + state.oi_state.value, state.volatility_regime.value, + state.price_structure_score.score, + entry_price, + outcomes.get("result_1d"), outcomes.get("result_3d"), + outcomes.get("result_5d"), outcomes.get("result_7d"), + outcomes.get("result_14d"), + outcomes.get("mfe"), outcomes.get("mae"), + outcomes.get("is_win_7d"), + )).lastrowid + + conn.commit() + conn.close() + + is_win = outcomes.get("is_win_7d", 0) + ret_7d = outcomes.get("result_7d", 0) or 0 + logger.info( + f"Recorded {signal_type} on {date} @ {entry_price:.0f} " + f"(regime={state.regime.value}, breadth={state.breadth_bucket.value}, " + f"oi={state.oi_state.value}) → 7d={ret_7d:+.1f}%" + ) + return record_id + + def _compute_outcomes(self, conn: sqlite3.Connection, date: Date, + entry_price: float) -> dict: + """ + Compute forward returns, MFE, MAE from OHLCV data. + + Queries future daily bars relative to the signal date. + """ + # Get future OHLCV data + df = pd.read_sql_query( + "SELECT date, high, low, close FROM ohlcv_daily " + "WHERE date > ? AND symbol = 'BTC/USDT:USDT' " + "ORDER BY date ASC LIMIT 20", + conn, params=(str(date),) + ) + + if df.empty: + return {} + + outcomes = {} + entry = entry_price + + # Forward returns + for horizon_days, col in [(1, "result_1d"), (3, "result_3d"), + (5, "result_5d"), (7, "result_7d"), + (14, "result_14d")]: + if len(df) >= horizon_days: + exit_price = float(df.iloc[horizon_days - 1]["close"]) + outcomes[col] = round((exit_price - entry) / entry * 100, 2) + + # MFE / MAE + if len(df) > 0: + highs = df["high"].astype(float).values[:14] + lows = df["low"].astype(float).values[:14] + outcomes["mfe"] = round((max(highs) - entry) / entry * 100, 2) + outcomes["mae"] = round((min(lows) - entry) / entry * 100, 2) + + # is_win_7d + outcomes["is_win_7d"] = 1 if outcomes.get("result_7d", 0) > 0 else 0 + + return outcomes + + def backfill_signals(self, signals: list[dict]) -> int: + """ + Backfill multiple signals from historical data. + + Each signal dict: + {"date": Date, "signal_type": str, "entry_price": float, + "signal_grade": str (optional), "signal_strength": float (optional)} + + This requires the scoring pipeline to have been run for those dates + (breadth_daily, ohlcv_daily, derivatives all populated). + """ + from scoring.price_structure import PriceStructureScorer + from scoring.breadth_scorer import BreadthScorer + from scoring.oi_matrix import OIMatrixScorer + from scoring.volatility_regime import VolatilityRegimeScorer + from regime_detector import RegimeDetector + + detector = RegimeDetector() + count = 0 + + for sig in signals: + target = sig["date"] + try: + # Compute market state for this date + ps = PriceStructureScorer(self.db_path).compute(target) + br = BreadthScorer(self.db_path).compute(target) + oi = OIMatrixScorer(self.db_path).compute(target) + vol = VolatilityRegimeScorer(self.db_path).compute(target) + + regime_result = detector.detect( + price_structure_score=ps.score, + breadth_score=br.breadth_top50, + volatility_regime=vol.vol_regime.value, + date=target, + ) + + state = MarketStateVector( + date=target, + regime=regime_result.regime, + regime_confidence=regime_result.confidence, + regime_version=regime_result.regime_version, + regime_maturity_score=regime_result.maturity_score, + breadth_top20=br.breadth_top20, + breadth_top30=br.breadth_top30, + breadth_top50=br.breadth_top50, + breadth_bucket=br.breadth_bucket, + breadth_divergence=br.breadth_divergence, + oi_state=oi.oi_state, + volatility_regime=vol.vol_regime, + price_structure_score=ps, + breadth_score=br, + oi_matrix_score=oi, + volatility_regime_score=vol, + ) + state.market_state_hash = state.compute_hash() + + self.record( + date=target, + signal_type=sig["signal_type"], + entry_price=sig["entry_price"], + state=state, + signal_grade=sig.get("signal_grade"), + signal_strength=sig.get("signal_strength"), + ) + count += 1 + except Exception as e: + logger.warning(f"Failed to backfill {sig['signal_type']} on {target}: {e}") + + return count + + def get_samples(self, signal_type: Optional[str] = None, + regime: Optional[str] = None, + breadth_bucket: Optional[str] = None, + oi_state: Optional[str] = None, + volatility_regime: Optional[str] = None, + limit: int = 5000) -> list[dict]: + """Query signal_features with optional filters.""" + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + + query = "SELECT * FROM signal_features WHERE 1=1" + params = [] + + if signal_type: + query += " AND signal_type = ?" + params.append(signal_type) + if regime: + query += " AND regime = ?" + params.append(regime) + if breadth_bucket: + query += " AND breadth_bucket = ?" + params.append(breadth_bucket) + if oi_state: + query += " AND oi_state = ?" + params.append(oi_state) + if volatility_regime: + query += " AND volatility_regime = ?" + params.append(volatility_regime) + + query += " ORDER BY date DESC LIMIT ?" + params.append(limit) + + rows = conn.execute(query, params).fetchall() + conn.close() + return [dict(r) for r in rows] + + def count_samples(self) -> dict: + """Count signal_features by signal_type and regime.""" + conn = sqlite3.connect(self.db_path) + rows = conn.execute(""" + SELECT signal_type, regime, COUNT(*) as cnt + FROM signal_features + GROUP BY signal_type, regime + ORDER BY signal_type, regime + """).fetchall() + conn.close() + return {f"{r[0]}/{r[1]}": r[2] for r in rows} diff --git a/ChanMacro/fetchers/__init__.py b/ChanMacro/fetchers/__init__.py new file mode 100644 index 0000000..da06603 --- /dev/null +++ b/ChanMacro/fetchers/__init__.py @@ -0,0 +1,5 @@ +"""Data fetchers — L0 raw data acquisition.""" +from .base import BaseFetcher +from .ohlcv import OHLCVFetcher +from .breadth import BreadthFetcher +from .derivatives import DerivativesFetcher diff --git a/ChanMacro/fetchers/base.py b/ChanMacro/fetchers/base.py new file mode 100644 index 0000000..21f2be5 --- /dev/null +++ b/ChanMacro/fetchers/base.py @@ -0,0 +1,69 @@ +""" +fetchers/base.py — Abstract base class for all macro data fetchers. + +Provides retry logic, rate limiting, and a common interface. +""" + +from abc import ABC, abstractmethod +from datetime import date as Date +from typing import Optional +import logging +import time +import requests + + +class BaseFetcher(ABC): + """Abstract base for all macro data fetchers.""" + + def __init__(self, timeout: int = 30, max_retries: int = 3): + self.timeout = timeout + self.max_retries = max_retries + self.logger = logging.getLogger(self.__class__.__name__) + + def _get(self, url: str, params: Optional[dict] = None, + headers: Optional[dict] = None) -> dict: + """GET with retry and exponential backoff.""" + for attempt in range(self.max_retries): + try: + resp = requests.get( + url, params=params, headers=headers, timeout=self.timeout + ) + resp.raise_for_status() + return resp.json() + except requests.RequestException as e: + wait = 2 ** attempt + self.logger.warning( + f"Request failed (attempt {attempt+1}/{self.max_retries}): {e}. " + f"Retrying in {wait}s" + ) + if attempt < self.max_retries - 1: + time.sleep(wait) + else: + raise + + def _get_raw(self, url: str, params: Optional[dict] = None, + headers: Optional[dict] = None) -> bytes: + """GET raw bytes with retry (for non-JSON endpoints).""" + for attempt in range(self.max_retries): + try: + resp = requests.get( + url, params=params, headers=headers, timeout=self.timeout + ) + resp.raise_for_status() + return resp.content + except requests.RequestException as e: + wait = 2 ** attempt + if attempt < self.max_retries - 1: + time.sleep(wait) + else: + raise + + @abstractmethod + def fetch(self, target_date: Optional[Date] = None) -> list[dict]: + """Fetch raw data. Returns list of record dicts.""" + ... + + @abstractmethod + def store(self, db_path: str, records: list[dict]) -> int: + """Store raw records into SQLite. Returns count of new rows.""" + ... diff --git a/ChanMacro/fetchers/breadth.py b/ChanMacro/fetchers/breadth.py new file mode 100644 index 0000000..642b66a --- /dev/null +++ b/ChanMacro/fetchers/breadth.py @@ -0,0 +1,189 @@ +""" +fetchers/breadth.py — Fetches TOP50 OHLCV and computes market breadth metrics. + +Multi-tier: Top20 / Top30 / Top50 for advance/decline, EMA20%, new highs, BTC.D. +""" + +from datetime import date as Date, datetime +from typing import Optional +import logging + +import pandas as pd +import numpy as np +import requests + +from .base import BaseFetcher +from config import config + + +class BreadthFetcher(BaseFetcher): + """Fetches TOP50 coin OHLCV data and computes breadth metrics.""" + + def __init__(self, provider_url: Optional[str] = None): + super().__init__(timeout=60, max_retries=3) + self.provider_url = provider_url or config.provider_url + self.symbols = config.top50_symbols + self.ema_period = config.breadth_ema_period + self.new_high_window = config.breadth_new_high_window + self.logger = logging.getLogger(__name__) + + def fetch(self, target_date: Optional[Date] = None) -> dict: + """ + Fetch daily OHLCV for all TOP50 symbols and compute breadth. + + Returns a dict suitable for storing in breadth_daily table. + """ + if target_date is None: + target_date = Date.today() + + # Fetch last 60 days of daily data for each symbol to compute EMAs and new highs + all_data = {} + for symbol in self.symbols: + try: + df = self._fetch_symbol(symbol) + if df is not None and not df.empty: + all_data[symbol] = df + except Exception as e: + self.logger.debug(f"Failed to fetch {symbol}: {e}") + + if not all_data: + self.logger.error("No symbol data fetched for breadth") + return {} + + # Compute breadth metrics for the target date + breadth = self._compute_breadth(all_data, target_date) + return breadth + + def _fetch_symbol(self, symbol: str) -> Optional[pd.DataFrame]: + """Fetch daily OHLCV for a single symbol.""" + url = f"{self.provider_url}/api/candles" + params = { + "symbol": symbol, + "tf": "1d", + "limit": 100, + } + try: + resp = requests.get(url, params=params, timeout=15) + resp.raise_for_status() + data = resp.json() + if not data: + return None + + df = pd.DataFrame(data) + df["timestamp"] = pd.to_datetime(df["timestamp"], unit="ms", utc=True) + df["date"] = df["timestamp"].dt.date + df = df.drop_duplicates(subset="date").sort_values("date").reset_index(drop=True) + df["close"] = df["close"].astype(float) + df["ema20"] = df["close"].ewm(span=self.ema_period, adjust=False).mean() + return df + except Exception: + return None + + def _compute_breadth(self, all_data: dict, target_date: Date) -> dict: + """Compute breadth metrics for a specific date across all symbols.""" + total = len(all_data) + + advances_50 = declines_50 = 0 + above_ema20_50 = 0 + new_highs_50 = 0 + advances_30 = declines_30 = 0 + above_ema20_30 = 0 + new_highs_30 = 0 + advances_20 = declines_20 = 0 + above_ema20_20 = 0 + new_highs_20 = 0 + + for i, (symbol, df) in enumerate(all_data.items()): + # Get data for target date + df["date_str"] = df["date"].astype(str) + target_str = str(target_date) + idx = df[df["date_str"] == target_str].index + + if len(idx) == 0: + continue + + row_idx = idx[0] + if row_idx < 1: + continue + + current_close = df.loc[row_idx, "close"] + prev_close = df.loc[row_idx - 1, "close"] + + # Advance/Decline + if current_close > prev_close: + if i < 50: advances_50 += 1 + if i < 30: advances_30 += 1 + if i < 20: advances_20 += 1 + elif current_close < prev_close: + if i < 50: declines_50 += 1 + if i < 30: declines_30 += 1 + if i < 20: declines_20 += 1 + + # Above EMA20 + ema20_val = df.loc[row_idx, "ema20"] + if not pd.isna(ema20_val) and current_close > ema20_val: + if i < 50: above_ema20_50 += 1 + if i < 30: above_ema20_30 += 1 + if i < 20: above_ema20_20 += 1 + + # New 20-day highs + lookback_start = max(0, row_idx - self.new_high_window) + recent_highs = df.loc[lookback_start:row_idx - 1, "high"].astype(float) + current_high = df.loc[row_idx, "high"] + if len(recent_highs) > 0 and float(current_high) > recent_highs.max(): + if i < 50: new_highs_50 += 1 + if i < 30: new_highs_30 += 1 + if i < 20: new_highs_20 += 1 + + return { + "date": str(target_date), + "total_tracked": total, + "advance_top50": advances_50, + "decline_top50": declines_50, + "above_ema20_top50": above_ema20_50, + "new_highs_20d_top50": new_highs_50, + "advance_top30": advances_30, + "advance_top20": advances_20, + "above_ema20_top30": above_ema20_30, + "above_ema20_top20": above_ema20_20, + "new_highs_20d_top30": new_highs_30, + "new_highs_20d_top20": new_highs_20, + "btc_dominance": None, # Reserved for Coinglass API integration + } + + def store(self, db_path: Optional[str] = None, record: Optional[dict] = None) -> int: + """Store a breadth record into SQLite. Returns 1 if inserted/updated.""" + import sqlite3 + db_path = db_path or config.db_path + conn = sqlite3.connect(db_path) + + if record is None: + conn.close() + return 0 + + try: + conn.execute(""" + INSERT OR REPLACE INTO breadth_daily + (date, total_tracked, + advance_top50, decline_top50, above_ema20_top50, new_highs_20d_top50, + advance_top30, advance_top20, + above_ema20_top30, above_ema20_top20, + new_highs_20d_top30, new_highs_20d_top20, + btc_dominance) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, ( + record["date"], record.get("total_tracked", 50), + record.get("advance_top50", 0), record.get("decline_top50", 0), + record.get("above_ema20_top50", 0), record.get("new_highs_20d_top50", 0), + record.get("advance_top30", 0), record.get("advance_top20", 0), + record.get("above_ema20_top30", 0), record.get("above_ema20_top20", 0), + record.get("new_highs_20d_top30", 0), record.get("new_highs_20d_top20", 0), + record.get("btc_dominance"), + )) + conn.commit() + return 1 + except Exception as e: + self.logger.error(f"Failed to store breadth: {e}") + return 0 + finally: + conn.close() diff --git a/ChanMacro/fetchers/derivatives.py b/ChanMacro/fetchers/derivatives.py new file mode 100644 index 0000000..01a768b --- /dev/null +++ b/ChanMacro/fetchers/derivatives.py @@ -0,0 +1,66 @@ +""" +fetchers/derivatives.py — Fetches derivatives data from data_provider API. + +Clean consumer: no direct ccxt dependency. Just HTTP GET /api/derivatives. +""" + +from datetime import date as Date +from typing import Optional + +import requests + +from .base import BaseFetcher +from config import config + + +class DerivativesFetcher(BaseFetcher): + """Fetches derivatives snapshot from data_provider /api/derivatives.""" + + def __init__(self, provider_url: Optional[str] = None): + super().__init__(timeout=15, max_retries=3) + self.provider_url = provider_url or config.provider_url + + def fetch(self, target_date: Optional[Date] = None) -> list[dict]: + """Fetch derivatives data. Returns list with one record dict.""" + url = f"{self.provider_url}/api/derivatives" + params = {"symbol": config.btc_symbol} + try: + data = self._get(url, params=params) + record = { + "date": str(target_date or Date.today()), + "symbol": config.btc_symbol, + "funding_rate": data.get("funding_rate"), + "open_interest": data.get("open_interest"), + "oi_24h_change_pct": data.get("oi_change_pct"), + "basis_annualised_pct": data.get("basis"), + "source": "data_provider", + } + return [record] + except Exception: + return [] + + def store(self, db_path: Optional[str] = None, records: Optional[list[dict]] = None) -> int: + """Store derivatives records into SQLite.""" + import sqlite3 + db_path = db_path or config.db_path + records = records or [] + conn = sqlite3.connect(db_path) + count = 0 + for r in records: + try: + conn.execute(""" + INSERT OR REPLACE INTO derivatives + (date, symbol, funding_rate, open_interest, oi_24h_change_pct, + long_liquidations, short_liquidations, basis_annualised_pct) + VALUES (?, ?, ?, ?, ?, NULL, NULL, ?) + """, ( + r["date"], r.get("symbol", config.btc_symbol), + r.get("funding_rate"), r.get("open_interest"), + r.get("oi_24h_change_pct"), r.get("basis_annualised_pct"), + )) + count += 1 + except Exception: + continue + conn.commit() + conn.close() + return count diff --git a/ChanMacro/fetchers/ohlcv.py b/ChanMacro/fetchers/ohlcv.py new file mode 100644 index 0000000..4847b8a --- /dev/null +++ b/ChanMacro/fetchers/ohlcv.py @@ -0,0 +1,157 @@ +""" +fetchers/ohlcv.py — Fetches BTC daily OHLCV from the existing data_provider service. + +Also pre-computes EMA20/60/120, ATR(14), BB width, ADX(14). +""" + +from datetime import date as Date, datetime, timedelta +from typing import Optional +import logging + +import pandas as pd +import numpy as np +import requests + +from .base import BaseFetcher +from config import config + + +class OHLCVFetcher(BaseFetcher): + """Fetches BTC daily K-line data from data_provider API.""" + + def __init__(self, provider_url: Optional[str] = None): + super().__init__(timeout=30, max_retries=3) + self.provider_url = provider_url or config.provider_url + self.symbol = config.btc_symbol + self.logger = logging.getLogger(__name__) + + def fetch(self, target_date: Optional[Date] = None) -> pd.DataFrame: + """ + Fetch daily OHLCV for BTC. Returns DataFrame with computed indicators. + + Fetches enough history (200 bars) to compute EMAs/ATR/BB/ADX accurately. + """ + url = f"{self.provider_url}/api/candles" + params = { + "symbol": self.symbol, + "tf": "1d", + "limit": 200, + } + resp = requests.get(url, params=params, timeout=self.timeout) + resp.raise_for_status() + data = resp.json() + + if not data: + self.logger.warning("OHLCV API returned empty data") + return pd.DataFrame() + + df = pd.DataFrame(data) + df["timestamp"] = pd.to_datetime(df["timestamp"], unit="ms", utc=True) + df["date"] = df["timestamp"].dt.date + df = df.drop_duplicates(subset="date").sort_values("date").reset_index(drop=True) + + # Rename columns to match expected format + df = df.rename(columns={ + "open": "open", "high": "high", "low": "low", "close": "close", + "volume": "volume", + }) + + # Compute indicators + df = self._add_indicators(df) + + return df + + def _add_indicators(self, df: pd.DataFrame) -> pd.DataFrame: + """Add EMA, ATR, BB, ADX indicators.""" + close = df["close"].astype(float) + high = df["high"].astype(float) + low = df["low"].astype(float) + + # EMAs + df["ema20"] = close.ewm(span=20, adjust=False).mean() + df["ema60"] = close.ewm(span=60, adjust=False).mean() + df["ema120"] = close.ewm(span=120, adjust=False).mean() + + # ATR(14) + tr1 = high - low + tr2 = (high - close.shift(1)).abs() + tr3 = (low - close.shift(1)).abs() + tr = pd.concat([tr1, tr2, tr3], axis=1).max(axis=1) + df["atr_14"] = tr.rolling(14).mean() + + # Bollinger Bands width + sma20 = close.rolling(20).mean() + std20 = close.rolling(20).std() + df["bb_width"] = (2 * std20) / sma20 * 100 # as percentage + + # ADX(14) + df["adx_14"] = self._compute_adx(df, period=14) + + return df + + @staticmethod + def _compute_adx(df: pd.DataFrame, period: int = 14) -> pd.Series: + """Compute ADX from OHLC data.""" + high = df["high"].astype(float) + low = df["low"].astype(float) + close = df["close"].astype(float) + + plus_dm = high.diff() + minus_dm = low.diff().abs() * -1 + plus_dm = plus_dm.where(plus_dm > 0, 0) + minus_dm = minus_dm.where(minus_dm < 0, 0).abs() + + tr1 = high - low + tr2 = (high - close.shift(1)).abs() + tr3 = (low - close.shift(1)).abs() + tr = pd.concat([tr1, tr2, tr3], axis=1).max(axis=1) + + atr = tr.rolling(period).mean() + plus_di = 100 * (plus_dm.rolling(period).mean() / atr) + minus_di = 100 * (minus_dm.rolling(period).mean() / atr) + + dx = (abs(plus_di - minus_di) / (plus_di + minus_di)) * 100 + adx = dx.rolling(period).mean() + return adx + + def store(self, db_path: str, records: list[dict]) -> int: + """Store OHLCV records into SQLite. Not used directly — see store_df.""" + return 0 + + def store_df(self, df: pd.DataFrame, db_path: Optional[str] = None) -> int: + """Store the DataFrame into the ohlcv_daily table.""" + import sqlite3 + db_path = db_path or config.db_path + conn = sqlite3.connect(db_path) + + count = 0 + for _, row in df.iterrows(): + if pd.isna(row.get("date")): + continue + date_str = str(row["date"]) + try: + conn.execute(""" + INSERT OR REPLACE INTO ohlcv_daily + (date, symbol, open, high, low, close, volume, + ema20, ema60, ema120, atr_14, bb_width, adx_14) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, ( + date_str, self.symbol, + float(row["open"]), float(row["high"]), + float(row["low"]), float(row["close"]), + float(row.get("volume", 0)), + float(row["ema20"]) if not pd.isna(row.get("ema20")) else None, + float(row["ema60"]) if not pd.isna(row.get("ema60")) else None, + float(row["ema120"]) if not pd.isna(row.get("ema120")) else None, + float(row["atr_14"]) if not pd.isna(row.get("atr_14")) else None, + float(row["bb_width"]) if not pd.isna(row.get("bb_width")) else None, + float(row["adx_14"]) if not pd.isna(row.get("adx_14")) else None, + )) + count += 1 + except Exception as e: + self.logger.debug(f"Skip row {date_str}: {e}") + + conn.commit() + conn.close() + self.logger.info(f"Stored {count} OHLCV rows") + return count diff --git a/ChanMacro/main.py b/ChanMacro/main.py new file mode 100644 index 0000000..4bdcf72 --- /dev/null +++ b/ChanMacro/main.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +""" +main.py — ChanMacro entry point. + +CLI: python main.py fetch|score|regime|serve +""" + +import sys +import os + +# Ensure package root is on path +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from cli import main + +if __name__ == "__main__": + main() diff --git a/ChanMacro/models.py b/ChanMacro/models.py new file mode 100644 index 0000000..e31f488 --- /dev/null +++ b/ChanMacro/models.py @@ -0,0 +1,370 @@ +""" +models.py — Pydantic v2 models and enums for ChanMacro. + +All market state types, factor scores, and database record models. +""" + +from datetime import date as Date +from enum import Enum +from typing import Optional + +from pydantic import BaseModel, Field, field_validator + + +# ═══════════════════════════════════════════════════════════════ +# Shared validators +# ═══════════════════════════════════════════════════════════════ + +def _parse_date(v): + """Reusable date-string parser for field_validator.""" + if isinstance(v, str): + return Date.fromisoformat(v) + return v + + +# ═══════════════════════════════════════════════════════════════ +# Enums +# ═══════════════════════════════════════════════════════════════ + +class MarketRegime(str, Enum): + """V1: 3-state regime (factor-locked: Price + Breadth + Vol).""" + TREND = "TREND" + RANGE = "RANGE" + PANIC = "PANIC" + + +class OIState(str, Enum): + """Discrete OI × Price state machine. NOT compressed into a score.""" + NEW_LONGS = "New Longs" + SHORT_COVERING = "Short Covering" + NEW_SHORTS = "New Shorts" + LONG_EXIT = "Long Exit" + NEUTRAL = "Neutral" + + +class BreadthBucket(str, Enum): + """Quantile-based breadth buckets — always have samples regardless of cycle.""" + EXTREME = "EXTREME" + STRONG = "STRONG" + NORMAL = "NORMAL" + WEAK = "WEAK" + PANIC = "PANIC" + + +class VolRegime(str, Enum): + """Volatility regime classification.""" + LOW_VOL = "LOW_VOL" + NORMAL_VOL = "NORMAL_VOL" + HIGH_VOL = "HIGH_VOL" + EXPLOSIVE_VOL = "EXPLOSIVE_VOL" + + +class MacroDirection(str, Enum): + BULLISH = "bullish" + NEUTRAL = "neutral" + BEARISH = "bearish" + + +class MarketEmotion(str, Enum): + EXTREME_FEAR = "Extreme Fear" + FEAR = "Fear" + NEUTRAL = "Neutral" + GREED = "Greed" + EXTREME_GREED = "Extreme Greed" + + +class FlowState(str, Enum): + STRONG_INFLOW = "Strong Inflow" + INFLOW = "Inflow" + NEUTRAL = "Neutral" + OUTFLOW = "Outflow" + STRONG_OUTFLOW = "Strong Outflow" + + +class CapitalState(str, Enum): + ENTERING = "Entering" + STABLE = "Stable" + EXITING = "Exiting" + + +class SufficiencyLevel(str, Enum): + HIGH = "HIGH" + MEDIUM = "MEDIUM" + LOW = "LOW" + INSUFFICIENT = "INSUFFICIENT" + + +class SignalGrade(str, Enum): + A = "A" + B = "B" + C = "C" + + +# ═══════════════════════════════════════════════════════════════ +# L0: Raw Data Models +# ═══════════════════════════════════════════════════════════════ + +class OHLCVDaily(BaseModel): + _parse_date = field_validator("date", mode="before")(_parse_date) + date: Date + symbol: str + open: float + high: float + low: float + close: float + volume: float + ema20: Optional[float] = None + ema60: Optional[float] = None + ema120: Optional[float] = None + atr_14: Optional[float] = None + bb_width: Optional[float] = None + adx_14: Optional[float] = None + + +class BreadthRecord(BaseModel): + _parse_date = field_validator("date", mode="before")(_parse_date) + date: Date + total_tracked: int = 50 + advance_top50: int = 0 + decline_top50: int = 0 + above_ema20_top50: int = 0 + new_highs_20d_top50: int = 0 + btc_dominance: Optional[float] = None + advance_top20: int = 0 + advance_top30: int = 0 + above_ema20_top20: int = 0 + above_ema20_top30: int = 0 + new_highs_20d_top20: int = 0 + new_highs_20d_top30: int = 0 + + +class DerivativesRecord(BaseModel): + _parse_date = field_validator("date", mode="before")(_parse_date) + date: Date + symbol: str = "BTC/USDT:USDT" + funding_rate: Optional[float] = None + open_interest: Optional[float] = None + oi_24h_change_pct: Optional[float] = None + long_liquidations: Optional[float] = None + short_liquidations: Optional[float] = None + basis_annualised_pct: Optional[float] = None + source: str = "binance" + + +class ETFFlowRecord(BaseModel): + _parse_date = field_validator("date", mode="before")(_parse_date) + date: Date + product: str + net_flow_million: float + price: Optional[float] = None + source: str = "farside" + + +class StablecoinSupplyRecord(BaseModel): + _parse_date = field_validator("date", mode="before")(_parse_date) + date: Date + token: str + chain: str = "all" + supply: float + source: str = "defillama" + + +# ═══════════════════════════════════════════════════════════════ +# L1: Factor Score Models +# ═══════════════════════════════════════════════════════════════ + +class FactorScore(BaseModel): + """Single factor scoring output.""" + name: str = "" + score: float = Field(default=50.0, ge=0.0, le=100.0) + label: str = "" + direction: MacroDirection = MacroDirection.NEUTRAL + sub_scores: dict = Field(default_factory=dict) + narrative: str = "" + + +class PriceStructureScore(FactorScore): + """Price Structure — 3 sub-dimensions.""" + trend_strength: float = 0.0 + volatility_compression: float = 0.0 + momentum: float = 0.0 + + +class BreadthScore(FactorScore): + """Breadth — multi-tier market diffusion.""" + breadth_top20: float = 0.0 + breadth_top30: float = 0.0 + breadth_top50: float = 0.0 + breadth_bucket: BreadthBucket = BreadthBucket.NORMAL + breadth_divergence: float = 0.0 + advance_pct_top50: float = 0.0 + above_ema20_pct_top50: float = 0.0 + new_highs_top50: int = 0 + btc_dominance_7d_chg: Optional[float] = None + + +class OIMatrixScore(FactorScore): + """OI Matrix — discrete state + continuous score.""" + oi_state: OIState = OIState.NEUTRAL + price_change_pct: float = 0.0 + oi_change_pct: float = 0.0 + + +class VolatilityRegimeScore(FactorScore): + """Volatility regime classification.""" + vol_regime: VolRegime = VolRegime.NORMAL_VOL + atr_pct: float = 0.0 + hv_ratio: float = 1.0 + bb_width_ratio: float = 1.0 + + +# ═══════════════════════════════════════════════════════════════ +# L4: Market State Vector (the final product) +# ═══════════════════════════════════════════════════════════════ + +class MarketStateVector(BaseModel): + """L4: Complete market state description. NOT compressed into one number.""" + _parse_date = field_validator("date", mode="before")(_parse_date) + + date: Date + symbol: str = "BTC/USDT:USDT" + + regime: MarketRegime + regime_confidence: float = Field(ge=0.0, le=1.0) + regime_version: str + regime_maturity_score: float = Field(ge=0.0, le=100.0, default=50.0) + + breadth_top20: float = Field(default=50.0, ge=0.0, le=100.0) + breadth_top30: float = Field(default=50.0, ge=0.0, le=100.0) + breadth_top50: float = Field(default=50.0, ge=0.0, le=100.0) + breadth_bucket: BreadthBucket = BreadthBucket.NORMAL + breadth_divergence: float = 0.0 + + oi_state: OIState = OIState.NEUTRAL + volatility_regime: VolRegime = VolRegime.NORMAL_VOL + + price_structure_score: FactorScore = Field(default_factory=FactorScore) + breadth_score: BreadthScore = Field(default_factory=BreadthScore) + oi_matrix_score: OIMatrixScore = Field(default_factory=OIMatrixScore) + volatility_regime_score: VolatilityRegimeScore = Field(default_factory=VolatilityRegimeScore) + + market_state_hash: str = "" + + def compute_hash(self) -> str: + import hashlib + key = f"{self.regime.value}|{self.breadth_bucket.value}|{self.oi_state.value}|{self.volatility_regime.value}" + return hashlib.md5(key.encode()).hexdigest()[:12] + + def state_embedding(self) -> list[float]: + return [ + self.breadth_top20, + self.breadth_top30, + self.breadth_top50, + self.regime_maturity_score, + self.price_structure_score.score, + ] + + +# ═══════════════════════════════════════════════════════════════ +# Factor Contribution +# ═══════════════════════════════════════════════════════════════ + +class FactorContribution(BaseModel): + """How much a factor contributed to the overall score.""" + factor: str + raw_score: float + weight: float + impact: float + direction: str # 'bullish' / 'bearish' / 'neutral' + + +# ═══════════════════════════════════════════════════════════════ +# Regime Result +# ═══════════════════════════════════════════════════════════════ + +class RegimeResult(BaseModel): + _parse_date = field_validator("date", mode="before")(_parse_date) + date: Date + regime: MarketRegime + confidence: float + regime_version: str + maturity_score: float + all_scores: dict = Field(default_factory=dict) + prior_regime: Optional[MarketRegime] = None + confirmation_days: int = 0 + + +class SignalFeatureRecord(BaseModel): + """A single signal → market state → outcome record.""" + _parse_date = field_validator("date", mode="before")(_parse_date) + date: Date + signal_type: str + signal_version: str = "b3_v1" + symbol: str = "BTC/USDT:USDT" + + regime_version: str + signal_grade: Optional[SignalGrade] = None + signal_strength: Optional[float] = None + + regime: MarketRegime + regime_confidence: float + regime_maturity_score: float + market_state_hash: str + state_embedding: str = "[]" + breadth_top20: float + breadth_top30: float + breadth_top50: float + breadth_bucket: BreadthBucket + breadth_divergence: float + oi_state: OIState + volatility_regime: VolRegime + price_structure_score: float + + chan_trend_direction: Optional[str] = None + chan_pivot_count: Optional[int] = None + chan_divergence_type: Optional[str] = None + + entry_price: Optional[float] = None + result_1d: Optional[float] = None + result_3d: Optional[float] = None + result_5d: Optional[float] = None + result_7d: Optional[float] = None + result_14d: Optional[float] = None + max_favorable_excursion: Optional[float] = None + max_adverse_excursion: Optional[float] = None + is_win_7d: Optional[int] = None + + +class ExpectancyLayer(BaseModel): + name: str + posterior_winrate: float + raw_winrate: Optional[float] = None + samples: int = 0 + effective_samples: float = 0.0 + avg_return: Optional[float] = None + + +class ExpectancyReport(BaseModel): + _parse_date = field_validator("date", mode="before")(_parse_date) + signal_type: str + date: Date + layers: list[ExpectancyLayer] = Field(default_factory=list) + final_estimate: float + sufficiency: SufficiencyLevel = SufficiencyLevel.INSUFFICIENT + prior_strength: int = 40 + half_life_days: int = 180 + source: str = "bayesian" + + avg_return_7d: Optional[float] = None + profit_factor: Optional[float] = None + max_adverse_excursion: Optional[float] = None + + +class DailyOutput(BaseModel): + """Final daily output: Market State + Expectancy.""" + _parse_date = field_validator("date", mode="before")(_parse_date) + date: Date + market_state: MarketStateVector + expectancy: dict[str, ExpectancyReport] = Field(default_factory=dict) + ai_report_en: Optional[str] = None + ai_report_zh: Optional[str] = None diff --git a/ChanMacro/regime_detector.py b/ChanMacro/regime_detector.py new file mode 100644 index 0000000..69194d7 --- /dev/null +++ b/ChanMacro/regime_detector.py @@ -0,0 +1,213 @@ +""" +regime_detector.py — Market regime detection (V1: 3 states). + +★ FACTOR-LOCKED: Regime = f(Price Structure, Breadth, Volatility) — forever. + Fear, Liquidation, ETF, Funding are Context, NOT regime inputs. + Adding new factors MUST NOT change regime definition. + +★ VERSIONED: regime_version = 'v1_price_breadth_vol'. + Weight changes → new version. Multiple versions coexist. + Query: WHERE regime_version = 'v1_price_breadth_vol'. + +★ CONFIDENCE-BASED: Each regime gets a continuous score. Highest wins. + No hard thresholds (prevents boundary oscillation). +""" + +from datetime import date as Date +from typing import Optional +from collections import deque + +from models import MarketRegime, RegimeResult +from config import config + + +class RegimeDetector: + """ + Detects market regime from Price + Breadth + Vol. + + V1: 3 regimes (TREND / RANGE / PANIC) + V2+: Can split TREND→TREND_UP/TREND_DOWN/EUPHORIA when samples > 500/regime. + """ + + def __init__(self, regime_version: Optional[str] = None): + self.version = regime_version or config.regime_version + self.w_price = config.regime_w_price + self.w_breadth = config.regime_w_breadth + self.w_vol = config.regime_w_vol + self.panic_w_anti_trend = config.regime_panic_w_anti_trend + self.panic_w_vol_extreme = config.regime_panic_w_vol_extreme + + # State persistence + self._current_regime: Optional[MarketRegime] = None + self._pending_regime: Optional[MarketRegime] = None + self._confirmation_count: int = 0 + self._consecutive_days: int = 0 + self._regime_history: deque = deque(maxlen=100) + + # Confirmation: 2 days minimum + self.MIN_CONFIRMATION = 2 + + def load_state(self, db_path: str): + """Restore regime state from the most recent regime_history record.""" + import sqlite3 + try: + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + row = conn.execute( + "SELECT regime, confidence, confirmation_days, maturity_score " + "FROM regime_history ORDER BY date DESC LIMIT 1" + ).fetchone() + conn.close() + + if row: + regime_str = row["regime"] + if regime_str in ("TREND", "RANGE", "PANIC"): + self._current_regime = MarketRegime(regime_str) + self._consecutive_days = row["confirmation_days"] or 1 + except Exception: + pass # DB not initialized yet, use defaults + + def detect(self, price_structure_score: float, breadth_score: float, + volatility_regime: str, date: Date) -> RegimeResult: + """ + Detect regime from the 3 locked factors. + + Args: + price_structure_score: 0-100 from PriceStructureScorer + breadth_score: 0-100 from BreadthScorer + volatility_regime: 'LOW_VOL'/'NORMAL_VOL'/'HIGH_VOL'/'EXPLOSIVE_VOL' + date: Target date + """ + # ── Compute regime scores ──────────────────────── + # TREND: strong price + strong breadth + non-extreme vol + trend_score = ( + price_structure_score * self.w_price + + breadth_score * self.w_breadth + + self._vol_to_trend(volatility_regime) * self.w_vol + ) + + # RANGE: neutral price + neutral breadth + low vol + # Score how "range-like" each dimension is + price_neutral = 60 - abs(price_structure_score - 50) + breadth_neutral = 60 - abs(breadth_score - 50) + vol_neutral = 80 if volatility_regime in ("LOW_VOL", "NORMAL_VOL") else 30 + range_score = ( + price_neutral * 0.40 + + breadth_neutral * 0.40 + + vol_neutral * 0.20 + ) + + # PANIC: very weak trend + extreme vol (NO Fear/Liquidation!) + anti_trend = 100 - trend_score + vol_extreme = 100 if volatility_regime == "EXPLOSIVE_VOL" else ( + 60 if volatility_regime == "HIGH_VOL" else 20 + ) + panic_score = ( + anti_trend * self.panic_w_anti_trend + + vol_extreme * self.panic_w_vol_extreme + ) + + scores = { + MarketRegime.TREND: round(trend_score, 1), + MarketRegime.RANGE: round(range_score, 1), + MarketRegime.PANIC: round(panic_score, 1), + } + + best_regime = max(scores, key=scores.get) + + # ── Persistence check ──────────────────────────── + prior_regime = self._current_regime + + if best_regime == self._current_regime: + self._consecutive_days += 1 + self._pending_regime = None + self._confirmation_count = 0 + elif best_regime == self._pending_regime: + self._confirmation_count += 1 + if self._confirmation_count >= self.MIN_CONFIRMATION: + # Transition confirmed + prior_regime = self._current_regime + self._current_regime = best_regime + self._consecutive_days = self.MIN_CONFIRMATION + self._pending_regime = None + self._confirmation_count = 0 + else: + self._pending_regime = best_regime + self._confirmation_count = 1 + + # Fallback: if no current regime yet (first run) + if self._current_regime is None: + self._current_regime = best_regime + self._consecutive_days = 1 + + # ── Confidence: for the CONFIRMED regime, not the raw best ── + confirmed_regime = self._current_regime + confidence = scores[confirmed_regime] / 100.0 + + # ── Maturity ───────────────────────────────────── + maturity = self._compute_maturity( + trend_score, breadth_score, volatility_regime + ) + + # Track history + self._regime_history.append({ + "date": date, + "regime": confirmed_regime.value, + "confidence": round(confidence, 3), + }) + + return RegimeResult( + date=date, + regime=confirmed_regime, + confidence=round(confidence, 3), + prior_regime=prior_regime, + regime_version=self.version, + maturity_score=round(maturity, 1), + all_scores={k.value: v for k, v in scores.items()}, + confirmation_days=self._consecutive_days, + ) + + @property + def current_regime(self) -> Optional[MarketRegime]: + return self._current_regime + + @property + def pending_regime(self) -> Optional[MarketRegime]: + return self._pending_regime + + @property + def confirmation_progress(self) -> tuple[int, int]: + """(confirmed_days, required_days) for pending transition.""" + return (self._confirmation_count, self.MIN_CONFIRMATION) + + @staticmethod + def _vol_to_trend(vol_regime: str) -> float: + """Convert volatility regime to trend-contributing score.""" + mapping = { + "LOW_VOL": 50, # Low vol: neutral for trend + "NORMAL_VOL": 70, # Normal vol: good for trend + "HIGH_VOL": 60, # High vol: trending but risky + "EXPLOSIVE_VOL": 30, # Explosive: anti-trend + } + return mapping.get(vol_regime, 50) + + @staticmethod + def _compute_maturity(trend_score: float, breadth_score: float, + vol_regime: str) -> float: + """ + Compute regime maturity: 0-100 continuous. + 0-30: EMERGING (trend accelerating, breadth expanding) + 30-70: CONFIRMED (stable) + 70-100: EXHAUSTING (decelerating, vol abnormal) + """ + # Trend strength contribution + trend_contrib = trend_score * 0.50 + + # Breadth contribution + breadth_contrib = breadth_score * 0.30 + + # Vol contribution (inverted: low vol = early, explosive = late) + vol_contrib = {"LOW_VOL": 20, "NORMAL_VOL": 40, "HIGH_VOL": 60, "EXPLOSIVE_VOL": 85} + vol_val = vol_contrib.get(vol_regime, 50) * 0.20 + + return trend_contrib + breadth_contrib + vol_val diff --git a/ChanMacro/report/__init__.py b/ChanMacro/report/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ChanMacro/requirements.txt b/ChanMacro/requirements.txt new file mode 100644 index 0000000..6a07a7d --- /dev/null +++ b/ChanMacro/requirements.txt @@ -0,0 +1,7 @@ +ccxt>=4.0.0 +pandas>=2.0.0 +numpy>=1.21.2 +pydantic>=2.0.0 +requests>=2.31.0 +python-dotenv>=1.0.0 +scipy>=1.10.0 diff --git a/ChanMacro/run_tests.sh b/ChanMacro/run_tests.sh new file mode 100755 index 0000000..5b23f38 --- /dev/null +++ b/ChanMacro/run_tests.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# run_tests.sh — Run the ChanMacro test suite. +# +# Usage: +# ./run_tests.sh # All tests +# ./run_tests.sh -v # Verbose +# ./run_tests.sh -k regime # Only regime tests +# ./run_tests.sh --cov # With coverage (requires pytest-cov) + +cd "$(dirname "$0")" +python -m pytest tests/ "$@" --tb=short diff --git a/ChanMacro/scoring/__init__.py b/ChanMacro/scoring/__init__.py new file mode 100644 index 0000000..771ff28 --- /dev/null +++ b/ChanMacro/scoring/__init__.py @@ -0,0 +1,6 @@ +"""Scoring engine — L1 factor computation.""" +from .base import BaseScorer +from .price_structure import PriceStructureScorer +from .breadth_scorer import BreadthScorer +from .oi_matrix import OIMatrixScorer +from .volatility_regime import VolatilityRegimeScorer diff --git a/ChanMacro/scoring/base.py b/ChanMacro/scoring/base.py new file mode 100644 index 0000000..a135d8d --- /dev/null +++ b/ChanMacro/scoring/base.py @@ -0,0 +1,28 @@ +""" +scoring/base.py — Abstract base class for all scoring modules. +""" + +from abc import ABC, abstractmethod +from datetime import date as Date +from typing import Optional +import sqlite3 + +from models import FactorScore +from config import config + + +class BaseScorer(ABC): + """Abstract base for all factor scorers.""" + + def __init__(self, db_path: Optional[str] = None): + self.db_path = db_path or config.db_path + + def get_connection(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + return conn + + @abstractmethod + def compute(self, target_date: Date) -> FactorScore: + """Compute factor score for a given date from database records.""" + ... diff --git a/ChanMacro/scoring/breadth_scorer.py b/ChanMacro/scoring/breadth_scorer.py new file mode 100644 index 0000000..a68f726 --- /dev/null +++ b/ChanMacro/scoring/breadth_scorer.py @@ -0,0 +1,218 @@ +""" +scoring/breadth_scorer.py — Market Breadth Score. + +The first citizen of the system. Diffusion always leads price. + +Multi-tier: Top20 / Top30 / Top50. +Quantile-based bucketing: EXTREME / STRONG / NORMAL / WEAK / PANIC. + +4 sub-indicators (equal weight): + 1. Advance/Decline ratio (30%) + 2. % above EMA20 (35%) + 3. New 20d highs (20%) + 4. BTC Dominance change (15%, inverted) +""" + +from datetime import date as Date +import sqlite3 +import numpy as np +import pandas as pd + +from .base import BaseScorer +from .constants import ( + BREADTH_W_ADVANCE, BREADTH_W_EMA20, BREADTH_W_NEW_HIGHS, BREADTH_W_BTC_DOM, +) +from models import FactorScore, BreadthScore, BreadthBucket, MacroDirection +from config import config + + +class BreadthScorer(BaseScorer): + """Scores market breadth with quantile-based bucketing.""" + + def compute(self, target_date: Date) -> BreadthScore: + conn = self.get_connection() + try: + row = conn.execute( + "SELECT * FROM breadth_daily WHERE date = ?", (str(target_date),) + ).fetchone() + + if row is None: + return BreadthScore( + name="Breadth", + score=50.0, + label="No Data", + breadth_bucket=BreadthBucket.NORMAL, + ) + + row = dict(row) + total = row.get("total_tracked", 50) or 50 + + # 1. Advance/Decline ratio + advance = row.get("advance_top50", 0) or 0 + decline = row.get("decline_top50", 0) or 0 + if advance + decline > 0: + ad_ratio = advance / (advance + decline) + else: + ad_ratio = 0.5 + ad_score = ad_ratio * 100 + + # 2. % above EMA20 + above_ema = row.get("above_ema20_top50", 0) or 0 + ema_pct = above_ema / total if total > 0 else 0.5 + ema_score = ema_pct * 100 + + # 3. New highs + new_highs = row.get("new_highs_20d_top50", 0) or 0 + highs_pct = new_highs / total if total > 0 else 0 + highs_score = highs_pct * 100 + + # 4. BTC Dominance (inverted: BTC.D up = bearish for alts) + btc_dom = row.get("btc_dominance") + btc_dom_score = 50.0 # neutral default + if btc_dom is not None: + # Placeholder — needs historical comparison + btc_dom_score = 50.0 + + # Weighted aggregate + score = ( + ad_score * BREADTH_W_ADVANCE + + ema_score * BREADTH_W_EMA20 + + highs_score * BREADTH_W_NEW_HIGHS + + btc_dom_score * BREADTH_W_BTC_DOM + ) + + # Multi-tier breadth + b20 = self._compute_tier_breadth(row, 20, total) + b30 = self._compute_tier_breadth(row, 30, total) + b50 = score # Top50 = full score + + # Quantile bucket + bucket = self._assign_bucket(score) + + # Divergence + divergence = b20 - b50 + + # Direction + if score >= 60: + direction = MacroDirection.BULLISH + elif score <= 40: + direction = MacroDirection.BEARISH + else: + direction = MacroDirection.NEUTRAL + + # Narrative + narrative = self._build_narrative(bucket, divergence, ema_pct, ad_ratio) + + return BreadthScore( + name="Breadth", + score=round(score, 1), + label=bucket.value, + direction=direction, + breadth_top20=round(b20, 1), + breadth_top30=round(b30, 1), + breadth_top50=round(b50, 1), + breadth_bucket=bucket, + breadth_divergence=round(divergence, 1), + advance_pct_top50=round(ad_ratio * 100, 1), + above_ema20_pct_top50=round(ema_pct * 100, 1), + new_highs_top50=new_highs, + sub_scores={ + "advance_decline": round(ad_score, 1), + "above_ema20": round(ema_score, 1), + "new_highs": round(highs_score, 1), + "btc_dominance": round(btc_dom_score, 1), + }, + narrative=narrative, + ) + finally: + conn.close() + + def _compute_tier_breadth(self, row: dict, tier: int, total: int) -> float: + """Compute breadth score for a specific tier (Top20 or Top30).""" + advance = row.get(f"advance_top{tier}", 0) or 0 + above_ema = row.get(f"above_ema20_top{tier}", 0) or 0 + new_highs = row.get(f"new_highs_20d_top{tier}", 0) or 0 + + tier_actual = min(tier, total) + if tier_actual == 0: + return 50.0 + + ad_ratio = advance / tier_actual if tier_actual > 0 else 0.5 + ema_ratio = above_ema / tier_actual if tier_actual > 0 else 0.5 + highs_ratio = new_highs / tier_actual if tier_actual > 0 else 0 + + return ( + ad_ratio * 100 * BREADTH_W_ADVANCE + + ema_ratio * 100 * BREADTH_W_EMA20 + + highs_ratio * 100 * BREADTH_W_NEW_HIGHS + + 50 * BREADTH_W_BTC_DOM # neutral for BTC.D + ) + + def _assign_bucket(self, score: float) -> BreadthBucket: + """Assign quantile-based bucket. V1 uses fixed thresholds until history accumulated.""" + # V1: fixed thresholds (will switch to quantile when enough history) + if score >= 80: + return BreadthBucket.EXTREME + elif score >= 60: + return BreadthBucket.STRONG + elif score >= 40: + return BreadthBucket.NORMAL + elif score >= 20: + return BreadthBucket.WEAK + else: + return BreadthBucket.PANIC + + @staticmethod + def compute_quantile_boundaries(db_path: str) -> dict: + """Compute quantile boundaries from historical breadth data. + + This should be called after accumulating enough history (> 1 year). + Returns boundaries for pd.qcut. + """ + conn = sqlite3.connect(db_path) + df = pd.read_sql_query( + "SELECT date, advance_top50, decline_top50, above_ema20_top50 FROM breadth_daily", + conn + ) + conn.close() + + if len(df) < 100: + return {"boundaries": [0, 20, 40, 60, 80, 100], "is_quantile": False} + + df["ad_ratio"] = df["advance_top50"] / (df["advance_top50"] + df["decline_top50"]) + df["ema_ratio"] = df["above_ema20_top50"] / 50 + df["breadth_raw"] = ( + df["ad_ratio"] * BREADTH_W_ADVANCE * 100 + + df["ema_ratio"] * BREADTH_W_EMA20 * 100 + + 40 * BREADTH_W_NEW_HIGHS + + 50 * BREADTH_W_BTC_DOM + ) + + boundaries = list(np.percentile(df["breadth_raw"].dropna(), [10, 30, 70, 90])) + return { + "boundaries": [0] + boundaries + [100], + "is_quantile": True, + "n_samples": len(df), + } + + @staticmethod + def _build_narrative(bucket: BreadthBucket, divergence: float, + ema_pct: float, ad_ratio: float) -> str: + parts = [] + if bucket == BreadthBucket.EXTREME: + parts.append(f"全市场极度扩散({ema_pct:.0%}站上EMA20)") + elif bucket == BreadthBucket.STRONG: + parts.append("市场广度强势") + elif bucket == BreadthBucket.NORMAL: + parts.append("市场广度中性") + elif bucket == BreadthBucket.WEAK: + parts.append("市场广度疲弱") + else: + parts.append("市场广度恐慌") + + if divergence > 10: + parts.append("资金集中于大市值(Top20>>Top50)") + elif divergence < -10: + parts.append("垃圾币狂欢(Top50>>Top20)") + + return ", ".join(parts) diff --git a/ChanMacro/scoring/constants.py b/ChanMacro/scoring/constants.py new file mode 100644 index 0000000..a3bb1d2 --- /dev/null +++ b/ChanMacro/scoring/constants.py @@ -0,0 +1,98 @@ +""" +scoring/constants.py — Scoring thresholds, scale factors, and reference values. + +All magic numbers in one place. Tune these via Phase 0 validation. +""" + +# ── Price Structure ────────────────────────────────────────── +# ADX thresholds +ADX_TREND_THRESHOLD = 25 # ADX > 25 = trending +ADX_STRONG_THRESHOLD = 40 # ADX > 40 = strong trend + +# EMA alignment +EMA_ALIGNMENT_BULLISH = 1.0 # EMA20 > EMA60 > EMA120 +EMA_ALIGNMENT_NEUTRAL = 0.5 # mixed +EMA_ALIGNMENT_BEARISH = 0.0 # EMA20 < EMA60 < EMA120 + +# Volatility compression (BB width relative to 20d average) +BB_COMPRESSION_LOW = 0.7 # < 70% of avg = compressing +BB_COMPRESSION_HIGH = 1.5 # > 150% of avg = expanding + +# Momentum (ROC annualized) +ROC_STRONG_BULLISH = 10.0 # % over period +ROC_STRONG_BEARISH = -10.0 + +# Consecutive candle threshold +CONSECUTIVE_CANDLES_SIGNAL = 4 + +# ── Breadth ────────────────────────────────────────────────── +# Quantile boundaries for breadth buckets +BREADTH_QUANTILES = [0, 0.1, 0.3, 0.7, 0.9, 1.0] # PANIC/WEAK/NORMAL/STRONG/EXTREME + +# Breadth score computation weights +BREADTH_W_ADVANCE = 0.30 # advance/decline ratio +BREADTH_W_EMA20 = 0.35 # % above EMA20 +BREADTH_W_NEW_HIGHS = 0.20 # new highs count +BREADTH_W_BTC_DOM = 0.15 # BTC dominance change (inverted) + +# ── OI Matrix ──────────────────────────────────────────────── +OI_PRICE_THRESHOLD = 0.5 # min |price_change%| to classify +OI_OI_THRESHOLD = 0.5 # min |OI_change%| to classify + +# Score mapping for OI states +OI_STATE_SCORES = { + "New Longs": 85, + "Short Covering": 60, + "New Shorts": 20, + "Long Exit": 35, + "Neutral": 50, +} + +# ── Volatility Regime ──────────────────────────────────────── +VOL_LOW = 2.0 # ATR/Close % below this = LOW_VOL +VOL_HIGH = 5.0 # ATR/Close % below this = HIGH_VOL (above = EXPLOSIVE) +HV_RATIO_LOW = 0.7 # HV(20)/HV(60) below this = compressing +HV_RATIO_HIGH = 1.5 # HV(20)/HV(60) above this = expanding + +# Score mapping +VOL_REGIME_SCORES = { + "LOW_VOL": 40, # Low vol → neutral with breakout potential + "NORMAL_VOL": 55, + "HIGH_VOL": 75, + "EXPLOSIVE_VOL": 90, +} + +# ── Regime ─────────────────────────────────────────────────── +REGIME_W_PRICE = 0.35 +REGIME_W_BREADTH = 0.50 +REGIME_W_VOL = 0.15 + +# PANIC: anti-trend + extreme vol (NO Fear/Liquidation) +PANIC_W_ANTI_TREND = 0.60 +PANIC_W_VOL_EXTREME = 0.40 + +# ── Trend (L2) ─────────────────────────────────────────────── +TREND_W_PRICE = 0.30 +TREND_W_BREADTH = 0.70 + +# ── Maturity ───────────────────────────────────────────────── +MATURITY_W_TREND = 0.50 +MATURITY_W_BREADTH = 0.30 +MATURITY_W_VOL = 0.20 + +# ── Expectancy ─────────────────────────────────────────────── +HALF_LIFE_DAYS = 180 +SUFFICIENCY_MIN = 30 +SUFFICIENCY_LOW = 50 +SUFFICIENCY_MEDIUM = 100 +LEVEL_MIN_SAMPLES = 50 +KNN_MAX_DISTANCE = 0.35 +KNN_K = 200 + +# ── Validation ─────────────────────────────────────────────── +MIN_AVG_DURATION = 5 +MAX_FLIP_RATE = 0.15 +MIN_IC_THRESHOLD = 0.03 +MIN_ICIR_THRESHOLD = 0.5 +MIN_IG_THRESHOLD = 0.1 # Information Gain for regime factors +MIN_KL_THRESHOLD = 0.5 # KL Divergence for regime separation diff --git a/ChanMacro/scoring/oi_matrix.py b/ChanMacro/scoring/oi_matrix.py new file mode 100644 index 0000000..71d1801 --- /dev/null +++ b/ChanMacro/scoring/oi_matrix.py @@ -0,0 +1,137 @@ +""" +scoring/oi_matrix.py — OI × Price 2×2 state machine. + +Discrete states, NOT a continuous score: + NEW_LONGS: Price↑ OI↑ → new money entering, trend continuation + SHORT_COVERING: Price↑ OI↓ → shorts covering, rally fragile + NEW_SHORTS: Price↓ OI↑ → new shorts entering, trend continuation + LONG_EXIT: Price↓ OI↓ → longs stopping out, panic (possible bottom) + NEUTRAL: flat → noise, don't force classification +""" + +from datetime import date as Date +import sqlite3 + +from .base import BaseScorer +from .constants import OI_PRICE_THRESHOLD, OI_OI_THRESHOLD, OI_STATE_SCORES +from models import FactorScore, OIMatrixScore, OIState, MacroDirection +from config import config + + +class OIMatrixScorer(BaseScorer): + """Classifies OI × Price state and assigns score.""" + + def compute(self, target_date: Date) -> OIMatrixScore: + conn = self.get_connection() + try: + row = conn.execute( + "SELECT * FROM derivatives WHERE date = ? AND symbol = 'BTC/USDT:USDT'", + (str(target_date),) + ).fetchone() + + if row is None: + return OIMatrixScore( + name="OI Matrix", + score=50.0, + label="No Data", + oi_state=OIState.NEUTRAL, + ) + + row = dict(row) + oi_change = row.get("oi_24h_change_pct") or 0 + + # Get price change from OHLCV + price_change = self._get_price_change(conn, str(target_date)) + + # Classify state + oi_state = self._classify(price_change, oi_change) + + # Score from state + score = OI_STATE_SCORES.get(oi_state.value, 50) + + # Direction + if oi_state == OIState.NEW_LONGS: + direction = MacroDirection.BULLISH + elif oi_state == OIState.SHORT_COVERING: + direction = MacroDirection.BULLISH # bullish but fragile + elif oi_state == OIState.NEW_SHORTS: + direction = MacroDirection.BEARISH + elif oi_state == OIState.LONG_EXIT: + direction = MacroDirection.BEARISH # bearish but possible bottom + else: + direction = MacroDirection.NEUTRAL + + # Narrative + narrative = self._build_narrative(oi_state, price_change, oi_change) + + return OIMatrixScore( + name="OI Matrix", + score=float(score), + label=oi_state.value, + direction=direction, + oi_state=oi_state, + price_change_pct=round(price_change, 2), + oi_change_pct=round(oi_change, 2), + sub_scores={ + "price_change_pct": round(price_change, 2), + "oi_change_pct": round(oi_change, 2), + }, + narrative=narrative, + ) + finally: + conn.close() + + def _get_price_change(self, conn: sqlite3.Connection, date_str: str) -> float: + """Get BTC 24h price change % for a given date.""" + row = conn.execute( + "SELECT close FROM ohlcv_daily WHERE date = ? AND symbol = 'BTC/USDT:USDT'", + (date_str,) + ).fetchone() + if row is None: + return 0.0 + + # Get previous day close + prev = conn.execute( + "SELECT close FROM ohlcv_daily WHERE date < ? AND symbol = 'BTC/USDT:USDT' ORDER BY date DESC LIMIT 1", + (date_str,) + ).fetchone() + + if prev is None: + return 0.0 + + current_close = float(row["close"]) + prev_close = float(prev["close"]) + if prev_close == 0: + return 0.0 + + return (current_close - prev_close) / prev_close * 100 + + @staticmethod + def _classify(price_change_pct: float, oi_change_pct: float) -> OIState: + """Classify OI × Price into discrete state.""" + price_up = price_change_pct > OI_PRICE_THRESHOLD + price_down = price_change_pct < -OI_PRICE_THRESHOLD + oi_up = oi_change_pct > OI_OI_THRESHOLD + oi_down = oi_change_pct < -OI_OI_THRESHOLD + + if price_up and oi_up: + return OIState.NEW_LONGS + elif price_up and oi_down: + return OIState.SHORT_COVERING + elif price_down and oi_up: + return OIState.NEW_SHORTS + elif price_down and oi_down: + return OIState.LONG_EXIT + else: + return OIState.NEUTRAL + + @staticmethod + def _build_narrative(state: OIState, price_chg: float, oi_chg: float) -> str: + mapping = { + OIState.NEW_LONGS: f"新多进场: 价格+{price_chg:.1f}%, OI+{oi_chg:.1f}%, 真金白银推动", + OIState.SHORT_COVERING: f"空头回补: 价格+{price_chg:.1f}%, OI{oi_chg:.1f}%, 上涨脆弱", + OIState.NEW_SHORTS: f"新空进场: 价格{price_chg:.1f}%, OI+{oi_chg:.1f}%, 趋势延续", + OIState.LONG_EXIT: f"多头止损: 价格{price_chg:.1f}%, OI{oi_chg:.1f}%, 恐慌(可能见底)", + OIState.NEUTRAL: "OI/价格变化不显著, 噪音区", + } + return mapping.get(state, "Unknown") diff --git a/ChanMacro/scoring/price_structure.py b/ChanMacro/scoring/price_structure.py new file mode 100644 index 0000000..743fbf7 --- /dev/null +++ b/ChanMacro/scoring/price_structure.py @@ -0,0 +1,248 @@ +""" +scoring/price_structure.py — Price Structure Score (OHLCV-only). + +Three sub-dimensions: + 1. Trend Strength (40%): EMA alignment + ADX + 2. Volatility Compression (30%): ATR + BB width + 3. Momentum (30%): ROC + consecutive candles + +This module works with zero external dependencies — just OHLCV data. +""" + +from datetime import date as Date +import sqlite3 +import math +import numpy as np +import pandas as pd + +from .base import BaseScorer +from .constants import ( + ADX_TREND_THRESHOLD, ADX_STRONG_THRESHOLD, + BB_COMPRESSION_LOW, BB_COMPRESSION_HIGH, + ROC_STRONG_BULLISH, ROC_STRONG_BEARISH, + CONSECUTIVE_CANDLES_SIGNAL, +) +from models import FactorScore, PriceStructureScore, MacroDirection +from config import config + + +class PriceStructureScorer(BaseScorer): + """Scores market structure from OHLCV data alone.""" + + def compute(self, target_date: Date) -> PriceStructureScore: + conn = self.get_connection() + try: + df = self._load_ohlcv(conn, str(target_date), lookback=120) + if df.empty: + return PriceStructureScore( + name="Price Structure", + score=50.0, + label="No Data", + ) + + trend = self._score_trend_strength(df) + vol_comp = self._score_volatility_compression(df) + momentum = self._score_momentum(df) + + # Weighted aggregate + score = trend * 0.40 + vol_comp * 0.30 + momentum * 0.30 + + # Determine direction + if trend > 60: + direction = MacroDirection.BULLISH + elif trend < 40: + direction = MacroDirection.BEARISH + else: + direction = MacroDirection.NEUTRAL + + # Build narrative + latest = df.iloc[-1] + narrative = self._build_narrative(trend, vol_comp, momentum, latest) + + return PriceStructureScore( + name="Price Structure", + score=round(score, 1), + label=self._label(score), + direction=direction, + trend_strength=round(trend, 1), + volatility_compression=round(vol_comp, 1), + momentum=round(momentum, 1), + sub_scores={ + "trend_strength": round(trend, 1), + "volatility_compression": round(vol_comp, 1), + "momentum": round(momentum, 1), + }, + narrative=narrative, + ) + finally: + conn.close() + + def _load_ohlcv(self, conn: sqlite3.Connection, date_str: str, + lookback: int = 120) -> pd.DataFrame: + """Load OHLCV data up to target_date.""" + df = pd.read_sql_query( + "SELECT * FROM ohlcv_daily WHERE date <= ? ORDER BY date DESC LIMIT ?", + conn, params=(date_str, lookback) + ) + if df.empty: + return df + return df.sort_values("date").reset_index(drop=True) + + def _score_trend_strength(self, df: pd.DataFrame) -> float: + """Score trend based on EMA alignment and ADX.""" + latest = df.iloc[-1] + + # EMA alignment + ema20 = latest.get("ema20") + ema60 = latest.get("ema60") + ema120 = latest.get("ema120") + + ema_score = 50.0 + if ema20 and ema60 and ema120 and not pd.isna(ema20) and not pd.isna(ema60) and not pd.isna(ema120): + alignments = 0 + if ema20 > ema60: alignments += 1 + if ema60 > ema120: alignments += 1 + if ema20 > ema120: alignments += 1 + + # Distance from EMAs + close = float(latest["close"]) + ema20_dist = abs(close - ema20) / ema20 * 100 if ema20 else 0 + + if alignments == 3: + ema_score = 80 + min(ema20_dist, 15) # strong bullish alignment + elif alignments == 0: + ema_score = 20 - min(ema20_dist, 15) # strong bearish alignment + elif alignments == 2: + ema_score = 65 + else: + ema_score = 35 + + # ADX + adx = latest.get("adx_14") + adx_score = 50.0 + if adx and not pd.isna(adx): + if adx > ADX_STRONG_THRESHOLD: + adx_score = 85 + elif adx > ADX_TREND_THRESHOLD: + adx_score = 65 + (adx - ADX_TREND_THRESHOLD) / (ADX_STRONG_THRESHOLD - ADX_TREND_THRESHOLD) * 20 + else: + adx_score = 50 - (ADX_TREND_THRESHOLD - adx) / ADX_TREND_THRESHOLD * 30 + + return ema_score * 0.55 + adx_score * 0.45 + + def _score_volatility_compression(self, df: pd.DataFrame) -> float: + """Score volatility compression — expansion = high, compression = low-mid.""" + latest = df.iloc[-1] + + bb_width = latest.get("bb_width") + if not bb_width or pd.isna(bb_width) or len(df) < 20: + return 50.0 + + # BB width relative to 20d average + recent_bb = df["bb_width"].dropna().tail(20) + if len(recent_bb) < 10: + return 50.0 + + bb_avg = recent_bb.mean() + bb_ratio = bb_width / bb_avg if bb_avg > 0 else 1.0 + + if bb_ratio < BB_COMPRESSION_LOW: + # Compression → potential breakout, neutral-bullish + return 45 + (BB_COMPRESSION_LOW - bb_ratio) * 30 + elif bb_ratio > BB_COMPRESSION_HIGH: + # Expansion → trending or chaotic + return 75 + min((bb_ratio - BB_COMPRESSION_HIGH) * 20, 20) + else: + # Normal + return 55 + + def _score_momentum(self, df: pd.DataFrame) -> float: + """Score momentum using ROC and consecutive candles.""" + if len(df) < 10: + return 50.0 + + closes = df["close"].astype(float) + latest = float(closes.iloc[-1]) + + # ROC (5-bar) + if len(closes) >= 6: + roc5 = (closes.iloc[-1] - closes.iloc[-6]) / closes.iloc[-6] * 100 + else: + roc5 = 0 + + # ROC (10-bar) + if len(closes) >= 11: + roc10 = (closes.iloc[-1] - closes.iloc[-11]) / closes.iloc[-11] * 100 + else: + roc10 = 0 + + # ROC (20-bar) + if len(closes) >= 21: + roc20 = (closes.iloc[-1] - closes.iloc[-21]) / closes.iloc[-21] * 100 + else: + roc20 = 0 + + # Score ROC: map to 0-100 + def roc_to_score(roc, scale=15): + return 50 + np.clip(roc / scale * 50, -50, 50) + + roc_score = roc_to_score(roc5, 10) * 0.4 + roc_to_score(roc10, 15) * 0.35 + roc_to_score(roc20, 20) * 0.25 + + # Consecutive candle direction + consec_score = 50.0 + consec_up = 0 + consec_down = 0 + for i in range(len(closes) - 1, max(0, len(closes) - 10), -1): + if closes.iloc[i] > closes.iloc[i - 1]: + consec_up += 1 + consec_down = 0 + elif closes.iloc[i] < closes.iloc[i - 1]: + consec_down += 1 + consec_up = 0 + else: + break + + if consec_up >= CONSECUTIVE_CANDLES_SIGNAL: + consec_score = 70 + min(consec_up * 5, 25) + elif consec_down >= CONSECUTIVE_CANDLES_SIGNAL: + consec_score = 30 - min(consec_down * 5, 25) + + return roc_score * 0.70 + consec_score * 0.30 + + def _build_narrative(self, trend: float, vol: float, momentum: float, + latest: pd.Series) -> str: + parts = [] + if trend > 65: + parts.append("EMA多头排列+ADX趋势明确") + elif trend > 50: + parts.append("趋势温和偏多") + elif trend < 35: + parts.append("EMA空头排列+ADX趋势明确") + elif trend < 50: + parts.append("趋势温和偏空") + else: + parts.append("趋势中性") + + if vol > 70: + parts.append("波动率扩张") + elif vol < 45: + parts.append("波动率压缩(突破前兆)") + + if momentum > 65: + parts.append("动量强劲") + elif momentum < 35: + parts.append("动量疲弱") + + return ", ".join(parts) if parts else "中性" + + @staticmethod + def _label(score: float) -> str: + if score >= 75: + return "Strong Bullish Structure" + elif score >= 60: + return "Bullish Structure" + elif score >= 40: + return "Neutral Structure" + elif score >= 25: + return "Bearish Structure" + return "Weak Bearish Structure" diff --git a/ChanMacro/scoring/volatility_regime.py b/ChanMacro/scoring/volatility_regime.py new file mode 100644 index 0000000..241c817 --- /dev/null +++ b/ChanMacro/scoring/volatility_regime.py @@ -0,0 +1,143 @@ +""" +scoring/volatility_regime.py — Volatility Regime Classification. + +4 regimes from OHLCV data: + LOW_VOL: ATR/Close < 2% → compression, breakout imminent + NORMAL_VOL: ATR/Close 2-5% → normal trading + HIGH_VOL: ATR/Close 5-10% → trend acceleration, wider stops + EXPLOSIVE_VOL: ATR/Close > 10% → extreme, reduce or wait + +Uses: ATR(14)/Close, HV(20)/HV(60) ratio, BB width ratio. +OHLCV-only — never goes offline. +""" + +from datetime import date as Date +import sqlite3 +import numpy as np +import pandas as pd + +from .base import BaseScorer +from .constants import ( + VOL_LOW, VOL_HIGH, VOL_REGIME_SCORES, HV_RATIO_LOW, HV_RATIO_HIGH, +) +from models import FactorScore, VolatilityRegimeScore, VolRegime, MacroDirection +from config import config + + +class VolatilityRegimeScorer(BaseScorer): + """Classifies volatility regime from OHLCV data.""" + + def compute(self, target_date: Date) -> VolatilityRegimeScore: + conn = self.get_connection() + try: + df = pd.read_sql_query( + "SELECT * FROM ohlcv_daily WHERE date <= ? ORDER BY date DESC LIMIT 120", + conn, params=(str(target_date),) + ) + if df.empty: + return VolatilityRegimeScore( + name="Volatility Regime", + score=50.0, + label="No Data", + ) + + df = df.sort_values("date").reset_index(drop=True) + + # 1. ATR/Close % + latest = df.iloc[-1] + atr = latest.get("atr_14") + close = float(latest["close"]) + atr_pct = (atr / close * 100) if atr and not pd.isna(atr) and close > 0 else 3.0 + + # 2. HV(20) / HV(60) ratio + hv_ratio = self._compute_hv_ratio(df) + + # 3. BB width ratio + bb_ratio = self._compute_bb_ratio(df) + + # Classify regime + regime = self._classify(atr_pct, hv_ratio, bb_ratio) + + # Score + score = VOL_REGIME_SCORES.get(regime.value, 50) + + # Narrative + narrative = self._build_narrative(regime, atr_pct, hv_ratio, bb_ratio) + + return VolatilityRegimeScore( + name="Volatility Regime", + score=float(score), + label=regime.value, + direction=MacroDirection.NEUTRAL, + vol_regime=regime, + atr_pct=round(atr_pct, 2), + hv_ratio=round(hv_ratio, 2), + bb_width_ratio=round(bb_ratio, 2), + sub_scores={ + "atr_pct": round(atr_pct, 2), + "hv_ratio": round(hv_ratio, 2), + "bb_width_ratio": round(bb_ratio, 2), + }, + narrative=narrative, + ) + finally: + conn.close() + + def _compute_hv_ratio(self, df: pd.DataFrame) -> float: + """Compute HV(20) / HV(60) ratio.""" + closes = df["close"].astype(float) + returns = closes.pct_change().dropna() + + if len(returns) < 60: + return 1.0 + + hv20 = returns.tail(20).std() * np.sqrt(365) * 100 + hv60 = returns.tail(60).std() * np.sqrt(365) * 100 + + if hv60 == 0: + return 1.0 + + return hv20 / hv60 + + def _compute_bb_ratio(self, df: pd.DataFrame) -> float: + """Compute current BB width / 20d average BB width.""" + bb_widths = df["bb_width"].dropna().tail(40) + if len(bb_widths) < 20: + return 1.0 + + current = bb_widths.iloc[-1] + avg = bb_widths.tail(20).mean() + if avg == 0: + return 1.0 + + return current / avg + + @staticmethod + def _classify(atr_pct: float, hv_ratio: float, bb_ratio: float) -> VolRegime: + """Classify volatility regime from multiple indicators.""" + # Primary: ATR/Close % + if atr_pct > 10.0: + return VolRegime.EXPLOSIVE_VOL + elif atr_pct > VOL_HIGH: + return VolRegime.HIGH_VOL + elif atr_pct < VOL_LOW: + return VolRegime.LOW_VOL + + # Secondary: HV ratio and BB ratio for edge cases + if hv_ratio > HV_RATIO_HIGH and bb_ratio > 1.3: + return VolRegime.HIGH_VOL + elif hv_ratio < HV_RATIO_LOW and bb_ratio < 0.8: + return VolRegime.LOW_VOL + + return VolRegime.NORMAL_VOL + + @staticmethod + def _build_narrative(regime: VolRegime, atr_pct: float, + hv_ratio: float, bb_ratio: float) -> str: + mapping = { + VolRegime.LOW_VOL: f"低波动(ATR={atr_pct:.1f}%), 布林带收窄, 突破前兆", + VolRegime.NORMAL_VOL: f"正常波动(ATR={atr_pct:.1f}%), 正常交易环境", + VolRegime.HIGH_VOL: f"高波动(ATR={atr_pct:.1f}%), 趋势加速, 放宽止损", + VolRegime.EXPLOSIVE_VOL: f"极端波动(ATR={atr_pct:.1f}%), 减仓或等待", + } + return mapping.get(regime, "Unknown") diff --git a/ChanMacro/tests/__init__.py b/ChanMacro/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ChanMacro/tests/conftest.py b/ChanMacro/tests/conftest.py new file mode 100644 index 0000000..35ca5cc --- /dev/null +++ b/ChanMacro/tests/conftest.py @@ -0,0 +1,134 @@ +""" +tests/conftest.py — Shared fixtures for ChanMacro tests. +""" + +import os +import sys +import pytest +import sqlite3 +import numpy as np +import pandas as pd +from datetime import date, timedelta +from pathlib import Path + +# Ensure package root on path +sys.path.insert(0, str(Path(__file__).parent.parent)) + + +@pytest.fixture +def db_path(tmp_path): + """Create a temporary SQLite database with full mock data.""" + db = str(tmp_path / "test_macro.db") + from database import init_db + conn = init_db(db) + + np.random.seed(42) + base = date(2025, 9, 1) + n_days = 300 + + # Generate realistic price series with 3 regime periods + prices = [90000] + regimes = [] + for i in range(n_days): + if i < 100: + ret = np.random.normal(0.003, 0.015) + regime = "TREND" + elif i < 200: + ret = np.random.normal(0.000, 0.012) + regime = "RANGE" + else: + ret = np.random.normal(-0.003, 0.025) + regime = "PANIC" + prices.append(prices[-1] * (1 + ret)) + regimes.append(regime) + + for i in range(n_days): + d = base + timedelta(days=i) + c = prices[i] + r = regimes[i] + + # OHLCV + conn.execute(""" + INSERT OR REPLACE INTO ohlcv_daily + (date,symbol,open,high,low,close,volume,ema20,ema60,ema120,atr_14,bb_width,adx_14) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) + """, ( + d.strftime("%Y-%m-%d"), "BTC/USDT:USDT", + c * 0.99, c * 1.03, c * 0.97, c, 1000, + c * (0.98 if r == "TREND" else 1.02 if r == "PANIC" else 1.0), + c * (0.95 if r == "TREND" else 1.05 if r == "PANIC" else 1.0), + c * (0.90 if r == "TREND" else 1.10 if r == "PANIC" else 1.0), + c * (0.02 if r == "PANIC" else 0.015), + 4.5, 28.0 if r == "TREND" else 18.0, + )) + + # Breadth + adv = 42 if r == "TREND" else 25 if r == "RANGE" else 8 + conn.execute(""" + INSERT OR REPLACE INTO breadth_daily + (date,total_tracked,advance_top50,decline_top50,above_ema20_top50, + new_highs_20d_top50,advance_top30,advance_top20, + above_ema20_top30,above_ema20_top20,new_highs_20d_top30,new_highs_20d_top20) + VALUES (?,50,?,?,?,?,?,?,?,?,?,?) + """, ( + d.strftime("%Y-%m-%d"), adv, 50 - adv, adv, min(adv, 15), + int(adv * 0.7), int(adv * 0.5), int(adv * 0.7), int(adv * 0.5), + min(int(adv * 0.7), 12), min(int(adv * 0.5), 8), + )) + + # Derivatives + oi_chg = 3.5 if r == "TREND" else 0.5 if r == "RANGE" else -2.0 + conn.execute(""" + INSERT OR REPLACE INTO derivatives + (date,symbol,funding_rate,open_interest,oi_24h_change_pct, + long_liquidations,short_liquidations,basis_annualised_pct) + VALUES (?,?,?,?,?,?,?,?) + """, ( + d.strftime("%Y-%m-%d"), "BTC/USDT:USDT", + 0.0001 + np.random.normal(0, 0.0002), + 35e9, oi_chg + np.random.normal(0, 1.0), + 50e6 * np.random.random(), 30e6 * np.random.random(), + 8.5 if r == "TREND" else 3.0, + )) + + # Regime history + conn.execute(""" + INSERT OR REPLACE INTO regime_history + (date,regime,confidence,regime_version,maturity_score,all_scores_json,confirmation_days) + VALUES (?,?,?,?,?,?,?) + """, (d.strftime("%Y-%m-%d"), r, 0.75, "v1_price_breadth_vol", 50, "{}", 1)) + + conn.commit() + conn.close() + + # Override config to use test DB + from config import config + old_db = config.db_path + config.db_path = db + yield db + config.db_path = old_db + + +@pytest.fixture +def sample_state(db_path): + """Build a MarketStateVector for a known test date.""" + from models import ( + MarketStateVector, MarketRegime, BreadthBucket, + OIState, VolRegime, + ) + state = MarketStateVector( + date=date(2026, 3, 15), + regime=MarketRegime.TREND, + regime_confidence=0.82, + regime_version="v1_price_breadth_vol", + regime_maturity_score=55.0, + breadth_top20=82.0, + breadth_top30=78.0, + breadth_top50=74.0, + breadth_bucket=BreadthBucket.STRONG, + breadth_divergence=8.0, + oi_state=OIState.NEW_LONGS, + volatility_regime=VolRegime.NORMAL_VOL, + ) + state.market_state_hash = state.compute_hash() + return state diff --git a/ChanMacro/tests/test_expectancy.py b/ChanMacro/tests/test_expectancy.py new file mode 100644 index 0000000..cb59a4d --- /dev/null +++ b/ChanMacro/tests/test_expectancy.py @@ -0,0 +1,173 @@ +"""Test SignalTracker, TimeDecay, and BayesianExpectancyEngine.""" +import pytest +from datetime import date, timedelta +import numpy as np + + +class TestTimeDecay: + def test_recent_weight_near_one(self): + from expectancy.decay import TimeDecay + d = TimeDecay(180) + w = d.weight(date(2026, 6, 20), date(2026, 6, 24)) + assert 0.95 < w < 1.0 + + def test_old_weight_decays(self): + from expectancy.decay import TimeDecay + d = TimeDecay(180) + w = d.weight(date(2025, 6, 24), date(2026, 6, 24)) + assert 0.2 < w < 0.3 # ~365 days at half_life=180 + + def test_effective_samples(self): + from expectancy.decay import TimeDecay + d = TimeDecay(180) + dates = [date(2026, 6, 24)] * 10 + weights = d.weights(dates, date(2026, 6, 24)) + eff = d.effective_samples(weights) + assert eff == pytest.approx(10.0, rel=0.01) + + def test_weighted_win_rate(self): + from expectancy.decay import TimeDecay + d = TimeDecay(180) + wins = np.array([1, 0, 1, 0]) + weights = np.array([1.0, 1.0, 1.0, 1.0]) + wr = d.weighted_win_rate(wins, weights) + assert wr == 0.5 + + def test_weight_at_age(self): + from expectancy.decay import TimeDecay + w = TimeDecay.weight_at_age(180, 180) + assert w == pytest.approx(0.5, rel=0.01) + + +class TestSignalTracker: + def test_record_signal(self, db_path, sample_state): + from expectancy.tracker import SignalTracker + tracker = SignalTracker() + rid = tracker.record( + date(2026, 3, 15), "B3", 98000.0, sample_state, + signal_grade="A", signal_strength=75.0, + ) + assert rid is not None + assert rid > 0 + + def test_get_samples(self, db_path, sample_state): + from expectancy.tracker import SignalTracker + tracker = SignalTracker() + tracker.record(date(2026, 3, 15), "B3", 98000.0, sample_state) + tracker.record(date(2026, 3, 16), "B2", 98500.0, sample_state) + + samples = tracker.get_samples(signal_type="B3") + assert len(samples) == 1 + assert samples[0]["signal_type"] == "B3" + + def test_count_samples(self, db_path, sample_state): + from expectancy.tracker import SignalTracker + tracker = SignalTracker() + tracker.record(date(2026, 3, 15), "B3", 98000.0, sample_state) + tracker.record(date(2026, 3, 16), "B3", 98500.0, sample_state) + + counts = tracker.count_samples() + assert "B3/TREND" in counts + assert counts["B3/TREND"] == 2 + + def test_filter_by_regime(self, db_path, sample_state): + from expectancy.tracker import SignalTracker + tracker = SignalTracker() + tracker.record(date(2026, 3, 15), "B3", 98000.0, sample_state) + + samples = tracker.get_samples(signal_type="B3", regime="TREND") + assert len(samples) == 1 + + samples = tracker.get_samples(signal_type="B3", regime="PANIC") + assert len(samples) == 0 + + def test_backfill_signals(self, db_path, sample_state): + from expectancy.tracker import SignalTracker + tracker = SignalTracker() + signals = [ + {"date": date(2026, 3, 15), "signal_type": "B3", "entry_price": 98000}, + {"date": date(2026, 3, 20), "signal_type": "B2", "entry_price": 99000}, + ] + count = tracker.backfill_signals(signals) + assert count == 2 + + +class TestBayesianExpectancyEngine: + def test_estimate_returns_report(self, db_path, sample_state): + from expectancy.tracker import SignalTracker + from expectancy.engine import BayesianExpectancyEngine + + # Record some signals first + tracker = SignalTracker() + for i in range(10): + tracker.record( + date(2026, 3, 15) + timedelta(days=i), + "B3", 98000.0, sample_state, + ) + + engine = BayesianExpectancyEngine(level_min_samples=3) + report = engine.estimate(sample_state, "B3", date(2026, 3, 25)) + assert report.signal_type == "B3" + assert len(report.layers) > 0 + assert report.source in ("bayesian", "insufficient") + + def test_insufficient_with_no_samples(self, db_path, sample_state): + from expectancy.engine import BayesianExpectancyEngine + engine = BayesianExpectancyEngine(level_min_samples=10) + report = engine.estimate(sample_state, "B1", date(2026, 3, 25)) + assert report.sufficiency.value in ("INSUFFICIENT", "LOW", "MEDIUM", "HIGH") + + def test_empirical_bayes_shrinks_small_samples(self, db_path, sample_state): + """With N=3, raw=100%, posterior should be pulled toward prior.""" + from expectancy.tracker import SignalTracker + from expectancy.engine import BayesianExpectancyEngine + + tracker = SignalTracker() + for i in range(3): + tracker.record( + date(2026, 3, 15) + timedelta(days=i), + "B3", 98000.0, sample_state, + ) + + engine = BayesianExpectancyEngine(level_min_samples=1) + report = engine.estimate(sample_state, "B3", date(2026, 3, 25)) + + # With small N, posterior should differ from raw + base_layer = report.layers[0] + if base_layer.raw_winrate and base_layer.samples < 50: + # Posterior should be pulled toward prior (50% or global rate) + if base_layer.raw_winrate > 0.8: + assert base_layer.posterior_winrate < base_layer.raw_winrate + + def test_leveled_fallback_stops_at_min_samples(self, db_path, sample_state): + from expectancy.tracker import SignalTracker + from expectancy.engine import BayesianExpectancyEngine + + tracker = SignalTracker() + for i in range(20): + tracker.record(date(2026, 3, 15) + timedelta(days=i), "B3", 98000.0, sample_state) + + engine = BayesianExpectancyEngine(level_min_samples=15) + report = engine.estimate(sample_state, "B3", date(2026, 3, 25)) + # Should have stopped at a level with >= 15 effective samples + assert report.final_estimate >= 0 + + +class TestSufficiencyGuard: + def test_insufficient(self): + from expectancy.engine import SufficiencyGuard + from models import SufficiencyLevel + g = SufficiencyGuard() + assert g.evaluate(10) == SufficiencyLevel.INSUFFICIENT + + def test_low(self): + from expectancy.engine import SufficiencyGuard + from models import SufficiencyLevel + g = SufficiencyGuard() + assert g.evaluate(40) == SufficiencyLevel.LOW + + def test_high(self): + from expectancy.engine import SufficiencyGuard + from models import SufficiencyLevel + g = SufficiencyGuard() + assert g.evaluate(200) == SufficiencyLevel.HIGH diff --git a/ChanMacro/tests/test_models.py b/ChanMacro/tests/test_models.py new file mode 100644 index 0000000..c92a871 --- /dev/null +++ b/ChanMacro/tests/test_models.py @@ -0,0 +1,130 @@ +"""Test all Pydantic models and enums.""" +import pytest +from datetime import date +from models import ( + MarketRegime, OIState, BreadthBucket, VolRegime, + MarketStateVector, FactorScore, RegimeResult, + SignalFeatureRecord, ExpectancyReport, DailyOutput, + FactorContribution, SufficiencyLevel, SignalGrade, +) + + +class TestEnums: + def test_regime_values(self): + assert MarketRegime.TREND.value == "TREND" + assert MarketRegime.RANGE.value == "RANGE" + assert MarketRegime.PANIC.value == "PANIC" + + def test_oi_state_has_neutral(self): + assert OIState.NEUTRAL.value == "Neutral" + assert len(OIState) == 5 + + def test_breadth_bucket_values(self): + assert BreadthBucket.EXTREME.value == "EXTREME" + assert len(BreadthBucket) == 5 + + def test_vol_regime_values(self): + assert VolRegime.LOW_VOL.value == "LOW_VOL" + assert VolRegime.EXPLOSIVE_VOL.value == "EXPLOSIVE_VOL" + + +class TestMarketStateVector: + def test_minimal_construction(self): + sv = MarketStateVector( + date="2026-06-24", + regime=MarketRegime.TREND, + regime_confidence=0.82, + regime_version="v1_price_breadth_vol", + ) + assert sv.date == date(2026, 6, 24) + assert sv.regime == MarketRegime.TREND + assert sv.breadth_top50 == 50.0 # default + + def test_date_string_parsing(self): + sv = MarketStateVector( + date="2026-01-15", + regime=MarketRegime.RANGE, + regime_confidence=0.55, + regime_version="v1_price_breadth_vol", + ) + assert sv.date == date(2026, 1, 15) + + def test_compute_hash(self): + sv = MarketStateVector( + date="2026-06-24", + regime=MarketRegime.TREND, + regime_confidence=0.82, + regime_version="v1_price_breadth_vol", + breadth_bucket=BreadthBucket.EXTREME, + oi_state=OIState.NEW_LONGS, + volatility_regime=VolRegime.NORMAL_VOL, + ) + h = sv.compute_hash() + assert len(h) == 12 + # Same state = same hash + sv2 = MarketStateVector( + date="2026-06-25", + regime=MarketRegime.TREND, + regime_confidence=0.80, + regime_version="v1_price_breadth_vol", + breadth_bucket=BreadthBucket.EXTREME, + oi_state=OIState.NEW_LONGS, + volatility_regime=VolRegime.NORMAL_VOL, + ) + assert sv2.compute_hash() == h + + def test_state_embedding(self): + sv = MarketStateVector( + date="2026-06-24", + regime=MarketRegime.TREND, + regime_confidence=0.82, + regime_version="v1_price_breadth_vol", + breadth_top20=80.0, + breadth_top30=75.0, + breadth_top50=70.0, + regime_maturity_score=60.0, + ) + emb = sv.state_embedding() + assert len(emb) == 5 + assert emb[0] == 80.0 + assert emb[3] == 60.0 + + +class TestRegimeResult: + def test_construction(self): + r = RegimeResult( + date="2026-06-24", + regime=MarketRegime.TREND, + confidence=0.82, + regime_version="v1_price_breadth_vol", + maturity_score=55.0, + all_scores={"TREND": 82.0, "RANGE": 45.0, "PANIC": 20.0}, + confirmation_days=5, + ) + assert r.regime == MarketRegime.TREND + assert r.confirmation_days == 5 + + +class TestExpectancyReport: + def test_insufficient(self): + r = ExpectancyReport( + signal_type="B3", + date="2026-06-24", + final_estimate=0.0, + sufficiency=SufficiencyLevel.INSUFFICIENT, + source="insufficient", + ) + assert r.final_estimate == 0.0 + assert r.sufficiency == SufficiencyLevel.INSUFFICIENT + + +class TestFactorContribution: + def test_construction(self): + fc = FactorContribution( + factor="ETF Flow", + raw_score=85.0, + weight=0.1925, + impact=6.7, + direction="bullish", + ) + assert fc.impact > 0 diff --git a/ChanMacro/tests/test_regime.py b/ChanMacro/tests/test_regime.py new file mode 100644 index 0000000..480dc34 --- /dev/null +++ b/ChanMacro/tests/test_regime.py @@ -0,0 +1,109 @@ +"""Test regime detector and validation.""" +import pytest +from datetime import date +import pandas as pd +import numpy as np + + +class TestRegimeDetector: + def test_detects_trend(self): + from regime_detector import RegimeDetector + from models import MarketRegime + d = RegimeDetector() + r = d.detect(75.0, 80.0, "NORMAL_VOL", date(2026, 6, 24)) + assert r.regime == MarketRegime.TREND + assert r.confidence > 0.5 + + def test_detects_range(self): + from regime_detector import RegimeDetector + from models import MarketRegime + d = RegimeDetector() + r = d.detect(50.0, 50.0, "LOW_VOL", date(2026, 6, 24)) + assert r.regime in (MarketRegime.RANGE, MarketRegime.TREND) + + def test_detects_panic(self): + from regime_detector import RegimeDetector + from models import MarketRegime + d = RegimeDetector() + r = d.detect(15.0, 10.0, "EXPLOSIVE_VOL", date(2026, 6, 24)) + assert r.regime == MarketRegime.PANIC + + def test_2day_confirmation(self): + from regime_detector import RegimeDetector + from models import MarketRegime + d = RegimeDetector() + # Day 1: RANGE + r1 = d.detect(50.0, 50.0, "LOW_VOL", date(2026, 6, 24)) + assert r1.regime == MarketRegime.RANGE # first run, no confirmation needed + # Day 2: still RANGE + r2 = d.detect(50.0, 50.0, "LOW_VOL", date(2026, 6, 25)) + assert r2.regime == MarketRegime.RANGE + assert r2.confirmation_days == 2 + + def test_transition_needs_confirmation(self): + from regime_detector import RegimeDetector + from models import MarketRegime + d = RegimeDetector() + # Establish TREND + d.detect(75.0, 80.0, "NORMAL_VOL", date(2026, 6, 24)) + d.detect(75.0, 80.0, "NORMAL_VOL", date(2026, 6, 25)) + # Day 3: weak scores → raw best = RANGE, but TREND should persist + r3 = d.detect(35.0, 40.0, "NORMAL_VOL", date(2026, 6, 26)) + # First day of pending transition — should still be TREND + assert r3.regime == MarketRegime.TREND + assert d.pending_regime is not None + + def test_version_is_stored(self): + from regime_detector import RegimeDetector + d = RegimeDetector(regime_version="v1_price_breadth_vol") + r = d.detect(75.0, 80.0, "NORMAL_VOL", date(2026, 6, 24)) + assert r.regime_version == "v1_price_breadth_vol" + + def test_load_state(self, db_path): + from regime_detector import RegimeDetector + from models import MarketRegime + d = RegimeDetector() + d.load_state(db_path) + # DB has TREND for first 100 days, so most recent should load + assert d.current_regime is not None + + def test_confidence_for_confirmed_regime(self): + """Confidence should be for the confirmed regime, not raw best.""" + from regime_detector import RegimeDetector + from models import MarketRegime + d = RegimeDetector() + # Establish TREND + d.detect(75.0, 80.0, "NORMAL_VOL", date(2026, 6, 24)) + d.detect(75.0, 80.0, "NORMAL_VOL", date(2026, 6, 25)) + # Now feed weak scores → raw best would be PANIC or RANGE + r = d.detect(15.0, 10.0, "EXPLOSIVE_VOL", date(2026, 6, 26)) + # Should still report TREND (need 2 confirmations to switch) + assert r.regime == MarketRegime.TREND + + +class TestTransitionValidator: + def test_stable_regime_passes(self): + from validation.transition_validator import TransitionValidator + # Create stable regime sequence: long periods + seq = pd.Series( + ["TREND"] * 50 + ["RANGE"] * 50 + ["PANIC"] * 40, + index=pd.date_range("2026-01-01", periods=140), + ) + tv = TransitionValidator() + report = tv.validate(seq) + assert report.is_stable + assert report.avg_duration > 20 + assert report.flip_rate < 0.05 + + def test_unstable_regime_fails(self): + from validation.transition_validator import TransitionValidator + # Create unstable sequence: flips every 2 days + seq = pd.Series( + ["TREND", "TREND", "RANGE", "RANGE", "TREND", "TREND", + "PANIC", "PANIC", "RANGE", "RANGE"] * 5, + index=pd.date_range("2026-01-01", periods=50), + ) + tv = TransitionValidator() + report = tv.validate(seq) + assert not report.is_stable + assert report.flip_rate > 0.15 diff --git a/ChanMacro/tests/test_scoring.py b/ChanMacro/tests/test_scoring.py new file mode 100644 index 0000000..7a5ce57 --- /dev/null +++ b/ChanMacro/tests/test_scoring.py @@ -0,0 +1,121 @@ +"""Test all 4 core scorers.""" +import pytest +from datetime import date + + +class TestPriceStructureScorer: + def test_computes_score(self, db_path): + from scoring.price_structure import PriceStructureScorer + scorer = PriceStructureScorer() + result = scorer.compute(date(2026, 3, 15)) + assert result.name == "Price Structure" + assert 0 <= result.score <= 100 + assert result.trend_strength >= 0 + assert result.volatility_compression >= 0 + assert result.momentum >= 0 + assert result.label + + def test_bullish_in_trend(self, db_path): + from scoring.price_structure import PriceStructureScorer + scorer = PriceStructureScorer() + result = scorer.compute(date(2025, 11, 15)) # TREND period + assert result.score > 50 # Should be bullish in uptrend + + def test_bearish_in_panic(self, db_path): + from scoring.price_structure import PriceStructureScorer + scorer = PriceStructureScorer() + result = scorer.compute(date(2026, 5, 15)) # PANIC period + # In panic period, EMA alignment should be bearish + assert result.trend_strength < 60 + + def test_no_data_handling(self, db_path): + from scoring.price_structure import PriceStructureScorer + scorer = PriceStructureScorer() + result = scorer.compute(date(2020, 1, 1)) + assert result.score == 50.0 + assert result.label == "No Data" + + +class TestBreadthScorer: + def test_computes_score(self, db_path): + from scoring.breadth_scorer import BreadthScorer + scorer = BreadthScorer() + result = scorer.compute(date(2026, 3, 15)) + assert result.name == "Breadth" + assert 0 <= result.score <= 100 + assert result.breadth_bucket + assert result.breadth_top20 >= 0 + assert result.breadth_top50 >= 0 + + def test_tier_values(self, db_path): + from scoring.breadth_scorer import BreadthScorer + scorer = BreadthScorer() + result = scorer.compute(date(2025, 11, 15)) # TREND period + # Top20 should generally be higher than Top50 (large caps lead) + assert result.breadth_top20 >= 0 + assert result.breadth_top50 >= 0 + + def test_bucket_assignment(self, db_path): + from scoring.breadth_scorer import BreadthScorer, BreadthBucket + scorer = BreadthScorer() + result = scorer.compute(date(2025, 11, 15)) # TREND: adv=42/50 + assert result.breadth_bucket in ( + BreadthBucket.EXTREME, BreadthBucket.STRONG, BreadthBucket.NORMAL + ) + + def test_no_data(self, db_path): + from scoring.breadth_scorer import BreadthScorer + scorer = BreadthScorer() + result = scorer.compute(date(2020, 1, 1)) + assert result.score == 50.0 + + +class TestOIMatrixScorer: + def test_computes_state(self, db_path): + from scoring.oi_matrix import OIMatrixScorer, OIState + scorer = OIMatrixScorer() + result = scorer.compute(date(2025, 11, 15)) # TREND period, oi_chg=+3.5 + assert result.oi_state in OIState + assert 0 <= result.score <= 100 + + def test_new_longs_in_trend(self, db_path): + from scoring.oi_matrix import OIMatrixScorer, OIState + scorer = OIMatrixScorer() + # Test multiple dates in TREND period — at least one should be NEW_LONGS or NEUTRAL + found_bullish = False + for d in ["2025-11-15", "2025-11-20", "2025-12-01", "2025-12-15"]: + result = scorer.compute(date.fromisoformat(d)) + if result.oi_state in (OIState.NEW_LONGS, OIState.SHORT_COVERING, OIState.NEUTRAL): + found_bullish = True + break + assert found_bullish, "No bullish OI state found in TREND period" + + def test_no_data(self, db_path): + from scoring.oi_matrix import OIMatrixScorer + scorer = OIMatrixScorer() + result = scorer.compute(date(2020, 1, 1)) + assert result.score == 50.0 + assert result.label == "No Data" + + +class TestVolatilityRegimeScorer: + def test_computes_regime(self, db_path): + from scoring.volatility_regime import VolatilityRegimeScorer, VolRegime + scorer = VolatilityRegimeScorer() + result = scorer.compute(date(2026, 3, 15)) + assert result.vol_regime in VolRegime + assert 0 <= result.score <= 100 + + def test_higher_vol_in_panic(self, db_path): + from scoring.volatility_regime import VolatilityRegimeScorer, VolRegime + scorer = VolatilityRegimeScorer() + trend_result = scorer.compute(date(2025, 11, 15)) + panic_result = scorer.compute(date(2026, 5, 15)) + # PANIC period has higher ATR → higher vol regime or score + assert panic_result.atr_pct >= trend_result.atr_pct * 0.5 # at least comparable + + def test_no_data(self, db_path): + from scoring.volatility_regime import VolatilityRegimeScorer + scorer = VolatilityRegimeScorer() + result = scorer.compute(date(2020, 1, 1)) + assert result.score == 50.0 diff --git a/ChanMacro/trend_detector.py b/ChanMacro/trend_detector.py new file mode 100644 index 0000000..ffe9bcc --- /dev/null +++ b/ChanMacro/trend_detector.py @@ -0,0 +1,81 @@ +""" +trend_detector.py — Trend strength and maturity helpers. + +Utility functions for computing trend alignment, acceleration, persistence. +Used by regime_detector and price_structure scorer. +""" + +import numpy as np +import pandas as pd + + +def ema_alignment_score(close: float, ema20: float, ema60: float, ema120: float) -> float: + """Score EMA alignment: 0=bearish, 50=neutral, 100=bullish.""" + if any(pd.isna(x) for x in [ema20, ema60, ema120]): + return 50.0 + + alignments = 0 + if ema20 > ema60: + alignments += 1 + if ema60 > ema120: + alignments += 1 + if ema20 > ema120: + alignments += 1 + + if alignments == 3: + return 85.0 + elif alignments == 2: + return 65.0 + elif alignments == 1: + return 35.0 + else: + return 15.0 + + +def adx_trend_score(adx: float) -> float: + """Convert ADX value to trend score: 0-100.""" + if pd.isna(adx): + return 50.0 + if adx > 40: + return 90.0 + elif adx > 25: + return 60.0 + (adx - 25) / 15 * 30 + elif adx > 15: + return 40.0 + (adx - 15) / 10 * 20 + else: + return max(10.0, adx / 15 * 40) + + +def breadth_persistence(breadth_scores: list[float], window: int = 5) -> float: + """How consistently has breadth stayed at its current level? 0-100.""" + if len(breadth_scores) < window: + return 50.0 + recent = breadth_scores[-window:] + mean_val = np.mean(recent) + std_val = np.std(recent) if len(recent) > 1 else 0 + # Low std = high persistence + persistence = 100 - min(std_val * 5, 100) + # Bias: higher breadth = higher persistence score + return persistence * 0.5 + mean_val * 0.5 + + +def trend_strength_composite(ema_score: float, adx_score: float, + breadth_score: float) -> float: + """Composite trend strength 0-100.""" + return ema_score * 0.25 + adx_score * 0.25 + breadth_score * 0.50 + + +def compute_maturity(trend_strength: float, breadth_persistence: float, + vol_expansion: float) -> float: + """ + Compute regime maturity score 0-100. + + EMERGING (0-30): trend accelerating, breadth expanding + CONFIRMED (30-70): trend stable, breadth stable + EXHAUSTING (70-100): trend decelerating, breadth contracting, vol abnormal + """ + return ( + trend_strength * 0.50 + + breadth_persistence * 0.30 + + (100 - vol_expansion) * 0.20 # inverted: low vol = early stage + ) diff --git a/ChanMacro/validation/__init__.py b/ChanMacro/validation/__init__.py new file mode 100644 index 0000000..23ee1ac --- /dev/null +++ b/ChanMacro/validation/__init__.py @@ -0,0 +1,5 @@ +"""Validation Framework — Phase 0: verify every factor before trusting it.""" +from .factor_validator import FactorValidator +from .regime_validator import RegimeValidator +from .transition_validator import TransitionValidator +from .reporter import ValidationReporter diff --git a/ChanMacro/validation/factor_validator.py b/ChanMacro/validation/factor_validator.py new file mode 100644 index 0000000..ca7ccc9 --- /dev/null +++ b/ChanMacro/validation/factor_validator.py @@ -0,0 +1,174 @@ +""" +validation/factor_validator.py — Validates a factor's predictive power. + +Tests: IC, ICIR, Hit Ratio, Quantile Spread, Lead-Lag analysis. +Answers: "Does this factor predict future returns?" +""" + +from datetime import date as Date +from typing import Optional +import sqlite3 +import logging + +import numpy as np +import pandas as pd + +from config import config +from .metrics import ( + information_coefficient, icir, hit_ratio, + quantile_spread, lead_lag_ic, +) + +logger = logging.getLogger(__name__) + + +class FactorReport: + """Structured report for a single factor's validation results.""" + + def __init__(self, factor_name: str): + self.factor_name = factor_name + self.ic_mean: float = 0.0 + self.ic_std: float = 0.0 + self.icir: float = 0.0 + self.hit_ratio: float = 0.0 + self.quantile_spread: float = 0.0 + self.is_leading: bool = False + self.lead_days: int = 0 + self.lead_ic: float = 0.0 + self.n_observations: int = 0 + self.conclusion: str = "" + + def summary(self) -> str: + lines = [ + f"Factor: {self.factor_name}", + f" N={self.n_observations}", + f" IC mean={self.ic_mean:.4f} std={self.ic_std:.4f} ICIR={self.icir:.2f}", + f" Hit Ratio={self.hit_ratio:.1%} Top-Bot Spread={self.quantile_spread:.4f}", + f" Best Lead: {self.lead_days}d (IC={self.lead_ic:.4f})" if self.is_leading else " Leading: No (synchronous/lagging)", + f" → {self.conclusion}", + ] + return "\n".join(lines) + + +class FactorValidator: + """ + Validates a factor's predictive power using standard quant metrics. + + For each forward horizon (1d, 3d, 5d, 7d, 14d), computes: + - IC (Spearman rank correlation) + - ICIR (IC stability) + - Hit Ratio (direction accuracy) + - Quantile spread (top vs bottom bucket) + - Lead-lag profile + + A factor is valid if IC > 0.03 and ICIR > 0.5. + For regime factors, also check regime_validator. + """ + + def __init__(self, db_path: Optional[str] = None): + self.db_path = db_path or config.db_path + + def validate(self, factor_name: str, factor_scores: pd.Series, + forward_returns: dict[str, pd.Series]) -> FactorReport: + """ + Args: + factor_name: Human-readable name + factor_scores: Series indexed by date, values 0-100 + forward_returns: Dict of horizon → Series indexed by date (e.g. "1d" → returns) + """ + report = FactorReport(factor_name) + + # Align series to common dates + common_idx = factor_scores.index + for ret in forward_returns.values(): + common_idx = common_idx.intersection(ret.index) + + if len(common_idx) < 30: + report.conclusion = "INSUFFICIENT DATA (< 30 observations)" + return report + + f = factor_scores[common_idx] + report.n_observations = len(common_idx) + + # Test against 7d forward returns (primary horizon) + primary_ret = forward_returns.get("7d") + if primary_ret is None: + # Use first available + primary_ret = list(forward_returns.values())[0] + + r = primary_ret[common_idx] + + # IC + ic = information_coefficient(f, r) + report.ic_mean = round(ic, 4) + + # Rolling IC for ICIR + rolling_ics = [] + for i in range(30, len(f)): + ic_i = information_coefficient(f.iloc[:i], r.iloc[:i]) + rolling_ics.append(ic_i) + ic_series = pd.Series(rolling_ics) + report.ic_std = round(ic_series.std(), 4) + report.icir = round(icir(ic_series), 2) + + # Hit ratio + report.hit_ratio = round(hit_ratio(f, r), 4) + + # Quantile spread + report.quantile_spread = round(quantile_spread(f, r), 4) + + # Lead-lag + lead = lead_lag_ic(f, r, max_lag=14) + report.is_leading = lead["is_leading"] + report.lead_days = lead["lead_days"] + report.lead_ic = round(lead["best_ic"], 4) + + # Conclusion + if abs(report.ic_mean) > 0.05 and report.icir > 1.0: + report.conclusion = "STRONG: significant predictive power" + elif abs(report.ic_mean) > 0.03 and report.icir > 0.5: + report.conclusion = "VALID: moderate predictive power" + elif abs(report.ic_mean) < 0.02: + report.conclusion = "CONFIRMING: describes current state, not predictive" + else: + report.conclusion = "WEAK: borderline, monitor or downweight" + + return report + + def validate_from_db(self, factor_name: str, + score_query: str, + horizon_days: int = 7) -> FactorReport: + """ + Convenience: load scores from DB and OHLCV returns, then validate. + + score_query: SQL that returns (date, score) pairs. + """ + conn = sqlite3.connect(self.db_path) + + scores_df = pd.read_sql_query(score_query, conn) + if scores_df.empty: + conn.close() + r = FactorReport(factor_name) + r.conclusion = "NO DATA" + return r + + scores_df["date"] = pd.to_datetime(scores_df["date"]) + scores = scores_df.set_index("date")["score"] + + # Load forward returns from OHLCV + ohlcv = pd.read_sql_query( + "SELECT date, close FROM ohlcv_daily WHERE symbol='BTC/USDT:USDT' ORDER BY date", + conn + ) + conn.close() + + ohlcv["date"] = pd.to_datetime(ohlcv["date"]) + ohlcv = ohlcv.set_index("date") + ohlcv["ret"] = ohlcv["close"].pct_change().shift(-1) # forward 1d + + # Build forward returns for multiple horizons + forward = {} + for h in [1, 3, 5, 7, 14]: + forward[str(h) + "d"] = ohlcv["close"].pct_change(periods=h).shift(-h) + + return self.validate(factor_name, scores, forward) diff --git a/ChanMacro/validation/metrics.py b/ChanMacro/validation/metrics.py new file mode 100644 index 0000000..0bff5a5 --- /dev/null +++ b/ChanMacro/validation/metrics.py @@ -0,0 +1,192 @@ +""" +validation/metrics.py — Shared statistical metrics for factor and regime validation. +""" + +import numpy as np +import pandas as pd +from scipy import stats +from typing import Optional + + +def information_coefficient(factor: pd.Series, forward_returns: pd.Series) -> float: + """Spearman rank IC between factor values and forward returns.""" + mask = factor.notna() & forward_returns.notna() + if mask.sum() < 10: + return 0.0 + ic, _ = stats.spearmanr(factor[mask], forward_returns[mask]) + return float(ic) if not np.isnan(ic) else 0.0 + + +def icir(ic_series: pd.Series) -> float: + """Information Coefficient IR = mean(IC) / std(IC).""" + if len(ic_series) < 5 or ic_series.std() == 0: + return 0.0 + return float(ic_series.mean() / ic_series.std()) + + +def hit_ratio(factor: pd.Series, forward_returns: pd.Series) -> float: + """Fraction of times factor direction matches return direction.""" + mask = factor.notna() & forward_returns.notna() + if mask.sum() < 10: + return 0.5 + # Compare sign of factor deviation from median vs sign of returns + factor_median = factor[mask].median() + factor_sign = np.sign(factor[mask] - factor_median) + return_sign = np.sign(forward_returns[mask]) + return float((factor_sign == return_sign).mean()) + + +def quantile_spread(factor: pd.Series, forward_returns: pd.Series, + n_quantiles: int = 5) -> float: + """Top vs bottom quantile return spread (分层回测).""" + mask = factor.notna() & forward_returns.notna() + if mask.sum() < n_quantiles * 3: + return 0.0 + f = factor[mask] + r = forward_returns[mask] + labels = pd.qcut(f, n_quantiles, labels=False, duplicates="drop") + top_ret = r[labels == labels.max()].mean() + bot_ret = r[labels == labels.min()].mean() + return float(top_ret - bot_ret) + + +def lead_lag_ic(factor: pd.Series, returns: pd.Series, + max_lag: int = 14) -> dict: + """Find the best leading/trailing relationship by computing IC at each lag.""" + results = {} + for lag in range(-max_lag, max_lag + 1): + if lag < 0: + shifted = factor.shift(abs(lag)) + ic = information_coefficient(shifted, returns) + results[f"lead_{abs(lag)}d"] = ic + elif lag > 0: + shifted = returns.shift(lag) + ic = information_coefficient(factor, shifted) + results[f"lag_{lag}d"] = ic + else: + ic = information_coefficient(factor, returns) + results["sync"] = ic + + # Find best lead period + lead_ics = {k: v for k, v in results.items() if k.startswith("lead_")} + best_lead = max(lead_ics, key=lead_ics.get) if lead_ics else "sync" + best_ic = lead_ics.get(best_lead, results.get("sync", 0)) + + return { + "best_lead": best_lead, + "best_ic": best_ic, + "ic_curve": results, + "is_leading": best_lead.startswith("lead_") and abs(best_ic) > 0.03, + "lead_days": int(best_lead.split("_")[1].rstrip("d")) if best_lead.startswith("lead_") else 0, + } + + +def mutual_information(factor: pd.Series, labels: pd.Series, + n_bins: int = 10) -> float: + """Mutual information between factor (binned) and discrete regime labels.""" + mask = factor.notna() & labels.notna() + if mask.sum() < 20: + return 0.0 + f = factor[mask] + l = labels[mask] + try: + f_binned = pd.qcut(f, n_bins, labels=False, duplicates="drop") + except ValueError: + f_binned = pd.cut(f, n_bins, labels=False) + mi = 0.0 + for fi in range(n_bins): + p_f = (f_binned == fi).mean() + if p_f == 0: + continue + for li in l.unique(): + p_l = (l == li).mean() + p_joint = ((f_binned == fi) & (l == li)).mean() + if p_joint > 0: + mi += p_joint * np.log(p_joint / (p_f * p_l)) + return float(mi) + + +def kl_divergence(factor: pd.Series, labels: pd.Series, + regime_a: str, regime_b: str, n_bins: int = 10) -> float: + """KL divergence between factor distributions in two regimes.""" + mask_a = (labels == regime_a) & factor.notna() + mask_b = (labels == regime_b) & factor.notna() + if mask_a.sum() < 10 or mask_b.sum() < 10: + return 0.0 + try: + hist_a, edges = np.histogram(factor[mask_a], bins=n_bins, density=True) + hist_b, _ = np.histogram(factor[mask_b], bins=edges, density=True) + except ValueError: + return 0.0 + hist_a = np.clip(hist_a, 1e-10, None) + hist_b = np.clip(hist_b, 1e-10, None) + return float((hist_a * np.log(hist_a / hist_b)).sum()) + + +def anova_f_score(factor: pd.Series, labels: pd.Series) -> float: + """ANOVA F-statistic: how well factor separates different regimes.""" + mask = factor.notna() & labels.notna() + if mask.sum() < 20: + return 0.0 + groups = [factor[mask][labels[mask] == lbl] for lbl in labels[mask].unique()] + groups = [g for g in groups if len(g) > 1] + if len(groups) < 2: + return 0.0 + f_stat, _ = stats.f_oneway(*groups) + return float(f_stat) if not np.isnan(f_stat) else 0.0 + + +def transition_matrix(labels: pd.Series) -> pd.DataFrame: + """Compute Markov transition matrix from regime sequence.""" + unique = sorted(labels.dropna().unique()) + n = len(unique) + matrix = np.zeros((n, n)) + seq = labels.dropna().values + for i in range(len(seq) - 1): + from_idx = unique.index(seq[i]) + to_idx = unique.index(seq[i + 1]) + matrix[from_idx][to_idx] += 1 + + # Row-normalize + row_sums = matrix.sum(axis=1, keepdims=True) + row_sums[row_sums == 0] = 1 + matrix = matrix / row_sums + + return pd.DataFrame(matrix, index=unique, columns=unique) + + +def regime_duration_stats(labels: pd.Series) -> dict: + """Compute average duration, flip rate, state entropy for regime sequence.""" + seq = labels.dropna().values + if len(seq) < 2: + return {"avg_duration": 0, "flip_rate": 0, "state_entropy": 0, "n_days": len(seq)} + + # Count durations + durations = [] + current = seq[0] + count = 1 + flips = 0 + for i in range(1, len(seq)): + if seq[i] == current: + count += 1 + else: + durations.append(count) + current = seq[i] + count = 1 + flips += 1 + durations.append(count) + + avg_dur = float(np.mean(durations)) if durations else 0 + flip_rate = flips / len(seq) + + # State entropy + _, counts = np.unique(seq, return_counts=True) + probs = counts / counts.sum() + entropy = float(-(probs * np.log2(probs + 1e-10)).sum()) + + return { + "avg_duration": round(avg_dur, 1), + "flip_rate": round(flip_rate, 3), + "state_entropy": round(entropy, 3), + "n_days": len(seq), + } diff --git a/ChanMacro/validation/regime_validator.py b/ChanMacro/validation/regime_validator.py new file mode 100644 index 0000000..be1f51b --- /dev/null +++ b/ChanMacro/validation/regime_validator.py @@ -0,0 +1,144 @@ +""" +validation/regime_validator.py — Validates factors as regime separators. + +Tests: Mutual Information, KL Divergence, ANOVA F-score. +Answers: "Does this factor distinguish different market regimes?" + +Key insight: a factor may have low IC (poor return predictor) but high +regime separation (good regime classifier). Breadth is the prime example. +""" + +from datetime import date as Date +from typing import Optional +import sqlite3 +import logging + +import numpy as np +import pandas as pd + +from config import config +from .metrics import ( + mutual_information, kl_divergence, anova_f_score, +) + +logger = logging.getLogger(__name__) + + +class RegimeReport: + """Structured report for regime separation validation.""" + + def __init__(self, factor_name: str): + self.factor_name = factor_name + self.mutual_info: float = 0.0 + self.anova_f: float = 0.0 + self.kl_pairs: dict = {} # (regime_a, regime_b) → KL divergence + self.best_separates: list[str] = [] + self.separation_score: float = 0.0 + self.is_regime_factor: bool = False + self.conclusion: str = "" + + def summary(self) -> str: + lines = [ + f"Factor: {self.factor_name}", + f" Mutual Information: {self.mutual_info:.4f}", + f" ANOVA F: {self.anova_f:.1f}", + f" Best separates: {', '.join(self.best_separates) if self.best_separates else 'none'}", + f" Regime Factor: {'YES' if self.is_regime_factor else 'No'}", + f" → {self.conclusion}", + ] + return "\n".join(lines) + + +class RegimeValidator: + """ + Validates a factor's ability to separate different market regimes. + + A good regime factor has: + - Mutual Information > 0.1 + - KL Divergence between regimes > 0.5 + - ANOVA F-score high + """ + + def __init__(self, db_path: Optional[str] = None): + self.db_path = db_path or config.db_path + + def validate(self, factor_name: str, factor_scores: pd.Series, + regime_labels: pd.Series) -> RegimeReport: + """ + Args: + factor_name: Human-readable name + factor_scores: Series indexed by date, values 0-100 + regime_labels: Series indexed by date, values = 'TREND'/'RANGE'/'PANIC' + """ + report = RegimeReport(factor_name) + + # Align + common_idx = factor_scores.index.intersection(regime_labels.index) + if len(common_idx) < 30: + report.conclusion = "INSUFFICIENT DATA" + return report + + f = factor_scores[common_idx] + labels = regime_labels[common_idx] + + # Mutual Information + report.mutual_info = round(mutual_information(f, labels), 4) + + # ANOVA + report.anova_f = round(anova_f_score(f, labels), 1) + + # KL Divergence between each pair of regimes + unique_regimes = sorted(labels.unique()) + for i, ra in enumerate(unique_regimes): + for rb in unique_regimes[i + 1:]: + kl = kl_divergence(f, labels, ra, rb) + report.kl_pairs[f"{ra}↔{rb}"] = round(kl, 4) + + # Best separation + if report.kl_pairs: + sorted_pairs = sorted(report.kl_pairs, key=report.kl_pairs.get, reverse=True) + report.best_separates = sorted_pairs[:2] + + # Separation score (0-1 composite) + mi_norm = min(report.mutual_info / 0.5, 1.0) + kl_avg = np.mean(list(report.kl_pairs.values())) if report.kl_pairs else 0 + kl_norm = min(kl_avg / 1.0, 1.0) + report.separation_score = round(0.5 * mi_norm + 0.5 * kl_norm, 2) + + # Is this a good regime factor? + report.is_regime_factor = ( + report.mutual_info > 0.1 and + kl_avg > 0.5 + ) + + if report.separation_score > 0.8: + report.conclusion = "EXCELLENT regime separator" + elif report.separation_score > 0.5: + report.conclusion = "GOOD regime separator" + elif report.separation_score > 0.3: + report.conclusion = "MODERATE — some regime separation" + else: + report.conclusion = "WEAK regime separator" + + return report + + def validate_from_db(self, factor_name: str, + score_query: str) -> RegimeReport: + """Load scores and regime labels from DB, then validate.""" + conn = sqlite3.connect(self.db_path) + + scores_df = pd.read_sql_query(score_query, conn) + regimes_df = pd.read_sql_query( + "SELECT date, regime FROM regime_history", conn + ) + conn.close() + + if scores_df.empty or regimes_df.empty: + r = RegimeReport(factor_name) + r.conclusion = "NO DATA" + return r + + scores = scores_df.set_index("date")["score"] + regimes = regimes_df.set_index("date")["regime"] + + return self.validate(factor_name, scores, regimes) diff --git a/ChanMacro/validation/reporter.py b/ChanMacro/validation/reporter.py new file mode 100644 index 0000000..607b182 --- /dev/null +++ b/ChanMacro/validation/reporter.py @@ -0,0 +1,120 @@ +""" +validation/reporter.py — Aggregates all validation reports into a unified summary. + +Used by: python main.py validate +""" + +from datetime import date as Date +from typing import Optional +import logging + +from .factor_validator import FactorValidator, FactorReport +from .regime_validator import RegimeValidator, RegimeReport +from .transition_validator import TransitionValidator, TransitionReport + +logger = logging.getLogger(__name__) + + +class ValidationReporter: + """ + Orchestrates full validation pipeline: + + 1. Factor validation (IC, ICIR, Hit Ratio) for each factor + 2. Regime validation (MI, KL, ANOVA) for each factor + 3. Transition validation (stability, flip rate) + """ + + def __init__(self, db_path: Optional[str] = None): + from config import config + self.db_path = db_path or config.db_path + self.factor_validator = FactorValidator(self.db_path) + self.regime_validator = RegimeValidator(self.db_path) + self.transition_validator = TransitionValidator(self.db_path) + + def run_all(self) -> str: + """Run all validations and return a formatted report string.""" + lines = [] + lines.append("=" * 70) + lines.append(f" ChanMacro Validation Report — {Date.today()}") + lines.append("=" * 70) + + # ── Factor Validation ────────────────────────── + lines.append("") + lines.append("─" * 50) + lines.append(" FACTOR VALIDATION (Predictive Power)") + lines.append("─" * 50) + + factor_queries = { + "Price Structure": "SELECT date, score FROM ohlcv_daily WHERE ema20 IS NOT NULL", + "Breadth": """ + SELECT bd.date, + (bd.advance_top50*1.0/(bd.advance_top50+bd.decline_top50+1)*100*0.30 + + bd.above_ema20_top50*1.0/50*100*0.35 + + bd.new_highs_20d_top50*1.0/50*100*0.20 + + 50*0.15) as score + FROM breadth_daily bd + """, + } + + factor_reports: list[FactorReport] = [] + for name, query in factor_queries.items(): + try: + report = self.factor_validator.validate_from_db(name, query) + factor_reports.append(report) + lines.append(report.summary()) + lines.append("") + except Exception as e: + logger.warning(f"Factor validation failed for {name}: {e}") + + # ── Regime Validation ────────────────────────── + lines.append("─" * 50) + lines.append(" REGIME VALIDATION (Regime Separation)") + lines.append("─" * 50) + + regime_reports: list[RegimeReport] = [] + for name, query in factor_queries.items(): + try: + report = self.regime_validator.validate_from_db(name, query) + regime_reports.append(report) + lines.append(report.summary()) + lines.append("") + except Exception as e: + logger.warning(f"Regime validation failed for {name}: {e}") + + # ── Transition Validation ────────────────────── + lines.append("─" * 50) + lines.append(" TRANSITION VALIDATION (Regime Stability)") + lines.append("─" * 50) + + try: + t_report = self.transition_validator.validate_from_db() + lines.append(t_report.summary()) + except Exception as e: + logger.warning(f"Transition validation failed: {e}") + + # ── Summary ──────────────────────────────────── + lines.append("") + lines.append("=" * 70) + lines.append(" SUMMARY") + lines.append("=" * 70) + + # Factor ranking by IC + if factor_reports: + ranked = sorted(factor_reports, key=lambda r: abs(r.ic_mean), reverse=True) + lines.append(" Factor Ranking (by |IC|):") + for i, r in enumerate(ranked): + tag = "★★★" if abs(r.ic_mean) > 0.05 else "★★" if abs(r.ic_mean) > 0.03 else "★" + lines.append(f" {i+1}. {r.factor_name:20s} IC={r.ic_mean:+.4f} {tag} {r.conclusion}") + + # Regime factor ranking + if regime_reports: + ranked_r = sorted(regime_reports, key=lambda r: r.separation_score, reverse=True) + lines.append("") + lines.append(" Regime Factor Ranking (by Separation Score):") + for i, r in enumerate(ranked_r): + lines.append(f" {i+1}. {r.factor_name:20s} Score={r.separation_score:.2f} {r.conclusion}") + + lines.append("") + lines.append("=" * 70) + + return "\n".join(lines) diff --git a/ChanMacro/validation/transition_validator.py b/ChanMacro/validation/transition_validator.py new file mode 100644 index 0000000..b5f7e41 --- /dev/null +++ b/ChanMacro/validation/transition_validator.py @@ -0,0 +1,131 @@ +""" +validation/transition_validator.py — Validates regime stability. + +Tests: Transition matrix, average duration, flip rate, state entropy. +Answers: "Does the regime design produce stable, persistent states?" + +Hard requirements: + - avg_duration > 5 days + - flip_rate < 15% + - Fails → regime definition needs redesign. +""" + +from typing import Optional +import sqlite3 +import logging + +import numpy as np +import pandas as pd + +from config import config +from .metrics import transition_matrix, regime_duration_stats + +logger = logging.getLogger(__name__) + + +class TransitionReport: + """Structured report for regime stability validation.""" + + def __init__(self): + self.avg_duration: float = 0.0 + self.flip_rate: float = 0.0 + self.state_entropy: float = 0.0 + self.n_days: int = 0 + self.transition_matrix: Optional[pd.DataFrame] = None + self.persistence_score: float = 0.0 + self.is_stable: bool = False + self.conclusion: str = "" + self.warnings: list[str] = [] + + def summary(self) -> str: + lines = [ + f"Regime Stability (N={self.n_days} days)", + f" Avg Duration: {self.avg_duration:.1f} days (need > {config.regime_min_avg_duration})", + f" Flip Rate: {self.flip_rate:.1%} (need < {config.regime_max_flip_rate:.0%})", + f" State Entropy: {self.state_entropy:.3f}", + f" Persistence Score: {self.persistence_score:.2f}", + f" Stable: {'YES' if self.is_stable else 'NO — redesign needed'}", + ] + if self.warnings: + lines.append(f" Warnings: {'; '.join(self.warnings)}") + if self.transition_matrix is not None: + lines.append(f" Transition Matrix:\n{self.transition_matrix.to_string()}") + lines.append(f" → {self.conclusion}") + return "\n".join(lines) + + +class TransitionValidator: + """ + Validates regime temporal stability. + + Regime must persist — not flip daily. + If flip_rate > 20% or avg_duration < 3 days → regime definition failed. + """ + + def __init__(self, db_path: Optional[str] = None): + self.db_path = db_path or config.db_path + + def validate(self, regime_labels: pd.Series) -> TransitionReport: + """Validate a regime sequence for stability.""" + report = TransitionReport() + report.n_days = len(regime_labels) + + if len(regime_labels) < 30: + report.conclusion = "INSUFFICIENT DATA (< 30 days)" + return report + + # Duration stats + stats = regime_duration_stats(regime_labels) + report.avg_duration = stats["avg_duration"] + report.flip_rate = stats["flip_rate"] + report.state_entropy = stats["state_entropy"] + + # Transition matrix + report.transition_matrix = transition_matrix(regime_labels) + + # Persistence: how often does regime stay the same? + diag = np.diag(report.transition_matrix.values) + report.persistence_score = round(float(np.mean(diag)), 2) + + # Stability check + report.is_stable = ( + report.avg_duration >= config.regime_min_avg_duration and + report.flip_rate <= config.regime_max_flip_rate + ) + + # Warnings + if report.avg_duration < 3: + report.warnings.append(f"CRITICAL: avg duration={report.avg_duration:.1f}d — regime flips too fast") + elif report.avg_duration < config.regime_min_avg_duration: + report.warnings.append(f"WARNING: avg duration={report.avg_duration:.1f}d < {config.regime_min_avg_duration}") + + if report.flip_rate > 0.20: + report.warnings.append(f"CRITICAL: flip rate={report.flip_rate:.1%} — regime unstable") + elif report.flip_rate > config.regime_max_flip_rate: + report.warnings.append(f"WARNING: flip rate={report.flip_rate:.1%} > {config.regime_max_flip_rate:.0%}") + + if report.state_entropy > 2.0: + report.warnings.append(f"NOTE: high state entropy={report.state_entropy:.2f}, regimes may be too fine-grained") + + if report.is_stable: + report.conclusion = "PASS: regime design is stable" + else: + report.conclusion = "FAIL: regime definition needs adjustment" + + return report + + def validate_from_db(self) -> TransitionReport: + """Load regime history from DB and validate stability.""" + conn = sqlite3.connect(self.db_path) + df = pd.read_sql_query( + "SELECT date, regime FROM regime_history ORDER BY date", conn + ) + conn.close() + + if df.empty: + r = TransitionReport() + r.conclusion = "NO DATA" + return r + + regimes = df.set_index("date")["regime"] + return self.validate(regimes) diff --git a/ChanMacro/web/__init__.py b/ChanMacro/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/data_provider/main.py b/data_provider/main.py index 91f5652..5aad9d1 100644 --- a/data_provider/main.py +++ b/data_provider/main.py @@ -260,6 +260,9 @@ class DataProvider: # 尝试加载历史恢复点 self._load_resume_since() self._update_callbacks: List = [] + # 衍生品数据: 与 K 线一样的模式 — symbol -> List[Dict], 按 timestamp 去重 + self._derivatives: Dict[str, List[Dict]] = {symbol: [] for symbol in self.symbols} + self._derivatives_dir: Path = self.data_dir / "derivatives" def _load_config(self) -> Dict[str, object]: """读取 JSON 配置文件。""" @@ -460,6 +463,117 @@ class DataProvider: symbol_safe = symbol.replace("/", "_").replace(":", "_") return self.data_dir / timeframe / f"{self.exchange.id}_{symbol_safe}_{timeframe}.csv" + def _derivatives_file_path(self, symbol: str) -> Path: + """衍生品 CSV 路径：data_dir/derivatives/exchange_symbol.csv。""" + symbol_safe = symbol.replace("/", "_").replace(":", "_") + return self._derivatives_dir / f"{self.exchange.id}_{symbol_safe}.csv" + + def _load_derivatives_local(self, symbol: str) -> List[Dict]: + """加载本地衍生品历史数据。""" + path = self._derivatives_file_path(symbol) + if not path.exists(): + return [] + loaded = [] + with path.open("r", encoding="utf-8", newline="") as fp: + for row in csv.DictReader(fp): + try: + loaded.append({ + "timestamp": int(row["timestamp"]), + "datetime": row.get("datetime", ""), + "funding_rate": float(row.get("funding_rate", 0)), + "open_interest": float(row.get("open_interest", 0)), + "oi_change_pct": float(row.get("oi_change_pct", 0)) if row.get("oi_change_pct") else None, + "basis": float(row.get("basis", 0)) if row.get("basis") else None, + }) + except (KeyError, ValueError): + continue + loaded.sort(key=lambda item: item["timestamp"]) + return loaded + + def _fetch_derivatives_sync(self, symbol: str) -> Optional[List[Dict]]: + """用 REST 获取单个币对的费率、OI 和基差（同步，供后台线程调用）。 + 返回单条记录列表，与 K 线 fetcher 返回格式一致。""" + try: + funding = self.exchange.fetch_funding_rate(symbol) + oi = self.exchange.fetch_open_interest(symbol) + ticker = self.exchange.fetch_ticker(symbol) + except Exception as e: + logger.warning("衍生品获取失败 %s: %s", symbol, e) + return None + + now_ms = int(time.time() * 1000) + record = { + "timestamp": now_ms, + "datetime": to_utc_iso(now_ms), + "funding_rate": float(funding.get("fundingRate", 0)) if funding else 0, + "open_interest": float(oi.get("openInterestAmount", 0)) if oi else 0, + "oi_change_pct": None, + "basis": None, + } + + # OI 变化 (与上一条对比) + with self._lock: + history = self._derivatives.get(symbol, []) + if history: + prev = history[-1] + prev_oi = prev.get("open_interest", 0) + if prev_oi > 0 and record["open_interest"] > 0: + record["oi_change_pct"] = round( + (record["open_interest"] - prev_oi) / prev_oi * 100, 2 + ) + + # 基差 (期货-现货)/现货 + if ticker: + spot_symbol = symbol.split(":")[0] + try: + spot_ticker = self.exchange.fetch_ticker(spot_symbol) + future_price = float(ticker.get("last", 0)) + spot_price = float(spot_ticker.get("last", 0)) + if spot_price > 0 and future_price > 0: + record["basis"] = round( + (future_price - spot_price) / spot_price * 100, 2 + ) + except Exception: + pass + + return [record] + + def _merge_derivatives( + self, + base: List[Dict], + new_records: List[Dict], + ) -> List[Dict]: + """按 timestamp 去重合并衍生品记录，新数据覆盖同时间戳旧数据。与 _merge_candles 模式一致。""" + merged = {entry["timestamp"]: entry for entry in base} + for record in new_records: + merged[record["timestamp"]] = record + return list(sorted(merged.values(), key=lambda item: item["timestamp"])) + + def _write_derivatives_to_disk(self, symbol: str, records: List[Dict]) -> None: + """将衍生品历史数据写入 CSV。与 _write_to_disk 模式一致：先写 tmp 再 replace。""" + if not records: + return + path = self._derivatives_file_path(symbol) + path.parent.mkdir(parents=True, exist_ok=True) + fieldnames = ["timestamp", "datetime", "funding_rate", "open_interest", + "oi_change_pct", "basis"] + tmp_path = path.with_suffix(".tmp") + try: + with tmp_path.open("w", encoding="utf-8", newline="") as fp: + writer = csv.DictWriter(fp, fieldnames=fieldnames, extrasaction="ignore") + writer.writeheader() + for r in records: + row = {} + for k in fieldnames: + val = r.get(k, "") + row[k] = "" if val is None else val + writer.writerow(row) + tmp_path.replace(path) + except Exception as e: + logger.error("衍生品落盘失败 %s: %s", symbol, e) + if tmp_path.exists(): + tmp_path.unlink() + def _load_local(self, symbol: str, timeframe: str) -> List[Dict[str, float]]: """启动时从磁盘加载已有 K 线，损坏行跳过，按时间排序。""" path = self._data_file_path(symbol, timeframe) @@ -799,14 +913,38 @@ class DataProvider: except Exception as exc: logger.error("数据更新回调异常: %s", exc) + def _derivatives_refresh_loop(self) -> None: + """后台线程：每 60 秒拉取一次衍生品数据。与 K 线 watch 模式一致：拉取 → 合并 → 写入内存。""" + # 首次启动先加载本地历史 + for symbol in self.symbols: + history = self._load_derivatives_local(symbol) + if history: + with self._lock: + self._derivatives[symbol] = history + logger.info("衍生品 %s 加载本地记录: %d 条", symbol, len(history)) + + DERIVATIVES_INTERVAL = 60 + while not self._stop_event.wait(DERIVATIVES_INTERVAL): + for symbol in self.symbols: + try: + new_records = self._fetch_derivatives_sync(symbol) + if new_records: + with self._lock: + base = self._derivatives.get(symbol, []) + self._derivatives[symbol] = self._merge_derivatives(base, new_records) + except Exception as e: + logger.debug("衍生品刷新失败 %s: %s", symbol, e) + def start_background_workers(self) -> None: - """启动后台线程：冷启动回填 + 周期性落盘。WebSocket 监听由 lifespan 异步启动。""" + """启动后台线程：冷启动回填 + 周期性落盘 + 衍生品刷新。WebSocket 监听由 lifespan 异步启动。""" self._stop_event.clear() self._backfill_thread = threading.Thread(target=self._cold_start_backfill, name="backfill-loop", daemon=True) self._persist_thread = threading.Thread(target=self._persist_loop, name="persist-loop", daemon=True) + self._derivatives_thread = threading.Thread(target=self._derivatives_refresh_loop, name="derivatives-loop", daemon=True) self._backfill_thread.start() self._persist_thread.start() - logger.info("后台线程已启动（回填 + 落盘）") + self._derivatives_thread.start() + logger.info("后台线程已启动（回填 + 落盘 + 衍生品）") def start_watch_tasks(self) -> None: """在当前 asyncio event loop 上启动 WebSocket 监听。必须在 lifespan 内调用。""" @@ -857,6 +995,15 @@ class DataProvider: if timeframe in MAX_CANDLES_IN_MEMORY: continue # 内存只有尾部 N 根，不覆盖 CSV 全量 self._write_to_disk(symbol, timeframe, data) + # 周期性落盘衍生品数据（与 K 线一致：锁内复制 → 完整覆写 CSV） + with self._lock: + deriv_snapshot = { + symbol: list(records) for symbol, records in self._derivatives.items() + } + for symbol, records in deriv_snapshot.items(): + if records: + self._write_derivatives_to_disk(symbol, records) + # 周期性也保存一次恢复点，保证一致性 self._save_resume_since() @@ -1068,6 +1215,26 @@ def create_app(provider: DataProvider) -> FastAPI: data = provider.get_klines(symbol=symbol, timeframe=tf, start_time=start, end_time=end, limit=limit) return data + @app.get("/api/derivatives") + async def api_derivatives( + symbol: str = Query("BTC/USDT:USDT", description="如 BTC/USDT:USDT"), + ): + """返回指定币对的衍生品数据快照（资金费率、OI、基差）。""" + with provider._lock: + records = list(provider._derivatives.get(symbol, [])) + if not records: + raise HTTPException(status_code=404, detail=f"衍生品数据不可用: {symbol}") + record = records[-1] # 最新一条 + return { + "symbol": symbol, + "timestamp": record.get("timestamp"), + "datetime": record.get("datetime"), + "funding_rate": record.get("funding_rate"), + "open_interest": record.get("open_interest"), + "oi_change_pct": record.get("oi_change_pct"), + "basis": record.get("basis"), + } + homepage_path = Path(__file__).resolve().parent / "homepage.html" docs_path = Path(__file__).resolve().parent / "api_docs.html"