feat: 新增 ChanMacro 宏观 regime 检测模块
This commit is contained in:
jackyu66git
@@ -0,0 +1,5 @@
|
||||
"""Data fetchers — L0 raw data acquisition."""
|
||||
from .base import BaseFetcher
|
||||
from .ohlcv import OHLCVFetcher
|
||||
from .breadth import BreadthFetcher
|
||||
from .derivatives import DerivativesFetcher
|
||||
@@ -0,0 +1,69 @@
|
||||
"""
|
||||
fetchers/base.py — Abstract base class for all macro data fetchers.
|
||||
|
||||
Provides retry logic, rate limiting, and a common interface.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import date as Date
|
||||
from typing import Optional
|
||||
import logging
|
||||
import time
|
||||
import requests
|
||||
|
||||
|
||||
class BaseFetcher(ABC):
|
||||
"""Abstract base for all macro data fetchers."""
|
||||
|
||||
def __init__(self, timeout: int = 30, max_retries: int = 3):
|
||||
self.timeout = timeout
|
||||
self.max_retries = max_retries
|
||||
self.logger = logging.getLogger(self.__class__.__name__)
|
||||
|
||||
def _get(self, url: str, params: Optional[dict] = None,
|
||||
headers: Optional[dict] = None) -> dict:
|
||||
"""GET with retry and exponential backoff."""
|
||||
for attempt in range(self.max_retries):
|
||||
try:
|
||||
resp = requests.get(
|
||||
url, params=params, headers=headers, timeout=self.timeout
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
except requests.RequestException as e:
|
||||
wait = 2 ** attempt
|
||||
self.logger.warning(
|
||||
f"Request failed (attempt {attempt+1}/{self.max_retries}): {e}. "
|
||||
f"Retrying in {wait}s"
|
||||
)
|
||||
if attempt < self.max_retries - 1:
|
||||
time.sleep(wait)
|
||||
else:
|
||||
raise
|
||||
|
||||
def _get_raw(self, url: str, params: Optional[dict] = None,
|
||||
headers: Optional[dict] = None) -> bytes:
|
||||
"""GET raw bytes with retry (for non-JSON endpoints)."""
|
||||
for attempt in range(self.max_retries):
|
||||
try:
|
||||
resp = requests.get(
|
||||
url, params=params, headers=headers, timeout=self.timeout
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
except requests.RequestException as e:
|
||||
wait = 2 ** attempt
|
||||
if attempt < self.max_retries - 1:
|
||||
time.sleep(wait)
|
||||
else:
|
||||
raise
|
||||
|
||||
@abstractmethod
|
||||
def fetch(self, target_date: Optional[Date] = None) -> list[dict]:
|
||||
"""Fetch raw data. Returns list of record dicts."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def store(self, db_path: str, records: list[dict]) -> int:
|
||||
"""Store raw records into SQLite. Returns count of new rows."""
|
||||
...
|
||||
@@ -0,0 +1,189 @@
|
||||
"""
|
||||
fetchers/breadth.py — Fetches TOP50 OHLCV and computes market breadth metrics.
|
||||
|
||||
Multi-tier: Top20 / Top30 / Top50 for advance/decline, EMA20%, new highs, BTC.D.
|
||||
"""
|
||||
|
||||
from datetime import date as Date, datetime
|
||||
from typing import Optional
|
||||
import logging
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import requests
|
||||
|
||||
from .base import BaseFetcher
|
||||
from config import config
|
||||
|
||||
|
||||
class BreadthFetcher(BaseFetcher):
|
||||
"""Fetches TOP50 coin OHLCV data and computes breadth metrics."""
|
||||
|
||||
def __init__(self, provider_url: Optional[str] = None):
|
||||
super().__init__(timeout=60, max_retries=3)
|
||||
self.provider_url = provider_url or config.provider_url
|
||||
self.symbols = config.top50_symbols
|
||||
self.ema_period = config.breadth_ema_period
|
||||
self.new_high_window = config.breadth_new_high_window
|
||||
self.logger = logging.getLogger(__name__)
|
||||
|
||||
def fetch(self, target_date: Optional[Date] = None) -> dict:
|
||||
"""
|
||||
Fetch daily OHLCV for all TOP50 symbols and compute breadth.
|
||||
|
||||
Returns a dict suitable for storing in breadth_daily table.
|
||||
"""
|
||||
if target_date is None:
|
||||
target_date = Date.today()
|
||||
|
||||
# Fetch last 60 days of daily data for each symbol to compute EMAs and new highs
|
||||
all_data = {}
|
||||
for symbol in self.symbols:
|
||||
try:
|
||||
df = self._fetch_symbol(symbol)
|
||||
if df is not None and not df.empty:
|
||||
all_data[symbol] = df
|
||||
except Exception as e:
|
||||
self.logger.debug(f"Failed to fetch {symbol}: {e}")
|
||||
|
||||
if not all_data:
|
||||
self.logger.error("No symbol data fetched for breadth")
|
||||
return {}
|
||||
|
||||
# Compute breadth metrics for the target date
|
||||
breadth = self._compute_breadth(all_data, target_date)
|
||||
return breadth
|
||||
|
||||
def _fetch_symbol(self, symbol: str) -> Optional[pd.DataFrame]:
|
||||
"""Fetch daily OHLCV for a single symbol."""
|
||||
url = f"{self.provider_url}/api/candles"
|
||||
params = {
|
||||
"symbol": symbol,
|
||||
"tf": "1d",
|
||||
"limit": 100,
|
||||
}
|
||||
try:
|
||||
resp = requests.get(url, params=params, timeout=15)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if not data:
|
||||
return None
|
||||
|
||||
df = pd.DataFrame(data)
|
||||
df["timestamp"] = pd.to_datetime(df["timestamp"], unit="ms", utc=True)
|
||||
df["date"] = df["timestamp"].dt.date
|
||||
df = df.drop_duplicates(subset="date").sort_values("date").reset_index(drop=True)
|
||||
df["close"] = df["close"].astype(float)
|
||||
df["ema20"] = df["close"].ewm(span=self.ema_period, adjust=False).mean()
|
||||
return df
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _compute_breadth(self, all_data: dict, target_date: Date) -> dict:
|
||||
"""Compute breadth metrics for a specific date across all symbols."""
|
||||
total = len(all_data)
|
||||
|
||||
advances_50 = declines_50 = 0
|
||||
above_ema20_50 = 0
|
||||
new_highs_50 = 0
|
||||
advances_30 = declines_30 = 0
|
||||
above_ema20_30 = 0
|
||||
new_highs_30 = 0
|
||||
advances_20 = declines_20 = 0
|
||||
above_ema20_20 = 0
|
||||
new_highs_20 = 0
|
||||
|
||||
for i, (symbol, df) in enumerate(all_data.items()):
|
||||
# Get data for target date
|
||||
df["date_str"] = df["date"].astype(str)
|
||||
target_str = str(target_date)
|
||||
idx = df[df["date_str"] == target_str].index
|
||||
|
||||
if len(idx) == 0:
|
||||
continue
|
||||
|
||||
row_idx = idx[0]
|
||||
if row_idx < 1:
|
||||
continue
|
||||
|
||||
current_close = df.loc[row_idx, "close"]
|
||||
prev_close = df.loc[row_idx - 1, "close"]
|
||||
|
||||
# Advance/Decline
|
||||
if current_close > prev_close:
|
||||
if i < 50: advances_50 += 1
|
||||
if i < 30: advances_30 += 1
|
||||
if i < 20: advances_20 += 1
|
||||
elif current_close < prev_close:
|
||||
if i < 50: declines_50 += 1
|
||||
if i < 30: declines_30 += 1
|
||||
if i < 20: declines_20 += 1
|
||||
|
||||
# Above EMA20
|
||||
ema20_val = df.loc[row_idx, "ema20"]
|
||||
if not pd.isna(ema20_val) and current_close > ema20_val:
|
||||
if i < 50: above_ema20_50 += 1
|
||||
if i < 30: above_ema20_30 += 1
|
||||
if i < 20: above_ema20_20 += 1
|
||||
|
||||
# New 20-day highs
|
||||
lookback_start = max(0, row_idx - self.new_high_window)
|
||||
recent_highs = df.loc[lookback_start:row_idx - 1, "high"].astype(float)
|
||||
current_high = df.loc[row_idx, "high"]
|
||||
if len(recent_highs) > 0 and float(current_high) > recent_highs.max():
|
||||
if i < 50: new_highs_50 += 1
|
||||
if i < 30: new_highs_30 += 1
|
||||
if i < 20: new_highs_20 += 1
|
||||
|
||||
return {
|
||||
"date": str(target_date),
|
||||
"total_tracked": total,
|
||||
"advance_top50": advances_50,
|
||||
"decline_top50": declines_50,
|
||||
"above_ema20_top50": above_ema20_50,
|
||||
"new_highs_20d_top50": new_highs_50,
|
||||
"advance_top30": advances_30,
|
||||
"advance_top20": advances_20,
|
||||
"above_ema20_top30": above_ema20_30,
|
||||
"above_ema20_top20": above_ema20_20,
|
||||
"new_highs_20d_top30": new_highs_30,
|
||||
"new_highs_20d_top20": new_highs_20,
|
||||
"btc_dominance": None, # Reserved for Coinglass API integration
|
||||
}
|
||||
|
||||
def store(self, db_path: Optional[str] = None, record: Optional[dict] = None) -> int:
|
||||
"""Store a breadth record into SQLite. Returns 1 if inserted/updated."""
|
||||
import sqlite3
|
||||
db_path = db_path or config.db_path
|
||||
conn = sqlite3.connect(db_path)
|
||||
|
||||
if record is None:
|
||||
conn.close()
|
||||
return 0
|
||||
|
||||
try:
|
||||
conn.execute("""
|
||||
INSERT OR REPLACE INTO breadth_daily
|
||||
(date, total_tracked,
|
||||
advance_top50, decline_top50, above_ema20_top50, new_highs_20d_top50,
|
||||
advance_top30, advance_top20,
|
||||
above_ema20_top30, above_ema20_top20,
|
||||
new_highs_20d_top30, new_highs_20d_top20,
|
||||
btc_dominance)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
record["date"], record.get("total_tracked", 50),
|
||||
record.get("advance_top50", 0), record.get("decline_top50", 0),
|
||||
record.get("above_ema20_top50", 0), record.get("new_highs_20d_top50", 0),
|
||||
record.get("advance_top30", 0), record.get("advance_top20", 0),
|
||||
record.get("above_ema20_top30", 0), record.get("above_ema20_top20", 0),
|
||||
record.get("new_highs_20d_top30", 0), record.get("new_highs_20d_top20", 0),
|
||||
record.get("btc_dominance"),
|
||||
))
|
||||
conn.commit()
|
||||
return 1
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to store breadth: {e}")
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -0,0 +1,66 @@
|
||||
"""
|
||||
fetchers/derivatives.py — Fetches derivatives data from data_provider API.
|
||||
|
||||
Clean consumer: no direct ccxt dependency. Just HTTP GET /api/derivatives.
|
||||
"""
|
||||
|
||||
from datetime import date as Date
|
||||
from typing import Optional
|
||||
|
||||
import requests
|
||||
|
||||
from .base import BaseFetcher
|
||||
from config import config
|
||||
|
||||
|
||||
class DerivativesFetcher(BaseFetcher):
|
||||
"""Fetches derivatives snapshot from data_provider /api/derivatives."""
|
||||
|
||||
def __init__(self, provider_url: Optional[str] = None):
|
||||
super().__init__(timeout=15, max_retries=3)
|
||||
self.provider_url = provider_url or config.provider_url
|
||||
|
||||
def fetch(self, target_date: Optional[Date] = None) -> list[dict]:
|
||||
"""Fetch derivatives data. Returns list with one record dict."""
|
||||
url = f"{self.provider_url}/api/derivatives"
|
||||
params = {"symbol": config.btc_symbol}
|
||||
try:
|
||||
data = self._get(url, params=params)
|
||||
record = {
|
||||
"date": str(target_date or Date.today()),
|
||||
"symbol": config.btc_symbol,
|
||||
"funding_rate": data.get("funding_rate"),
|
||||
"open_interest": data.get("open_interest"),
|
||||
"oi_24h_change_pct": data.get("oi_change_pct"),
|
||||
"basis_annualised_pct": data.get("basis"),
|
||||
"source": "data_provider",
|
||||
}
|
||||
return [record]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def store(self, db_path: Optional[str] = None, records: Optional[list[dict]] = None) -> int:
|
||||
"""Store derivatives records into SQLite."""
|
||||
import sqlite3
|
||||
db_path = db_path or config.db_path
|
||||
records = records or []
|
||||
conn = sqlite3.connect(db_path)
|
||||
count = 0
|
||||
for r in records:
|
||||
try:
|
||||
conn.execute("""
|
||||
INSERT OR REPLACE INTO derivatives
|
||||
(date, symbol, funding_rate, open_interest, oi_24h_change_pct,
|
||||
long_liquidations, short_liquidations, basis_annualised_pct)
|
||||
VALUES (?, ?, ?, ?, ?, NULL, NULL, ?)
|
||||
""", (
|
||||
r["date"], r.get("symbol", config.btc_symbol),
|
||||
r.get("funding_rate"), r.get("open_interest"),
|
||||
r.get("oi_24h_change_pct"), r.get("basis_annualised_pct"),
|
||||
))
|
||||
count += 1
|
||||
except Exception:
|
||||
continue
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return count
|
||||
@@ -0,0 +1,157 @@
|
||||
"""
|
||||
fetchers/ohlcv.py — Fetches BTC daily OHLCV from the existing data_provider service.
|
||||
|
||||
Also pre-computes EMA20/60/120, ATR(14), BB width, ADX(14).
|
||||
"""
|
||||
|
||||
from datetime import date as Date, datetime, timedelta
|
||||
from typing import Optional
|
||||
import logging
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import requests
|
||||
|
||||
from .base import BaseFetcher
|
||||
from config import config
|
||||
|
||||
|
||||
class OHLCVFetcher(BaseFetcher):
|
||||
"""Fetches BTC daily K-line data from data_provider API."""
|
||||
|
||||
def __init__(self, provider_url: Optional[str] = None):
|
||||
super().__init__(timeout=30, max_retries=3)
|
||||
self.provider_url = provider_url or config.provider_url
|
||||
self.symbol = config.btc_symbol
|
||||
self.logger = logging.getLogger(__name__)
|
||||
|
||||
def fetch(self, target_date: Optional[Date] = None) -> pd.DataFrame:
|
||||
"""
|
||||
Fetch daily OHLCV for BTC. Returns DataFrame with computed indicators.
|
||||
|
||||
Fetches enough history (200 bars) to compute EMAs/ATR/BB/ADX accurately.
|
||||
"""
|
||||
url = f"{self.provider_url}/api/candles"
|
||||
params = {
|
||||
"symbol": self.symbol,
|
||||
"tf": "1d",
|
||||
"limit": 200,
|
||||
}
|
||||
resp = requests.get(url, params=params, timeout=self.timeout)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
if not data:
|
||||
self.logger.warning("OHLCV API returned empty data")
|
||||
return pd.DataFrame()
|
||||
|
||||
df = pd.DataFrame(data)
|
||||
df["timestamp"] = pd.to_datetime(df["timestamp"], unit="ms", utc=True)
|
||||
df["date"] = df["timestamp"].dt.date
|
||||
df = df.drop_duplicates(subset="date").sort_values("date").reset_index(drop=True)
|
||||
|
||||
# Rename columns to match expected format
|
||||
df = df.rename(columns={
|
||||
"open": "open", "high": "high", "low": "low", "close": "close",
|
||||
"volume": "volume",
|
||||
})
|
||||
|
||||
# Compute indicators
|
||||
df = self._add_indicators(df)
|
||||
|
||||
return df
|
||||
|
||||
def _add_indicators(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Add EMA, ATR, BB, ADX indicators."""
|
||||
close = df["close"].astype(float)
|
||||
high = df["high"].astype(float)
|
||||
low = df["low"].astype(float)
|
||||
|
||||
# EMAs
|
||||
df["ema20"] = close.ewm(span=20, adjust=False).mean()
|
||||
df["ema60"] = close.ewm(span=60, adjust=False).mean()
|
||||
df["ema120"] = close.ewm(span=120, adjust=False).mean()
|
||||
|
||||
# ATR(14)
|
||||
tr1 = high - low
|
||||
tr2 = (high - close.shift(1)).abs()
|
||||
tr3 = (low - close.shift(1)).abs()
|
||||
tr = pd.concat([tr1, tr2, tr3], axis=1).max(axis=1)
|
||||
df["atr_14"] = tr.rolling(14).mean()
|
||||
|
||||
# Bollinger Bands width
|
||||
sma20 = close.rolling(20).mean()
|
||||
std20 = close.rolling(20).std()
|
||||
df["bb_width"] = (2 * std20) / sma20 * 100 # as percentage
|
||||
|
||||
# ADX(14)
|
||||
df["adx_14"] = self._compute_adx(df, period=14)
|
||||
|
||||
return df
|
||||
|
||||
@staticmethod
|
||||
def _compute_adx(df: pd.DataFrame, period: int = 14) -> pd.Series:
|
||||
"""Compute ADX from OHLC data."""
|
||||
high = df["high"].astype(float)
|
||||
low = df["low"].astype(float)
|
||||
close = df["close"].astype(float)
|
||||
|
||||
plus_dm = high.diff()
|
||||
minus_dm = low.diff().abs() * -1
|
||||
plus_dm = plus_dm.where(plus_dm > 0, 0)
|
||||
minus_dm = minus_dm.where(minus_dm < 0, 0).abs()
|
||||
|
||||
tr1 = high - low
|
||||
tr2 = (high - close.shift(1)).abs()
|
||||
tr3 = (low - close.shift(1)).abs()
|
||||
tr = pd.concat([tr1, tr2, tr3], axis=1).max(axis=1)
|
||||
|
||||
atr = tr.rolling(period).mean()
|
||||
plus_di = 100 * (plus_dm.rolling(period).mean() / atr)
|
||||
minus_di = 100 * (minus_dm.rolling(period).mean() / atr)
|
||||
|
||||
dx = (abs(plus_di - minus_di) / (plus_di + minus_di)) * 100
|
||||
adx = dx.rolling(period).mean()
|
||||
return adx
|
||||
|
||||
def store(self, db_path: str, records: list[dict]) -> int:
|
||||
"""Store OHLCV records into SQLite. Not used directly — see store_df."""
|
||||
return 0
|
||||
|
||||
def store_df(self, df: pd.DataFrame, db_path: Optional[str] = None) -> int:
|
||||
"""Store the DataFrame into the ohlcv_daily table."""
|
||||
import sqlite3
|
||||
db_path = db_path or config.db_path
|
||||
conn = sqlite3.connect(db_path)
|
||||
|
||||
count = 0
|
||||
for _, row in df.iterrows():
|
||||
if pd.isna(row.get("date")):
|
||||
continue
|
||||
date_str = str(row["date"])
|
||||
try:
|
||||
conn.execute("""
|
||||
INSERT OR REPLACE INTO ohlcv_daily
|
||||
(date, symbol, open, high, low, close, volume,
|
||||
ema20, ema60, ema120, atr_14, bb_width, adx_14)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
date_str, self.symbol,
|
||||
float(row["open"]), float(row["high"]),
|
||||
float(row["low"]), float(row["close"]),
|
||||
float(row.get("volume", 0)),
|
||||
float(row["ema20"]) if not pd.isna(row.get("ema20")) else None,
|
||||
float(row["ema60"]) if not pd.isna(row.get("ema60")) else None,
|
||||
float(row["ema120"]) if not pd.isna(row.get("ema120")) else None,
|
||||
float(row["atr_14"]) if not pd.isna(row.get("atr_14")) else None,
|
||||
float(row["bb_width"]) if not pd.isna(row.get("bb_width")) else None,
|
||||
float(row["adx_14"]) if not pd.isna(row.get("adx_14")) else None,
|
||||
))
|
||||
count += 1
|
||||
except Exception as e:
|
||||
self.logger.debug(f"Skip row {date_str}: {e}")
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
self.logger.info(f"Stored {count} OHLCV rows")
|
||||
return count
|
||||
Reference in New Issue
Block a user