chanmacro: Signal Expectancy Engine V1 — Market Memory System
Phase A-C complete: 4 core factors, regime detection, signal tracking, Bayesian expectancy. chanmacro/ (32 files, ~4000 lines): - models: 12 enums + 15 Pydantic v2 models (DateAwareModel, MarketStateVector, etc.) - fetchers: OHLCV + Breadth (from data_provider) + Derivatives (new endpoint) - scoring: Price Structure / Breadth (quantile buckets) / OI Matrix (5 discrete states) / Volatility Regime - regime_detector: 3-state (TREND/RANGE/PANIC), factor-locked (Price+Breadth+Vol), versioned, 2-day confirmation - expectancy: SignalTracker (record+outcomes), TimeDecay (half-life=180d), BayesianExpectancyEngine (Empirical Bayes, Leveled, SufficiencyGuard) - validation: FactorValidator (IC/ICIR/Hit Ratio), RegimeValidator (MI/KL/ANOVA), TransitionValidator (stability) - CLI: fetch|score|regime|track|backfill|expectancy|validate|serve - tests: 52 passing (models, scoring, regime, expectancy) data_provider: - /api/derivatives endpoint: funding rate, OI, OI change, basis - _derivatives storage: same persist pattern as K-line (merge→lock→snapshot→atomic write) - background refresh every 60s Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
jackyu66git
+169
-2
@@ -260,6 +260,9 @@ class DataProvider:
|
||||
# 尝试加载历史恢复点
|
||||
self._load_resume_since()
|
||||
self._update_callbacks: List = []
|
||||
# 衍生品数据: 与 K 线一样的模式 — symbol -> List[Dict], 按 timestamp 去重
|
||||
self._derivatives: Dict[str, List[Dict]] = {symbol: [] for symbol in self.symbols}
|
||||
self._derivatives_dir: Path = self.data_dir / "derivatives"
|
||||
|
||||
def _load_config(self) -> Dict[str, object]:
|
||||
"""读取 JSON 配置文件。"""
|
||||
@@ -460,6 +463,117 @@ class DataProvider:
|
||||
symbol_safe = symbol.replace("/", "_").replace(":", "_")
|
||||
return self.data_dir / timeframe / f"{self.exchange.id}_{symbol_safe}_{timeframe}.csv"
|
||||
|
||||
def _derivatives_file_path(self, symbol: str) -> Path:
|
||||
"""衍生品 CSV 路径：data_dir/derivatives/exchange_symbol.csv。"""
|
||||
symbol_safe = symbol.replace("/", "_").replace(":", "_")
|
||||
return self._derivatives_dir / f"{self.exchange.id}_{symbol_safe}.csv"
|
||||
|
||||
def _load_derivatives_local(self, symbol: str) -> List[Dict]:
|
||||
"""加载本地衍生品历史数据。"""
|
||||
path = self._derivatives_file_path(symbol)
|
||||
if not path.exists():
|
||||
return []
|
||||
loaded = []
|
||||
with path.open("r", encoding="utf-8", newline="") as fp:
|
||||
for row in csv.DictReader(fp):
|
||||
try:
|
||||
loaded.append({
|
||||
"timestamp": int(row["timestamp"]),
|
||||
"datetime": row.get("datetime", ""),
|
||||
"funding_rate": float(row.get("funding_rate", 0)),
|
||||
"open_interest": float(row.get("open_interest", 0)),
|
||||
"oi_change_pct": float(row.get("oi_change_pct", 0)) if row.get("oi_change_pct") else None,
|
||||
"basis": float(row.get("basis", 0)) if row.get("basis") else None,
|
||||
})
|
||||
except (KeyError, ValueError):
|
||||
continue
|
||||
loaded.sort(key=lambda item: item["timestamp"])
|
||||
return loaded
|
||||
|
||||
def _fetch_derivatives_sync(self, symbol: str) -> Optional[List[Dict]]:
|
||||
"""用 REST 获取单个币对的费率、OI 和基差（同步，供后台线程调用）。
|
||||
返回单条记录列表，与 K 线 fetcher 返回格式一致。"""
|
||||
try:
|
||||
funding = self.exchange.fetch_funding_rate(symbol)
|
||||
oi = self.exchange.fetch_open_interest(symbol)
|
||||
ticker = self.exchange.fetch_ticker(symbol)
|
||||
except Exception as e:
|
||||
logger.warning("衍生品获取失败 %s: %s", symbol, e)
|
||||
return None
|
||||
|
||||
now_ms = int(time.time() * 1000)
|
||||
record = {
|
||||
"timestamp": now_ms,
|
||||
"datetime": to_utc_iso(now_ms),
|
||||
"funding_rate": float(funding.get("fundingRate", 0)) if funding else 0,
|
||||
"open_interest": float(oi.get("openInterestAmount", 0)) if oi else 0,
|
||||
"oi_change_pct": None,
|
||||
"basis": None,
|
||||
}
|
||||
|
||||
# OI 变化 (与上一条对比)
|
||||
with self._lock:
|
||||
history = self._derivatives.get(symbol, [])
|
||||
if history:
|
||||
prev = history[-1]
|
||||
prev_oi = prev.get("open_interest", 0)
|
||||
if prev_oi > 0 and record["open_interest"] > 0:
|
||||
record["oi_change_pct"] = round(
|
||||
(record["open_interest"] - prev_oi) / prev_oi * 100, 2
|
||||
)
|
||||
|
||||
# 基差 (期货-现货)/现货
|
||||
if ticker:
|
||||
spot_symbol = symbol.split(":")[0]
|
||||
try:
|
||||
spot_ticker = self.exchange.fetch_ticker(spot_symbol)
|
||||
future_price = float(ticker.get("last", 0))
|
||||
spot_price = float(spot_ticker.get("last", 0))
|
||||
if spot_price > 0 and future_price > 0:
|
||||
record["basis"] = round(
|
||||
(future_price - spot_price) / spot_price * 100, 2
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return [record]
|
||||
|
||||
def _merge_derivatives(
|
||||
self,
|
||||
base: List[Dict],
|
||||
new_records: List[Dict],
|
||||
) -> List[Dict]:
|
||||
"""按 timestamp 去重合并衍生品记录，新数据覆盖同时间戳旧数据。与 _merge_candles 模式一致。"""
|
||||
merged = {entry["timestamp"]: entry for entry in base}
|
||||
for record in new_records:
|
||||
merged[record["timestamp"]] = record
|
||||
return list(sorted(merged.values(), key=lambda item: item["timestamp"]))
|
||||
|
||||
def _write_derivatives_to_disk(self, symbol: str, records: List[Dict]) -> None:
|
||||
"""将衍生品历史数据写入 CSV。与 _write_to_disk 模式一致：先写 tmp 再 replace。"""
|
||||
if not records:
|
||||
return
|
||||
path = self._derivatives_file_path(symbol)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fieldnames = ["timestamp", "datetime", "funding_rate", "open_interest",
|
||||
"oi_change_pct", "basis"]
|
||||
tmp_path = path.with_suffix(".tmp")
|
||||
try:
|
||||
with tmp_path.open("w", encoding="utf-8", newline="") as fp:
|
||||
writer = csv.DictWriter(fp, fieldnames=fieldnames, extrasaction="ignore")
|
||||
writer.writeheader()
|
||||
for r in records:
|
||||
row = {}
|
||||
for k in fieldnames:
|
||||
val = r.get(k, "")
|
||||
row[k] = "" if val is None else val
|
||||
writer.writerow(row)
|
||||
tmp_path.replace(path)
|
||||
except Exception as e:
|
||||
logger.error("衍生品落盘失败 %s: %s", symbol, e)
|
||||
if tmp_path.exists():
|
||||
tmp_path.unlink()
|
||||
|
||||
def _load_local(self, symbol: str, timeframe: str) -> List[Dict[str, float]]:
|
||||
"""启动时从磁盘加载已有 K 线，损坏行跳过，按时间排序。"""
|
||||
path = self._data_file_path(symbol, timeframe)
|
||||
@@ -799,14 +913,38 @@ class DataProvider:
|
||||
except Exception as exc:
|
||||
logger.error("数据更新回调异常: %s", exc)
|
||||
|
||||
def _derivatives_refresh_loop(self) -> None:
|
||||
"""后台线程：每 60 秒拉取一次衍生品数据。与 K 线 watch 模式一致：拉取 → 合并 → 写入内存。"""
|
||||
# 首次启动先加载本地历史
|
||||
for symbol in self.symbols:
|
||||
history = self._load_derivatives_local(symbol)
|
||||
if history:
|
||||
with self._lock:
|
||||
self._derivatives[symbol] = history
|
||||
logger.info("衍生品 %s 加载本地记录: %d 条", symbol, len(history))
|
||||
|
||||
DERIVATIVES_INTERVAL = 60
|
||||
while not self._stop_event.wait(DERIVATIVES_INTERVAL):
|
||||
for symbol in self.symbols:
|
||||
try:
|
||||
new_records = self._fetch_derivatives_sync(symbol)
|
||||
if new_records:
|
||||
with self._lock:
|
||||
base = self._derivatives.get(symbol, [])
|
||||
self._derivatives[symbol] = self._merge_derivatives(base, new_records)
|
||||
except Exception as e:
|
||||
logger.debug("衍生品刷新失败 %s: %s", symbol, e)
|
||||
|
||||
def start_background_workers(self) -> None:
|
||||
"""启动后台线程：冷启动回填 + 周期性落盘。WebSocket 监听由 lifespan 异步启动。"""
|
||||
"""启动后台线程：冷启动回填 + 周期性落盘 + 衍生品刷新。WebSocket 监听由 lifespan 异步启动。"""
|
||||
self._stop_event.clear()
|
||||
self._backfill_thread = threading.Thread(target=self._cold_start_backfill, name="backfill-loop", daemon=True)
|
||||
self._persist_thread = threading.Thread(target=self._persist_loop, name="persist-loop", daemon=True)
|
||||
self._derivatives_thread = threading.Thread(target=self._derivatives_refresh_loop, name="derivatives-loop", daemon=True)
|
||||
self._backfill_thread.start()
|
||||
self._persist_thread.start()
|
||||
logger.info("后台线程已启动（回填 + 落盘）")
|
||||
self._derivatives_thread.start()
|
||||
logger.info("后台线程已启动（回填 + 落盘 + 衍生品）")
|
||||
|
||||
def start_watch_tasks(self) -> None:
|
||||
"""在当前 asyncio event loop 上启动 WebSocket 监听。必须在 lifespan 内调用。"""
|
||||
@@ -857,6 +995,15 @@ class DataProvider:
|
||||
if timeframe in MAX_CANDLES_IN_MEMORY:
|
||||
continue # 内存只有尾部 N 根，不覆盖 CSV 全量
|
||||
self._write_to_disk(symbol, timeframe, data)
|
||||
# 周期性落盘衍生品数据（与 K 线一致：锁内复制 → 完整覆写 CSV）
|
||||
with self._lock:
|
||||
deriv_snapshot = {
|
||||
symbol: list(records) for symbol, records in self._derivatives.items()
|
||||
}
|
||||
for symbol, records in deriv_snapshot.items():
|
||||
if records:
|
||||
self._write_derivatives_to_disk(symbol, records)
|
||||
|
||||
# 周期性也保存一次恢复点，保证一致性
|
||||
self._save_resume_since()
|
||||
|
||||
@@ -1068,6 +1215,26 @@ def create_app(provider: DataProvider) -> FastAPI:
|
||||
data = provider.get_klines(symbol=symbol, timeframe=tf, start_time=start, end_time=end, limit=limit)
|
||||
return data
|
||||
|
||||
@app.get("/api/derivatives")
|
||||
async def api_derivatives(
|
||||
symbol: str = Query("BTC/USDT:USDT", description="如 BTC/USDT:USDT"),
|
||||
):
|
||||
"""返回指定币对的衍生品数据快照（资金费率、OI、基差）。"""
|
||||
with provider._lock:
|
||||
records = list(provider._derivatives.get(symbol, []))
|
||||
if not records:
|
||||
raise HTTPException(status_code=404, detail=f"衍生品数据不可用: {symbol}")
|
||||
record = records[-1] # 最新一条
|
||||
return {
|
||||
"symbol": symbol,
|
||||
"timestamp": record.get("timestamp"),
|
||||
"datetime": record.get("datetime"),
|
||||
"funding_rate": record.get("funding_rate"),
|
||||
"open_interest": record.get("open_interest"),
|
||||
"oi_change_pct": record.get("oi_change_pct"),
|
||||
"basis": record.get("basis"),
|
||||
}
|
||||
|
||||
homepage_path = Path(__file__).resolve().parent / "homepage.html"
|
||||
docs_path = Path(__file__).resolve().parent / "api_docs.html"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user