import asyncio import csv import json import logging import os import threading import time from contextlib import asynccontextmanager from datetime import datetime, timezone from pathlib import Path from typing import Dict, Iterable, List, Optional import ccxt # type: ignore import pandas as pd # type: ignore from fastapi import FastAPI, HTTPException, Query from fastapi.middleware.cors import CORSMiddleware import uvicorn from technical.util import resample_to_interval # docker compose logs --tail=200 # docker compose down && docker compose build --no-cache && docker compose up -d TIMEFRAME_ORDER = ["1m", "1h", "1d", "1w"] ALLOWED_TIMEFRAMES = set(TIMEFRAME_ORDER) TIMEFRAME_TO_MS: Dict[str, int] = { "1m": 60_000, "1h": 3_600_000, "1d": 86_400_000, "1w": 604_800_000, } DERIVED_TIMEFRAME_PLAN: Dict[str, List[str]] = { "1m": ["2m", "3m", "4m", "5m", "10m", "15m", "20m", "25m", "30m"], "1h": ["2h", "3h", "4h", "6h", "8h", "12h", "16h"], "1d": ["2d", "3d", "4d", "5d", "6d"], "1w": ["2w"], } CSV_FIELDNAMES = ["timestamp", "datetime", "open", "high", "low", "close", "volume"] DEFAULT_LIMIT = 500 RECENT_CANDLE_LIMIT = 10 RECENT_FETCH_INTERVAL = 5 PERSIST_INTERVAL = 600 logger = logging.getLogger("data_provider") logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", ) def to_utc_iso(timestamp_ms: int) -> str: dt = datetime.fromtimestamp(timestamp_ms / 1000, tz=timezone.utc) return dt.isoformat().replace("+00:00", "Z") def parse_timestamp(value: Optional[object]) -> Optional[int]: if value is None: return None if isinstance(value, (int, float)): return int(value) if isinstance(value, str): text = value.strip() if not text: return None if text.isdigit(): return int(text) if text.endswith("Z"): text = text[:-1] + "+00:00" try: dt = datetime.fromisoformat(text) except ValueError as exc: # pragma: no cover - informative logging raise ValueError(f"无法解析时间字符串: {value}") from exc if dt.tzinfo is None: dt = dt.replace(tzinfo=timezone.utc) else: dt = dt.astimezone(timezone.utc) return int(dt.timestamp() * 1000) raise ValueError(f"不支持的时间格式: {value}") def candle_to_dict(candle: Iterable[float]) -> Dict[str, float]: ts = int(candle[0]) return { "timestamp": ts, "datetime": to_utc_iso(ts), "open": float(candle[1]), "high": float(candle[2]), "low": float(candle[3]), "close": float(candle[4]), "volume": float(candle[5]), } def timeframe_to_minutes(tf: str) -> Optional[int]: if not tf: return None unit = tf[-1] try: value = int(tf[:-1]) except ValueError: return None multiplier = { "m": 1, "h": 60, "d": 1_440, "w": 10_080, }.get(unit) if multiplier is None: return None return value * multiplier class DataProvider: def __init__(self, config_path: Path) -> None: self.config_path = config_path self.config = self._load_config() self.exchange_name: str = self.config["exchange"] self.symbols: List[str] = self._load_symbols(self.config) self.timeframes: List[str] = self._validate_timeframes(self.config.get("timeframes")) self.data_dir = Path(self.config.get("data_dir", "./data")).expanduser() start = parse_timestamp(self.config.get("start_time")) if start is None: raise ValueError("配置文件必须包含 start_time 字段") self.start_time_ms: int = start self.exchange = self._init_exchange() self.data: Dict[str, Dict[str, List[Dict[str, float]]]] = { symbol: {tf: [] for tf in self.timeframes} for symbol in self.symbols } self.derived_map: Dict[str, str] = {} for base_tf in self.timeframes: for derived_tf in DERIVED_TIMEFRAME_PLAN.get(base_tf, []): self.derived_map.setdefault(derived_tf, base_tf) derived_order: List[str] = [] for base_tf in TIMEFRAME_ORDER: if base_tf not in self.timeframes: continue for derived_tf in DERIVED_TIMEFRAME_PLAN.get(base_tf, []): if derived_tf in self.derived_map and derived_tf not in derived_order: derived_order.append(derived_tf) self.available_timeframes: List[str] = list(self.timeframes) + derived_order self._lock = threading.RLock() self._ready = threading.Event() self._stop_event = threading.Event() self._fetch_thread: Optional[threading.Thread] = None self._persist_thread: Optional[threading.Thread] = None # 记录断线后需要从哪个 since 重新拉取（symbol -> timeframe -> since_ms） self._resume_since: Dict[str, Dict[str, int]] = {} # 恢复点持久化文件 self._resume_file: Path = self.data_dir / "resume_since.json" # 尝试加载历史恢复点 self._load_resume_since() def _load_config(self) -> Dict[str, object]: if not self.config_path.exists(): raise FileNotFoundError(f"未找到配置文件: {self.config_path}") with self.config_path.open("r", encoding="utf-8") as fp: return json.load(fp) def _load_symbols(self, config: Dict[str, object]) -> List[str]: raw_symbols: List[str] = [] symbols_value = config.get("symbols") if isinstance(symbols_value, list): raw_symbols = [str(item).strip() for item in symbols_value if isinstance(item, str) and item.strip()] elif isinstance(symbols_value, str) and symbols_value.strip(): raw_symbols = [item.strip() for item in symbols_value.split(",") if item.strip()] symbol_single = config.get("symbol") if not raw_symbols and isinstance(symbol_single, str) and symbol_single.strip(): raw_symbols = [symbol_single.strip()] if not raw_symbols: raise ValueError("配置文件必须提供 symbols（列表或逗号分隔字符串）或 symbol 字段") unique: List[str] = [] for item in raw_symbols: if item not in unique: unique.append(item) return unique def _validate_timeframes(self, configured: Optional[Iterable[str]]) -> List[str]: if not configured: return list(TIMEFRAME_ORDER) invalid = [tf for tf in configured if tf not in ALLOWED_TIMEFRAMES] if invalid: raise ValueError(f"不支持的时间周期: {invalid}. 允许值: {sorted(ALLOWED_TIMEFRAMES)}") unique = [] seen = set() for tf in TIMEFRAME_ORDER: if tf in configured and tf not in seen: unique.append(tf) seen.add(tf) for tf in configured: if tf not in seen: unique.append(tf) seen.add(tf) return unique def _init_exchange(self): if not hasattr(ccxt, self.exchange_name): raise ValueError(f"不支持的交易所: {self.exchange_name}") exchange_class = getattr(ccxt, self.exchange_name) exchange = exchange_class({"enableRateLimit": True}) if exchange.id == "binance": exchange.options.setdefault("defaultType", "future") exchange.load_markets() logger.info("已初始化交易所 %s", exchange.id) return exchange def _data_file_path(self, symbol: str, timeframe: str) -> Path: symbol_safe = symbol.replace("/", "_").replace(":", "_") return self.data_dir / timeframe / f"{self.exchange.id}_{symbol_safe}_{timeframe}.csv" def _load_local(self, symbol: str, timeframe: str) -> List[Dict[str, float]]: path = self._data_file_path(symbol, timeframe) if not path.exists(): return [] loaded: List[Dict[str, float]] = [] with path.open("r", encoding="utf-8", newline="") as fp: reader = csv.DictReader(fp) for row in reader: try: loaded.append( { "timestamp": int(row["timestamp"]), "datetime": row.get("datetime") or to_utc_iso(int(row["timestamp"])), "open": float(row["open"]), "high": float(row["high"]), "low": float(row["low"]), "close": float(row["close"]), "volume": float(row["volume"]), } ) except (KeyError, ValueError): logger.warning("忽略损坏的行: %s", row) loaded.sort(key=lambda item: item["timestamp"]) logger.info("交易对 %s 时间周期 %s 加载本地K线数量: %s", symbol, timeframe, len(loaded)) return loaded def _merge_candles( self, timeframe: str, base: List[Dict[str, float]], new_candles: Iterable[Iterable[float]], ) -> List[Dict[str, float]]: merged = {entry["timestamp"]: entry for entry in base} for candle in new_candles: entry = candle_to_dict(candle) merged[entry["timestamp"]] = entry ordered = list(sorted(merged.values(), key=lambda item: item["timestamp"])) logger.debug("时间周期 %s 合并后K线数量: %s", timeframe, len(ordered)) return ordered def _write_to_disk(self, symbol: str, timeframe: str, data: List[Dict[str, float]]) -> None: path = self._data_file_path(symbol, timeframe) path.parent.mkdir(parents=True, exist_ok=True) tmp_path = path.with_suffix(path.suffix + ".tmp") try: with tmp_path.open("w", encoding="utf-8", newline="") as fp: writer = csv.DictWriter(fp, fieldnames=CSV_FIELDNAMES) writer.writeheader() writer.writerows(data) os.replace(tmp_path, path) finally: if tmp_path.exists(): try: tmp_path.unlink() except OSError: pass logger.info("交易对 %s 时间周期 %s 已写入磁盘 (%s 根K线)", symbol, timeframe, len(data)) def _fetch_history(self, symbol: str, timeframe: str, since_ms: int) -> List[List[float]]: results: List[List[float]] = [] limit = 1500 now_ms = self.exchange.milliseconds() tf_ms = TIMEFRAME_TO_MS[timeframe] fetch_since = since_ms max_rounds = 5000 rounds = 0 while fetch_since < now_ms and rounds < max_rounds: rounds += 1 try: candles = self.exchange.fetch_ohlcv( symbol, timeframe=timeframe, since=fetch_since, limit=limit, ) except ccxt.RateLimitExceeded as exc: logger.warning("触发频率限制，等待: %s", exc) time.sleep(self.exchange.rateLimit / 1000 if self.exchange.rateLimit else 1) continue except ccxt.BaseError as exc: logger.error("拉取历史K线失败 (%s, %s): %s", timeframe, fetch_since, exc) time.sleep(5) continue if not candles: break results.extend(candles) last_ts = candles[-1][0] fetch_since = last_ts + tf_ms if last_ts >= now_ms - tf_ms: break time.sleep(self.exchange.rateLimit / 1000 if self.exchange.rateLimit else 0.2) logger.info("交易对 %s 时间周期 %s 拉取历史K线数量: %s", symbol, timeframe, len(results)) return results def initialize(self) -> None: logger.info("开始初始化数据提供商") for symbol in self.symbols: for timeframe in self.timeframes: existing = self._load_local(symbol, timeframe) tf_ms = TIMEFRAME_TO_MS[timeframe] last_ts = existing[-1]["timestamp"] if existing else None if last_ts is not None: if len(existing) >= 2: fetch_since = existing[-2]["timestamp"] else: fetch_since = max(0, last_ts - tf_ms) else: fetch_since = self.start_time_ms logger.debug( "初始化拉取参数", extra={ "symbol": symbol, "timeframe": timeframe, "existing_last": last_ts, "fetch_since": fetch_since, "tf_ms": tf_ms, }, ) history = self._fetch_history(symbol, timeframe, fetch_since) merged = self._merge_candles(timeframe, existing, history) with self._lock: self.data.setdefault(symbol, {})[timeframe] = merged self._write_to_disk(symbol, timeframe, merged) self._ready.set() logger.info("数据初始化完成") def resample_df(self, df: pd.DataFrame, interval: int) -> pd.DataFrame: return resample_to_interval(df, interval) def _save_resume_since(self) -> None: path = self._resume_file path.parent.mkdir(parents=True, exist_ok=True) tmp_path = path.with_suffix(path.suffix + ".tmp") with self._lock: snapshot = { symbol: {tf: int(since) for tf, since in tf_map.items()} for symbol, tf_map in self._resume_since.items() } try: with tmp_path.open("w", encoding="utf-8") as fp: json.dump(snapshot, fp, ensure_ascii=False, separators=(",", ":")) os.replace(tmp_path, path) finally: if tmp_path.exists(): try: tmp_path.unlink() except OSError: pass logger.debug("恢复点已保存到磁盘: %s", path) def _load_resume_since(self) -> None: path = self._resume_file if not path.exists(): return try: with path.open("r", encoding="utf-8") as fp: raw = json.load(fp) except Exception as exc: logger.warning("恢复点文件读取失败，忽略: %s (%s)", path, exc) return if not isinstance(raw, dict): logger.warning("恢复点文件格式错误，忽略: %s", path) return loaded: Dict[str, Dict[str, int]] = {} for symbol, tf_map in raw.items(): if not isinstance(tf_map, dict): continue per_symbol: Dict[str, int] = {} for timeframe, since in tf_map.items(): try: per_symbol[str(timeframe)] = int(since) except Exception: continue if per_symbol: loaded[str(symbol)] = per_symbol if not loaded: return with self._lock: # 合并为更早的 since，避免遗漏 for symbol, tf_map in loaded.items(): cur = self._resume_since.setdefault(symbol, {}) for timeframe, since in tf_map.items(): prev = cur.get(timeframe) if prev is None or since < prev: cur[timeframe] = since logger.info("已加载恢复点: %s", path) def _get_resume_since(self, symbol: str, timeframe: str) -> Optional[int]: with self._lock: return self._resume_since.get(symbol, {}).get(timeframe) def _set_resume_since(self, symbol: str, timeframe: str, since_ms: int) -> None: with self._lock: per_symbol = self._resume_since.setdefault(symbol, {}) prev = per_symbol.get(timeframe) # 取更早的 since，避免跳过数据 if prev is None or since_ms < prev: per_symbol[timeframe] = since_ms logger.warning( "记录断线恢复点: %s %s since=%s (%s)", symbol, timeframe, since_ms, to_utc_iso(since_ms), ) # 同步写盘 self._save_resume_since() def _clear_resume_since(self, symbol: str, timeframe: str) -> None: with self._lock: if symbol in self._resume_since and timeframe in self._resume_since[symbol]: del self._resume_since[symbol][timeframe] if not self._resume_since[symbol]: del self._resume_since[symbol] logger.info("清除断线恢复点: %s %s", symbol, timeframe) # 同步写盘 self._save_resume_since() def start_background_workers(self) -> None: if self._fetch_thread and self._fetch_thread.is_alive(): return self._stop_event.clear() self._fetch_thread = threading.Thread(target=self._refresh_loop, name="refresh-loop", daemon=True) self._persist_thread = threading.Thread(target=self._persist_loop, name="persist-loop", daemon=True) self._fetch_thread.start() self._persist_thread.start() logger.info("后台线程已启动") def stop(self) -> None: self._stop_event.set() if self._fetch_thread: self._fetch_thread.join(timeout=5) if self._persist_thread: self._persist_thread.join(timeout=5) logger.info("数据提供商已停止") def _refresh_loop(self) -> None: while not self._stop_event.is_set(): for symbol in self.symbols: for timeframe in self.timeframes: try: # 若存在断线恢复点，则优先从该 since 补齐历史数据 resume_since = self._get_resume_since(symbol, timeframe) if resume_since is not None: logger.info( "开始断线后补数: %s %s since=%s (%s)", symbol, timeframe, resume_since, to_utc_iso(resume_since), ) history = self._fetch_history(symbol, timeframe, resume_since) with self._lock: current = self.data.setdefault(symbol, {}).get(timeframe, []) merged = self._merge_candles(timeframe, current, history) self.data[symbol][timeframe] = merged self._clear_resume_since(symbol, timeframe) else: # 正常增量获取最近若干根K线 candles = self.exchange.fetch_ohlcv( symbol, timeframe=timeframe, limit=RECENT_CANDLE_LIMIT, ) if not candles: continue with self._lock: current = self.data.setdefault(symbol, {}).get(timeframe, []) merged = self._merge_candles(timeframe, current, candles) self.data[symbol][timeframe] = merged except ccxt.BaseError as exc: logger.error("更新最新K线失败 (%s %s): %s", symbol, timeframe, exc) # 记录应当从何时恢复拉取，避免重连后从当前时间开始导致丢K with self._lock: current = self.data.get(symbol, {}).get(timeframe, []) if current: last_ts = int(current[-1]["timestamp"]) else: last_ts = self.start_time_ms tf_ms = TIMEFRAME_TO_MS[timeframe] # 回退一个周期，确保包含可能未完全收盘的K线，去重由 _merge_candles 处理 since_ms = max(self.start_time_ms, last_ts - tf_ms) self._set_resume_since(symbol, timeframe, since_ms) time.sleep(2) continue if self._stop_event.wait(RECENT_FETCH_INTERVAL): break def _persist_loop(self) -> None: while not self._stop_event.wait(PERSIST_INTERVAL): self._persist_all() def _persist_all(self) -> None: if not self._ready.is_set(): return with self._lock: snapshot = { symbol: {tf: list(data) for tf, data in tf_map.items()} for symbol, tf_map in self.data.items() } for symbol, tf_map in snapshot.items(): for timeframe, data in tf_map.items(): self._write_to_disk(symbol, timeframe, data) # 周期性也保存一次恢复点，保证一致性 self._save_resume_since() def is_ready(self) -> bool: return self._ready.is_set() def wait_ready(self, timeout: Optional[float] = None) -> bool: return self._ready.wait(timeout) def get_available_timeframes(self) -> List[str]: return list(self.available_timeframes) def get_derived_timeframes(self) -> List[str]: return list(self.derived_map.keys()) def _get_base_klines( self, symbol: str, timeframe: str, start_ms: Optional[int], end_ms: Optional[int], limit: Optional[int], ) -> List[Dict[str, float]]: with self._lock: candles = list(self.data.get(symbol, {}).get(timeframe, [])) if start_ms is not None: candles = [row for row in candles if row["timestamp"] >= start_ms] if end_ms is not None: candles = [row for row in candles if row["timestamp"] <= end_ms] if limit: candles = candles[-limit:] return candles def get_klines( self, symbol: str, timeframe: str, start_time: Optional[object] = None, end_time: Optional[object] = None, limit: Optional[int] = None, ) -> List[Dict[str, float]]: if symbol not in self.symbols: raise HTTPException(status_code=404, detail=f"symbol {symbol} 不可用") self.wait_ready() start_ms = parse_timestamp(start_time) end_ms = parse_timestamp(end_time) if timeframe in self.timeframes: return self._get_base_klines(symbol, timeframe, start_ms, end_ms, limit) base_tf = self.derived_map.get(timeframe) if not base_tf: raise HTTPException(status_code=404, detail=f"{symbol} 时间周期 {timeframe} 不可用") target_minutes = timeframe_to_minutes(timeframe) if target_minutes is None: raise HTTPException(status_code=400, detail=f"不支持的时间周期: {timeframe}") target_ms = target_minutes * 60_000 adjusted_start = None if start_ms is None else max(0, start_ms - target_ms) base_candles = self._get_base_klines(symbol, base_tf, adjusted_start, end_ms, None) if not base_candles: return [] df = pd.DataFrame(base_candles) if df.empty: return [] df = df.drop_duplicates(subset=["timestamp"], keep="last").sort_values("timestamp") df["date"] = pd.to_datetime(df["timestamp"], unit="ms", utc=True) resampled = self.resample_df(df, target_minutes) if resampled is None or resampled.empty: return [] if "timestamp" in resampled.columns: resampled_df = resampled.copy() else: resampled_df = resampled.copy() if "date" in resampled_df.columns: dates = pd.to_datetime(resampled_df["date"], utc=True, errors="coerce") resampled_df["timestamp"] = (dates.view("int64") // 1_000_000).astype("int64") elif isinstance(resampled_df.index, pd.DatetimeIndex): idx = resampled_df.index if idx.tz is None: idx = idx.tz_localize("UTC") else: idx = idx.tz_convert("UTC") resampled_df["timestamp"] = (idx.view("int64") // 1_000_000).astype("int64") else: raise HTTPException(status_code=500, detail=f"聚合结果缺少 timestamp 列 ({timeframe})") resampled_df = resampled_df.dropna(subset=["timestamp"]).sort_values("timestamp") if start_ms is not None: resampled_df = resampled_df[resampled_df["timestamp"] >= start_ms] if end_ms is not None: resampled_df = resampled_df[resampled_df["timestamp"] <= end_ms] if resampled_df.empty: return [] resampled_df["datetime"] = resampled_df["timestamp"].apply(to_utc_iso) for column in ["open", "high", "low", "close", "volume"]: if column not in resampled_df.columns: resampled_df[column] = 0.0 resampled_df = resampled_df[["timestamp", "datetime", "open", "high", "low", "close", "volume"]] result = resampled_df.to_dict("records") if limit: result = result[-limit:] logger.debug( "衍生周期返回", extra={ "symbol": symbol, "timeframe": timeframe, "base_timeframe": base_tf, "count": len(result), }, ) return result def create_app(provider: DataProvider) -> FastAPI: @asynccontextmanager async def lifespan(app: FastAPI): loop = asyncio.get_running_loop() await loop.run_in_executor(None, provider.initialize) provider.start_background_workers() try: yield finally: provider.stop() app = FastAPI(title="Chan 数据提供商", version="1.0.0", lifespan=lifespan) app.add_middleware( CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) @app.get("/health") async def health() -> Dict[str, object]: return { "status": "ok", "exchange": provider.exchange_name, "symbols": provider.symbols, "base_timeframes": provider.timeframes, "derived_timeframes": provider.get_derived_timeframes(), "timeframes": provider.get_available_timeframes(), "ready": provider.is_ready(), } @app.get("/timeframes") async def list_timeframes() -> Dict[str, List[str]]: provider.wait_ready() return { "base_timeframes": provider.timeframes, "derived_timeframes": provider.get_derived_timeframes(), "timeframes": provider.get_available_timeframes(), } @app.get("/api/candles") async def api_candles( symbol: str = Query(..., description="如 BTC/USDT"), tf: str = Query("1m", description="时间周期"), start: Optional[int] = Query(None, description="开始时间戳(ms)"), end: Optional[int] = Query(None, description="结束时间戳(ms)"), limit: Optional[int] = Query(None, description="可选，限制返回数量"), ): data = provider.get_klines(symbol=symbol, timeframe=tf, start_time=start, end_time=end, limit=limit) return data @app.get("/") async def root() -> Dict[str, object]: return { "service": "Data Provider", "exchange": provider.exchange_name, "symbols": provider.symbols, "base_timeframes": provider.timeframes, "derived_timeframes": provider.get_derived_timeframes(), "timeframes": provider.get_available_timeframes(), "ready": provider.is_ready(), } return app def build_app() -> FastAPI: config_path = Path(os.getenv("CONFIG_PATH", "config.json")) provider = DataProvider(config_path) return create_app(provider) app = build_app() def main() -> None: host = os.getenv("UVICORN_HOST", "0.0.0.0") port = int(os.getenv("UVICORN_PORT", "9009")) uvicorn.run(app, host=host, port=port, log_level=os.getenv("UVICORN_LOG_LEVEL", "info")) if __name__ == "__main__": main()