"""影子交易器:在 Hummingbot 运行时上测 1m 腿的真实入场滑点。 不下单。用 Hummingbot 的 Bitget 连接器取真实盘口,按信号方向和仓位吃单深度 算出「若此刻市价单进场会成交在哪」,再与回测假设的成交价相减。 为什么必须跑在 Hummingbot 上而不是自写脚本:要测的是**生产路径**的滑点。 决定成交价的是实际执行链路的延迟,换个运行时测出来的数就不作数了。 （连接器的换根解析 bug 见 patched_candles.py,已修,拿回约 1.06 秒。） 为什么不能用 paper trade 的成交:那是 Hummingbot 自己的撮合模型模拟的, 测出来是模型行为不是市场行为。 口径对齐 aggregate_robustness.py:回测假设成交在**信号次根的开盘价** (entry_delay=1),所以基准价就是换根后新一根的 open。滑点为正表示比回测差。 ### 盘口滚动缓冲把延迟变成自变量 每 100ms 存一份盘口。信号触发后,不只记「我们实际算完时」的滑点,而是回查 t_close+0.5s / 1s / 2s / 5s 各一个。这样即使本机算得慢,也能读出「若延迟为 X 秒,滑点是多少」,决策不被自身实现拖累。 ### 滑点分解 延迟漂移 中间价相对次根开盘价的偏移——主项,且入场方向上系统性追价 盘口价差 最优价相对中间价 深度冲击 吃单加权价相对最优价 Bitget 永续实测价差仅约 0.01bp、100 档深度,故预期延迟漂移占绝大部分。 docker run -d --name shadow -w /home/hummingbot \\ -e PYTHONPATH=/home/hummingbot:/repo/research:/repo/research/live:/repo \\ -v $PWD:/repo:ro -v $PWD/research/out:/out \\ --entrypoint /opt/conda/envs/hummingbot/bin/python \\ hummingbot/hummingbot:latest /repo/research/live/shadow_hb.py --hours 24 """ from __future__ import annotations import argparse import asyncio import csv import time from collections import deque from concurrent.futures import ProcessPoolExecutor from pathlib import Path import numpy as np import pandas as pd SYMS = ("BTC", "ETH", "SOL") # 多存一根:deque 尾部是尚未收盘的当前根,剔除后正好剩 step39 定下的窗口 LTF_BARS, HTF_BARS = 2001, 801 # 有效窗口 2000 / 800,命中率在此饱和 BOOK_HZ = 10 # 盘口采样 10Hz BOOK_KEEP_S = 30 # 缓冲保留 30 秒,够回查到 +5s BOOK_DEPTH = 25 DELAYS_S = (0.5, 1.0, 2.0, 5.0) # 回查点 NOTIONALS = (1_000.0, 5_000.0, 20_000.0) NUM_COLS = ["timestamp", "open", "high", "low", "close", "volume"] def out_dir() -> Path: p = Path("/out") return p if p.is_dir() else Path(__file__).resolve().parents[1] / "out" def hb_to_research(cdf: pd.DataFrame) -> pd.DataFrame: """Hummingbot 的 candles_df 转成 research/lib/data.py 的列结构。 HB 的 timestamp 是秒且无 date 列;chanlun 的 kline builder 需要真 datetime, 时区跟 lib/data.py 取 Asia/Shanghai,保证与回测同一口径。 """ ts_ms = (cdf["timestamp"].astype("int64") * 1000) date = pd.to_datetime(ts_ms, unit="ms", utc=True).dt.tz_convert("Asia/Shanghai") out = pd.DataFrame({"timestamp": ts_ms.astype("int64"), "date": date}) for c in ("open", "high", "low", "close", "volume"): out[c] = pd.to_numeric(cdf[c], errors="coerce") return out.dropna().drop_duplicates(subset=["timestamp"]) \ .sort_values("timestamp").reset_index(drop=True) def walk_book(levels: list[tuple[float, float]], notional: float ) -> tuple[float, float]: """吃单到 notional(计价币)为止,返回 (加权成交价, 实际吃到的额度)。 深度不足时返回吃到的部分,由调用方按 filled < notional 判断是否可信。 """ if not levels: return float("nan"), 0.0 got = 0.0 cost = 0.0 qty = 0.0 for px, sz in levels: avail = px * sz take = min(avail, notional - got) if take <= 0: break q = take / px cost += q * px qty += q got += take if got >= notional - 1e-9: break if qty <= 0: return float("nan"), 0.0 return cost / qty, got class BookBuffer: """每币一份滚动盘口。按时间戳回查,取第一个不早于目标时刻的快照。""" def __init__(self) -> None: self.buf: dict[str, deque] = {s: deque() for s in SYMS} def push(self, sym: str, t_ms: int, bids: list, asks: list) -> None: d = self.buf[sym] d.append((t_ms, bids, asks)) cutoff = t_ms - BOOK_KEEP_S * 1000 while d and d[0][0] < cutoff: d.popleft() def at(self, sym: str, t_ms: int) -> tuple | None: best = None for snap in self.buf[sym]: if snap[0] >= t_ms: best = snap break return best class Shadow: def __init__(self, workers: int, hours: float) -> None: self.workers = workers self.deadline = time.time() + hours * 3600 self.books = BookBuffer() self.pool: ProcessPoolExecutor | None = None self.feeds_l: dict = {} self.feeds_h: dict = {} self.connector = None self.stop = asyncio.Event() self.n_signal = 0 self.n_bars = 0 d = out_dir() # 追加模式:长跑期间若重启,已收集的样本不该被清掉 p_sig = d / "shadow_signals.csv" new_sig = not p_sig.exists() or p_sig.stat().st_size == 0 self.f_sig = p_sig.open("a", newline="") self.w_sig = csv.DictWriter(self.f_sig, fieldnames=[ "sym", "kline_ts", "direction", "h1_agree", "t_close_ms", "t_data_ms", "t_signal_ms", "lag_data_ms", "lag_signal_ms", "delay_label", "delay_ms", "notional", "baseline_px", "mid", "best_px", "fill_px", "filled", "slip_bp", "drift_bp", "spread_bp", "impact_bp"]) if new_sig: self.w_sig.writeheader() p_lat = d / "shadow_latency.csv" new_lat = not p_lat.exists() or p_lat.stat().st_size == 0 self.f_lat = p_lat.open("a", newline="") self.w_lat = csv.DictWriter(self.f_lat, fieldnames=[ "sym", "kline_ts", "t_close_ms", "t_data_ms", "t_signal_ms", "lag_data_ms", "lag_signal_ms", "compute_ms", "n_bars", "n_hits"]) if new_lat: self.w_lat.writeheader() # ---------- 启动 ---------- async def start(self) -> None: from hummingbot.connector.derivative.bitget_perpetual.bitget_perpetual_derivative import ( BitgetPerpetualDerivative, ) from patched_candles import PatchedBitgetPerpetualCandles for s in SYMS: self.feeds_l[s] = PatchedBitgetPerpetualCandles( f"{s}-USDT", "1m", LTF_BARS) self.feeds_h[s] = PatchedBitgetPerpetualCandles( f"{s}-USDT", "5m", HTF_BARS) self.feeds_l[s].start() self.feeds_h[s].start() print(f"[影子] {SYMS} · 1m×{LTF_BARS} + 5m×{HTF_BARS} · " f"{self.workers} 个计算进程", flush=True) # 只取公开数据:无密钥 + trading_required=False self.connector = BitgetPerpetualDerivative( bitget_perpetual_api_key="", bitget_perpetual_secret_key="", bitget_perpetual_passphrase="", trading_pairs=[f"{s}-USDT" for s in SYMS], trading_required=False) await self.connector.start_network() print(" 连接器已启动,等盘口与历史回填", flush=True) t0 = time.time() while time.time() - t0 < 600: ready = all(f.ready for f in list(self.feeds_l.values()) + list(self.feeds_h.values())) books = all(self._snapshot(s) is not None for s in SYMS) if ready and books: break await asyncio.sleep(1) print(f" 就绪 {time.time() - t0:.1f}s · " f"1m {[len(self.feeds_l[s]._candles) for s in SYMS]} 根 · " f"5m {[len(self.feeds_h[s]._candles) for s in SYMS]} 根", flush=True) def _snapshot(self, sym: str): try: ob = self.connector.get_order_book(f"{sym}-USDT") except Exception: return None if ob is None: return None bids = [(float(r.price), float(r.amount)) for r, _ in zip(ob.bid_entries(), range(BOOK_DEPTH))] asks = [(float(r.price), float(r.amount)) for r, _ in zip(ob.ask_entries(), range(BOOK_DEPTH))] if not bids or not asks: return None return bids, asks # ---------- 三个循环 ---------- async def sample_books(self) -> None: period = 1.0 / BOOK_HZ while not self.stop.is_set(): t = int(time.time() * 1000) for s in SYMS: snap = self._snapshot(s) if snap: self.books.push(s, t, snap[0], snap[1]) await asyncio.sleep(period) async def watch_bars(self) -> None: last = {s: (int(self.feeds_l[s]._candles[-1][0]) if len(self.feeds_l[s]._candles) else None) for s in SYMS} while not self.stop.is_set(): for s in SYMS: c = self.feeds_l[s]._candles if not len(c): continue newest = int(c[-1][0]) if last[s] is not None and newest > last[s]: t_data = int(time.time() * 1000) kts = newest * 1000 if newest < 1e12 else newest asyncio.create_task(self.on_bar(s, kts, t_data)) last[s] = newest await asyncio.sleep(0.01) async def on_bar(self, sym: str, kline_ts: int, t_data: int) -> None: """kline_ts 是新一根的开盘时刻,也就是上一根的收盘时刻 t_close。""" df_l = hb_to_research(self.feeds_l[sym].candles_df) df_h = hb_to_research(self.feeds_h[sym].candles_df) # 末行是刚开始的那根,未收盘,必须剔除,否则等于用未来数据 df_l = df_l[df_l["timestamp"] < kline_ts] df_h = df_h[df_h["timestamp"] < kline_ts] baseline = self._new_bar_open(sym, kline_ts) t0 = time.perf_counter() payload = (df_l[NUM_COLS].values.tolist(), df_h[NUM_COLS].values.tolist()) loop = asyncio.get_running_loop() from shadow_signal import compute_packed res = await loop.run_in_executor(self.pool, compute_packed, payload) compute_ms = int((time.perf_counter() - t0) * 1000) t_signal = int(time.time() * 1000) self.n_bars += 1 self.w_lat.writerow({ "sym": sym, "kline_ts": kline_ts, "t_close_ms": kline_ts, "t_data_ms": t_data, "t_signal_ms": t_signal, "lag_data_ms": t_data - kline_ts, "lag_signal_ms": t_signal - kline_ts, "compute_ms": compute_ms, "n_bars": res.get("n_bars", 0), "n_hits": len(res.get("hits", []))}) self.f_lat.flush() if res.get("error"): print(f" [{sym}] 信号计算出错 {res['error']}", flush=True) return hits = res.get("hits", []) if not hits: return if baseline is None or not np.isfinite(baseline): print(f" [{sym}] 有信号但拿不到次根开盘价,跳过", flush=True) return for h in hits: self.n_signal += 1 print(f" ★ [{sym}] {kline_ts} 方向 {h['direction']:+d} " f"h1_agree={h['h1_agree']} · 数据 {t_data - kline_ts}ms " f"信号 {t_signal - kline_ts}ms", flush=True) # 最远的回查点在 t_close+5s，此刻尚未发生；等它过去再一次性落盘 asyncio.create_task( self._record_later(sym, kline_ts, h, t_data, t_signal, baseline)) async def _record_later(self, sym: str, kline_ts: int, hit: dict, t_data: int, t_signal: int, baseline: float) -> None: target = kline_ts + int(max(DELAYS_S) * 1000) + 500 wait = target / 1000.0 - time.time() if wait > 0: await asyncio.sleep(wait) self._record(sym, kline_ts, hit, t_data, t_signal, baseline) def _new_bar_open(self, sym: str, kline_ts: int) -> float | None: """次根开盘价 = 回测假设的成交价。""" c = self.feeds_l[sym]._candles if not len(c): return None row = c[-1] ts = int(row[0]) ts = ts * 1000 if ts < 1e12 else ts return float(row[1]) if ts == kline_ts else None def _record(self, sym: str, kline_ts: int, hit: dict, t_data: int, t_signal: int, baseline: float) -> None: points = [("actual", t_signal - kline_ts)] points += [(f"{d}s", int(d * 1000)) for d in DELAYS_S] d_sign = hit["direction"] for label, delay_ms in points: snap = self.books.at(sym, kline_ts + delay_ms) if snap is None: continue _, bids, asks = snap best_bid, best_ask = bids[0][0], asks[0][0] mid = (best_bid + best_ask) / 2.0 # 多头吃卖盘,空头吃买盘 side = asks if d_sign > 0 else bids best_px = best_ask if d_sign > 0 else best_bid for notional in NOTIONALS: fill, filled = walk_book(side, notional) if not np.isfinite(fill): continue slip = d_sign * (fill - baseline) / baseline * 1e4 drift = d_sign * (mid - baseline) / baseline * 1e4 spread = d_sign * (best_px - mid) / mid * 1e4 impact = d_sign * (fill - best_px) / best_px * 1e4 self.w_sig.writerow({ "sym": sym, "kline_ts": kline_ts, "direction": d_sign, "h1_agree": hit["h1_agree"], "t_close_ms": kline_ts, "t_data_ms": t_data, "t_signal_ms": t_signal, "lag_data_ms": t_data - kline_ts, "lag_signal_ms": t_signal - kline_ts, "delay_label": label, "delay_ms": delay_ms, "notional": notional, "baseline_px": baseline, "mid": mid, "best_px": best_px, "fill_px": fill, "filled": round(filled, 2), "slip_bp": round(slip, 4), "drift_bp": round(drift, 4), "spread_bp": round(spread, 4), "impact_bp": round(impact, 4)}) self.f_sig.flush() async def heartbeat(self) -> None: while not self.stop.is_set(): await asyncio.sleep(300) depth = {s: len(self.books.buf[s]) for s in SYMS} print(f" [心跳] 已处理 {self.n_bars} 根 · 命中 {self.n_signal} 个 " f"· 盘口缓冲 {depth}", flush=True) async def run(self) -> None: await self.start() tasks = [asyncio.create_task(self.sample_books()), asyncio.create_task(self.watch_bars()), asyncio.create_task(self.heartbeat())] while time.time() < self.deadline: await asyncio.sleep(5) self.stop.set() for t in tasks: t.cancel() await asyncio.gather(*tasks, return_exceptions=True) for f in list(self.feeds_l.values()) + list(self.feeds_h.values()): f.stop() await self.connector.stop_network() self.f_sig.close() self.f_lat.close() print(f"\n收工:{self.n_bars} 根 · {self.n_signal} 个信号", flush=True) async def main_async(workers: int, hours: float, pool) -> None: sh = Shadow(workers, hours) sh.pool = pool await sh.run() def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--hours", type=float, default=24.0) ap.add_argument("--workers", type=int, default=2) a = ap.parse_args() # 进程池必须在事件循环和任何 WS 连接之前建好:fork 一个已带活跃 socket # 的进程会把连接状态一起复制过去,后果不可预测 with ProcessPoolExecutor(max_workers=a.workers) as pool: asyncio.run(main_async(a.workers, a.hours, pool)) if __name__ == "__main__": main()