Compare commits
4
Commits
6259f7380b
...
631d97e493
| Author | SHA1 | Date | |
|---|---|---|---|
 jackandCursor
|
631d97e493 | ||
|
jackandCursor
|
01c4a4ab4d | ||
|
jackandCursor
|
28075173af | ||
|
jackandCursor
|
181bca303f |
@@ -42,3 +42,6 @@ venv/
|
||||
|
||||
# Local tooling
|
||||
.gstack/
|
||||
research/out/*.jsonl.gz
|
||||
research/out/penetration.csv
|
||||
research/out/shadow_*.csv
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
"""出场模拟，但止盈按限价单的**真实成交量**结算，而非假定全额成交。
|
||||
|
||||
## 为什么要另写一份
|
||||
|
||||
`exit_model.walk_exits` 给止盈记的毛收益是 `target * a / entry`，即假定挂在
|
||||
目标价的限价单全额成交在目标价。影子交易的成交流数据显示这个假定在真实
|
||||
仓位上不成立：止盈位被首次触及那一根，限价落在该根价格区间中的位置中位
|
||||
k≈0.28，而该位置之上可供成交的主动买量，对 32 万仓位只够覆盖 30%/16%/1.5%
|
||||
（BTC/ETH/SOL）。
|
||||
|
||||
不成交不等于仓位消失——它继续持有，结果从「继续走下去」的分布里抽，其中
|
||||
包含反转打到止损。所以这不是给预算打折能修的事，是出场规则变了。
|
||||
|
||||
## 模型
|
||||
|
||||
两张挂单常驻：半仓在 `scale_at`、半仓在 `runner`。每根按该根在限价之上的
|
||||
可成交量逐步吃进，未成交部分继续持有；整仓止损始终有效，触发时未成交的
|
||||
部分市价平掉；到 `maxb` 根仍未了结的按收盘市价平。
|
||||
|
||||
某根在限价 P 之上的可成交量：
|
||||
|
||||
P ≤ low 整根成交量都在限价之上 avail = V
|
||||
P > high 该根没到限价 avail = 0
|
||||
否则 k = (high−P)/(high−low) avail = f(k) × V
|
||||
|
||||
`V` 是该根的**主动买**成交额（多头出场靠主动买盘打上来）。实测买卖大致
|
||||
均衡，取总成交额的一半；以 BTC 校验，历史 `volume×close` 中位与影子成交流
|
||||
实测差 0.3%。`f` 由成交流定，实测几乎是线性（f(k)≈k，即区间内均匀分布），
|
||||
所以结论对形状假设不敏感。
|
||||
|
||||
## 仍然乐观的两处
|
||||
|
||||
1. **未计排队**。我们的单排在该价位既有挂单之后，真实成交更少。
|
||||
2. **未计自身的流动性效应**。大单挂在 3ATR 会吸收本该冲到 8ATR 的买盘，
|
||||
即两张挂单在真实市场里互相竞争，此处按独立处理。
|
||||
|
||||
两处都指向同一方向：真实成交率比本模型更低。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from lib.exit_model import FEE_MAKER, FEE_TAKER, SLIP
|
||||
|
||||
# 成交流实测的区间内成交分布形状（BTC/ETH/SOL 均值，见 shadow_depth.tape_shape）
|
||||
SHAPE_K = np.linspace(0.0, 1.0, 21)
|
||||
SHAPE_F = np.array([0.044, 0.088, 0.110, 0.179, 0.204, 0.240, 0.282, 0.316,
|
||||
0.390, 0.430, 0.465, 0.537, 0.583, 0.617, 0.662, 0.714,
|
||||
0.761, 0.804, 0.857, 0.904, 1.000])
|
||||
TAKER_SHARE = 0.5
|
||||
|
||||
|
||||
def avail_at(price: float, hi: float, lo: float, vol_notional: float,
|
||||
is_long: bool, kgrid=SHAPE_K, f=SHAPE_F) -> float:
|
||||
"""该根里能打到限价 `price` 的对手方成交额。
|
||||
|
||||
多头在 `price` 挂卖出，靠价格 ≥ price 的主动买成交；空头挂买回，靠
|
||||
价格 ≤ price 的主动卖成交。方向用显式参数而非「把价格取负」——取负会
|
||||
让所有价格变成负数，任何对价格正负的假设都会静默失效。
|
||||
"""
|
||||
if vol_notional <= 0 or not (np.isfinite(hi) and np.isfinite(lo)):
|
||||
return 0.0
|
||||
if hi <= lo:
|
||||
# 该根无波动：只要限价被覆盖就算整根可成交
|
||||
return vol_notional if (hi >= price if is_long else lo <= price) \
|
||||
else 0.0
|
||||
if is_long:
|
||||
if price > hi:
|
||||
return 0.0 # 该根没涨到限价
|
||||
if price <= lo:
|
||||
return vol_notional # 整根都在限价之上
|
||||
k = (hi - price) / (hi - lo)
|
||||
else:
|
||||
if price < lo:
|
||||
return 0.0 # 该根没跌到限价
|
||||
if price >= hi:
|
||||
return vol_notional # 整根都在限价之下
|
||||
k = (price - lo) / (hi - lo)
|
||||
return float(np.interp(k, kgrid, f)) * vol_notional
|
||||
|
||||
|
||||
def walk_filled(cdf: pd.DataFrame, sig: pd.DataFrame, notional: float,
|
||||
sl: float = 2.0, scale_at: float = 3.0, runner: float = 8.0,
|
||||
runner_stop: float = 2.0, maxb: int = 48,
|
||||
taker_share: float = TAKER_SHARE) -> pd.DataFrame:
|
||||
"""前推每笔信号，返回按成交量结算的出场权重与毛收益。
|
||||
|
||||
每行的 `w_*` 是各出场去向占**全仓名义额**的比例，四者相加为 1。
|
||||
"""
|
||||
high = cdf["high"].to_numpy(float)
|
||||
low = cdf["low"].to_numpy(float)
|
||||
close = cdf["close"].to_numpy(float)
|
||||
open_ = cdf["open"].to_numpy(float)
|
||||
vol = cdf["volume"].to_numpy(float)
|
||||
atr = cdf["atr"].to_numpy(float)
|
||||
n = len(cdf)
|
||||
out = []
|
||||
|
||||
for s, d in zip(sig["entry_idx"].astype(int), sig["direction"].astype(int)):
|
||||
e = s + 1
|
||||
if e >= n - 1:
|
||||
continue
|
||||
a = atr[s]
|
||||
if not np.isfinite(a) or a <= 0:
|
||||
continue
|
||||
entry = open_[e]
|
||||
cap = min(e + maxb, n - 1)
|
||||
|
||||
p_stop = entry - d * sl * a
|
||||
p_scale = entry + d * scale_at * a
|
||||
p_run = entry + d * runner * a
|
||||
|
||||
# 两张挂单各半仓，单位是「占全仓的比例」
|
||||
rem_scale, rem_run = 0.5, 0.5
|
||||
w_scale = w_run = w_stop = w_time = 0.0
|
||||
scaled_any = False
|
||||
stop_bar = None
|
||||
|
||||
for j in range(e, cap + 1):
|
||||
hi, lo = high[j], low[j]
|
||||
# 同根内止损优先，与 walk_exits 一致，宁可低估
|
||||
hit_stop = (lo <= p_stop) if d == 1 else (hi >= p_stop)
|
||||
if hit_stop:
|
||||
stop_bar = j
|
||||
w_stop = rem_scale + rem_run
|
||||
rem_scale = rem_run = 0.0
|
||||
break
|
||||
|
||||
v = vol[j] * close[j] * taker_share
|
||||
is_long = d == 1
|
||||
if rem_scale > 0:
|
||||
got = avail_at(p_scale, hi, lo, v, is_long)
|
||||
fill = min(rem_scale, got / notional) if notional > 0 else \
|
||||
rem_scale
|
||||
if fill > 0:
|
||||
rem_scale -= fill
|
||||
w_scale += fill
|
||||
scaled_any = True
|
||||
if rem_run > 0:
|
||||
got = avail_at(p_run, hi, lo, v, is_long)
|
||||
fill = min(rem_run, got / notional) if notional > 0 else \
|
||||
rem_run
|
||||
if fill > 0:
|
||||
rem_run -= fill
|
||||
w_run += fill
|
||||
if rem_scale <= 1e-12 and rem_run <= 1e-12:
|
||||
break
|
||||
|
||||
# 减仓成交后，剩余半仓的止损位可以另设；此处 runner_stop 等于初始 SL
|
||||
# 即止损不动，与 step42 的 k=2.0 一致，故上面那个统一止损已覆盖
|
||||
left = rem_scale + rem_run
|
||||
if left > 1e-12 and stop_bar is None:
|
||||
w_time = left
|
||||
r_scale = d * (p_scale - entry) / entry
|
||||
r_run = d * (p_run - entry) / entry
|
||||
r_stop = d * (p_stop - entry) / entry
|
||||
r_time = d * (close[cap] - entry) / entry
|
||||
|
||||
gross = (w_scale * r_scale + w_run * r_run
|
||||
+ w_stop * r_stop + w_time * r_time)
|
||||
# 入场整仓 taker；两张挂单成交的部分是 maker；止损与超时是 taker
|
||||
fee = (FEE_TAKER * 1.0 + FEE_MAKER * (w_scale + w_run)
|
||||
+ FEE_TAKER * (w_stop + w_time))
|
||||
tk = 1.0 + w_stop + w_time
|
||||
out.append({"sig_idx": s, "direction": d, "atr_pct": a / entry,
|
||||
"w_scale": w_scale, "w_run": w_run,
|
||||
"w_stop": w_stop, "w_time": w_time,
|
||||
"maker_share": w_scale + w_run,
|
||||
"gross": gross, "fee": fee, "taker_notional": tk,
|
||||
"scaled": int(scaled_any)})
|
||||
return pd.DataFrame(out)
|
||||
|
||||
|
||||
def budget_bp(r: pd.DataFrame) -> float:
|
||||
"""盈亏平衡的单边滑点上限（bp）。与 exit_model.slip_budget 同口径。"""
|
||||
if r.empty:
|
||||
return float("nan")
|
||||
net = r["gross"].mean() - r["fee"].mean()
|
||||
return net / r["taker_notional"].mean() * 1e4
|
||||
|
||||
|
||||
def assert_converges(cdf: pd.DataFrame, sig: pd.DataFrame, sl: float = 2.0,
|
||||
scale_at: float = 3.0, runner: float = 8.0,
|
||||
runner_stop: float = 2.0, maxb: int = 48,
|
||||
tol: float = 1e-9) -> None:
|
||||
"""仓位趋近 0 时必须逐笔收敛到 exit_model.walk_exits，否则抛错。
|
||||
|
||||
这个断言是必需的。首版实现里空头的可成交量恒为 0（负价格空间踩到了一个
|
||||
`hi <= 0` 的守卫），后果是空头全被拖到超时收盘——而下跌段里那比 3ATR
|
||||
目标赚得多，于是预算反而**偏高** 0.76bp，看上去像个合理的模型差异。
|
||||
没有这条断言，这种错只会表现为「数字有点不一样」。
|
||||
"""
|
||||
from lib.exit_model import cfg_name, walk_exits
|
||||
|
||||
old = walk_exits(cdf, sig, [sl], [scale_at], [maxb], scale_at,
|
||||
[runner], [runner_stop])
|
||||
c = cfg_name(sl, runner, maxb, runner_stop)
|
||||
new = walk_filled(cdf, sig, 1e-12, sl, scale_at, runner, runner_stop, maxb)
|
||||
j = old[["sig_idx"]].copy()
|
||||
j["g_old"] = old[f"{c}_g"].to_numpy()
|
||||
j = j.merge(new[["sig_idx", "gross"]], on="sig_idx")
|
||||
d = (j["gross"] - j["g_old"]).abs()
|
||||
bad = int((d > tol).sum())
|
||||
if bad:
|
||||
worst = j.loc[d.idxmax()]
|
||||
raise AssertionError(
|
||||
f"仓位趋近 0 时应与 walk_exits 一致，但 {bad}/{len(j)} 笔不符；"
|
||||
f"最大差 {d.max():.3e}（sig_idx {int(worst['sig_idx'])}："
|
||||
f"老 {worst['g_old']:.6f} 新 {worst['gross']:.6f}）")
|
||||
|
||||
|
||||
def net_bp(r: pd.DataFrame, slip: float = SLIP) -> float:
|
||||
"""扣掉手续费与滑点后的净均收益（bp）。"""
|
||||
if r.empty:
|
||||
return float("nan")
|
||||
net = r["gross"] - r["fee"] - slip * r["taker_notional"]
|
||||
return float(net.mean() * 1e4)
|
||||
@@ -0,0 +1,184 @@
|
||||
"""对比两个（或多个）采集站点的数据差异。
|
||||
|
||||
部署第二台机器的目的就是这个：延迟是「本地接收 − 交易所 K 线收盘」，直接
|
||||
取决于机器到交易所的网络距离，换个地理位置这个数会变。而滑点里最大的一项
|
||||
是延迟漂移，所以站点选址本身就是一个可优化的参数。
|
||||
|
||||
## 判读前必须先看的两件事
|
||||
|
||||
1. **时钟。** 两台机器的时钟偏移差多少，延迟对比就凭空差多少，且不报错。
|
||||
run_meta_*.json 里有各站启动时的 chrony 偏移，先确认都在 10ms 内。
|
||||
2. **同期。** 只比两站都有数据的那些 kline_ts。不取交集的话，比的可能是
|
||||
不同时段的市场状态，而延迟对市场活跃度是敏感的。
|
||||
|
||||
## 用法
|
||||
|
||||
把各站的 research/out/ 收到一处（文件名相同会覆盖，所以先按站点改名或
|
||||
分目录放），然后：
|
||||
|
||||
python research/live/compare_sites.py --glob 'collected/*/shadow_latency.csv'
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def load(patterns: list[str]) -> pd.DataFrame:
|
||||
paths: list[str] = []
|
||||
for p in patterns:
|
||||
paths.extend(sorted(glob.glob(p)))
|
||||
if not paths:
|
||||
raise SystemExit(f"没有匹配到文件：{patterns}")
|
||||
frames = []
|
||||
for p in paths:
|
||||
df = pd.read_csv(p)
|
||||
if "site" not in df.columns:
|
||||
raise SystemExit(
|
||||
f"{p} 没有 site 列。这是 2026-08-28 之前采的旧数据，"
|
||||
f"无法确定来源，不能用于跨地对比")
|
||||
df["_src"] = p
|
||||
frames.append(df)
|
||||
out = pd.concat(frames, ignore_index=True)
|
||||
print(f"读入 {len(paths)} 个文件、{len(out):,} 行、"
|
||||
f"站点 {sorted(out['site'].unique())}")
|
||||
return out
|
||||
|
||||
|
||||
def show_meta(out_dirs: list[Path]) -> None:
|
||||
print("\n########## 一、运行元数据 ##########")
|
||||
metas = []
|
||||
for d in out_dirs:
|
||||
metas.extend(sorted(d.glob("run_meta_*.json")))
|
||||
if not metas:
|
||||
print(" 没找到 run_meta_*.json。时钟偏移与代码版本无法核对——")
|
||||
print(" 两站数据若有差异，分不清是地理位置还是环境不同造成的")
|
||||
return
|
||||
rows = []
|
||||
for m in metas:
|
||||
try:
|
||||
rows.append(json.loads(m.read_text()))
|
||||
except Exception as e:
|
||||
print(f" {m.name} 读取失败：{e!r}")
|
||||
if not rows:
|
||||
return
|
||||
df = pd.DataFrame(rows)
|
||||
keep = [c for c in ("site", "clock_offset_ms", "git_commit", "git_dirty",
|
||||
"image_digest", "nproc", "mem_gb", "tz",
|
||||
"started_utc") if c in df.columns]
|
||||
print(df[keep].to_string(index=False))
|
||||
if "clock_offset_ms" in df and df["clock_offset_ms"].notna().any():
|
||||
o = df["clock_offset_ms"].astype(float)
|
||||
spread = float(o.max() - o.min())
|
||||
flag = "" if spread < 5 else " ⚠ 这个差会直接叠加到延迟对比上"
|
||||
print(f"\n 站点间时钟偏移极差 {spread:.3f}ms{flag}")
|
||||
if "git_commit" in df and df["git_commit"].nunique() > 1:
|
||||
print(" ⚠ 各站代码版本不同，差异可能来自代码而非地理位置")
|
||||
if "git_dirty" in df and df["git_dirty"].any():
|
||||
print(" ⚠ 有站点带未提交改动，无法复现")
|
||||
|
||||
|
||||
def compare_latency(df: pd.DataFrame, col: str = "lag_data_ms") -> None:
|
||||
"""延迟对比。只取各站都有的 kline_ts，避免比到不同时段。"""
|
||||
if col not in df.columns:
|
||||
print(f"\n没有 {col} 列")
|
||||
return
|
||||
sites = sorted(df["site"].unique())
|
||||
if len(sites) < 2:
|
||||
print(f"\n只有一个站点（{sites[0]}），无从对比。"
|
||||
f"等第二台机器的数据到齐")
|
||||
return
|
||||
|
||||
print(f"\n########## 二、到达延迟（{col}） ##########")
|
||||
print("\n 全量（各站各自的样本，时段可能不同）")
|
||||
for s in sites:
|
||||
x = df[df["site"] == s][col].dropna().astype(float)
|
||||
print(f" {s:<16} n={len(x):>6} 中位 {x.median():>7.0f}ms "
|
||||
f"P90 {np.percentile(x, 90):>7.0f}ms "
|
||||
f"P99 {np.percentile(x, 99):>7.0f}ms")
|
||||
|
||||
# 取交集：同一根 K 线在各站都有记录
|
||||
key = ["sym", "kline_ts"]
|
||||
piv = df.pivot_table(index=key, columns="site", values=col,
|
||||
aggfunc="first")
|
||||
both = piv.dropna()
|
||||
if both.empty:
|
||||
print("\n 各站没有共同的 K 线。可能是采集时段不重叠，")
|
||||
print(" 或 kline_ts 对不上（先查两站时区与时钟）")
|
||||
return
|
||||
print(f"\n 同根对比（{len(both):,} 根 K 线，各站都有）")
|
||||
for s in sites:
|
||||
x = both[s].astype(float)
|
||||
print(f" {s:<16} 中位 {x.median():>7.0f}ms "
|
||||
f"P90 {np.percentile(x, 90):>7.0f}ms")
|
||||
base = sites[0]
|
||||
for s in sites[1:]:
|
||||
d = (both[s] - both[base]).astype(float)
|
||||
# 配对差的符号检验：同根配对消掉了市场状态，比两个中位数相减干净
|
||||
n_pos = int((d > 0).sum())
|
||||
print(f"\n {s} − {base}：中位差 {d.median():+.0f}ms "
|
||||
f"· 均值差 {d.mean():+.0f}ms")
|
||||
print(f" {s} 更慢的根占 {n_pos / len(d) * 100:.1f}%"
|
||||
f"（50% 表示无系统性差异）")
|
||||
for sym in sorted(both.index.get_level_values("sym").unique()):
|
||||
ds = d.xs(sym, level="sym")
|
||||
print(f" {sym:<5} 中位差 {ds.median():+7.0f}ms (n={len(ds)})")
|
||||
|
||||
|
||||
def compare_drift(patterns: list[str]) -> None:
|
||||
"""漂移对比。延迟差若能兑换成漂移差，才是钱上的差别。"""
|
||||
paths: list[str] = []
|
||||
for p in patterns:
|
||||
paths.extend(sorted(glob.glob(p)))
|
||||
if not paths:
|
||||
return
|
||||
frames = []
|
||||
for p in paths:
|
||||
d = pd.read_csv(p)
|
||||
if "site" in d.columns:
|
||||
frames.append(d)
|
||||
if not frames:
|
||||
return
|
||||
df = pd.concat(frames, ignore_index=True)
|
||||
if df["site"].nunique() < 2:
|
||||
return
|
||||
print("\n########## 三、延迟漂移（无条件，每根都记） ##########")
|
||||
for label in sorted(df["delay_label"].dropna().unique()):
|
||||
sub = df[df["delay_label"] == label]
|
||||
line = f" {label:>7}"
|
||||
for s in sorted(sub["site"].unique()):
|
||||
x = sub[sub["site"] == s]["drift_bp_long"].dropna().astype(float)
|
||||
if len(x):
|
||||
line += f" · {s} {x.abs().median():.3f}bp(n={len(x)})"
|
||||
print(line)
|
||||
print("\n 同一个固定延迟点上，两站的漂移应当几乎相同——漂移是市场性质，")
|
||||
print(" 与机器位置无关。若差异明显，先查时钟与采集时段是否对齐")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--glob", action="append", default=None,
|
||||
help="shadow_latency.csv 的路径模式，可给多次")
|
||||
ap.add_argument("--drift-glob", action="append", default=None)
|
||||
ap.add_argument("--meta-dir", action="append", default=None)
|
||||
a = ap.parse_args()
|
||||
|
||||
lat = a.glob or ["research/out/shadow_latency.csv",
|
||||
"collected/*/shadow_latency.csv"]
|
||||
drf = a.drift_glob or ["research/out/shadow_drift.csv",
|
||||
"collected/*/shadow_drift.csv"]
|
||||
metas = [Path(p) for p in (a.meta_dir or ["research/out", "collected"])]
|
||||
|
||||
show_meta([p for p in metas if p.is_dir()])
|
||||
df = load(lat)
|
||||
compare_latency(df)
|
||||
compare_drift(drf)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,97 @@
|
||||
# 影子采集器部署
|
||||
|
||||
在第二台机器上跑一套完全相同的采集，用来看不同地理位置的数据差异。
|
||||
|
||||
## 为什么要跨地采集
|
||||
|
||||
滑点里最大的一项是**延迟漂移**：从 K 线收盘到实际下单之间，价格已经走掉的
|
||||
那部分。而延迟 = 交易所出包 + 网络传输 + 本地处理。本机（新加坡）实测到达
|
||||
延迟中位 350~650ms，其中网络传输占多少、换个机房能压掉多少，只有实测。
|
||||
|
||||
延迟压下来直接等于滑点下降，所以机房选址是个可优化参数，不是固定成本。
|
||||
|
||||
## 三个必须一致，一个必须不同
|
||||
|
||||
必须一致，否则差异分不清是地理位置还是环境造成的：
|
||||
|
||||
- **代码版本**（`git_commit`）——同一个 commit
|
||||
- **镜像摘要**（`image_digest`）——同一个 hummingbot 镜像
|
||||
- **时钟**——两台都同步到 NTP，偏移都在 10ms 内
|
||||
|
||||
必须不同：
|
||||
|
||||
- **`SHADOW_SITE`**——写进每一行数据，是合并后区分来源的唯一依据
|
||||
|
||||
`start.sh` 会把这四项连同内核、核数、内存一起写进
|
||||
`research/out/run_meta_<site>.json`。两地数据对不上时先看这个文件。
|
||||
|
||||
## 时钟为什么是硬门槛
|
||||
|
||||
所有延迟数字都是「本地时钟 − 交易所 K 线收盘时间戳」。时钟偏 50ms，全部
|
||||
延迟就同向偏 50ms，而且**不会有任何报错**——只会让跨地对比得出一个干净、
|
||||
自信、且完全错误的结论。所以 `start.sh` 在时钟未同步或偏移超阈值时直接
|
||||
拒绝启动，而不是打个警告了事。
|
||||
|
||||
## 步骤
|
||||
|
||||
在新机器上：
|
||||
|
||||
```bash
|
||||
git clone ssh://jack@git.jackyu66.com:2222/jack/chan.git
|
||||
cd chan && git checkout chan
|
||||
|
||||
bash research/live/deploy/setup.sh # 装 docker + chrony，拉镜像
|
||||
SHADOW_SITE=aws-tokyo bash research/live/deploy/start.sh
|
||||
```
|
||||
|
||||
确认健康：
|
||||
|
||||
```bash
|
||||
bash research/live/deploy/status.sh
|
||||
```
|
||||
|
||||
启动日志里应当看到：
|
||||
|
||||
```
|
||||
[补丁] 覆盖生效：基类取首元素 … 本地取末元素 …
|
||||
成交流已挂 ['BTC', 'ETH', 'SOL']
|
||||
就绪 3.0s · 1m [2001, 2001, 2001] 根 · 5m [801, 801, 801] 根
|
||||
```
|
||||
|
||||
第一行尤其重要。上游 Bitget 连接器的换根解析有 bug（只取多根消息的首元素），
|
||||
补丁把它修掉拿回约 1.06 秒。补丁若失效是静默的——不崩不报错，只是延迟悄悄
|
||||
退回 1.4 秒，所以启动时做了断言。
|
||||
|
||||
## 对比
|
||||
|
||||
把两站的 `research/out/` 收到一处（同名文件会覆盖，所以分目录放）：
|
||||
|
||||
```bash
|
||||
mkdir -p collected/sg collected/aws
|
||||
rsync -av sg-box:chan/research/out/ collected/sg/
|
||||
rsync -av aws-box:chan/research/out/ collected/aws/
|
||||
|
||||
python research/live/compare_sites.py \
|
||||
--glob 'collected/*/shadow_latency.csv' \
|
||||
--drift-glob 'collected/*/shadow_drift.csv' \
|
||||
--meta-dir collected/sg --meta-dir collected/aws
|
||||
```
|
||||
|
||||
对比脚本做两件事值得说明：
|
||||
|
||||
- **只取各站都有的 K 线**做配对比较。不取交集就可能在比不同时段，而延迟对
|
||||
市场活跃度敏感。
|
||||
- 报**配对差的符号占比**而不只是两个中位数相减。同根配对消掉了市场状态，
|
||||
「A 比 B 慢的根占多少」比「两个中位数差多少」更能说明有无系统性差异。
|
||||
|
||||
判读上有一条自检：**同一个固定延迟点上，两站的漂移应当几乎相同**——漂移是
|
||||
市场性质，与机器位置无关。若漂移也差很多，先怀疑时钟或时段没对齐，而不是
|
||||
急着下结论。
|
||||
|
||||
## 资源占用
|
||||
|
||||
本机实测：内存约 1.5GB（两个计算进程 + 盘口缓冲），CPU 单核不满。
|
||||
盘口与成交流落盘约 15MB/天（gzip）。一周 168 小时的量级在百 MB 内。
|
||||
|
||||
`--workers 2` 是因为信号计算走独立进程池、不能阻塞事件循环。核数少的机型
|
||||
可以给 1，但要看心跳里的 `compute_ms`：若接近 60 秒就会开始堆积。
|
||||
Executable
+87
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env bash
|
||||
# 影子采集器的机器初始化。幂等，可重复跑。
|
||||
#
|
||||
# 用途：在另一台机器（如 AWS）上部署一套完全相同的采集，用来看不同地理位置
|
||||
# 的数据采集有无差异。要比的主要是延迟——「本地接收 − K线收盘」这个量直接
|
||||
# 取决于机器到交易所的网络距离。
|
||||
#
|
||||
# 时钟同步是硬前置条件，不是可选项。所有延迟数字都是本地时钟减交易所时间戳，
|
||||
# 时钟偏 50ms 就等于所有延迟凭空多（或少）50ms，而且不会有任何报错。所以这里
|
||||
# 装并启用 NTP，start.sh 里还会再校验一次、不合格拒绝启动。
|
||||
#
|
||||
# bash research/live/deploy/setup.sh
|
||||
set -euo pipefail
|
||||
|
||||
IMAGE="${SHADOW_IMAGE:-hummingbot/hummingbot:latest}"
|
||||
|
||||
say() { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
|
||||
|
||||
say "系统信息"
|
||||
uname -a
|
||||
echo "内存：$(free -g | awk '/^Mem:/{print $2"GB 总 / "$7"GB 可用"}')"
|
||||
echo "CPU：$(nproc) 核"
|
||||
|
||||
say "安装 docker"
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
echo "已有 docker $(docker --version)"
|
||||
else
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq ca-certificates curl gnupg
|
||||
sudo install -m 0755 -d /etc/apt/keyrings
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg \
|
||||
| sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
|
||||
sudo chmod a+r /etc/apt/keyrings/docker.gpg
|
||||
. /etc/os-release
|
||||
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \
|
||||
https://download.docker.com/linux/ubuntu ${VERSION_CODENAME} stable" \
|
||||
| sudo tee /etc/apt/sources.list.d/docker.list >/dev/null
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq docker-ce docker-ce-cli containerd.io
|
||||
elif command -v dnf >/dev/null 2>&1; then
|
||||
# Amazon Linux 2023
|
||||
sudo dnf install -y -q docker
|
||||
sudo systemctl enable --now docker
|
||||
else
|
||||
echo "不认识的包管理器，请手动装 docker" >&2
|
||||
exit 1
|
||||
fi
|
||||
sudo usermod -aG docker "$USER" || true
|
||||
echo "已装 docker。若本次 shell 无权限，重新登录后再跑 start.sh"
|
||||
fi
|
||||
|
||||
say "启用时钟同步（延迟测量的前置条件）"
|
||||
if command -v chronyc >/dev/null 2>&1; then
|
||||
echo "已有 chrony"
|
||||
elif command -v apt-get >/dev/null 2>&1; then
|
||||
sudo apt-get install -y -qq chrony
|
||||
elif command -v dnf >/dev/null 2>&1; then
|
||||
sudo dnf install -y -q chrony
|
||||
fi
|
||||
sudo systemctl enable --now chrony 2>/dev/null \
|
||||
|| sudo systemctl enable --now chronyd 2>/dev/null || true
|
||||
sleep 3
|
||||
if command -v chronyc >/dev/null 2>&1; then
|
||||
chronyc tracking | grep -E 'Reference ID|System time|Last offset' || true
|
||||
else
|
||||
timedatectl 2>/dev/null | grep -i synchron || true
|
||||
fi
|
||||
|
||||
say "拉镜像 $IMAGE"
|
||||
docker pull "$IMAGE"
|
||||
docker image inspect "$IMAGE" --format '摘要 {{index .RepoDigests 0}}' 2>/dev/null || true
|
||||
|
||||
say "准备输出目录"
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||
mkdir -p "$REPO_ROOT/research/out"
|
||||
echo "$REPO_ROOT/research/out"
|
||||
|
||||
say "完成"
|
||||
cat <<'EOF'
|
||||
下一步：
|
||||
|
||||
SHADOW_SITE=aws-tokyo bash research/live/deploy/start.sh
|
||||
|
||||
SHADOW_SITE 必须显式给且两台机器不能相同——它会写进每一行数据，
|
||||
是之后区分数据来源的唯一依据。
|
||||
EOF
|
||||
Executable
+115
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env bash
|
||||
# 启动影子采集器。跑之前先 setup.sh。
|
||||
#
|
||||
# SHADOW_SITE=aws-tokyo bash research/live/deploy/start.sh
|
||||
# SHADOW_SITE=aws-tokyo HOURS=168 WORKERS=2 bash research/live/deploy/start.sh
|
||||
#
|
||||
# 为什么 SHADOW_SITE 必填：它写进每一行数据，是两台机器的数据合起来之后
|
||||
# 唯一的来源区分。缺了就只能靠文件路径猜，一合并就分不清了。
|
||||
#
|
||||
# 为什么时钟不同步就拒绝启动：所有延迟数字都是「本地时钟 − 交易所 K 线收盘
|
||||
# 时间戳」。时钟偏 50ms，全部延迟就凭空偏 50ms，而这**不会有任何报错**，
|
||||
# 只会让跨地对比得出一个完全错误的结论。这类静默错误必须在启动就挡掉。
|
||||
set -euo pipefail
|
||||
|
||||
NAME="${NAME:-shadow}"
|
||||
IMAGE="${SHADOW_IMAGE:-hummingbot/hummingbot:latest}"
|
||||
HOURS="${HOURS:-168}"
|
||||
WORKERS="${WORKERS:-2}"
|
||||
MAX_OFFSET_MS="${MAX_OFFSET_MS:-10}"
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||
OUT="$REPO_ROOT/research/out"
|
||||
|
||||
die() { printf '\033[31m错误：%s\033[0m\n' "$*" >&2; exit 1; }
|
||||
say() { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
|
||||
|
||||
[[ -n "${SHADOW_SITE:-}" ]] || die "必须设 SHADOW_SITE，例如 SHADOW_SITE=aws-tokyo。
|
||||
它写进每一行数据，是跨地对比时区分来源的唯一依据。"
|
||||
|
||||
say "站点 $SHADOW_SITE"
|
||||
|
||||
say "校验时钟同步"
|
||||
offset_ms=""
|
||||
if command -v chronyc >/dev/null 2>&1; then
|
||||
if ! chronyc tracking >/dev/null 2>&1; then
|
||||
die "chrony 没在跑。先 sudo systemctl start chrony（或 chronyd）"
|
||||
fi
|
||||
# System time 那行形如 "0.000058703 seconds fast of NTP time"
|
||||
line="$(chronyc tracking | grep '^System time' || true)"
|
||||
secs="$(awk '{print $4}' <<<"$line")"
|
||||
offset_ms="$(awk -v s="$secs" 'BEGIN{printf "%.3f", s*1000}')"
|
||||
echo "$line"
|
||||
leap="$(chronyc tracking | awk -F': *' '/Leap status/{print $2}')"
|
||||
[[ "$leap" == "Normal" ]] || die "chrony leap status = $leap，尚未收敛。等几分钟再试"
|
||||
over="$(awk -v o="$offset_ms" -v m="$MAX_OFFSET_MS" 'BEGIN{print (o>m)?1:0}')"
|
||||
[[ "$over" == "0" ]] || die "时钟偏移 ${offset_ms}ms 超过阈值 ${MAX_OFFSET_MS}ms。
|
||||
所有延迟测量都会同向偏这么多且不报错，跨地对比会得出错误结论。
|
||||
先等 chrony 收敛，或调 MAX_OFFSET_MS（不建议）。"
|
||||
echo "偏移 ${offset_ms}ms，在 ${MAX_OFFSET_MS}ms 阈值内"
|
||||
elif command -v timedatectl >/dev/null 2>&1; then
|
||||
timedatectl | grep -qi 'synchronized: yes' \
|
||||
|| die "系统时钟未同步。装 chrony：见 setup.sh"
|
||||
echo "timedatectl 报已同步（无 chronyc，拿不到具体偏移）"
|
||||
else
|
||||
die "既无 chronyc 也无 timedatectl，无法确认时钟。装 chrony 后再启动"
|
||||
fi
|
||||
|
||||
say "检查镜像"
|
||||
docker image inspect "$IMAGE" >/dev/null 2>&1 || die "没有镜像 $IMAGE，先跑 setup.sh"
|
||||
digest="$(docker image inspect "$IMAGE" --format '{{if .RepoDigests}}{{index .RepoDigests 0}}{{end}}' 2>/dev/null || true)"
|
||||
|
||||
say "停掉旧容器"
|
||||
docker rm -f "$NAME" >/dev/null 2>&1 || true
|
||||
|
||||
mkdir -p "$OUT"
|
||||
|
||||
# 运行元数据。两地数据对不上时，先看这个文件——镜像摘要、代码版本、时钟偏移
|
||||
# 三者任一不同都足以解释差异，不必去猜。
|
||||
meta="$OUT/run_meta_${SHADOW_SITE}.json"
|
||||
cat >"$meta" <<EOF
|
||||
{
|
||||
"site": "$SHADOW_SITE",
|
||||
"hostname": "$(hostname)",
|
||||
"started_utc": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
||||
"tz": "$(date +%Z%z)",
|
||||
"clock_offset_ms": ${offset_ms:-null},
|
||||
"git_commit": "$(git -C "$REPO_ROOT" rev-parse --short HEAD 2>/dev/null || echo unknown)",
|
||||
"git_dirty": $(git -C "$REPO_ROOT" diff --quiet 2>/dev/null && echo false || echo true),
|
||||
"image": "$IMAGE",
|
||||
"image_digest": "${digest:-unknown}",
|
||||
"hours": $HOURS,
|
||||
"workers": $WORKERS,
|
||||
"kernel": "$(uname -r)",
|
||||
"nproc": $(nproc),
|
||||
"mem_gb": $(free -g | awk '/^Mem:/{print $2}')
|
||||
}
|
||||
EOF
|
||||
echo "元数据已写 $meta"
|
||||
|
||||
say "启动容器 $NAME"
|
||||
docker run -d --name "$NAME" -w /home/hummingbot \
|
||||
--restart unless-stopped \
|
||||
-e PYTHONPATH=/home/hummingbot:/repo/research:/repo/research/live:/repo \
|
||||
-e SHADOW_SITE="$SHADOW_SITE" \
|
||||
-v "$REPO_ROOT:/repo:ro" \
|
||||
-v "$OUT:/out" \
|
||||
--entrypoint /opt/conda/envs/hummingbot/bin/python \
|
||||
"$IMAGE" /repo/research/live/shadow_hb.py \
|
||||
--hours "$HOURS" --workers "$WORKERS" >/dev/null
|
||||
|
||||
echo "已启动。等启动自检（补丁断言 + 历史回填，约 60 秒）…"
|
||||
sleep 45
|
||||
docker logs "$NAME" 2>&1 | tail -12
|
||||
|
||||
cat <<EOF
|
||||
|
||||
看日志： docker logs -f $NAME
|
||||
看状态： bash research/live/deploy/status.sh
|
||||
停止： docker rm -f $NAME
|
||||
|
||||
启动日志里应能看到：
|
||||
[补丁] 覆盖生效 —— 换根解析补丁有效（缺了会静默慢 1.06 秒）
|
||||
成交流已挂 [...] —— maker 成交率要用
|
||||
就绪 … 根 —— 历史回填完成
|
||||
EOF
|
||||
Executable
+67
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env bash
|
||||
# 采集器健康速查。跨地部署时两台都跑一遍，对着看。
|
||||
set -uo pipefail
|
||||
|
||||
NAME="${NAME:-shadow}"
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||
OUT="$REPO_ROOT/research/out"
|
||||
|
||||
say() { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
|
||||
|
||||
say "容器"
|
||||
docker ps -a --filter "name=^${NAME}$" \
|
||||
--format 'table {{.Names}}\t{{.Status}}\t{{.RunningFor}}' || true
|
||||
|
||||
say "时钟"
|
||||
if command -v chronyc >/dev/null 2>&1; then
|
||||
chronyc tracking | grep -E 'System time|Last offset|Leap status'
|
||||
fi
|
||||
|
||||
say "最近心跳"
|
||||
docker logs "$NAME" 2>&1 | grep '\[心跳\]' | tail -3 || echo "还没到第一次心跳（每 5 分钟一次）"
|
||||
|
||||
say "告警与异常"
|
||||
docker logs "$NAME" 2>&1 \
|
||||
| grep -E '⚠|错误|失效|损坏|停滞|Traceback|退化' | tail -10 \
|
||||
|| echo "无"
|
||||
|
||||
say "输出规模"
|
||||
for f in shadow_latency.csv shadow_drift.csv shadow_signals.csv; do
|
||||
p="$OUT/$f"
|
||||
if [[ -s "$p" ]]; then
|
||||
printf ' %-22s %8d 行\n' "$f" "$(( $(wc -l <"$p") - 1 ))"
|
||||
elif [[ -f "$p" ]]; then
|
||||
# 已建但表头还没冲刷：signals 只在有信号时才 flush，门控后每天仅 4~6 个
|
||||
printf ' %-22s %8s\n' "$f" "0（待首条）"
|
||||
else
|
||||
printf ' %-22s %8s\n' "$f" "无"
|
||||
fi
|
||||
done
|
||||
for f in shadow_books.jsonl.gz shadow_tape.jsonl.gz; do
|
||||
p="$OUT/$f"
|
||||
[[ -f "$p" ]] && printf ' %-22s %8s\n' "$f" "$(du -h "$p" | cut -f1)" \
|
||||
|| printf ' %-22s %8s\n' "$f" "无"
|
||||
done
|
||||
|
||||
say "各站点行数（确认 site 列生效）"
|
||||
p="$OUT/shadow_latency.csv"
|
||||
if [[ -f "$p" ]]; then
|
||||
awk -F, 'NR>1{c[$1]++} END{for(s in c) printf " %-16s %8d 行\n", s, c[s]}' "$p"
|
||||
else
|
||||
echo " 尚无数据"
|
||||
fi
|
||||
|
||||
say "到达延迟中位（本站，跨地对比的主指标）"
|
||||
# 按列名取下标，不写死数字：加了 site 列之后字段整体右移过一次，
|
||||
# 写死 $8 会静默变成读 lag_signal_ms
|
||||
if [[ -s "$p" ]]; then
|
||||
for sym in BTC ETH SOL; do
|
||||
med=$(awk -F, -v s="$sym" '
|
||||
NR==1 { for (i=1;i<=NF;i++) { if ($i=="sym") si=i; if ($i=="lag_data_ms") li=i } ; next }
|
||||
$si==s && $li!="" { print $li }' "$p" | sort -n | awk '
|
||||
{ v[NR]=$1 } END { if (NR) printf "%.0f %d", v[int((NR+1)/2)], NR }')
|
||||
[[ -n "$med" ]] && printf ' %-5s 中位 %6sms (n=%s)\n' "$sym" ${med} \
|
||||
|| printf ' %-5s 尚无数据\n' "$sym"
|
||||
done
|
||||
echo " 参考：本机（新加坡）补丁后实测 350~650ms，理论下限约 500ms"
|
||||
fi
|
||||
@@ -5,9 +5,11 @@
|
||||
的标准差。入场方向上还有系统性追价（信号触发往往伴随同向动量），所以随机
|
||||
漂移只是下限，真实成本更高——这也是为什么最终仍要用真实盘口测滑点。
|
||||
|
||||
余量（bitget_baseline.py 得出，Bitget 原生基线减去手续费后剩下的空间）：
|
||||
BTC -0.13bp ETH +4.02bp SOL +2.92bp
|
||||
BTC 本就为负，留着只作延迟测量的参照物，不作交易标的。
|
||||
预算从 `lib/shadow_budget` import，不在这里写死。曾经写死的
|
||||
`{BTC: -0.13, ETH: 4.02, SOL: 2.92}` 是错的——那是 `bitget_baseline.py`
|
||||
按「毛均 − 6bp 双边 taker」算的，六处口径叠加（费率档位记高、余量没除
|
||||
taker 名义额、用了 5m~30m 的出场参数、只有同向没有阶梯与 ATR 门控）。
|
||||
正确值是 8.58 / 20.64 / 16.83，ETH 差了五倍。
|
||||
|
||||
.venv/bin/python research/live/latency_compare.py
|
||||
"""
|
||||
@@ -23,8 +25,9 @@ HERE = Path(__file__).resolve().parent
|
||||
RESEARCH = HERE.parent
|
||||
sys.path.insert(0, str(RESEARCH))
|
||||
|
||||
from lib.shadow_budget import budget_of # noqa: E402
|
||||
|
||||
OUT = RESEARCH / "out"
|
||||
BUDGET_BP = {"BTC": -0.13, "ETH": 4.02, "SOL": 2.92}
|
||||
SYMS = ("BTC", "ETH", "SOL")
|
||||
|
||||
|
||||
@@ -67,8 +70,8 @@ def describe(df: pd.DataFrame, name: str, vols: dict) -> None:
|
||||
med, p90 = float(np.median(v)), float(np.percentile(v, 90))
|
||||
vol = vols.get(s)
|
||||
d = drift_bp(med, vol) if vol else float("nan")
|
||||
b = BUDGET_BP[s]
|
||||
share = f"{d / b * 100:.0f}%" if b > 0 else "—(负)"
|
||||
b = budget_of(s)
|
||||
share = f"{d / b * 100:.0f}%" if np.isfinite(b) and b > 0 else "—"
|
||||
print(f"{s:<5}{len(v):>5}{med:>9.0f}{p90:>9.0f}{v.max():>9.0f}"
|
||||
f"{d:>12.2f}{b:>9.2f}{share:>9}")
|
||||
|
||||
|
||||
@@ -14,9 +14,21 @@
|
||||
再把最后一根交给基类走正常的 append 流程。
|
||||
|
||||
已向上游反馈前，本地用子类覆盖，不改动镜像。
|
||||
|
||||
## 为什么必须有启动断言
|
||||
|
||||
子类覆盖的失效方式是**静默**的：上游若把 `_parse_websocket_message` 改名、
|
||||
或改走别的钩子，我们的覆盖就成了死代码，行情悄悄退回慢 1.06 秒，不崩、
|
||||
不报错、不留日志，只会让收益慢慢变差，几周后才从统计里看出来。
|
||||
|
||||
`assert_patch_effective()` 不做名字检查——名字对不上未必失效，名字对得上
|
||||
也未必生效。它喂一条合成的两元素消息，直接验证行为：基类返回首元素（bug
|
||||
仍在、覆盖仍有必要），子类返回末元素（覆盖确实生效）。再加一条源码检查
|
||||
确认基类的收包循环还在调这个钩子。任一不满足就在启动时抛错。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import numpy as np
|
||||
@@ -24,6 +36,7 @@ import numpy as np
|
||||
from hummingbot.data_feed.candles_feed.bitget_perpetual_candles import (
|
||||
BitgetPerpetualCandles,
|
||||
)
|
||||
from hummingbot.data_feed.candles_feed.candles_base import CandlesBase
|
||||
|
||||
|
||||
def _row_to_dict(row: list, ensure_s) -> Dict[str, Any]:
|
||||
@@ -63,3 +76,51 @@ class PatchedBitgetPerpetualCandles(BitgetPerpetualCandles):
|
||||
d["volume"], d["quote_asset_volume"], d["n_trades"],
|
||||
d["taker_buy_base_volume"], d["taker_buy_quote_volume"]]
|
||||
).astype(float)
|
||||
|
||||
|
||||
# 换根时 Bitget 推的就是这个形状：[上一根, 新一根]
|
||||
_PROBE = {
|
||||
"action": "update",
|
||||
"arg": {"instType": "USDT-FUTURES", "channel": "candle1m",
|
||||
"instId": "BTCUSDT"},
|
||||
"data": [
|
||||
["1700000040000", "1", "1", "1", "1", "1", "1", "1"],
|
||||
["1700000100000", "2", "2", "2", "2", "2", "2", "2"],
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def assert_patch_effective() -> None:
|
||||
"""启动即验证覆盖真的生效，否则抛错。让静默失效变成启动失败。"""
|
||||
src = inspect.getsource(CandlesBase._process_websocket_messages_task)
|
||||
if "_parse_websocket_message" not in src:
|
||||
raise RuntimeError(
|
||||
"上游收包循环已不再调用 _parse_websocket_message，"
|
||||
"patched_candles 的覆盖失效。需重新定位钩子后再启动。")
|
||||
|
||||
stock = BitgetPerpetualCandles("BTC-USDT", "1m", 20)
|
||||
ours = PatchedBitgetPerpetualCandles("BTC-USDT", "1m", 20)
|
||||
got_stock = stock._parse_websocket_message(_PROBE)
|
||||
got_ours = ours._parse_websocket_message(_PROBE)
|
||||
|
||||
head_ts = stock.ensure_timestamp_in_seconds(int(_PROBE["data"][0][0]))
|
||||
tail_ts = stock.ensure_timestamp_in_seconds(int(_PROBE["data"][-1][0]))
|
||||
|
||||
if not got_ours or int(got_ours["timestamp"]) != int(tail_ts):
|
||||
raise RuntimeError(
|
||||
f"覆盖未生效：子类返回 {got_ours and got_ours.get('timestamp')}，"
|
||||
f"应为末元素 {tail_ts}。")
|
||||
if got_stock and int(got_stock["timestamp"]) == int(tail_ts):
|
||||
# 上游自己修好了。此时覆盖无害但已多余，明确说出来，免得以后
|
||||
# 有人以为那 1.06 秒还是靠我们拿回来的
|
||||
print(" [补丁] 上游已自行修正换根解析，本地覆盖现为冗余，可移除",
|
||||
flush=True)
|
||||
elif not got_stock or int(got_stock["timestamp"]) != int(head_ts):
|
||||
raise RuntimeError(
|
||||
f"基类行为与预期不符：返回 "
|
||||
f"{got_stock and got_stock.get('timestamp')}，"
|
||||
f"既非首元素 {head_ts} 也非末元素 {tail_ts}。"
|
||||
f"上游改了解析逻辑，补丁的前提需重新确认。")
|
||||
else:
|
||||
print(f" [补丁] 覆盖生效：基类取首元素 {int(head_ts)}、"
|
||||
f"本地取末元素 {int(tail_ts)}", flush=True)
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
"""止盈位被首次触及那一根，价格穿透了多深。
|
||||
|
||||
为什么要这个数：影子成交流显示，限价单若正好落在某根的最高价，该价位之上
|
||||
的主动买成交额只有几十到几千美元——对十万量级的仓位等于不成交。但那是
|
||||
最坏情形。真实成交率取决于**止盈位被穿透了多深**：若价格一路冲过目标，
|
||||
成交没问题；若只是上影线点一下就回落，就成交不了。
|
||||
|
||||
这个分布不需要再采数据，历史 K 线里就有：给定入场价与 ATR，目标位是
|
||||
`entry + T×ATR`，找到首次 `high ≥ target` 的那根，穿透深度就是
|
||||
`high − target`。把它折成「占该根价格区间的比例」，就能直接对上成交流
|
||||
那条「≥ 限价的成交额 vs 限价在区间中的位置」曲线。
|
||||
|
||||
口径与 lib/exit_model.walk_exits 对齐：入场取信号次根开盘价，ATR 取信号
|
||||
根的 Wilder ATR-14，上限 48 根。
|
||||
|
||||
**取样方式的局限**：这里用全体 K 线做候选入场点，而非真实的三滤网信号。
|
||||
真实信号是按结构条件挑出来的，入场时刻可能与波动率状态相关。以 ATR 归一
|
||||
后的穿透深度对波动率状态应当不敏感，但要精确到信号级别，得重跑一次
|
||||
step42 的缠论链路（单币约 370s、峰值 24.5GB）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
SCALE_AT = 3.0 # 分批减仓位
|
||||
RUNNER = 8.0 # 剩余半仓目标
|
||||
MAX_BARS = 48
|
||||
|
||||
|
||||
def wilder_atr(high, low, close, period: int = 14) -> np.ndarray:
|
||||
"""与 chanlun.indicators.ta.ATR 逐位一致的 Wilder ATR。"""
|
||||
n = high.size
|
||||
out = np.full(n, np.nan)
|
||||
if n <= period:
|
||||
return out
|
||||
prev_close = close[:-1]
|
||||
tr = np.maximum.reduce([high[1:] - low[1:],
|
||||
np.abs(high[1:] - prev_close),
|
||||
np.abs(low[1:] - prev_close)])
|
||||
sm = np.empty(tr.size)
|
||||
sm[period - 1] = tr[:period].mean()
|
||||
a = 1.0 / period
|
||||
for k in range(period, tr.size):
|
||||
sm[k] = sm[k - 1] + a * (tr[k] - sm[k - 1])
|
||||
out[period:] = sm[period - 1:]
|
||||
return out
|
||||
|
||||
|
||||
def penetration(df: pd.DataFrame, target_atr: float,
|
||||
stride: int = 1) -> pd.DataFrame:
|
||||
"""对每个候选入场点，求首次触及 `target_atr` 时的穿透深度。
|
||||
|
||||
只统计**触及了**的那些（未触及的属止损或超时出场，不涉及 maker 腿）。
|
||||
"""
|
||||
high = df["high"].to_numpy(float)
|
||||
low = df["low"].to_numpy(float)
|
||||
close = df["close"].to_numpy(float)
|
||||
open_ = df["open"].to_numpy(float)
|
||||
atr = wilder_atr(high, low, close)
|
||||
n = len(df)
|
||||
rows = []
|
||||
for i in range(20, n - MAX_BARS - 2, stride):
|
||||
a = atr[i]
|
||||
if not np.isfinite(a) or a <= 0:
|
||||
continue
|
||||
e = i + 1
|
||||
entry = open_[e]
|
||||
# 多头：目标在上方。空头对称，穿透深度分布按对称性等价，故只算一边
|
||||
target = entry + target_atr * a
|
||||
end = e + MAX_BARS
|
||||
seg_hi = high[e:end + 1]
|
||||
hit = np.flatnonzero(seg_hi >= target)
|
||||
if not hit.size:
|
||||
continue
|
||||
j = e + int(hit[0])
|
||||
rng = high[j] - low[j]
|
||||
if rng <= 0:
|
||||
continue
|
||||
pen = high[j] - target
|
||||
rows.append({
|
||||
"bar": j,
|
||||
# 限价在该根价格区间中的位置：0 = 正好在最高价（最坏），
|
||||
# 1 = 在最低价（该根全部成交都在限价之上）
|
||||
"k": min(1.0, pen / rng),
|
||||
"pen_bp": pen / target * 1e4,
|
||||
"pen_atr": pen / a,
|
||||
"range_bp": rng / target * 1e4,
|
||||
})
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def report(sym: str, df: pd.DataFrame) -> dict:
|
||||
print(f"\n{'=' * 68}\n{sym} 共 {len(df):,} 根 1m")
|
||||
out = {}
|
||||
for tgt, name in ((SCALE_AT, f"减仓位 {SCALE_AT:g}ATR"),
|
||||
(RUNNER, f"目标位 {RUNNER:g}ATR")):
|
||||
p = penetration(df, tgt)
|
||||
if p.empty:
|
||||
print(f" {name}: 无触及样本")
|
||||
continue
|
||||
k = p["k"].to_numpy()
|
||||
print(f"\n {name} · 触及 {len(p):,} 次")
|
||||
print(f" 穿透深度 中位 {p['pen_bp'].median():.2f}bp "
|
||||
f"({p['pen_atr'].median():.2f} ATR) · "
|
||||
f"P25 {p['pen_bp'].quantile(.25):.2f}bp · "
|
||||
f"P75 {p['pen_bp'].quantile(.75):.2f}bp")
|
||||
print(f" 限价在区间中的位置 k（0=正好在最高价，越大越靠下越易成交）")
|
||||
print(f" 中位 {np.median(k):.3f} · P10 {np.percentile(k, 10):.3f}"
|
||||
f" · P25 {np.percentile(k, 25):.3f}"
|
||||
f" · P75 {np.percentile(k, 75):.3f}")
|
||||
for thr in (0.05, 0.10, 0.25, 0.50):
|
||||
print(f" k ≤ {thr:.2f}（限价挤在该根顶部 {thr * 100:.0f}% 内）："
|
||||
f"{float((k <= thr).mean()) * 100:5.1f}% 的触及")
|
||||
out[tgt] = p
|
||||
return out
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--syms", default="BTC,ETH,SOL")
|
||||
ap.add_argument("--cache", default="research/live/cache")
|
||||
ap.add_argument("--save", default="research/out/penetration.csv")
|
||||
a = ap.parse_args()
|
||||
|
||||
root = Path(a.cache)
|
||||
allp = []
|
||||
for sym in a.syms.split(","):
|
||||
cands = sorted(root.glob(f"bitget_{sym}_1m_*.feather"),
|
||||
key=lambda p: p.stat().st_size, reverse=True)
|
||||
if not cands:
|
||||
print(f"{sym}: 找不到 1m 缓存，跳过")
|
||||
continue
|
||||
df = pd.read_feather(cands[0])
|
||||
got = report(sym, df)
|
||||
for tgt, p in got.items():
|
||||
p = p.copy()
|
||||
p["sym"], p["target_atr"] = sym, tgt
|
||||
allp.append(p)
|
||||
if allp:
|
||||
out = pd.concat(allp, ignore_index=True)
|
||||
Path(a.save).parent.mkdir(parents=True, exist_ok=True)
|
||||
out.to_csv(a.save, index=False)
|
||||
print(f"\n已存 {a.save}（{len(out):,} 行），"
|
||||
f"供 shadow_depth.py 合并成交量曲线")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,68 @@
|
||||
"""真实 Bitget 盘口在 BOOK_DEPTH 档内能不能吃下各个名义额档位。
|
||||
|
||||
shadow_hb 把吃单换成了框架的 get_vwap_for_volume，深度不足时它返回 nan、
|
||||
整行标 depth_ok=0。所以「档数够不够」直接决定某个仓位档会不会整段丢失，
|
||||
不是个可以事后补救的参数——先量出来再定 BOOK_DEPTH。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, "/repo/research/live")
|
||||
|
||||
from shadow_hb import BOOK_DEPTH, NOTIONALS, SYMS, book_from
|
||||
|
||||
|
||||
async def run() -> None:
|
||||
from hummingbot.connector.derivative.bitget_perpetual.bitget_perpetual_derivative import (
|
||||
BitgetPerpetualDerivative,
|
||||
)
|
||||
conn = BitgetPerpetualDerivative(
|
||||
bitget_perpetual_api_key="", bitget_perpetual_secret_key="",
|
||||
bitget_perpetual_passphrase="",
|
||||
trading_pairs=[f"{s}-USDT" for s in SYMS], trading_required=False)
|
||||
await conn.start_network()
|
||||
print(f"连接器已启动，等盘口（档数上限 {BOOK_DEPTH}）")
|
||||
for _ in range(60):
|
||||
await asyncio.sleep(1)
|
||||
try:
|
||||
if all(conn.get_order_book(f"{s}-USDT") is not None for s in SYMS):
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
for s in SYMS:
|
||||
ob = conn.get_order_book(f"{s}-USDT")
|
||||
bids = np.array([(float(r.price), float(r.amount), i)
|
||||
for i, (r, _) in enumerate(
|
||||
zip(ob.bid_entries(), range(BOOK_DEPTH)))])
|
||||
asks = np.array([(float(r.price), float(r.amount), i)
|
||||
for i, (r, _) in enumerate(
|
||||
zip(ob.ask_entries(), range(BOOK_DEPTH)))])
|
||||
mid = (bids[0][0] + asks[0][0]) / 2.0
|
||||
snap = book_from(bids, asks)
|
||||
ask_notional = float((asks[:, 0] * asks[:, 1]).sum())
|
||||
print(f"\n{s} 中价 {mid:.2f} · 取到 {len(asks)} 档 · "
|
||||
f"卖盘 {len(asks)} 档合计 {ask_notional:,.0f} USDT")
|
||||
print(f" 最深一档距中价 "
|
||||
f"{(asks[-1][0] / mid - 1) * 1e4:.1f}bp")
|
||||
for notional in NOTIONALS:
|
||||
base = notional / mid
|
||||
r = snap.get_vwap_for_volume(True, base)
|
||||
px = float(r.result_price)
|
||||
ok = float(r.result_volume) >= base * 0.999
|
||||
if ok:
|
||||
print(f" 名义 {notional:>7,.0f} → {base:.6f} 币 · "
|
||||
f"冲击 {(px / mid - 1) * 1e4:6.2f}bp · 吃得下")
|
||||
else:
|
||||
print(f" 名义 {notional:>7,.0f} → {base:.6f} 币 · "
|
||||
f"深度不足，仅 {r.result_volume:.6f} 币 · "
|
||||
f"这一档会整段丢失")
|
||||
await conn.stop_network()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run())
|
||||
@@ -0,0 +1,348 @@
|
||||
"""从落盘的完整盘口与成交流，算资金容量、排队量与单根成交率。
|
||||
|
||||
## 结论先行（2026-08-28）
|
||||
|
||||
主口径仓位 **10 万 USDT**。该规模下两条约束都不绑定：
|
||||
|
||||
冲击 单边 0.01~1.87bp，占预算 0.1~11.1%，冲击反推上限 100~500 万
|
||||
成交率 按逐根累积结算，预算降幅 BTC −0% / ETH −0% / SOL −1.1%
|
||||
|
||||
⚠ 本文件里的 `composite_fill` 曾给出「10 万仓位全额成交率仅 7~63%」这种数，
|
||||
那个口径只算**首次触及那一根**的可成交量，系统性偏悲观，已不作为结论。
|
||||
真实成交率见 lib/exit_fill.py（挂单常驻多根逐步成交）。
|
||||
|
||||
|
||||
这两个数都不该等实盘暴露：
|
||||
|
||||
**容量**。预算 20bp 意味着存在一个资金上限，超过它策略就不工作。既然完整
|
||||
深度已落盘，任意仓位的冲击都能重算——一次采集回答所有资金量级，换个规模
|
||||
不必重测一周。绑定约束是**薄盘时段**而非中位盘口，所以按分位数报。
|
||||
|
||||
**maker 成交率**。回测假设 3ATR / 8ATR 的限价单全额成交。深度回答不了这个
|
||||
问题：深度说的是「现在挂着多少」，成交率问的是「之后打过来多少」。只有
|
||||
成交流能回答，而且买卖必须分开——多头在 3ATR 挂卖出，靠主动买盘成交。
|
||||
|
||||
读 gzip 时必须容忍末尾成员不完整：采集进程还在写，最后一个 gzip 成员没有
|
||||
结尾标记，直接遍历会在文件尾抛 EOFError 而丢掉**全部**已读记录。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def out_dir() -> Path:
|
||||
p = Path("/out")
|
||||
return p if p.is_dir() else Path(__file__).resolve().parents[1] / "out"
|
||||
|
||||
|
||||
def read_jsonl_gz(path: Path):
|
||||
"""逐行读 gzip JSONL，末尾截断则静默停止。
|
||||
|
||||
采集仍在进行时，最后一个 gzip 成员缺结尾标记；不接这个异常的话，
|
||||
整个分析会因为文件尾而失败，前面几万条完好记录一起丢掉。
|
||||
"""
|
||||
if not path.exists():
|
||||
return
|
||||
n_ok = 0
|
||||
try:
|
||||
with gzip.open(path, "rt", encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
break # 半行，说明写到这里被打断
|
||||
n_ok += 1
|
||||
yield rec
|
||||
except (EOFError, OSError, gzip.BadGzipFile):
|
||||
# 采集进程正在写，尾部不完整属正常
|
||||
pass
|
||||
|
||||
|
||||
def impact_bp(levels: list, notional: float, mid: float) -> float | None:
|
||||
"""吃掉 notional 计价币后的加权均价相对中间价，bp。深度不足返回 None。"""
|
||||
need = notional
|
||||
cost = 0.0
|
||||
qty = 0.0
|
||||
for px, amt in levels:
|
||||
avail = px * amt
|
||||
take = min(avail, need)
|
||||
q = take / px
|
||||
cost += q * px
|
||||
qty += q
|
||||
need -= take
|
||||
if need <= 1e-9:
|
||||
break
|
||||
if need > 1e-9 or qty <= 0:
|
||||
return None
|
||||
return (cost / qty / mid - 1.0) * 1e4
|
||||
|
||||
|
||||
def queue_ahead(books_path: Path, notional: float = 1e5) -> None:
|
||||
"""限价单排在多少量之后。
|
||||
|
||||
这是唯一还没建模的成本项。exit_fill 假定我们能吃到该价位的全部对手方
|
||||
成交量，实际上我们排在该价位既有挂单之后。
|
||||
|
||||
这里量不了 3ATR 处的排队——3ATR 约 30bp，而 50 档盘口只覆盖 1.4~4.5bp，
|
||||
那个价位远在可见深度之外。但价格走到我们的限价时，我们的单就成了盘口
|
||||
最优档附近的一员，所以「最优档通常趴着多少量」是个有用的上界参照。
|
||||
"""
|
||||
per: dict[str, dict[str, list[float]]] = {}
|
||||
for r in read_jsonl_gz(books_path):
|
||||
asks, bids = r["asks"], r["bids"]
|
||||
if not asks or not bids:
|
||||
continue
|
||||
mid = (asks[0][0] + bids[0][0]) / 2.0
|
||||
d = per.setdefault(r["sym"], {"top": [], "b1": [], "b5": []})
|
||||
d["top"].append(asks[0][0] * asks[0][1])
|
||||
for tag, bp in (("b1", 1.0), ("b5", 5.0)):
|
||||
lim = mid * (1 + bp / 1e4)
|
||||
d[tag].append(sum(p * a for p, a in asks if p <= lim))
|
||||
if not per:
|
||||
return
|
||||
print(f"\n\n########## 限价单前方的排队量 ##########")
|
||||
print(f" 主口径仓位 {notional:,.0f} USDT\n")
|
||||
for sym, d in sorted(per.items()):
|
||||
top = np.array(d["top"]); b1 = np.array(d["b1"])
|
||||
b5 = np.array(d["b5"])
|
||||
print(f" {sym} 最优档 {np.median(top):>10,.0f} · "
|
||||
f"1bp 内累计 {np.median(b1):>10,.0f} · "
|
||||
f"5bp 内累计 {np.median(b5):>10,.0f}")
|
||||
print(f" 我们的 {notional:,.0f} 相当于最优档的 "
|
||||
f"{notional / max(np.median(top), 1):.1f} 倍、"
|
||||
f"1bp 内总量的 {notional / max(np.median(b1), 1):.2f} 倍")
|
||||
print("\n 倍数远小于 1 则排队可忽略；接近或超过 1 则我们本身就是那一档的")
|
||||
print(" 主要挂单，exit_fill 的成交量假设需要打折。")
|
||||
print(" SOL 的最优档倍数畸高是 tick 更细所致（Bitget 的 SOL tick 比同类")
|
||||
print(" 细约 10 倍，同样的量摊到 10 倍多的价位上），所以对 SOL 该看 5bp")
|
||||
print(" 档而非最优档；但即便如此它仍是三个币里排队压力最大的一个")
|
||||
|
||||
|
||||
def capacity(books_path: Path, budgets: dict[str, float],
|
||||
pctl: float = 10.0) -> None:
|
||||
"""报各币的深度曲线与「冲击吃掉预算多少」的资金上限。"""
|
||||
grid = [1e4, 2.5e4, 5e4, 1e5, 2e5, 5e5, 1e6, 2e6]
|
||||
per: dict[str, dict[float, list[float]]] = {}
|
||||
n = 0
|
||||
for r in read_jsonl_gz(books_path):
|
||||
asks, bids = r["asks"], r["bids"]
|
||||
if not asks or not bids:
|
||||
continue
|
||||
mid = (asks[0][0] + bids[0][0]) / 2.0
|
||||
d = per.setdefault(r["sym"], {g: [] for g in grid})
|
||||
for g in grid:
|
||||
v = impact_bp(asks, g, mid)
|
||||
d[g].append(np.nan if v is None else v)
|
||||
n += 1
|
||||
|
||||
if not n:
|
||||
print("没有盘口快照，先跑采集")
|
||||
return
|
||||
|
||||
print(f"\n########## 资金容量 ##########")
|
||||
print(f" 基于 {n:,} 份完整盘口快照（单边买入方向）\n")
|
||||
for sym, d in per.items():
|
||||
b = budgets.get(sym)
|
||||
print(f" {sym} 预算 {b:.2f}bp" if b else f" {sym}")
|
||||
print(f" {'名义额':>12} {'冲击中位':>10} {'冲击P90':>10} "
|
||||
f"{'吃满深度率':>10} {'占预算':>8}")
|
||||
for g in grid:
|
||||
a = np.array(d[g], dtype=float)
|
||||
fill = float(np.isfinite(a).mean())
|
||||
if fill == 0:
|
||||
print(f" {g:>12,.0f} {'—— 50 档吃不下 ——':>30}")
|
||||
continue
|
||||
med = float(np.nanmedian(a))
|
||||
p90 = float(np.nanpercentile(a, 90))
|
||||
share = f"{med / b * 100:6.1f}%" if b else " na"
|
||||
print(f" {g:>12,.0f} {med:>10.2f} {p90:>10.2f} "
|
||||
f"{fill * 100:>9.1f}% {share:>8}")
|
||||
if b:
|
||||
# 上限：冲击的 P90（薄盘时段）吃掉预算三成为止。三成是留给
|
||||
# 漂移与价差的余地——它们才是主项，冲击不该独占预算
|
||||
cap = None
|
||||
for g in grid:
|
||||
a = np.array(d[g], dtype=float)
|
||||
if not np.isfinite(a).any():
|
||||
break
|
||||
if float(np.nanpercentile(a, 90)) > b * 0.30:
|
||||
break
|
||||
cap = g
|
||||
if cap is None:
|
||||
print(f" → 连最小档 {grid[0]:,.0f} 的薄盘冲击都超预算三成")
|
||||
else:
|
||||
print(f" → 资金上限约 {cap:,.0f} USDT"
|
||||
f"（薄盘 P90 冲击 ≤ 预算 30%）")
|
||||
print()
|
||||
|
||||
|
||||
def maker_fill(tape_path: Path, mults=(3.0, 8.0),
|
||||
notionals=(5e4, 1e5, 2e5)) -> None:
|
||||
"""限价单挂在离场目标位，本根内有多少主动量打到那里。
|
||||
|
||||
这里只回答「量够不够」。真实成交还要看排队位置——我们的单排在该价位
|
||||
已有挂单之后，所以这是**上界**：量不够则必然不能全成交，量够也未必成交。
|
||||
"""
|
||||
rows = list(read_jsonl_gz(tape_path))
|
||||
if not rows:
|
||||
print("没有成交流数据，先跑采集")
|
||||
return
|
||||
print(f"\n########## maker 腿成交量上界 ##########")
|
||||
print(f" 基于 {len(rows):,} 根的逐价位成交聚合")
|
||||
print(f" 多头在目标位挂卖出，成交靠主动**买**盘，故只计买方向\n")
|
||||
|
||||
# 限价单只能被**价格 ≥ 限价**的主动买成交打到。而止盈位被触及的那一根，
|
||||
# 限价往往就落在该根价格区间的顶部——最高价刚好碰到目标位是最典型的
|
||||
# 情形。所以按「限价距最高价多近」分层：depth=0 表示限价正好在最高价
|
||||
# （只有打在最高价那一档的量算数），depth=0.25 表示限价在区间顶部 25% 处
|
||||
depths = (0.0, 0.10, 0.25, 1.0)
|
||||
per: dict[str, dict[float, list[float]]] = {}
|
||||
for r in rows:
|
||||
buys = {float(p): v for p, v in r["buys"].items()}
|
||||
d = per.setdefault(r["sym"], {k: [] for k in depths})
|
||||
if not buys:
|
||||
for k in depths:
|
||||
d[k].append(0.0)
|
||||
continue
|
||||
hi, lo = max(buys), min(buys)
|
||||
rng = hi - lo
|
||||
for k in depths:
|
||||
floor_px = hi - k * rng
|
||||
d[k].append(sum(p * v for p, v in buys.items() if p >= floor_px))
|
||||
|
||||
for sym, d in per.items():
|
||||
print(f" {sym} ≥ 限价的主动买成交额（USDT），按限价所处位置分层")
|
||||
print(f" {'限价位置':>16} {'中位':>12} {'P25':>12} "
|
||||
+ " ".join(f"{n:>9,.0f}全仓" for n in notionals))
|
||||
for k in depths:
|
||||
a = np.array(d[k], dtype=float)
|
||||
where = ("正好在最高价" if k == 0 else
|
||||
"整根全部成交" if k == 1.0 else
|
||||
f"区间顶部 {k * 100:.0f}%")
|
||||
cells = " ".join(f"{float((a >= n).mean()) * 100:8.1f}%"
|
||||
for n in notionals)
|
||||
print(f" {where:>16} {np.median(a):>12,.0f} "
|
||||
f"{np.percentile(a, 25):>12,.0f} {cells}")
|
||||
print()
|
||||
print(" 「正好在最高价」那一行才是止盈被刚好触及时的真实处境；")
|
||||
print(" 「整根全部成交」是最宽松的上界。两行差多少，就是回测那个")
|
||||
print(" 「限价单全额成交」假设虚了多少。而且这仍未计排队——我们的单")
|
||||
print(" 排在该价位既有挂单之后，所以真实成交率比表里更低")
|
||||
|
||||
|
||||
def tape_shape(tape_path: Path, kgrid: np.ndarray) -> dict[str, np.ndarray]:
|
||||
"""成交流给「形状」：一根的主动买成交额里，有多少比例落在区间顶部 k 之内。
|
||||
|
||||
形状与规模分开是为了绕开成交流样本小的限制——形状是微观结构性质，
|
||||
几十根就相当稳定；规模（每根成交多少钱）则由 210 天历史成交量提供。
|
||||
"""
|
||||
acc: dict[str, list[np.ndarray]] = {}
|
||||
for r in read_jsonl_gz(tape_path):
|
||||
buys = {float(p): v for p, v in r["buys"].items()}
|
||||
if len(buys) < 2:
|
||||
continue
|
||||
hi, lo = max(buys), min(buys)
|
||||
rng = hi - lo
|
||||
if rng <= 0:
|
||||
continue
|
||||
tot = sum(p * v for p, v in buys.items())
|
||||
if tot <= 0:
|
||||
continue
|
||||
frac = np.array([sum(p * v for p, v in buys.items()
|
||||
if p >= hi - k * rng) / tot for k in kgrid])
|
||||
acc.setdefault(r["sym"], []).append(frac)
|
||||
return {s: np.mean(np.vstack(v), axis=0) for s, v in acc.items() if v}
|
||||
|
||||
|
||||
def composite_fill(tape_path: Path, pen_path: Path, cache: Path,
|
||||
notionals=(5e4, 1e5, 2e5)) -> None:
|
||||
"""把穿透深度分布与成交量曲线合并，出**单根**内的 maker 成交率。
|
||||
|
||||
⚠ 这个数只回答「限价单若仅有首次触及那一根可以成交，能否成交」。真实的
|
||||
挂单是常驻的：它在那儿放最多 48 根，每根都在成交，且价格决定性穿过限价
|
||||
时整根成交量都可用。所以本函数系统性**偏悲观**，不能当作成交率结论。
|
||||
真实成交率见 lib/exit_fill.walk_filled 与 step43_fill_aware_budget.py，
|
||||
那里按逐根累积结算，10 万仓位下预算降幅不足 1%。
|
||||
"""
|
||||
if not pen_path.exists():
|
||||
print("\n没有 penetration.csv，先跑 penetration.py")
|
||||
return
|
||||
kgrid = np.linspace(0.0, 1.0, 51)
|
||||
shape = tape_shape(tape_path, kgrid)
|
||||
if not shape:
|
||||
print("\n成交流样本不足，无法定形状")
|
||||
return
|
||||
pen = pd.read_csv(pen_path)
|
||||
|
||||
print(f"\n\n########## maker 腿真实成交率 ##########")
|
||||
print(f" 穿透深度分布（历史 63 万次触及）× 每根成交额（210 天）")
|
||||
print(f" × 区间内成交分布形状（影子成交流）\n")
|
||||
|
||||
for sym in sorted(shape):
|
||||
cands = sorted(cache.glob(f"bitget_{sym}_1m_*.feather"),
|
||||
key=lambda p: p.stat().st_size, reverse=True)
|
||||
if not cands:
|
||||
continue
|
||||
bars = pd.read_feather(cands[0])
|
||||
# 每根的主动买成交额。取总成交额的一半——买卖大致均衡，且这与
|
||||
# 成交流实测的买卖比一致
|
||||
bar_notional = (bars["volume"].to_numpy(float)
|
||||
* bars["close"].to_numpy(float)) * 0.5
|
||||
bar_notional = bar_notional[np.isfinite(bar_notional)
|
||||
& (bar_notional > 0)]
|
||||
f = shape[sym]
|
||||
for tgt in sorted(pen["target_atr"].unique()):
|
||||
k = pen[(pen["sym"] == sym)
|
||||
& (pen["target_atr"] == tgt)]["k"].to_numpy(float)
|
||||
if not k.size:
|
||||
continue
|
||||
# 独立配对：穿透位置与该根成交额各自抽样。真实触及根多为放量根，
|
||||
# 故此处偏**保守**（低估可成交量）
|
||||
rng = np.random.default_rng(0)
|
||||
m = 200_000
|
||||
ks = rng.choice(k, m)
|
||||
ns = rng.choice(bar_notional, m)
|
||||
avail = np.interp(ks, kgrid, f) * ns
|
||||
print(f" {sym} · 目标 {tgt:g}ATR · 每根主动买额中位 "
|
||||
f"{np.median(bar_notional):,.0f} USDT")
|
||||
for nt in notionals:
|
||||
full = float((avail >= nt).mean())
|
||||
half = float((avail >= nt / 2).mean())
|
||||
print(f" 仓位 {nt:>9,.0f}：全额成交 {full * 100:5.1f}%"
|
||||
f" · 至少半额 {half * 100:5.1f}%"
|
||||
f" · 可成交额中位 {np.median(avail):>10,.0f}")
|
||||
# 成交率反推的资金上限。这才是绑定约束——它比冲击反推的上限
|
||||
# 低一到两个数量级，而后者才是通常被当作「容量」的那个数
|
||||
for want in (0.80, 0.90):
|
||||
cap = float(np.quantile(avail, 1.0 - want))
|
||||
print(f" → 要 {want * 100:.0f}% 的止盈全额成交，"
|
||||
f"仓位须 ≤ {cap:,.0f} USDT")
|
||||
print()
|
||||
print(" 未计排队（我们的单排在该价位既有挂单之后），故仍是上界。")
|
||||
print(" 回测把这些止盈按「全额成交在目标价」计，差多少就是收益虚多少")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--books", default=None)
|
||||
ap.add_argument("--tape", default=None)
|
||||
a = ap.parse_args()
|
||||
d = out_dir()
|
||||
from lib.shadow_budget import BUDGET_BP
|
||||
tape = Path(a.tape) if a.tape else d / "shadow_tape.jsonl.gz"
|
||||
books = Path(a.books) if a.books else d / "shadow_books.jsonl.gz"
|
||||
capacity(books, BUDGET_BP)
|
||||
queue_ahead(books)
|
||||
maker_fill(tape)
|
||||
composite_fill(tape, d / "penetration.csv",
|
||||
Path(__file__).resolve().parent / "cache")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+484
-86
@@ -10,8 +10,25 @@
|
||||
为什么不能用 paper trade 的成交:那是 Hummingbot 自己的撮合模型模拟的,
|
||||
测出来是模型行为不是市场行为。
|
||||
|
||||
口径对齐 aggregate_robustness.py:回测假设成交在**信号次根的开盘价**
|
||||
口径对齐 step42_exit_tp_1m.py:回测假设成交在**信号次根的开盘价**
|
||||
(entry_delay=1),所以基准价就是换根后新一根的 open。滑点为正表示比回测差。
|
||||
滤网(同向 + 中枢阶梯 + ATR 门控)在 shadow_signal.py 里,必须与预算同源。
|
||||
|
||||
### 统计口径三条硬要求
|
||||
|
||||
1. **主口径只用 pass_all 的信号根**。未过滤的照记但只作提前读数——
|
||||
在我们根本不会下单的根上测滑点会把判据算宽
|
||||
2. **条件漂移与无条件漂移分开报**。所以每根 K 线都记一份漂移
|
||||
(shadow_drift.csv),不只信号根。两者的差就是「系统性追价」的大小
|
||||
3. **出场腿按 maker/taker 分开**。止盈挂限价不吃滑点,把那 60% 混进
|
||||
平均值会低估真实成本。出场腿属持仓管理,尚未实现
|
||||
|
||||
### lag 探针:超阈值要停开仓,不能只打日志
|
||||
|
||||
补丁只防得住「上游代码变了」,防不住 Bitget 再改一次消息格式。每根记
|
||||
本地接收 − K 线收盘,近 30 根取中位数,超 800ms 即判该币不健康。
|
||||
要停开仓是因为这种退化是**经济性且静默**的:不崩不报错,只让收益慢慢
|
||||
变差,几周后才从统计里看得出来。影子期不下单,故落到 lag_ok 字段上。
|
||||
|
||||
### 盘口滚动缓冲把延迟变成自变量
|
||||
|
||||
@@ -38,22 +55,39 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import asyncio
|
||||
import csv
|
||||
import gzip
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import socket
|
||||
import time
|
||||
from collections import deque
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from concurrent.futures.process import BrokenProcessPool
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from lib.shadow_budget import LAG_ALARM_MS, LAG_WINDOW, lag_healthy
|
||||
|
||||
# 站点标识。跨地对比时两台机器的 CSV 要能合起来读，没有这一列就分不清哪行
|
||||
# 来自哪台。默认取主机名，部署脚本会显式传 SHADOW_SITE（如 sg-hetzner）
|
||||
SITE = os.environ.get("SHADOW_SITE") or socket.gethostname()
|
||||
|
||||
SYMS = ("BTC", "ETH", "SOL")
|
||||
# 多存一根:deque 尾部是尚未收盘的当前根,剔除后正好剩 step39 定下的窗口
|
||||
LTF_BARS, HTF_BARS = 2001, 801 # 有效窗口 2000 / 800,命中率在此饱和
|
||||
BOOK_HZ = 10 # 盘口采样 10Hz
|
||||
BOOK_KEEP_S = 30 # 缓冲保留 30 秒,够回查到 +5s
|
||||
BOOK_DEPTH = 25
|
||||
BOOK_TOL_MS = 250 # 回查容差:10Hz 正常 ≤100ms,留些余量
|
||||
BOOK_DEPTH = 50 # 双边各 50 档，实测能撑 78 万~261 万美元
|
||||
DELAYS_S = (0.5, 1.0, 2.0, 5.0) # 回查点
|
||||
NOTIONALS = (1_000.0, 5_000.0, 20_000.0)
|
||||
# 主口径 10 万名义额（2026-08-28 定：不会有更大资金）。上下各留两档是为了
|
||||
# 读出局部斜率——单点看不出「再大一倍会怎样」。
|
||||
# **真正的答案在 shadow_books.jsonl.gz 里**：完整盘口已落盘，任意资金量级的
|
||||
# 冲击都能离线重算，换规模不必重测，这里的档位只为让 CSV 直接可读
|
||||
NOTIONALS = (25_000.0, 50_000.0, 100_000.0, 200_000.0)
|
||||
NUM_COLS = ["timestamp", "open", "high", "low", "close", "volume"]
|
||||
|
||||
|
||||
@@ -62,6 +96,137 @@ def out_dir() -> Path:
|
||||
return p if p.is_dir() else Path(__file__).resolve().parents[1] / "out"
|
||||
|
||||
|
||||
def _writer(path: Path, cols: list[str]):
|
||||
"""追加模式打开；表头对不上就先把旧文件归档。
|
||||
|
||||
不校验的话，列一改，DictWriter 会按新顺序把行写到旧表头下面——
|
||||
读出来整片错位，而且没有任何报错。长跑靠追加续命，这个校验是必需的。
|
||||
"""
|
||||
if path.exists() and path.stat().st_size > 0:
|
||||
with path.open(newline="") as fh:
|
||||
old = next(csv.reader(fh), [])
|
||||
if old != cols:
|
||||
arch = path.parent / "archive"
|
||||
arch.mkdir(exist_ok=True)
|
||||
dst = arch / f"{path.stem}_{time.strftime('%Y%m%d_%H%M%S')}.csv"
|
||||
path.rename(dst)
|
||||
print(f" [CSV] {path.name} 表头已变，旧数据归档为 {dst.name}",
|
||||
flush=True)
|
||||
fresh = not path.exists() or path.stat().st_size == 0
|
||||
f = path.open("a", newline="")
|
||||
w = _SiteWriter(csv.DictWriter(f, fieldnames=cols))
|
||||
if fresh:
|
||||
w.writeheader()
|
||||
return f, w
|
||||
|
||||
|
||||
class _SiteWriter:
|
||||
"""DictWriter 的薄包装，自动补上 site 列。
|
||||
|
||||
逐个 writerow 手加 site 有四处，漏一处就是静默的空值，而跨地对比正是靠
|
||||
这一列区分数据来源。在这里注入，漏不掉。
|
||||
"""
|
||||
|
||||
def __init__(self, w: csv.DictWriter) -> None:
|
||||
self._w = w
|
||||
|
||||
def writeheader(self) -> None:
|
||||
self._w.writeheader()
|
||||
|
||||
def writerow(self, row: dict) -> None:
|
||||
row.setdefault("site", SITE)
|
||||
self._w.writerow(row)
|
||||
|
||||
|
||||
class BookLog:
|
||||
"""把完整盘口快照落成 gzip JSONL。
|
||||
|
||||
只记「某几个仓位档的成交价」的话，这批数据的寿命就等于那几个档位的寿命：
|
||||
换一次资金规模就得重跑一周。存完整深度后，任意仓位的冲击都能离线重算，
|
||||
一次采集回答所有资金量级的问题——包括容量上限那个必须现在就算、
|
||||
不该等实盘暴露的数。
|
||||
|
||||
用 gzip 追加（多个 gzip 成员首尾相接仍可正常解压），进程被杀也只丢最后
|
||||
一个缓冲块，不会毁掉整个文件。
|
||||
"""
|
||||
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.path = path
|
||||
self.fh = gzip.open(path, "at", encoding="utf-8")
|
||||
self.n = 0
|
||||
|
||||
def write(self, sym: str, kline_ts: int, label: str, delay_ms: int,
|
||||
target: int, book_ts: int, bids: np.ndarray,
|
||||
asks: np.ndarray) -> None:
|
||||
# 只留价与量两列，update_id 对离线分析没用。round 到 10 位避免
|
||||
# float repr 把文件撑大一倍
|
||||
rec = {"site": SITE, "sym": sym, "kline_ts": kline_ts, "label": label,
|
||||
"delay_ms": delay_ms, "target": target, "book_ts": book_ts,
|
||||
"bids": [[round(float(p), 10), round(float(a), 10)]
|
||||
for p, a, *_ in bids],
|
||||
"asks": [[round(float(p), 10), round(float(a), 10)]
|
||||
for p, a, *_ in asks]}
|
||||
self.fh.write(json.dumps(rec, separators=(",", ":")) + "\n")
|
||||
self.n += 1
|
||||
|
||||
def flush(self) -> None:
|
||||
self.fh.flush()
|
||||
|
||||
def close(self) -> None:
|
||||
try:
|
||||
self.fh.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class TapeLog:
|
||||
"""按 K 线、按价位聚合成交量，用来判 maker 腿能不能全额成交。
|
||||
|
||||
回测假设 3ATR 和 8ATR 的限价单全额成交。十万量级挂在那里，全成交还是
|
||||
部分成交是完全不同的事——部分成交会把分批出场的收益结构改掉，而这个
|
||||
问题盘口深度回答不了：深度说的是「现在有多少人挂着」，成交率问的是
|
||||
「之后有多少人打过来」。只有成交流能回答。
|
||||
|
||||
**买卖必须分开存。** 多头在 3ATR 挂卖出止盈，成交靠的是主动**买盘**
|
||||
打上来；把双边成交量合在一起会把成交率高估约一倍。
|
||||
|
||||
聚合到「根 × 价位」而不是逐笔：判据是「本根内有多少量在 ≥ 限价处成交」，
|
||||
逐笔的时序对这个判据没有增量信息，而聚合能把体量压下两个数量级。
|
||||
"""
|
||||
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.fh = gzip.open(path, "at", encoding="utf-8")
|
||||
# sym -> side('b'/'s') -> price -> 累计基础币量
|
||||
self.acc: dict[str, dict[str, dict[float, float]]] = {}
|
||||
self.n_trades = 0
|
||||
|
||||
def add(self, sym: str, is_buy: bool, price: float, amount: float) -> None:
|
||||
d = self.acc.setdefault(sym, {"b": {}, "s": {}})
|
||||
side = d["b"] if is_buy else d["s"]
|
||||
side[price] = side.get(price, 0.0) + amount
|
||||
self.n_trades += 1
|
||||
|
||||
def flush_bar(self, sym: str, bar_ts: int) -> None:
|
||||
"""一根走完就把这根的聚合结果落盘并清空。"""
|
||||
d = self.acc.get(sym)
|
||||
if not d or (not d["b"] and not d["s"]):
|
||||
return
|
||||
rec = {"site": SITE, "sym": sym, "bar_ts": bar_ts,
|
||||
"buys": {f"{p:.10g}": round(v, 10)
|
||||
for p, v in sorted(d["b"].items())},
|
||||
"sells": {f"{p:.10g}": round(v, 10)
|
||||
for p, v in sorted(d["s"].items())}}
|
||||
self.fh.write(json.dumps(rec, separators=(",", ":")) + "\n")
|
||||
self.fh.flush()
|
||||
self.acc[sym] = {"b": {}, "s": {}}
|
||||
|
||||
def close(self) -> None:
|
||||
try:
|
||||
self.fh.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def hb_to_research(cdf: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Hummingbot 的 candles_df 转成 research/lib/data.py 的列结构。
|
||||
|
||||
@@ -77,38 +242,32 @@ def hb_to_research(cdf: pd.DataFrame) -> pd.DataFrame:
|
||||
.sort_values("timestamp").reset_index(drop=True)
|
||||
|
||||
|
||||
def walk_book(levels: list[tuple[float, float]], notional: float
|
||||
) -> tuple[float, float]:
|
||||
"""吃单到 notional(计价币)为止,返回 (加权成交价, 实际吃到的额度)。
|
||||
def book_from(bids: np.ndarray, asks: np.ndarray):
|
||||
"""用缓冲里的快照临时搭一个 OrderBook，以便调用框架自带的吃单查询。
|
||||
|
||||
深度不足时返回吃到的部分,由调用方按 filled < notional 判断是否可信。
|
||||
自己手写吃单曾经踩过两个坑，框架版都没有：`get_vwap_for_volume` 返回的
|
||||
是真加权均价（市价单的实际成交价），而 `get_price_for_quote_volume` 返回
|
||||
的是**边际价**，用后者会高估冲击；深度不足时框架返回 nan 而不是一个
|
||||
「看起来很正常」的部分成交均价，靠 query_volume/result_volume 判断。
|
||||
"""
|
||||
if not levels:
|
||||
return float("nan"), 0.0
|
||||
got = 0.0
|
||||
cost = 0.0
|
||||
qty = 0.0
|
||||
for px, sz in levels:
|
||||
avail = px * sz
|
||||
take = min(avail, notional - got)
|
||||
if take <= 0:
|
||||
break
|
||||
q = take / px
|
||||
cost += q * px
|
||||
qty += q
|
||||
got += take
|
||||
if got >= notional - 1e-9:
|
||||
break
|
||||
if qty <= 0:
|
||||
return float("nan"), 0.0
|
||||
return cost / qty, got
|
||||
from hummingbot.core.data_type.order_book import OrderBook
|
||||
ob = OrderBook()
|
||||
ob.apply_numpy_snapshot(bids, asks)
|
||||
return ob
|
||||
|
||||
|
||||
class BookBuffer:
|
||||
"""每币一份滚动盘口。按时间戳回查,取第一个不早于目标时刻的快照。"""
|
||||
"""每币一份滚动盘口。按时间戳回查,取第一个不早于目标时刻的快照。
|
||||
|
||||
回查必须有容差上界。10Hz 下正常落在目标后 100ms 内（所有延迟点同向
|
||||
偏约 +50ms，不影响曲线形状），但采样一旦卡顿，标着「0.5s」的那行可能
|
||||
用的是 +3s 的盘口——数据看不出异常，判读却已经错了。超容差宁可丢弃，
|
||||
并且把快照实际时刻写进 CSV，让这件事事后可查。
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.buf: dict[str, deque] = {s: deque() for s in SYMS}
|
||||
self.n_stale = 0 # 因超容差被丢弃的回查次数
|
||||
|
||||
def push(self, sym: str, t_ms: int, bids: list, asks: list) -> None:
|
||||
d = self.buf[sym]
|
||||
@@ -118,12 +277,13 @@ class BookBuffer:
|
||||
d.popleft()
|
||||
|
||||
def at(self, sym: str, t_ms: int) -> tuple | None:
|
||||
best = None
|
||||
for snap in self.buf[sym]:
|
||||
if snap[0] >= t_ms:
|
||||
best = snap
|
||||
break
|
||||
return best
|
||||
if snap[0] - t_ms > BOOK_TOL_MS:
|
||||
self.n_stale += 1
|
||||
return None
|
||||
return snap
|
||||
return None
|
||||
|
||||
|
||||
class Shadow:
|
||||
@@ -136,30 +296,84 @@ class Shadow:
|
||||
self.feeds_h: dict = {}
|
||||
self.connector = None
|
||||
self.stop = asyncio.Event()
|
||||
# 持有 fire-and-forget 任务的强引用。只 create_task 不留引用的话，
|
||||
# 任务可能在完成前被 GC 掉，asyncio 官方文档明确警告过这一点
|
||||
self._tasks: set = set()
|
||||
self.n_broken = 0
|
||||
self._hb_last_bars = 0
|
||||
# 成交监听：已挂上的币，以及必须持有的 forwarder 强引用
|
||||
# （PubSub 只存弱引用，不持有的话监听会被 GC 静默摘掉）
|
||||
self._hooked: set[str] = set()
|
||||
self._trade_fwd: dict = {}
|
||||
self.n_signal = 0
|
||||
self.n_pass = 0
|
||||
self.n_bars = 0
|
||||
# lag 探针的滚动窗口，逐币独立：一个币的行情退化不该连累其他币
|
||||
self.lag_hist: dict[str, deque] = {
|
||||
s: deque(maxlen=LAG_WINDOW) for s in SYMS}
|
||||
self.lag_ok: dict[str, bool] = {s: True for s in SYMS}
|
||||
d = out_dir()
|
||||
# 追加模式:长跑期间若重启,已收集的样本不该被清掉
|
||||
p_sig = d / "shadow_signals.csv"
|
||||
new_sig = not p_sig.exists() or p_sig.stat().st_size == 0
|
||||
self.f_sig = p_sig.open("a", newline="")
|
||||
self.w_sig = csv.DictWriter(self.f_sig, fieldnames=[
|
||||
"sym", "kline_ts", "direction", "h1_agree",
|
||||
self.f_sig, self.w_sig = _writer(d / "shadow_signals.csv", [
|
||||
"site", "sym", "kline_ts", "direction",
|
||||
"h1_agree", "ladder_ok", "gate_ok", "pass_all", "lag_ok",
|
||||
"atr_pct", "atr_bp",
|
||||
"t_close_ms", "t_data_ms", "t_signal_ms",
|
||||
"lag_data_ms", "lag_signal_ms",
|
||||
"delay_label", "delay_ms", "notional",
|
||||
"baseline_px", "mid", "best_px", "fill_px", "filled",
|
||||
"delay_label", "delay_ms", "book_ts", "book_lag_ms",
|
||||
"notional", "base_amt",
|
||||
"baseline_px", "mid", "best_px", "fill_px", "filled", "depth_ok",
|
||||
"slip_bp", "drift_bp", "spread_bp", "impact_bp"])
|
||||
if new_sig:
|
||||
self.w_sig.writeheader()
|
||||
p_lat = d / "shadow_latency.csv"
|
||||
new_lat = not p_lat.exists() or p_lat.stat().st_size == 0
|
||||
self.f_lat = p_lat.open("a", newline="")
|
||||
self.w_lat = csv.DictWriter(self.f_lat, fieldnames=[
|
||||
"sym", "kline_ts", "t_close_ms", "t_data_ms", "t_signal_ms",
|
||||
"lag_data_ms", "lag_signal_ms", "compute_ms", "n_bars", "n_hits"])
|
||||
if new_lat:
|
||||
self.w_lat.writeheader()
|
||||
self.f_lat, self.w_lat = _writer(d / "shadow_latency.csv", [
|
||||
"site", "sym", "kline_ts", "t_close_ms", "t_data_ms", "t_signal_ms",
|
||||
"lag_data_ms", "lag_signal_ms", "compute_ms", "n_bars", "n_hits",
|
||||
"n_pass", "atr_bp", "lag_med_ms", "lag_ok"])
|
||||
# 无条件漂移：每根都记，用来和信号根上的条件漂移对照
|
||||
self.f_drf, self.w_drf = _writer(d / "shadow_drift.csv", [
|
||||
"site", "sym", "kline_ts", "delay_label", "delay_ms",
|
||||
"book_ts", "book_lag_ms", "baseline_px", "mid", "drift_bp_long"])
|
||||
# 完整深度。挂在无条件漂移那条路径上，所以每根 K 线的四个固定延迟点
|
||||
# 都有一份，信号根上再补一份 actual 点
|
||||
self.blog = BookLog(d / "shadow_books.jsonl.gz")
|
||||
self.tape = TapeLog(d / "shadow_tape.jsonl.gz")
|
||||
|
||||
def _spawn(self, coro, what: str) -> None:
|
||||
"""起一个后台任务，但异常要吼出来。
|
||||
|
||||
裸 create_task 的异常只在对象被 GC 时才由 asyncio 打一句
|
||||
「Task exception was never retrieved」，很容易整晚没人发现。
|
||||
这套东西最怕的就是不崩不报错的静默退化。
|
||||
"""
|
||||
async def guard():
|
||||
try:
|
||||
await coro
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e:
|
||||
import traceback
|
||||
print(f" [异常] {what}: {type(e).__name__}: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
|
||||
t = asyncio.create_task(guard())
|
||||
self._tasks.add(t)
|
||||
t.add_done_callback(self._tasks.discard)
|
||||
|
||||
def _restart_pool(self) -> None:
|
||||
"""进程池坏了之后重建。
|
||||
|
||||
用 spawn 而非 fork：此刻进程里已经有活跃的 WS 连接，fork 会把连接
|
||||
状态一起复制进子进程。spawn 启动慢几秒，但只在故障时走这条路。
|
||||
"""
|
||||
import multiprocessing
|
||||
self.n_broken += 1
|
||||
try:
|
||||
self.pool.shutdown(wait=False, cancel_futures=True)
|
||||
except Exception:
|
||||
pass
|
||||
self.pool = ProcessPoolExecutor(
|
||||
max_workers=self.workers,
|
||||
mp_context=multiprocessing.get_context("spawn"))
|
||||
print(f" [进程池] 已重建（第 {self.n_broken} 次）", flush=True)
|
||||
|
||||
# ---------- 启动 ----------
|
||||
|
||||
@@ -167,7 +381,11 @@ class Shadow:
|
||||
from hummingbot.connector.derivative.bitget_perpetual.bitget_perpetual_derivative import (
|
||||
BitgetPerpetualDerivative,
|
||||
)
|
||||
from patched_candles import PatchedBitgetPerpetualCandles
|
||||
from patched_candles import (PatchedBitgetPerpetualCandles,
|
||||
assert_patch_effective)
|
||||
|
||||
# 覆盖失效是静默的（悄悄退回慢 1.06 秒，不报错），所以在启动就验一次
|
||||
assert_patch_effective()
|
||||
|
||||
for s in SYMS:
|
||||
self.feeds_l[s] = PatchedBitgetPerpetualCandles(
|
||||
@@ -187,6 +405,7 @@ class Shadow:
|
||||
trading_required=False)
|
||||
await self.connector.start_network()
|
||||
print(" 连接器已启动,等盘口与历史回填", flush=True)
|
||||
self._hook_trades()
|
||||
|
||||
t0 = time.time()
|
||||
while time.time() - t0 < 600:
|
||||
@@ -200,6 +419,39 @@ class Shadow:
|
||||
f"1m {[len(self.feeds_l[s]._candles) for s in SYMS]} 根 · "
|
||||
f"5m {[len(self.feeds_h[s]._candles) for s in SYMS]} 根", flush=True)
|
||||
|
||||
def _hook_trades(self) -> None:
|
||||
"""给每个盘口挂成交监听。
|
||||
|
||||
盘口对象可能还没建好（订阅是异步的），所以挂不上的先记下来，由
|
||||
watch_bars 那圈重试；一直挂不上会在心跳里显示成交笔数为 0。
|
||||
"""
|
||||
from hummingbot.core.event.event_forwarder import EventForwarder
|
||||
from hummingbot.core.event.events import OrderBookEvent
|
||||
from hummingbot.core.data_type.common import TradeType
|
||||
|
||||
def make(sym: str):
|
||||
def cb(ev) -> None:
|
||||
self.tape.add(sym, ev.type == TradeType.BUY,
|
||||
float(ev.price), float(ev.amount))
|
||||
return EventForwarder(cb)
|
||||
|
||||
for s in SYMS:
|
||||
if s in self._hooked: # 重复挂会让同一笔成交被记两次
|
||||
continue
|
||||
try:
|
||||
ob = self.connector.get_order_book(f"{s}-USDT")
|
||||
except Exception:
|
||||
ob = None
|
||||
if ob is None:
|
||||
continue
|
||||
fwd = make(s)
|
||||
ob.add_listener(OrderBookEvent.TradeEvent, fwd)
|
||||
self._trade_fwd[s] = fwd
|
||||
self._hooked.add(s)
|
||||
miss = [s for s in SYMS if s not in self._hooked]
|
||||
print(f" 成交流已挂 {sorted(self._hooked)}"
|
||||
+ (f"，待重试 {miss}" if miss else ""), flush=True)
|
||||
|
||||
def _snapshot(self, sym: str):
|
||||
try:
|
||||
ob = self.connector.get_order_book(f"{sym}-USDT")
|
||||
@@ -207,25 +459,41 @@ class Shadow:
|
||||
return None
|
||||
if ob is None:
|
||||
return None
|
||||
bids = [(float(r.price), float(r.amount))
|
||||
for r, _ in zip(ob.bid_entries(), range(BOOK_DEPTH))]
|
||||
asks = [(float(r.price), float(r.amount))
|
||||
for r, _ in zip(ob.ask_entries(), range(BOOK_DEPTH))]
|
||||
if not bids or not asks:
|
||||
# 存成 apply_numpy_snapshot 要的 [价, 量, update_id] 三列，
|
||||
# 回查时才能直接搭 OrderBook 调框架的吃单查询
|
||||
bids = np.array([(float(r.price), float(r.amount), i)
|
||||
for i, (r, _) in enumerate(
|
||||
zip(ob.bid_entries(), range(BOOK_DEPTH)))])
|
||||
asks = np.array([(float(r.price), float(r.amount), i)
|
||||
for i, (r, _) in enumerate(
|
||||
zip(ob.ask_entries(), range(BOOK_DEPTH)))])
|
||||
if not len(bids) or not len(asks):
|
||||
return None
|
||||
return bids, asks
|
||||
|
||||
# ---------- 三个循环 ----------
|
||||
|
||||
async def sample_books(self) -> None:
|
||||
"""按截止时刻补睡，且对齐到墙钟 100ms 网格。
|
||||
|
||||
补睡是因为「干完活再睡固定时长」的实际周期是 100ms 加采样耗时，
|
||||
名义 10Hz 到不了 10Hz。
|
||||
|
||||
对齐是因为回查目标都是 `kline_ts + n×500ms`，而 kline_ts 是整分钟，
|
||||
所以目标必然落在墙钟 100ms 的整数倍上。采样相位若随启动时刻漂移，
|
||||
每个回查点就会固定晚半个采样周期（实测 52ms）——四个固定延迟点
|
||||
同向偏置，虽不改曲线形状，但白白多算了 50ms 的漂移。
|
||||
"""
|
||||
period = 1.0 / BOOK_HZ
|
||||
nxt = math.ceil(time.time() / period) * period
|
||||
while not self.stop.is_set():
|
||||
t = int(time.time() * 1000)
|
||||
for s in SYMS:
|
||||
snap = self._snapshot(s)
|
||||
if snap:
|
||||
self.books.push(s, t, snap[0], snap[1])
|
||||
await asyncio.sleep(period)
|
||||
nxt += period
|
||||
await asyncio.sleep(max(0.0, nxt - time.time()))
|
||||
|
||||
async def watch_bars(self) -> None:
|
||||
last = {s: (int(self.feeds_l[s]._candles[-1][0])
|
||||
@@ -239,7 +507,11 @@ class Shadow:
|
||||
if last[s] is not None and newest > last[s]:
|
||||
t_data = int(time.time() * 1000)
|
||||
kts = newest * 1000 if newest < 1e12 else newest
|
||||
asyncio.create_task(self.on_bar(s, kts, t_data))
|
||||
# 刚收盘那根的成交聚合先落盘，再算信号
|
||||
self.tape.flush_bar(s, kts)
|
||||
if len(self._hooked) < len(SYMS):
|
||||
self._hook_trades() # 换根时才重试，避免重复挂
|
||||
self._spawn(self.on_bar(s, kts, t_data), f"on_bar {s}")
|
||||
last[s] = newest
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
@@ -252,15 +524,28 @@ class Shadow:
|
||||
df_h = df_h[df_h["timestamp"] < kline_ts]
|
||||
baseline = self._new_bar_open(sym, kline_ts)
|
||||
|
||||
lag_med, lag_ok = self._probe_lag(sym, t_data - kline_ts)
|
||||
|
||||
t0 = time.perf_counter()
|
||||
payload = (df_l[NUM_COLS].values.tolist(),
|
||||
df_h[NUM_COLS].values.tolist())
|
||||
df_h[NUM_COLS].values.tolist(), baseline)
|
||||
loop = asyncio.get_running_loop()
|
||||
from shadow_signal import compute_packed
|
||||
res = await loop.run_in_executor(self.pool, compute_packed, payload)
|
||||
try:
|
||||
res = await loop.run_in_executor(self.pool, compute_packed, payload)
|
||||
except BrokenProcessPool as e:
|
||||
# 不重建的话，之后每一根都会走到这里，采集静默停摆到跑完为止
|
||||
print(f" [{sym}] 进程池损坏 {e}，重建后跳过本根", flush=True)
|
||||
self._restart_pool()
|
||||
return
|
||||
compute_ms = int((time.perf_counter() - t0) * 1000)
|
||||
t_signal = int(time.time() * 1000)
|
||||
|
||||
hits = res.get("hits", [])
|
||||
atr_pct = res.get("atr_pct")
|
||||
atr_bp = round(atr_pct * 1e4, 3) if atr_pct else ""
|
||||
n_pass = sum(h["pass_all"] for h in hits)
|
||||
|
||||
self.n_bars += 1
|
||||
self.w_lat.writerow({
|
||||
"sym": sym, "kline_ts": kline_ts, "t_close_ms": kline_ts,
|
||||
@@ -268,13 +553,18 @@ class Shadow:
|
||||
"lag_data_ms": t_data - kline_ts,
|
||||
"lag_signal_ms": t_signal - kline_ts,
|
||||
"compute_ms": compute_ms, "n_bars": res.get("n_bars", 0),
|
||||
"n_hits": len(res.get("hits", []))})
|
||||
"n_hits": len(hits), "n_pass": n_pass, "atr_bp": atr_bp,
|
||||
"lag_med_ms": lag_med, "lag_ok": int(lag_ok)})
|
||||
self.f_lat.flush()
|
||||
|
||||
if baseline is not None and np.isfinite(baseline):
|
||||
# 无条件漂移：每根都记，不管有没有信号
|
||||
self._spawn(self._drift_later(sym, kline_ts, baseline),
|
||||
f"drift {sym}")
|
||||
|
||||
if res.get("error"):
|
||||
print(f" [{sym}] 信号计算出错 {res['error']}", flush=True)
|
||||
return
|
||||
hits = res.get("hits", [])
|
||||
if not hits:
|
||||
return
|
||||
if baseline is None or not np.isfinite(baseline):
|
||||
@@ -283,20 +573,73 @@ class Shadow:
|
||||
|
||||
for h in hits:
|
||||
self.n_signal += 1
|
||||
print(f" ★ [{sym}] {kline_ts} 方向 {h['direction']:+d} "
|
||||
f"h1_agree={h['h1_agree']} · 数据 {t_data - kline_ts}ms "
|
||||
self.n_pass += h["pass_all"]
|
||||
mark = "★" if h["pass_all"] else "·"
|
||||
print(f" {mark} [{sym}] {kline_ts} 方向 {h['direction']:+d} "
|
||||
f"同向{h['h1_agree']} 阶梯{h['ladder_ok']} 门控{h['gate_ok']} "
|
||||
f"(ATR {atr_bp or 'na'}bp) · 数据 {t_data - kline_ts}ms "
|
||||
f"信号 {t_signal - kline_ts}ms", flush=True)
|
||||
# 最远的回查点在 t_close+5s，此刻尚未发生；等它过去再一次性落盘
|
||||
asyncio.create_task(
|
||||
self._record_later(sym, kline_ts, h, t_data, t_signal, baseline))
|
||||
self._spawn(
|
||||
self._record_later(sym, kline_ts, h, t_data, t_signal,
|
||||
baseline, atr_pct, lag_ok),
|
||||
f"record {sym}")
|
||||
|
||||
async def _record_later(self, sym: str, kline_ts: int, hit: dict,
|
||||
t_data: int, t_signal: int, baseline: float) -> None:
|
||||
target = kline_ts + int(max(DELAYS_S) * 1000) + 500
|
||||
wait = target / 1000.0 - time.time()
|
||||
def _probe_lag(self, sym: str, lag_ms: int) -> tuple[float, bool]:
|
||||
"""记一根的到达延迟，返回 (滚动中位数, 该币是否健康)。
|
||||
|
||||
不健康时应停止开新仓；影子期不下单，故只落到 lag_ok 字段并告警。
|
||||
"""
|
||||
self.lag_hist[sym].append(lag_ms)
|
||||
ok = lag_healthy(self.lag_hist[sym])
|
||||
med = float(np.median(self.lag_hist[sym]))
|
||||
if ok != self.lag_ok[sym]:
|
||||
state = "恢复" if ok else f"退化，超 {LAG_ALARM_MS:.0f}ms 阈值，停开新仓"
|
||||
print(f" [lag] {sym} {state}：近 {len(self.lag_hist[sym])} 根"
|
||||
f"中位 {med:.0f}ms", flush=True)
|
||||
self.lag_ok[sym] = ok
|
||||
return round(med, 1), ok
|
||||
|
||||
async def _wait_for_delays(self, kline_ts: int) -> None:
|
||||
"""最远回查点是 t_close+5s，等它过去（多留 0.5s 给采样）。"""
|
||||
wait = (kline_ts + int(max(DELAYS_S) * 1000) + 500) / 1000.0 - time.time()
|
||||
if wait > 0:
|
||||
await asyncio.sleep(wait)
|
||||
self._record(sym, kline_ts, hit, t_data, t_signal, baseline)
|
||||
|
||||
async def _record_later(self, sym: str, kline_ts: int, hit: dict,
|
||||
t_data: int, t_signal: int, baseline: float,
|
||||
atr_pct: float | None, lag_ok: bool) -> None:
|
||||
await self._wait_for_delays(kline_ts)
|
||||
self._record(sym, kline_ts, hit, t_data, t_signal, baseline,
|
||||
atr_pct, lag_ok)
|
||||
|
||||
async def _drift_later(self, sym: str, kline_ts: int,
|
||||
baseline: float) -> None:
|
||||
await self._wait_for_delays(kline_ts)
|
||||
for label, delay_ms in self._points(None):
|
||||
target = kline_ts + delay_ms
|
||||
snap = self.books.at(sym, target)
|
||||
if snap is None:
|
||||
continue
|
||||
book_ts, bids, asks = snap
|
||||
mid = (float(bids[0][0]) + float(asks[0][0])) / 2.0
|
||||
self.blog.write(sym, kline_ts, label, delay_ms, target,
|
||||
book_ts, bids, asks)
|
||||
self.w_drf.writerow({
|
||||
"sym": sym, "kline_ts": kline_ts, "delay_label": label,
|
||||
"delay_ms": delay_ms, "book_ts": book_ts,
|
||||
"book_lag_ms": book_ts - target, "baseline_px": baseline,
|
||||
"mid": mid,
|
||||
"drift_bp_long": round((mid - baseline) / baseline * 1e4, 4)})
|
||||
self.f_drf.flush()
|
||||
self.blog.flush() # 每根冲刷一次，进程被杀最多丢一根
|
||||
|
||||
@staticmethod
|
||||
def _points(t_signal_delay: int | None) -> list[tuple[str, int]]:
|
||||
pts = [(f"{d}s", int(d * 1000)) for d in DELAYS_S]
|
||||
if t_signal_delay is not None:
|
||||
pts.insert(0, ("actual", t_signal_delay))
|
||||
return pts
|
||||
|
||||
def _new_bar_open(self, sym: str, kline_ts: int) -> float | None:
|
||||
"""次根开盘价 = 回测假设的成交价。"""
|
||||
@@ -309,25 +652,56 @@ class Shadow:
|
||||
return float(row[1]) if ts == kline_ts else None
|
||||
|
||||
def _record(self, sym: str, kline_ts: int, hit: dict,
|
||||
t_data: int, t_signal: int, baseline: float) -> None:
|
||||
points = [("actual", t_signal - kline_ts)]
|
||||
points += [(f"{d}s", int(d * 1000)) for d in DELAYS_S]
|
||||
t_data: int, t_signal: int, baseline: float,
|
||||
atr_pct: float | None, lag_ok: bool) -> None:
|
||||
d_sign = hit["direction"]
|
||||
|
||||
for label, delay_ms in points:
|
||||
snap = self.books.at(sym, kline_ts + delay_ms)
|
||||
for label, delay_ms in self._points(t_signal - kline_ts):
|
||||
target = kline_ts + delay_ms
|
||||
snap = self.books.at(sym, target)
|
||||
if snap is None:
|
||||
continue
|
||||
_, bids, asks = snap
|
||||
best_bid, best_ask = bids[0][0], asks[0][0]
|
||||
book_ts, bids, asks = snap
|
||||
best_bid, best_ask = float(bids[0][0]), float(asks[0][0])
|
||||
mid = (best_bid + best_ask) / 2.0
|
||||
# 多头吃卖盘,空头吃买盘
|
||||
side = asks if d_sign > 0 else bids
|
||||
best_px = best_ask if d_sign > 0 else best_bid
|
||||
ob = book_from(bids, asks)
|
||||
if label == "actual":
|
||||
# 四个固定点已由无条件漂移那条路径落过，只补这一个
|
||||
self.blog.write(sym, kline_ts, label, delay_ms, target,
|
||||
book_ts, bids, asks)
|
||||
|
||||
for notional in NOTIONALS:
|
||||
fill, filled = walk_book(side, notional)
|
||||
# 名义额按基准价折成基础币再下单——真实委托是基础币计价的，
|
||||
# 框架的 get_vwap_for_volume 也收基础币量。名义额那一栏留着
|
||||
# 是为了跨币可比（1 BTC 和 1 SOL 没法横向比）
|
||||
base_amt = notional / baseline
|
||||
r = ob.get_vwap_for_volume(d_sign > 0, base_amt)
|
||||
fill = float(r.result_price)
|
||||
depth_ok = int(float(r.result_volume) >= base_amt * 0.999)
|
||||
if not np.isfinite(fill):
|
||||
# 25 档吃不下这个量，框架直接给 nan。记一行标明深度不足，
|
||||
# 免得「某个仓位档在薄盘时段整段消失」看不出来
|
||||
self.w_sig.writerow({
|
||||
"sym": sym, "kline_ts": kline_ts, "direction": d_sign,
|
||||
"h1_agree": hit["h1_agree"],
|
||||
"ladder_ok": hit["ladder_ok"],
|
||||
"gate_ok": hit["gate_ok"], "pass_all": hit["pass_all"],
|
||||
"lag_ok": int(lag_ok),
|
||||
"atr_pct": atr_pct if atr_pct else "",
|
||||
"atr_bp": round(atr_pct * 1e4, 3) if atr_pct else "",
|
||||
"t_close_ms": kline_ts, "t_data_ms": t_data,
|
||||
"t_signal_ms": t_signal,
|
||||
"lag_data_ms": t_data - kline_ts,
|
||||
"lag_signal_ms": t_signal - kline_ts,
|
||||
"delay_label": label, "delay_ms": delay_ms,
|
||||
"book_ts": book_ts, "book_lag_ms": book_ts - target,
|
||||
"notional": notional, "base_amt": round(base_amt, 8),
|
||||
"baseline_px": baseline, "mid": mid,
|
||||
"best_px": best_px, "fill_px": "",
|
||||
"filled": round(float(r.result_volume), 8),
|
||||
"depth_ok": 0, "slip_bp": "", "drift_bp": "",
|
||||
"spread_bp": "", "impact_bp": ""})
|
||||
continue
|
||||
slip = d_sign * (fill - baseline) / baseline * 1e4
|
||||
drift = d_sign * (mid - baseline) / baseline * 1e4
|
||||
@@ -336,14 +710,21 @@ class Shadow:
|
||||
self.w_sig.writerow({
|
||||
"sym": sym, "kline_ts": kline_ts,
|
||||
"direction": d_sign, "h1_agree": hit["h1_agree"],
|
||||
"ladder_ok": hit["ladder_ok"], "gate_ok": hit["gate_ok"],
|
||||
"pass_all": hit["pass_all"], "lag_ok": int(lag_ok),
|
||||
"atr_pct": atr_pct if atr_pct else "",
|
||||
"atr_bp": round(atr_pct * 1e4, 3) if atr_pct else "",
|
||||
"t_close_ms": kline_ts, "t_data_ms": t_data,
|
||||
"t_signal_ms": t_signal,
|
||||
"lag_data_ms": t_data - kline_ts,
|
||||
"lag_signal_ms": t_signal - kline_ts,
|
||||
"delay_label": label, "delay_ms": delay_ms,
|
||||
"notional": notional, "baseline_px": baseline,
|
||||
"book_ts": book_ts, "book_lag_ms": book_ts - target,
|
||||
"notional": notional, "base_amt": round(base_amt, 8),
|
||||
"baseline_px": baseline,
|
||||
"mid": mid, "best_px": best_px, "fill_px": fill,
|
||||
"filled": round(filled, 2),
|
||||
"filled": round(float(r.result_volume), 8),
|
||||
"depth_ok": depth_ok,
|
||||
"slip_bp": round(slip, 4), "drift_bp": round(drift, 4),
|
||||
"spread_bp": round(spread, 4),
|
||||
"impact_bp": round(impact, 4)})
|
||||
@@ -353,8 +734,22 @@ class Shadow:
|
||||
while not self.stop.is_set():
|
||||
await asyncio.sleep(300)
|
||||
depth = {s: len(self.books.buf[s]) for s in SYMS}
|
||||
print(f" [心跳] 已处理 {self.n_bars} 根 · 命中 {self.n_signal} 个 "
|
||||
f"· 盘口缓冲 {depth}", flush=True)
|
||||
lag = {s: (f"{np.median(h):.0f}ms" if h else "na")
|
||||
+ ("" if self.lag_ok[s] else "!")
|
||||
for s, h in self.lag_hist.items()}
|
||||
print(f" [心跳] 已处理 {self.n_bars} 根 · 命中 {self.n_signal} 个"
|
||||
f"(过全部滤网 {self.n_pass}) · lag {lag} · 盘口缓冲 {depth}"
|
||||
f" · 回查超容差 {self.books.n_stale} 次"
|
||||
f" · 在途任务 {len(self._tasks)}"
|
||||
f" · 盘口落盘 {self.blog.n} 份"
|
||||
f" · 成交 {self.tape.n_trades} 笔{'' if self.tape.n_trades else ' ⚠监听未生效'}",
|
||||
flush=True)
|
||||
# 五分钟一根都没进来，说明管道断了。不喊一声就只能靠人翻日志
|
||||
if self.n_bars == self._hb_last_bars:
|
||||
print(f" ⚠ [停滞] 距上次心跳未处理任何 K 线"
|
||||
f"（进程池重建 {self.n_broken} 次），管道可能已断",
|
||||
flush=True)
|
||||
self._hb_last_bars = self.n_bars
|
||||
|
||||
async def run(self) -> None:
|
||||
await self.start()
|
||||
@@ -370,9 +765,12 @@ class Shadow:
|
||||
for f in list(self.feeds_l.values()) + list(self.feeds_h.values()):
|
||||
f.stop()
|
||||
await self.connector.stop_network()
|
||||
self.f_sig.close()
|
||||
self.f_lat.close()
|
||||
print(f"\n收工:{self.n_bars} 根 · {self.n_signal} 个信号", flush=True)
|
||||
for f in (self.f_sig, self.f_lat, self.f_drf):
|
||||
f.close()
|
||||
self.blog.close()
|
||||
self.tape.close()
|
||||
print(f"\n收工:{self.n_bars} 根 · {self.n_signal} 个信号"
|
||||
f"(过全部滤网 {self.n_pass})", flush=True)
|
||||
|
||||
|
||||
async def main_async(workers: int, hours: float, pool) -> None:
|
||||
|
||||
+179
-53
@@ -1,27 +1,44 @@
|
||||
"""影子交易器的报表:首日延迟门槛 + 滑点对延迟曲线。
|
||||
"""影子交易器的报表:延迟门槛 + 滑点对延迟曲线。
|
||||
|
||||
两份产物对应计划里的两件事。
|
||||
### 判据常数一律从研究侧 import，不在这里写死
|
||||
|
||||
### 延迟门槛(提前止损用)
|
||||
`BUDGET_BP` 等常数留在 `research/lib/shadow_budget.py`。理由是这些数会变——
|
||||
2026-08-27 一天之内预算就动了四次（3.91 → 11.06 → 14.25 → 15.19bp），
|
||||
费率也改了一次。本文件曾经写死过 BTC −0.13 / ETH 4.02 / SOL 2.92，那三个数
|
||||
由六处差异叠加而来（只有同向没有阶梯、费率按 6bp 双边 taker、TP=3.0、
|
||||
余量没除 taker 名义额、无 ATR 门控，且 BTC/ETH/SOL 恰是 ATR 最低的三个币）。
|
||||
正确值是 8.58 / 20.64 / 16.83——**ETH 差了五倍**。
|
||||
|
||||
跑满 24 小时先看这个。若**总延迟已令预期漂移超过余量**,说明方案在这台机器
|
||||
上就不成立,不必等两周样本再停。余量取 bitget_baseline.py 的实测值:
|
||||
BTC −0.13bp(本就为负,只作参照)、ETH +4.02bp、SOL +2.92bp。
|
||||
这件事要紧是因为下面的判读是自动停机开关：用 4.02 当 ETH 的预算，真实滑点
|
||||
只要到 2.4bp 就会报「需要压延迟或放弃」，会误杀一个可行的策略。
|
||||
|
||||
漂移按随机游走折算:σ_1m · √(t/60)。这是下限——入场条件是「收盘突破转强」,
|
||||
那一刻价格正朝我们方向跑,延迟造成的是系统性追价,不会正负抵消。所以实测
|
||||
滑点理应比这个折算值更差,两者对照本身就是个校验。
|
||||
### 什么时候能判什么
|
||||
|
||||
| | 一天的样本量 | 够不够 |
|
||||
|---|---|---|
|
||||
| 延迟 | 1440 根/币 | 够，统计上很厚 |
|
||||
| 滑点 | 门控后 4~6 笔/天 | **不够**，判据要 30 笔以上，即一周起步 |
|
||||
|
||||
所以首日只能判延迟和管道通不通。滑点那一节在样本不足时会明说。
|
||||
|
||||
### 延迟门槛（提前止损用）
|
||||
|
||||
若**总延迟已令预期漂移超过预算**，说明方案在这台机器上就不成立，不必等
|
||||
两周样本再停。漂移按随机游走折算 σ_1m · √(t/60)。这是下限——入场条件是
|
||||
「收盘突破转强」，那一刻价格正朝我们方向跑，延迟造成的是系统性追价，
|
||||
不会正负抵消。所以实测滑点理应比折算值更差，两者对照本身就是个校验。
|
||||
|
||||
### 滑点对延迟曲线
|
||||
|
||||
把延迟当自变量:0.5s / 1s / 2s / 5s 各一个滑点值,外加「actual」= 本机实际
|
||||
算完的时刻。这样即便本机算得慢,也能读出「若延迟压到 X 秒,滑点是多少」,
|
||||
算完的时刻。同信号内的受控对比，能直接读出「若延迟压到 X 秒，滑点是多少」,
|
||||
决策不被当前实现拖累。
|
||||
|
||||
.venv/bin/python research/live/shadow_report.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
@@ -29,8 +46,16 @@ import pandas as pd
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
OUT = HERE.parent / "out"
|
||||
sys.path.insert(0, str(HERE.parent))
|
||||
|
||||
from lib.shadow_budget import ( # noqa: E402
|
||||
ATR_GATE_BP, BUDGET_PORTFOLIO_2026, LAG_ALARM_MS, budget_of, lag_healthy,
|
||||
verdict,
|
||||
)
|
||||
|
||||
SYMS = ("BTC", "ETH", "SOL")
|
||||
BUDGET_BP = {"BTC": -0.13, "ETH": 4.02, "SOL": 2.92}
|
||||
ORDER = ["0.5s", "1.0s", "2.0s", "5.0s", "actual"]
|
||||
MIN_N = 30 # 滑点判据的最低笔数，低于此只报数不下结论
|
||||
|
||||
|
||||
def vol_bp() -> dict[str, float]:
|
||||
@@ -46,13 +71,38 @@ def vol_bp() -> dict[str, float]:
|
||||
return v
|
||||
|
||||
|
||||
def lag_health(lat: pd.DataFrame) -> None:
|
||||
"""运行时 lag 探针的回看。补丁后实测 506~642ms，理论下限约 500ms。"""
|
||||
print("\n\n########## 二、lag 探针（>%.0fms 该停开仓）##########"
|
||||
% LAG_ALARM_MS)
|
||||
print(f"{'币':<5}{'根数':>6}{'中位ms':>9}{'P90ms':>8}{'最差30根中位':>14}"
|
||||
f"{'超阈根数':>10}{'判定':>8}")
|
||||
for s in SYMS:
|
||||
g = lat[lat["sym"] == s].sort_values("kline_ts")
|
||||
if g.empty:
|
||||
continue
|
||||
x = g["lag_data_ms"].to_numpy(float)
|
||||
roll = pd.Series(x).rolling(30).median()
|
||||
worst = float(np.nanmax(roll)) if roll.notna().any() else float("nan")
|
||||
ok = lag_healthy(x)
|
||||
print(f"{s:<5}{len(g):>6}{np.median(x):>9.0f}"
|
||||
f"{np.percentile(x, 90):>8.0f}{worst:>14.0f}"
|
||||
f"{int((x > LAG_ALARM_MS).sum()):>10}"
|
||||
f"{'健康' if ok else '退化':>8}")
|
||||
if "lag_ok" in lat.columns:
|
||||
bad = int((lat["lag_ok"] == 0).sum())
|
||||
if bad:
|
||||
print(f"\n 采集期间有 {bad} 根被判不健康，那些根上的信号"
|
||||
f"（lag_ok=0）在真实运行下不会开仓，统计时应排除")
|
||||
|
||||
|
||||
def latency_gate(lat: pd.DataFrame, vols: dict) -> None:
|
||||
print("########## 一、延迟门槛 ##########")
|
||||
span_h = (lat["t_close_ms"].max() - lat["t_close_ms"].min()) / 3.6e6
|
||||
print(f"样本跨度 {span_h:.1f} 小时 · 共 {len(lat)} 根\n")
|
||||
print(f"{'币':<5}{'根数':>6}{'数据ms':>9}{'计算ms':>9}{'总延迟ms':>10}"
|
||||
f"{'P90ms':>8}{'折算漂移bp':>12}{'余量bp':>9}{'占余量':>9}")
|
||||
verdicts = {}
|
||||
f"{'P90ms':>8}{'折算漂移bp':>12}{'预算bp':>9}{'占预算':>9}")
|
||||
rows = {}
|
||||
for s in SYMS:
|
||||
g = lat[lat["sym"] == s]
|
||||
if g.empty:
|
||||
@@ -63,48 +113,110 @@ def latency_gate(lat: pd.DataFrame, vols: dict) -> None:
|
||||
p90 = float(np.percentile(g["lag_signal_ms"], 90))
|
||||
vol = vols.get(s)
|
||||
drift = vol * np.sqrt(t / 60_000) if vol else float("nan")
|
||||
b = BUDGET_BP[s]
|
||||
share = drift / b if b > 0 else float("nan")
|
||||
verdicts[s] = (drift, b)
|
||||
txt = f"{share * 100:.0f}%" if b > 0 else "—(负)"
|
||||
b = budget_of(s)
|
||||
share = drift / b if np.isfinite(b) and b > 0 else float("nan")
|
||||
rows[s] = (drift, b)
|
||||
txt = f"{share * 100:.0f}%" if np.isfinite(share) else "—"
|
||||
print(f"{s:<5}{len(g):>6}{d:>9.0f}{c:>9.0f}{t:>10.0f}{p90:>8.0f}"
|
||||
f"{drift:>12.2f}{b:>9.2f}{txt:>9}")
|
||||
|
||||
print("\n判读：")
|
||||
for s, (drift, b) in verdicts.items():
|
||||
if b <= 0:
|
||||
print(f" {s}: 余量本就为负，不作交易标的，仅作延迟参照")
|
||||
print("\n判读（预算来自 lib/shadow_budget，2026 年口径）：")
|
||||
for s, (drift, b) in rows.items():
|
||||
if not np.isfinite(b):
|
||||
print(f" {s}: 当前环境预算不足，不作交易标的，仅作延迟参照")
|
||||
elif not np.isfinite(drift):
|
||||
# 不特判的话 nan > b 是 False，会一路落到「尚有空间」说反话
|
||||
print(f" {s}: 缺 1m 波动率缓存，折算不出漂移，无法判读")
|
||||
elif drift > b:
|
||||
print(f" {s}: 折算漂移 {drift:.2f}bp 已超余量 {b:.2f}bp —— 停下改方案")
|
||||
print(f" {s}: 折算漂移 {drift:.2f}bp 已超预算 {b:.2f}bp —— 停下改方案")
|
||||
elif drift > b * 0.6:
|
||||
print(f" {s}: 折算漂移 {drift:.2f}bp 吃掉余量 {b:.2f}bp 的六成以上，"
|
||||
print(f" {s}: 折算漂移 {drift:.2f}bp 吃掉预算 {b:.2f}bp 的六成以上，"
|
||||
f"需要压延迟或放弃")
|
||||
else:
|
||||
print(f" {s}: 折算漂移 {drift:.2f}bp 对余量 {b:.2f}bp 尚有空间，继续收集")
|
||||
print(f" {s}: 折算漂移 {drift:.2f}bp 对预算 {b:.2f}bp 尚有空间，继续收集")
|
||||
|
||||
|
||||
def book_quality(sig: pd.DataFrame, drf: pd.DataFrame) -> None:
|
||||
"""回查到的盘口比目标时刻晚多少。晚太多的已在采集侧丢弃，这里做复核。"""
|
||||
frames = [d for d in (sig, drf) if not d.empty and "book_lag_ms" in d]
|
||||
if not frames:
|
||||
return
|
||||
x = pd.concat([d["book_lag_ms"] for d in frames]).astype(float)
|
||||
print("\n\n########## 二·五、盘口回查质量 ##########")
|
||||
print(f" 回查 {len(x)} 次 · 中位 {x.median():.0f}ms · "
|
||||
f"P90 {np.percentile(x, 90):.0f}ms · 最大 {x.max():.0f}ms")
|
||||
print(f" 10Hz 采样下这个值应在 0~100ms。它是所有延迟点的同向偏置，"
|
||||
f"不改变曲线形状，但要确认没有异常长尾")
|
||||
|
||||
|
||||
def drift_split(sig: pd.DataFrame, drf: pd.DataFrame) -> None:
|
||||
"""条件漂移 vs 无条件漂移。两者的差就是「系统性追价」的大小。"""
|
||||
print("\n\n########## 三、条件漂移 vs 无条件漂移 ##########")
|
||||
if drf.empty:
|
||||
print(" 尚无逐根漂移数据（shadow_drift.csv 由本轮起才开始记）")
|
||||
return
|
||||
print(" 无条件 = 每根 K 线，方向未知故取 |漂移|；"
|
||||
"条件 = 信号根按下单方向定号")
|
||||
print(f"\n{'延迟':<8}{'无条件n':>9}{'无条件|漂移|':>14}"
|
||||
f"{'条件n':>7}{'条件漂移':>10}{'追价差':>9}")
|
||||
cond = sig[(sig["notional"] == sig["notional"].min())] if not sig.empty \
|
||||
else sig
|
||||
for lb in ORDER:
|
||||
u = drf[drf["delay_label"] == lb]["drift_bp_long"].abs()
|
||||
c = cond[cond["delay_label"] == lb]["drift_bp"] if not cond.empty \
|
||||
else pd.Series(dtype=float)
|
||||
if u.empty and c.empty:
|
||||
continue
|
||||
um = u.mean() if not u.empty else float("nan")
|
||||
cm = c.mean() if not c.empty else float("nan")
|
||||
print(f"{lb:<8}{len(u):>9}{um:>14.2f}{len(c):>7}{cm:>10.2f}"
|
||||
f"{cm - um:>9.2f}")
|
||||
if len(cond) and len(cond[cond["delay_label"] == "1.0s"]) < MIN_N:
|
||||
print(f"\n 条件侧样本不足 {MIN_N}，差值还读不出方向")
|
||||
|
||||
|
||||
def slippage_curve(sig: pd.DataFrame) -> None:
|
||||
print("\n\n########## 二、滑点对延迟曲线 ##########")
|
||||
print("\n\n########## 四、滑点对延迟曲线 ##########")
|
||||
if sig.empty:
|
||||
print(" 尚无信号样本")
|
||||
return
|
||||
n_sig = sig.groupby(["sym", "kline_ts", "direction"]).ngroups
|
||||
n_agree = sig[sig["h1_agree"] == 1].groupby(
|
||||
["sym", "kline_ts", "direction"]).ngroups
|
||||
print(f"信号总数 {n_sig}（其中 h1_agree=1 的 {n_agree} 个）\n")
|
||||
|
||||
order = ["0.5s", "1.0s", "2.0s", "5.0s", "actual"]
|
||||
for scope, sub in (("全部信号", sig),
|
||||
("仅 h1_agree=1（主口径）", sig[sig["h1_agree"] == 1])):
|
||||
def n_of(df):
|
||||
return df.groupby(["sym", "kline_ts", "direction"]).ngroups
|
||||
|
||||
has_flags = "pass_all" in sig.columns
|
||||
if not has_flags:
|
||||
print(" ⚠ 数据来自旧版采集（只有 h1_agree，无阶梯与 ATR 门控）。"
|
||||
"这批不是我们要交易的那批信号，只能作管道验证，不能对预算判读。\n")
|
||||
main_scope, main_name = sig[sig["h1_agree"] == 1], "仅 h1_agree=1（旧口径）"
|
||||
else:
|
||||
# depth_ok=0 是 25 档吃不满该仓位，均价按部分成交算会**低估**冲击
|
||||
ok = (sig["pass_all"] == 1) & (sig["lag_ok"] == 1)
|
||||
if "depth_ok" in sig.columns:
|
||||
thin = int((sig["depth_ok"] == 0).sum())
|
||||
ok &= sig["depth_ok"] == 1
|
||||
if thin:
|
||||
print(f" （{thin} 行深度吃不满，已排除；这些行会低估冲击）")
|
||||
print(f"信号总数 {n_of(sig)} · 同向 {n_of(sig[sig['h1_agree'] == 1])}"
|
||||
f" · 同向+阶梯 "
|
||||
f"{n_of(sig[(sig['h1_agree'] == 1) & (sig['ladder_ok'] == 1)])}"
|
||||
f" · 三项全过 {n_of(sig[sig['pass_all'] == 1])}"
|
||||
f" · 再要求 lag 健康 {n_of(sig[ok])}")
|
||||
print(f"（门控阈值 ATR ≥ {ATR_GATE_BP:.0f}bp，是费率的函数不是市场常数）\n")
|
||||
main_scope = sig[ok]
|
||||
main_name = "三项滤网全过 + lag 健康 + 深度吃满（主口径）"
|
||||
|
||||
scopes = [("全部信号（含不会下单的，仅作提前读数）", sig),
|
||||
(main_name, main_scope)]
|
||||
for scope, sub in scopes:
|
||||
if sub.empty:
|
||||
print(f"--- {scope} ---\n 尚无样本\n")
|
||||
continue
|
||||
print(f"--- {scope} ---")
|
||||
print(f"{'延迟':<8}{'仓位':>9}{'n':>5}{'滑点均值bp':>12}"
|
||||
f"{'中位bp':>9}{'漂移bp':>9}{'价差bp':>9}{'冲击bp':>9}")
|
||||
for lb in order:
|
||||
for lb in ORDER:
|
||||
g0 = sub[sub["delay_label"] == lb]
|
||||
if g0.empty:
|
||||
continue
|
||||
for nt in sorted(sub["notional"].unique()):
|
||||
g = g0[g0["notional"] == nt]
|
||||
if g.empty:
|
||||
@@ -117,19 +229,36 @@ def slippage_curve(sig: pd.DataFrame) -> None:
|
||||
f"{g['impact_bp'].mean():>9.2f}")
|
||||
print()
|
||||
|
||||
print("--- 分币种（仅 h1_agree=1，仓位 5000）---")
|
||||
m = sig[(sig["h1_agree"] == 1) & (sig["notional"] == 5000.0)]
|
||||
print("--- 分币种判读（主口径，仓位 5000）---")
|
||||
m = main_scope[main_scope["notional"] == 5000.0] if not main_scope.empty \
|
||||
else main_scope
|
||||
if m.empty:
|
||||
print(" 尚无样本")
|
||||
return
|
||||
print(f"{'币':<5}{'延迟':<8}{'n':>5}{'滑点均值bp':>12}{'余量bp':>9}")
|
||||
print(f"{'币':<5}{'延迟':<8}{'n':>5}{'滑点中位bp':>12}{'预算bp':>9} 判读")
|
||||
for s in SYMS:
|
||||
for lb in order:
|
||||
for lb in ORDER:
|
||||
g = m[(m["sym"] == s) & (m["delay_label"] == lb)]
|
||||
if g.empty:
|
||||
continue
|
||||
print(f"{s:<5}{lb:<8}{len(g):>5}{g['slip_bp'].mean():>12.2f}"
|
||||
f"{BUDGET_BP[s]:>9.2f}")
|
||||
med = float(g["slip_bp"].median())
|
||||
b = budget_of(s)
|
||||
note = verdict(med, s) if len(g) >= MIN_N \
|
||||
else f"n={len(g)} < {MIN_N}，不下结论"
|
||||
print(f"{s:<5}{lb:<8}{len(g):>5}{med:>12.2f}{b:>9.2f} {note}")
|
||||
|
||||
n_main = len(m[m["delay_label"] == "actual"])
|
||||
if n_main < MIN_N:
|
||||
print(f"\n ⚠ 主口径仅 {n_main} 笔。门控后约 4~6 笔/天/全部币种，"
|
||||
f"滑点判据要 {MIN_N} 笔以上——**一周起步**。首日只能判延迟和管道。")
|
||||
print(f" 单币样本薄时可先看组合口径：2026 预算 {BUDGET_PORTFOLIO_2026}bp")
|
||||
|
||||
|
||||
def _load(name: str) -> pd.DataFrame:
|
||||
f = OUT / name
|
||||
if not f.exists() or f.stat().st_size == 0:
|
||||
return pd.DataFrame()
|
||||
return pd.read_csv(f)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
@@ -137,20 +266,17 @@ def main() -> None:
|
||||
print("1m 收益标准差（bp/分钟，Bitget 实测）："
|
||||
+ " ".join(f"{s} {v:.2f}" for s, v in vols.items()) + "\n")
|
||||
|
||||
f_lat = OUT / "shadow_latency.csv"
|
||||
if f_lat.exists() and f_lat.stat().st_size > 0:
|
||||
lat = pd.read_csv(f_lat)
|
||||
if not lat.empty:
|
||||
latency_gate(lat, vols)
|
||||
else:
|
||||
lat = _load("shadow_latency.csv")
|
||||
if lat.empty:
|
||||
print("尚无延迟数据")
|
||||
|
||||
f_sig = OUT / "shadow_signals.csv"
|
||||
if f_sig.exists() and f_sig.stat().st_size > 0:
|
||||
sig = pd.read_csv(f_sig)
|
||||
slippage_curve(sig)
|
||||
else:
|
||||
print("\n尚无信号数据")
|
||||
latency_gate(lat, vols)
|
||||
lag_health(lat)
|
||||
|
||||
sig, drf = _load("shadow_signals.csv"), _load("shadow_drift.csv")
|
||||
book_quality(sig, drf)
|
||||
drift_split(sig, drf)
|
||||
slippage_curve(sig)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -3,15 +3,30 @@
|
||||
单次调用约 0.26s 的纯 CPU，且 chanlun 是纯 Python 受 GIL 限制，放进
|
||||
Hummingbot 的 asyncio 循环里会把行情处理一起卡住，所以必须隔离到独立进程。
|
||||
|
||||
口径与 [aggregate_robustness.run_once] 逐行对齐：同样的 build_htf_zones →
|
||||
find_fast_bsp3 → attach_htf_context(h1) 链路，同样的 h1_agree 过滤。两边
|
||||
必须一致，否则影子测出来的滑点没法和 3.9bp 预算对照。
|
||||
## 口径必须与预算同源，缺一项数就不可比
|
||||
|
||||
与回测的唯一差别是这里只关心**最后一根已收盘 K 线**上有没有信号——
|
||||
实盘只能在当下下单，历史信号无意义。
|
||||
预算（`lib/shadow_budget.BUDGET_BP`）算在 step42 的这套滤网上，
|
||||
本文件逐行对齐 `step42_exit_tp_1m.run_one`：
|
||||
|
||||
未过滤信号也一并返回：过滤后样本太稀，先用未过滤的当提前读数，
|
||||
两者都记，靠 h1_agree 字段区分。
|
||||
同向 h1_agree == 1
|
||||
中枢阶梯 多头要求当前中枢整体高于前一个（zd > 前 zg），空头反之
|
||||
ATR 门控 atr_pct ≥ ATR_GATE_BP（当前 8bp）
|
||||
|
||||
早先这里只有 h1_agree。缺阶梯与门控测的就不是我们要交易的那批信号，
|
||||
而这一项改常数解决不了——必须改信号路径本身。
|
||||
|
||||
门控阈值是**费率的函数**不是市场常数（低 ATR 信号的毛质量反而最好，
|
||||
断崖只在扣费后出现），换 VIP 档或换交易所要重扫，不要抄 8bp。
|
||||
|
||||
## 与回测的两点差别
|
||||
|
||||
其一，这里只关心**最后一根已收盘 K 线**上有没有信号——实盘只能在当下下单。
|
||||
其二，`atr_pct` 的分母取次根开盘价，与 `exit_model.walk_exits` 一致，
|
||||
所以调用方必须把次根开盘价传进来。
|
||||
|
||||
未通过滤网的信号也一并返回并打上标志：过滤后样本很稀（门控后 8 个币
|
||||
合计约 38 笔/周），未过滤的可作提前读数。但**统计主口径只能用 pass_all**，
|
||||
在我们根本不会下单的根上测滑点会把判据算宽。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -23,15 +38,20 @@ for _v in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS"):
|
||||
os.environ.setdefault(_v, "1")
|
||||
|
||||
|
||||
def compute(df_l, df_h) -> dict:
|
||||
def compute(df_l, df_h, entry_px: float | None = None) -> dict:
|
||||
"""在 df_l 的最后一根上找信号。df_l/df_h 都只含已收盘 K 线。
|
||||
|
||||
entry_px 是次根开盘价（回测 entry_delay=1 的成交价），用作 atr_pct 的
|
||||
分母。取不到时退回用信号根收盘价，并在返回里标 atr_ref="close"。
|
||||
|
||||
返回 dict：
|
||||
last_idx 最后一根在 chanlun 处理后 dataframe 里的下标
|
||||
n_bars 实际参与计算的根数
|
||||
hits 命中列表，每项 {direction, h1_agree}
|
||||
atr_pct 信号根 ATR / 次根开盘价
|
||||
hits 命中列表，每项含方向与三个滤网标志、pass_all
|
||||
error 出错时的说明，正常为 None
|
||||
"""
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
try:
|
||||
@@ -40,45 +60,76 @@ def compute(df_l, df_h) -> dict:
|
||||
from lib.fx_signal import extract_fx_signals, signals_to_frame
|
||||
from lib.nested_bsp import attach_htf_context, htf_fx_timeline
|
||||
from lib.nested_level import build_htf_zones
|
||||
from lib.shadow_budget import ATR_GATE_BP
|
||||
|
||||
chan_l = TF_DF(df_l, 1, "1m")
|
||||
cdf = chan_l.dataframe
|
||||
last = len(cdf) - 1
|
||||
base = {"last_idx": last, "n_bars": int(len(df_l)), "hits": [],
|
||||
"error": None}
|
||||
"atr_pct": None, "atr_ref": None, "error": None}
|
||||
|
||||
zones = build_htf_zones(cdf, "1m", chan=chan_l)
|
||||
# ATR 门控。分母与 exit_model.walk_exits 一致，取次根开盘价。
|
||||
# 放在任何早退之前——无信号的根也要记，才能在线看到门控的真实刷除率
|
||||
atr = float(cdf["atr"].to_numpy(dtype=float)[last]) \
|
||||
if "atr" in cdf.columns else float("nan")
|
||||
ref = entry_px if (entry_px and np.isfinite(entry_px)) else \
|
||||
float(cdf["close"].to_numpy(dtype=float)[last])
|
||||
atr_pct = atr / ref if (np.isfinite(atr) and ref) else float("nan")
|
||||
gate_ok = bool(np.isfinite(atr_pct) and atr_pct * 1e4 >= ATR_GATE_BP)
|
||||
base["atr_pct"] = None if not np.isfinite(atr_pct) else float(atr_pct)
|
||||
base["atr_ref"] = "next_open" if (entry_px and np.isfinite(entry_px)) \
|
||||
else "close"
|
||||
|
||||
zones = build_htf_zones(cdf, "1m", chan=chan_l).reset_index(drop=True)
|
||||
if zones.empty:
|
||||
return base
|
||||
sig = find_fast_bsp3(cdf, zones.reset_index(drop=True))
|
||||
|
||||
# 中枢阶梯：当前中枢是否整体脱离前一个。与 step42 同一算法
|
||||
z = zones.copy()
|
||||
prev_zg, prev_zd = z["zg"].shift(), z["zd"].shift()
|
||||
z["z_above"], z["z_below"] = z["zd"] > prev_zg, z["zg"] < prev_zd
|
||||
z["zone_i"] = np.arange(len(z))
|
||||
|
||||
sig = find_fast_bsp3(cdf, zones)
|
||||
if sig is None or sig.empty:
|
||||
return base
|
||||
sig = sig.merge(z[["zone_i", "z_above", "z_below"]],
|
||||
on="zone_i", how="left")
|
||||
|
||||
# 只留落在最后一根上的信号，其余是历史，实盘下不了
|
||||
cur = sig[sig["entry_idx"].astype(int) == last]
|
||||
if cur.empty:
|
||||
return base
|
||||
|
||||
# 5m 同向过滤：算得出就标 h1_agree，算不出就当未过滤照记
|
||||
agree_map: dict[int, int] = {}
|
||||
# 5m 同向。算不出时 h1_agree 记 0，该信号自然不会通过 pass_all
|
||||
if df_h is not None and len(df_h) > 0:
|
||||
chan_h = TF_DF(df_h, 1, "5m")
|
||||
hdf = chan_h.dataframe
|
||||
tl = htf_fx_timeline(
|
||||
signals_to_frame(extract_fx_signals(chan_h, hdf)), hdf)
|
||||
full = attach_htf_context(sig, cdf, tl, "h1")
|
||||
f_cur = full[full["entry_idx"].astype(int) == last]
|
||||
for _, r in f_cur.iterrows():
|
||||
agree_map[int(r["direction"])] = int(r.get("h1_agree", 0))
|
||||
sig = attach_htf_context(sig, cdf, tl, "h1")
|
||||
else:
|
||||
sig["h1_agree"] = 0
|
||||
|
||||
base["hits"] = [{"direction": int(r["direction"]),
|
||||
"h1_agree": agree_map.get(int(r["direction"]), 0)}
|
||||
for _, r in cur.iterrows()]
|
||||
cur = sig[sig["entry_idx"].astype(int) == last]
|
||||
if cur.empty:
|
||||
return base
|
||||
|
||||
hits = []
|
||||
for _, r in cur.iterrows():
|
||||
d = int(r["direction"])
|
||||
push = r["z_above"] if d == 1 else r["z_below"]
|
||||
ladder_ok = bool(pd.notna(push) and bool(push))
|
||||
# attach_htf_context 在入场时刻之前没有大级别分型时写 NaN。
|
||||
# 不能写成 `int(x or 0)`——NaN 是真值，会走到 int(nan) 抛异常，
|
||||
# 整根的信号就此丢掉，只留一行报错
|
||||
raw = r.get("h1_agree", 0)
|
||||
agree = int(raw) if pd.notna(raw) else 0
|
||||
hits.append({"direction": d, "h1_agree": agree,
|
||||
"ladder_ok": int(ladder_ok), "gate_ok": int(gate_ok),
|
||||
"pass_all": int(agree == 1 and ladder_ok and gate_ok)})
|
||||
base["hits"] = hits
|
||||
return base
|
||||
except Exception as e: # 子进程里异常必须带回主进程，否则只见超时不见原因
|
||||
import traceback
|
||||
return {"last_idx": -1, "n_bars": int(len(df_l)) if df_l is not None else 0,
|
||||
"hits": [], "error": f"{type(e).__name__}: {e}",
|
||||
"hits": [], "atr_pct": None, "atr_ref": None,
|
||||
"error": f"{type(e).__name__}: {e}",
|
||||
"traceback": traceback.format_exc()}
|
||||
|
||||
|
||||
@@ -102,8 +153,8 @@ def _rebuild(rows) -> "object":
|
||||
|
||||
|
||||
def compute_packed(payload: tuple) -> dict:
|
||||
"""ProcessPoolExecutor 的入口：收 (l_rows, h_rows) 两组数值行。"""
|
||||
l_rows, h_rows = payload
|
||||
"""ProcessPoolExecutor 的入口：收 (l_rows, h_rows, entry_px)。"""
|
||||
l_rows, h_rows, entry_px = payload
|
||||
df_l = _rebuild(l_rows)
|
||||
df_h = _rebuild(h_rows) if h_rows else None
|
||||
return compute(df_l, df_h)
|
||||
return compute(df_l, df_h, entry_px)
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
"""验证 live 信号路径与 step42 的批量过滤等价。
|
||||
|
||||
live 侧每根只看最后一根、且只喂 2000 根窗口；研究侧一次性跑全量。两者
|
||||
用同一套滤网（同向 + 中枢阶梯 + ATR 门控），但**不保证逐笔一致**——
|
||||
缠论结构依赖历史，2000 根窗口是 step39 定的命中率饱和点，不是无损截断。
|
||||
|
||||
每个信号根上比三种口径：
|
||||
|
||||
批量 全量历史 + 完整 5m。这是预算的来源，是基准
|
||||
剔partial 窗口 2000 根 1m + 800 根**已收盘** 5m
|
||||
含partial 窗口，5m 末尾保留那根**尚未收盘**的。这是 live 现行做法
|
||||
|
||||
## 结论：partial 根要保留，不能剔
|
||||
|
||||
live 的 `df_h[df_h["timestamp"] < kline_ts]` 里 kline_ts 是 1m 的收盘时刻，
|
||||
而正在走的那根 5m 开盘更早，于是被保留下来——五根里有四根如此。乍看像是
|
||||
「把未收盘的根当完整根用」的口径错误，实测**反过来**：
|
||||
|
||||
BTC 25/25、ETH 24/25、SOL 23/25 与批量一致（合计 96%）
|
||||
剔掉则只有 21/25、21/25、18/25（合计 80%）
|
||||
|
||||
原因是批量口径里那根 5m 是存在的。缠论的包含处理与分型检测吃整条序列，
|
||||
凭空少一根会把结构整体挪位；保留一根「开盘价正确、高低点尚不完整」的
|
||||
近似根，比直接删掉更接近批量。
|
||||
|
||||
这也暴露了研究侧的一处残留：批量的 HTF 结构用到了那根 5m 的**最终**高低点，
|
||||
而实盘在该时刻不可能知道。`htf_fx_timeline` 的 `confirm_ts += period` 只挡住了
|
||||
分型**选取**上的未来函数，挡不住结构构建。live 用 partial 根逼近，落在 96%，
|
||||
差的那 4% 是这条残留的下界，不是可以修掉的 bug。
|
||||
|
||||
.venv/bin/python research/live/verify_signal_path.py --symbol BTC
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
sys.path.insert(0, str(HERE.parent))
|
||||
sys.path.insert(0, str(HERE.parents[1]))
|
||||
|
||||
import numpy as np # noqa: E402
|
||||
import pandas as pd # noqa: E402
|
||||
|
||||
WINDOW_L, WINDOW_H = 2000, 800
|
||||
|
||||
|
||||
HTF_MS = 300_000
|
||||
|
||||
|
||||
def partial_htf_bar(df_l: pd.DataFrame, kline_ts: int) -> dict | None:
|
||||
"""用 1m 合成「此刻正在走的那根 5m」，复现 live 曾经喂进去的 partial 根。"""
|
||||
bucket = kline_ts // HTF_MS * HTF_MS
|
||||
if bucket >= kline_ts: # 正好落在 5m 边界，没有未收盘的根
|
||||
return None
|
||||
part = df_l[(df_l["timestamp"] >= bucket) & (df_l["timestamp"] < kline_ts)]
|
||||
if part.empty:
|
||||
return None
|
||||
date = pd.to_datetime(bucket, unit="ms", utc=True) \
|
||||
.tz_convert("Asia/Shanghai")
|
||||
return {"timestamp": bucket, "date": date,
|
||||
"open": float(part["open"].iloc[0]),
|
||||
"high": float(part["high"].max()), "low": float(part["low"].min()),
|
||||
"close": float(part["close"].iloc[-1]),
|
||||
"volume": float(part["volume"].sum())}
|
||||
|
||||
|
||||
def load(sym: str, tf: str, days: int) -> pd.DataFrame:
|
||||
f = HERE / "cache" / f"bitget_{sym}_{tf}_{days}d.feather"
|
||||
if not f.exists():
|
||||
raise SystemExit(f"缺数据 {f}")
|
||||
return pd.read_feather(f)
|
||||
|
||||
|
||||
def batch_flags(df_l: pd.DataFrame, df_h: pd.DataFrame) -> pd.DataFrame:
|
||||
"""step42_exit_tp_1m.run_one 的滤网，逐行照搬。"""
|
||||
from chanlun import TF_DF
|
||||
from lib.fast_bsp3 import find_fast_bsp3
|
||||
from lib.fx_signal import extract_fx_signals, signals_to_frame
|
||||
from lib.nested_bsp import attach_htf_context, htf_fx_timeline
|
||||
from lib.nested_level import build_htf_zones
|
||||
from lib.shadow_budget import ATR_GATE_BP
|
||||
|
||||
chan_l = TF_DF(df_l, 1, "1m")
|
||||
cdf = chan_l.dataframe
|
||||
zones = build_htf_zones(cdf, "1m", chan=chan_l).reset_index(drop=True)
|
||||
z = zones.copy()
|
||||
pg, pdn = z["zg"].shift(), z["zd"].shift()
|
||||
z["z_above"], z["z_below"] = z["zd"] > pg, z["zg"] < pdn
|
||||
z["zone_i"] = np.arange(len(z))
|
||||
|
||||
chan_h = TF_DF(df_h, 1, "5m")
|
||||
hdf = chan_h.dataframe
|
||||
tl = htf_fx_timeline(
|
||||
signals_to_frame(extract_fx_signals(chan_h, hdf)), hdf)
|
||||
|
||||
sig = find_fast_bsp3(cdf, zones)
|
||||
sig = sig.merge(z[["zone_i", "z_above", "z_below"]], on="zone_i", how="left")
|
||||
sig = attach_htf_context(sig, cdf, tl, "h1")
|
||||
|
||||
d = sig["direction"].astype(int)
|
||||
push = np.where(d == 1, sig["z_above"], sig["z_below"])
|
||||
idx = sig["entry_idx"].astype(int).to_numpy()
|
||||
entry = cdf["open"].to_numpy(float)[np.minimum(idx + 1, len(cdf) - 1)]
|
||||
atr_pct = cdf["atr"].to_numpy(float)[idx] / entry
|
||||
|
||||
out = pd.DataFrame({
|
||||
"entry_idx": idx,
|
||||
"ts": cdf["timestamp"].to_numpy()[idx],
|
||||
"direction": d.to_numpy(),
|
||||
"h1_agree": sig["h1_agree"].fillna(0).astype(int).to_numpy(),
|
||||
"ladder_ok": pd.Series(push).fillna(False).astype(int).to_numpy(),
|
||||
"atr_bp": atr_pct * 1e4,
|
||||
})
|
||||
out["gate_ok"] = (out["atr_bp"] >= ATR_GATE_BP).astype(int)
|
||||
out["pass_all"] = ((out["h1_agree"] == 1) & (out["ladder_ok"] == 1)
|
||||
& (out["gate_ok"] == 1)).astype(int)
|
||||
return out, cdf
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--symbol", default="BTC")
|
||||
ap.add_argument("--days", type=int, default=30)
|
||||
ap.add_argument("--checks", type=int, default=12)
|
||||
a = ap.parse_args()
|
||||
|
||||
df_l, df_h = load(a.symbol, "1m", a.days), load(a.symbol, "5m", a.days)
|
||||
print(f"[{a.symbol}] 1m {len(df_l)} 根 / 5m {len(df_h)} 根，跑批量滤网…",
|
||||
flush=True)
|
||||
batch, cdf = batch_flags(df_l, df_h)
|
||||
n = len(batch)
|
||||
print(f" 原始 B4/S4 {n} 个 · 同向 {int((batch.h1_agree == 1).sum())}"
|
||||
f" · 同向+阶梯 "
|
||||
f"{int(((batch.h1_agree == 1) & (batch.ladder_ok == 1)).sum())}"
|
||||
f" · 三项全过 {int(batch.pass_all.sum())}")
|
||||
print(f" ATR 中位 {batch.atr_bp.median():.2f}bp · "
|
||||
f"门控刷掉 {(1 - batch.gate_ok.mean()) * 100:.1f}%\n")
|
||||
|
||||
# 挑最近的若干个信号根做窗口复现
|
||||
from shadow_signal import compute
|
||||
cand = batch[batch["entry_idx"] >= WINDOW_L].tail(a.checks)
|
||||
if cand.empty:
|
||||
raise SystemExit("窗口内没有可核对的信号")
|
||||
|
||||
def run_window(r, with_partial: bool):
|
||||
i = int(r["entry_idx"])
|
||||
kline_ts = int(r["ts"]) + 60_000 # 信号根的收盘时刻
|
||||
wl = df_l[df_l["timestamp"] < kline_ts].tail(WINDOW_L)
|
||||
wh = df_h[df_h["timestamp"] + HTF_MS <= kline_ts].tail(WINDOW_H)
|
||||
if with_partial:
|
||||
p = partial_htf_bar(df_l, kline_ts)
|
||||
if p is not None:
|
||||
wh = pd.concat([wh, pd.DataFrame([p])], ignore_index=True)
|
||||
entry_px = float(cdf["open"].to_numpy(float)[min(i + 1, len(cdf) - 1)])
|
||||
res = compute(wl.reset_index(drop=True), wh.reset_index(drop=True),
|
||||
entry_px)
|
||||
if res.get("error"):
|
||||
raise SystemExit(f"compute 报错，测试本身有问题：{res['error']}\n"
|
||||
f"{res.get('traceback', '')}")
|
||||
return next((h for h in res["hits"]
|
||||
if h["direction"] == int(r["direction"])), None)
|
||||
|
||||
def fmt(h):
|
||||
if h is None:
|
||||
return f"{'未复现':>18}"
|
||||
return (f"{h['h1_agree']:>6}{h['ladder_ok']:>5}{h['gate_ok']:>5}")
|
||||
|
||||
print(f"{'K线时刻':<15}{'方向':>4}{' 批量':>18}{' 窗口':>18}"
|
||||
f"{' 含未收盘':>18}{' 截断':>7}{'partial':>9}")
|
||||
print(f"{'':<15}{'':>4}{'同向 阶梯 门控':>20}{'同向 阶梯 门控':>20}"
|
||||
f"{'同向 阶梯 门控':>20}")
|
||||
n_trunc = n_part = n_ok = n_bad = 0
|
||||
for _, r in cand.iterrows():
|
||||
h_ok = run_window(r, False)
|
||||
h_bad = run_window(r, True)
|
||||
t = pd.to_datetime(r["ts"], unit="ms", utc=True) \
|
||||
.tz_convert("Asia/Shanghai").strftime("%m-%d %H:%M")
|
||||
ref = (int(r.h1_agree), int(r.ladder_ok), int(r.gate_ok))
|
||||
got = None if h_ok is None else (h_ok["h1_agree"], h_ok["ladder_ok"],
|
||||
h_ok["gate_ok"])
|
||||
bad = None if h_bad is None else (h_bad["h1_agree"], h_bad["ladder_ok"],
|
||||
h_bad["gate_ok"])
|
||||
n_trunc += got != ref
|
||||
n_part += bad != got
|
||||
n_ok += got == ref
|
||||
n_bad += bad == ref
|
||||
print(f"{t:<15}{int(r.direction):>+4}"
|
||||
f"{ref[0]:>6}{ref[1]:>5}{ref[2]:>5}"
|
||||
f"{fmt(h_ok):>18}{fmt(h_bad):>18}"
|
||||
f"{'' if got == ref else '差':>7}"
|
||||
f"{'' if bad == got else '差':>9}")
|
||||
n = len(cand)
|
||||
print(f"\n 与批量（预算口径）一致：")
|
||||
print(f" 剔掉未收盘 5m 根 {n_ok}/{n}")
|
||||
print(f" 保留未收盘 5m 根 {n_bad}/{n} ← live 现行做法")
|
||||
print(f" 两种窗口口径互不相同 {n_part}/{n}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,28 @@
|
||||
sym,notional,budget_bp,budget_bp_old,maker_share,w_stop,w_time,net_bp,n
|
||||
BTC,5000.0,10.971497515124305,10.971497515124295,0.4230769230769231,0.3516483516483517,0.22527472527472528,15.724284543080632,91
|
||||
BTC,10000.0,10.971497515124305,10.971497515124295,0.4230769230769231,0.3516483516483517,0.22527472527472528,15.724284543080632,91
|
||||
BTC,20000.0,10.971497515124305,10.971497515124295,0.4230769230769231,0.3516483516483517,0.22527472527472528,15.724284543080632,91
|
||||
BTC,50000.0,10.971497515124305,10.971497515124295,0.4230769230769231,0.3516483516483517,0.22527472527472528,15.724284543080632,91
|
||||
BTC,100000.0,10.971497515124305,10.971497515124295,0.4230769230769231,0.3516483516483517,0.22527472527472528,15.724284543080632,91
|
||||
BTC,200000.0,10.971497515124305,10.971497515124295,0.4230769230769231,0.3516483516483517,0.22527472527472528,15.724284543080632,91
|
||||
BTC,320000.0,10.971497515124305,10.971497515124295,0.4230769230769231,0.3516483516483517,0.22527472527472528,15.724284543080632,91
|
||||
BTC,530000.0,10.908448857487189,10.971497515124295,0.4214204503001539,0.3524912405185926,0.22608830918125347,15.641274735676083,91
|
||||
BTC,1000000.0,10.690710190313572,10.971497515124295,0.41456048099090254,0.35467750033199696,0.23076201867710053,15.36403490298731,91
|
||||
ETH,5000.0,15.336810540229132,15.336810540229127,0.4722222222222222,0.3253968253968254,0.20238095238095238,21.90346054757228,126
|
||||
ETH,10000.0,15.336810540229132,15.336810540229127,0.4722222222222222,0.3253968253968254,0.20238095238095238,21.90346054757228,126
|
||||
ETH,20000.0,15.336810540229132,15.336810540229127,0.4722222222222222,0.3253968253968254,0.20238095238095238,21.90346054757228,126
|
||||
ETH,50000.0,15.336810540229132,15.336810540229127,0.4722222222222222,0.3253968253968254,0.20238095238095238,21.90346054757228,126
|
||||
ETH,100000.0,15.336810540229132,15.336810540229127,0.4722222222222222,0.3253968253968254,0.20238095238095238,21.90346054757228,126
|
||||
ETH,200000.0,15.336810540229132,15.336810540229127,0.4722222222222222,0.3253968253968254,0.20238095238095238,21.90346054757228,126
|
||||
ETH,320000.0,15.336810540229132,15.336810540229127,0.4722222222222222,0.3253968253968254,0.20238095238095238,21.90346054757228,126
|
||||
ETH,530000.0,15.336810540229132,15.336810540229127,0.4722222222222222,0.3253968253968254,0.20238095238095238,21.90346054757228,126
|
||||
ETH,1000000.0,15.19839735083419,15.336810540229127,0.46778822061185865,0.3266775971833763,0.20553418220476508,21.75495166938152,126
|
||||
SOL,5000.0,17.20729581305582,17.207295813055815,0.49310344827586206,0.2655172413793103,0.2413793103448276,24.42271817346687,145
|
||||
SOL,10000.0,17.20729581305582,17.207295813055815,0.49310344827586206,0.2655172413793103,0.2413793103448276,24.42271817346687,145
|
||||
SOL,20000.0,17.20729581305582,17.207295813055815,0.49310344827586206,0.2655172413793103,0.2413793103448276,24.42271817346687,145
|
||||
SOL,50000.0,17.12169148018871,17.207295813055815,0.49112524587410755,0.26723407222095347,0.24164068190493895,24.325613268263233,145
|
||||
SOL,100000.0,17.024617862994077,17.207295813055815,0.4874068783573466,0.2680997947311664,0.24449332691148706,24.238726756516837,145
|
||||
SOL,200000.0,16.83524812354264,17.207295813055815,0.4813611824188418,0.26935037627246794,0.24928844130869035,24.048022486441045,145
|
||||
SOL,320000.0,16.524586307202075,17.207295813055815,0.4732031722818229,0.27277272324336915,0.254024104474808,23.702889125473174,145
|
||||
SOL,530000.0,16.275686387067427,17.207295813055815,0.4660589094844057,0.2753631016069268,0.2585779889086674,23.432003034952427,145
|
||||
SOL,1000000.0,15.829170691791866,17.207295813055815,0.4547125585779213,0.27881565544002057,0.2664717859820582,22.915331236730324,145
|
||||
|
@@ -0,0 +1,161 @@
|
||||
"""Step 43：把「限价单全额成交」的假设换成按成交量结算，重算滑点预算。
|
||||
|
||||
step42 的预算（BTC 8.58 / ETH 20.64 / SOL 16.83bp）建立在一个假设上：挂在
|
||||
3ATR 与 8ATR 的止盈限价单全额成交在目标价。影子交易的成交流数据推翻了它——
|
||||
真实仓位下全额成交率只有 30%/16%/1.5%（32 万仓位）。
|
||||
|
||||
预算因此不再是一个常数，而是**仓位规模的函数**。规模越大，止盈越难成交，
|
||||
越多仓位被拖到止损或超时（taker，且吃滑点），预算越低。这条曲线与「冲击
|
||||
反推的容量」是两个不同的约束，而后者宽松得多（100~500 万 vs 数万）。
|
||||
|
||||
数据用 Bitget 210 天 1m，与影子测量同源同交易所。全量 366 万根峰值 24.5GB，
|
||||
本机 15GB 跑不动；210 天 30 万根峰值约 2GB。
|
||||
|
||||
python research/step43_fill_aware_budget.py --syms BTC,ETH,SOL
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
for v in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS"):
|
||||
os.environ.setdefault(v, "1")
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
sys.path.insert(0, str(HERE.parent))
|
||||
pd.set_option("display.width", 400)
|
||||
|
||||
LTF, HTF = "1m", "5m"
|
||||
SL, SCALE_AT, RUNNER, RUNNER_STOP, MAXB = 2.0, 3.0, 8.0, 2.0, 48
|
||||
NOTIONALS = [5e3, 1e4, 2e4, 5e4, 1e5, 2e5, 3.2e5, 5.3e5, 1e6]
|
||||
|
||||
|
||||
def signals_for(sym: str, cache: Path) -> tuple[pd.DataFrame, pd.DataFrame]:
|
||||
"""跑缠论链路，返回 (cdf, 过完三滤网的信号)。口径抄 step42.run_one。"""
|
||||
from chanlun import TF_DF
|
||||
from lib.fast_bsp3 import find_fast_bsp3
|
||||
from lib.fx_signal import extract_fx_signals, signals_to_frame
|
||||
from lib.nested_bsp import attach_htf_context, htf_fx_timeline
|
||||
from lib.nested_level import build_htf_zones
|
||||
from lib.shadow_budget import ATR_GATE_BP
|
||||
|
||||
def load(tf: str) -> pd.DataFrame:
|
||||
c = sorted(cache.glob(f"bitget_{sym}_{tf}_*.feather"),
|
||||
key=lambda p: p.stat().st_size, reverse=True)
|
||||
if not c:
|
||||
raise FileNotFoundError(f"没有 {sym} {tf} 缓存")
|
||||
return pd.read_feather(c[0])
|
||||
|
||||
chan_l = TF_DF(load(LTF), 1, LTF)
|
||||
cdf = chan_l.dataframe
|
||||
zones = build_htf_zones(cdf, LTF, chan=chan_l).reset_index(drop=True)
|
||||
if zones.empty:
|
||||
raise RuntimeError("无中枢")
|
||||
z = zones.copy()
|
||||
pg, pdn = z["zg"].shift(), z["zd"].shift()
|
||||
z["z_above"], z["z_below"] = z["zd"] > pg, z["zg"] < pdn
|
||||
z["zone_i"] = np.arange(len(z))
|
||||
|
||||
chan_h = TF_DF(load(HTF), 1, HTF)
|
||||
tl = htf_fx_timeline(
|
||||
signals_to_frame(extract_fx_signals(chan_h, chan_h.dataframe)),
|
||||
chan_h.dataframe)
|
||||
del chan_h
|
||||
|
||||
sig = find_fast_bsp3(cdf, zones)
|
||||
if sig.empty:
|
||||
raise RuntimeError("无信号")
|
||||
sig = sig.merge(z[["zone_i", "z_above", "z_below"]], on="zone_i",
|
||||
how="left")
|
||||
sig = attach_htf_context(sig, cdf, tl, "h1")
|
||||
push = np.where(sig["direction"] == 1, sig["z_above"], sig["z_below"])
|
||||
keep = ((sig["h1_agree"] == 1)
|
||||
& pd.Series(push, index=sig.index).fillna(False).astype(bool))
|
||||
sig = sig[keep].copy()
|
||||
|
||||
# ATR 门控：与 shadow_signal 同源，分母取次根开盘价
|
||||
idx = sig["entry_idx"].astype(int).to_numpy()
|
||||
atr = cdf["atr"].to_numpy(float)
|
||||
op = cdf["open"].to_numpy(float)
|
||||
ref = op[np.minimum(idx + 1, len(op) - 1)]
|
||||
with np.errstate(invalid="ignore", divide="ignore"):
|
||||
atr_bp = atr[idx] / ref * 1e4
|
||||
sig = sig[np.isfinite(atr_bp) & (atr_bp >= ATR_GATE_BP)]
|
||||
return cdf, sig
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--syms", default="BTC,ETH,SOL")
|
||||
ap.add_argument("--cache", default="research/live/cache")
|
||||
ap.add_argument("--save", default="research/out/step43_fill_budget.csv")
|
||||
a = ap.parse_args()
|
||||
|
||||
from lib.exit_fill import (assert_converges, budget_bp, net_bp,
|
||||
walk_filled)
|
||||
from lib.exit_model import cfg_name, slip_budget
|
||||
from lib.exit_model import walk_exits
|
||||
|
||||
cache = Path(a.cache)
|
||||
rows = []
|
||||
for sym in a.syms.split(","):
|
||||
print(f"\n{'=' * 74}\n{sym}")
|
||||
try:
|
||||
cdf, sig = signals_for(sym, cache)
|
||||
except Exception as e:
|
||||
print(f" 跳过：{e!r}")
|
||||
continue
|
||||
print(f" {len(cdf):,} 根 1m · 过三滤网 {len(sig)} 笔信号")
|
||||
if len(sig) < 25:
|
||||
print(" 样本不足 25 笔，不出统计")
|
||||
continue
|
||||
|
||||
# 老口径：假定限价全额成交
|
||||
old = walk_exits(cdf, sig, [SL], [SCALE_AT], [MAXB], SCALE_AT,
|
||||
[RUNNER], [RUNNER_STOP])
|
||||
c = cfg_name(SL, RUNNER, MAXB, RUNNER_STOP)
|
||||
b_old = slip_budget(old[f"{c}_g"].to_numpy(),
|
||||
old[f"{c}_r"].to_numpy(),
|
||||
old[f"{c}_c"].to_numpy())
|
||||
# 先证明两套实现在「仓位趋近 0」这个极限上逐笔一致，
|
||||
# 否则后面看到的差异分不清是成交量效应还是实现 bug
|
||||
assert_converges(cdf, sig, SL, SCALE_AT, RUNNER, RUNNER_STOP, MAXB)
|
||||
print(f"\n 老口径（限价全额成交）预算 {b_old:.2f}bp"
|
||||
f" [极限一致性断言通过]")
|
||||
print(f"\n {'仓位':>10} {'预算bp':>9} {'maker占比':>10} "
|
||||
f"{'止损占比':>9} {'超时占比':>9} {'净收益bp':>10}")
|
||||
for nt in NOTIONALS:
|
||||
r = walk_filled(cdf, sig, nt, SL, SCALE_AT, RUNNER,
|
||||
RUNNER_STOP, MAXB)
|
||||
if r.empty:
|
||||
continue
|
||||
b = budget_bp(r)
|
||||
print(f" {nt:>10,.0f} {b:>9.2f} "
|
||||
f"{r['maker_share'].mean() * 100:>9.1f}% "
|
||||
f"{r['w_stop'].mean() * 100:>8.1f}% "
|
||||
f"{r['w_time'].mean() * 100:>8.1f}% "
|
||||
f"{net_bp(r):>10.2f}")
|
||||
rows.append({"sym": sym, "notional": nt, "budget_bp": b,
|
||||
"budget_bp_old": b_old,
|
||||
"maker_share": r["maker_share"].mean(),
|
||||
"w_stop": r["w_stop"].mean(),
|
||||
"w_time": r["w_time"].mean(),
|
||||
"net_bp": net_bp(r), "n": len(r)})
|
||||
del cdf
|
||||
if rows:
|
||||
out = pd.DataFrame(rows)
|
||||
Path(a.save).parent.mkdir(parents=True, exist_ok=True)
|
||||
out.to_csv(a.save, index=False)
|
||||
print(f"\n已存 {a.save}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user