加跨地部署脚本与站点对比,数据全表加 site 列
目的是在第二台机器(AWS)上跑一套完全相同的采集,看不同地理位置的数据差异。 滑点里最大的一项是延迟漂移,而延迟含网络传输,所以机房选址是可优化参数。 数据侧两处必需改动: - 全部输出加 site 列(三张 CSV 加 gzip 里的盘口与成交流)。没有这一列,两台 机器的数据合起来就分不清来源。为免四处 writerow 漏加一处产生静默空值, 改在 _SiteWriter 里统一注入。 - start.sh 在时钟未同步或偏移超 10ms 时**拒绝启动**。所有延迟数字都是 「本地时钟 − 交易所 K 线收盘」,时钟偏 50ms 就全部同向偏 50ms,且不报错, 只会让跨地对比得出一个干净且完全错误的结论。 deploy/ 下四个文件:setup.sh(docker + chrony + 拉镜像)、start.sh(校验时钟、 写运行元数据、起容器)、status.sh(健康速查)、README。运行元数据记 git commit、 镜像摘要、时钟偏移——两地数据对不上时,这三项任一不同都足以解释差异。 compare_sites.py 做配对对比:只取各站都有的 K 线(不取交集可能在比不同时段, 而延迟对市场活跃度敏感),并报配对差的符号占比而非两个中位数相减。已用注入 120ms 的合成数据验证能精确还原。另有一条自检:同一固定延迟点上两站漂移应当 相同——漂移是市场性质,若也差很多则先查时钟与时段对齐。 status.sh 里按列名取字段下标而非写死数字:加 site 列时字段整体右移过一次, 写死 $8 会静默变成读 lag_signal_ms 而非 lag_data_ms。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -58,6 +58,8 @@ import csv
|
||||
import gzip
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import socket
|
||||
import time
|
||||
from collections import deque
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
@@ -69,6 +71,10 @@ import pandas as pd
|
||||
|
||||
from lib.shadow_budget import LAG_ALARM_MS, LAG_WINDOW, lag_healthy
|
||||
|
||||
# 站点标识。跨地对比时两台机器的 CSV 要能合起来读,没有这一列就分不清哪行
|
||||
# 来自哪台。默认取主机名,部署脚本会显式传 SHADOW_SITE(如 sg-hetzner)
|
||||
SITE = os.environ.get("SHADOW_SITE") or socket.gethostname()
|
||||
|
||||
SYMS = ("BTC", "ETH", "SOL")
|
||||
# 多存一根:deque 尾部是尚未收盘的当前根,剔除后正好剩 step39 定下的窗口
|
||||
LTF_BARS, HTF_BARS = 2001, 801 # 有效窗口 2000 / 800,命中率在此饱和
|
||||
@@ -108,12 +114,30 @@ def _writer(path: Path, cols: list[str]):
|
||||
flush=True)
|
||||
fresh = not path.exists() or path.stat().st_size == 0
|
||||
f = path.open("a", newline="")
|
||||
w = csv.DictWriter(f, fieldnames=cols)
|
||||
w = _SiteWriter(csv.DictWriter(f, fieldnames=cols))
|
||||
if fresh:
|
||||
w.writeheader()
|
||||
return f, w
|
||||
|
||||
|
||||
class _SiteWriter:
|
||||
"""DictWriter 的薄包装,自动补上 site 列。
|
||||
|
||||
逐个 writerow 手加 site 有四处,漏一处就是静默的空值,而跨地对比正是靠
|
||||
这一列区分数据来源。在这里注入,漏不掉。
|
||||
"""
|
||||
|
||||
def __init__(self, w: csv.DictWriter) -> None:
|
||||
self._w = w
|
||||
|
||||
def writeheader(self) -> None:
|
||||
self._w.writeheader()
|
||||
|
||||
def writerow(self, row: dict) -> None:
|
||||
row.setdefault("site", SITE)
|
||||
self._w.writerow(row)
|
||||
|
||||
|
||||
class BookLog:
|
||||
"""把完整盘口快照落成 gzip JSONL。
|
||||
|
||||
@@ -136,7 +160,7 @@ class BookLog:
|
||||
asks: np.ndarray) -> None:
|
||||
# 只留价与量两列,update_id 对离线分析没用。round 到 10 位避免
|
||||
# float repr 把文件撑大一倍
|
||||
rec = {"sym": sym, "kline_ts": kline_ts, "label": label,
|
||||
rec = {"site": SITE, "sym": sym, "kline_ts": kline_ts, "label": label,
|
||||
"delay_ms": delay_ms, "target": target, "book_ts": book_ts,
|
||||
"bids": [[round(float(p), 10), round(float(a), 10)]
|
||||
for p, a, *_ in bids],
|
||||
@@ -187,7 +211,7 @@ class TapeLog:
|
||||
d = self.acc.get(sym)
|
||||
if not d or (not d["b"] and not d["s"]):
|
||||
return
|
||||
rec = {"sym": sym, "bar_ts": bar_ts,
|
||||
rec = {"site": SITE, "sym": sym, "bar_ts": bar_ts,
|
||||
"buys": {f"{p:.10g}": round(v, 10)
|
||||
for p, v in sorted(d["b"].items())},
|
||||
"sells": {f"{p:.10g}": round(v, 10)
|
||||
@@ -291,7 +315,7 @@ class Shadow:
|
||||
d = out_dir()
|
||||
# 追加模式:长跑期间若重启,已收集的样本不该被清掉
|
||||
self.f_sig, self.w_sig = _writer(d / "shadow_signals.csv", [
|
||||
"sym", "kline_ts", "direction",
|
||||
"site", "sym", "kline_ts", "direction",
|
||||
"h1_agree", "ladder_ok", "gate_ok", "pass_all", "lag_ok",
|
||||
"atr_pct", "atr_bp",
|
||||
"t_close_ms", "t_data_ms", "t_signal_ms",
|
||||
@@ -301,12 +325,12 @@ class Shadow:
|
||||
"baseline_px", "mid", "best_px", "fill_px", "filled", "depth_ok",
|
||||
"slip_bp", "drift_bp", "spread_bp", "impact_bp"])
|
||||
self.f_lat, self.w_lat = _writer(d / "shadow_latency.csv", [
|
||||
"sym", "kline_ts", "t_close_ms", "t_data_ms", "t_signal_ms",
|
||||
"site", "sym", "kline_ts", "t_close_ms", "t_data_ms", "t_signal_ms",
|
||||
"lag_data_ms", "lag_signal_ms", "compute_ms", "n_bars", "n_hits",
|
||||
"n_pass", "atr_bp", "lag_med_ms", "lag_ok"])
|
||||
# 无条件漂移:每根都记,用来和信号根上的条件漂移对照
|
||||
self.f_drf, self.w_drf = _writer(d / "shadow_drift.csv", [
|
||||
"sym", "kline_ts", "delay_label", "delay_ms",
|
||||
"site", "sym", "kline_ts", "delay_label", "delay_ms",
|
||||
"book_ts", "book_lag_ms", "baseline_px", "mid", "drift_bp_long"])
|
||||
# 完整深度。挂在无条件漂移那条路径上,所以每根 K 线的四个固定延迟点
|
||||
# 都有一份,信号根上再补一份 actual 点
|
||||
|
||||
Reference in New Issue
Block a user