Snapshot nautilus_mm after EXP_002 Phase 1 stop.
Keep frozen research conclusions and code; raw ledgers and secrets stay out of git. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,41 @@
|
||||
# Binance Futures API (prefer TESTNET first)
|
||||
BINANCE_API_KEY=
|
||||
BINANCE_API_SECRET=
|
||||
# Must be exactly TESTNET or LIVE (case-insensitive). Anything else exits.
|
||||
BINANCE_ENVIRONMENT=TESTNET
|
||||
# Required only for LIVE:
|
||||
# I_UNDERSTAND_LIVE=yes
|
||||
|
||||
# Proxy: 本地可开;服务器请留空(直连)
|
||||
# 本地: USE_PROXY=true ./scripts/run_probe.sh
|
||||
HTTP_PROXY=
|
||||
HTTPS_PROXY=
|
||||
|
||||
# Instrument
|
||||
SYMBOL=BTCUSDT-PERP
|
||||
ORDER_QTY=0.001
|
||||
# Default false — set true explicitly to place post-only quotes
|
||||
ENABLE_TRADING=false
|
||||
COOLDOWN_SECS=60
|
||||
OBI_ENTER=0.25
|
||||
QUOTE_TTL_SECS=30
|
||||
# Absolute inventory circuit breaker (BTC). Beyond: reduce-only side.
|
||||
MAX_ABS_INVENTORY=0.005
|
||||
|
||||
# Logs (shared schema with Freqtrade MakerEdgeProbe)
|
||||
MAKER_EDGE_LOG_DIR=
|
||||
|
||||
# Experiment identity (bind every fill + every report)
|
||||
EXPERIMENT_ID=MM_EDGE_EXP_001
|
||||
PROBE_VERSION=probe_v0.1
|
||||
EXCHANGE_NAME=binance_usdm
|
||||
|
||||
# --- MM_EDGE_EXP_002 (Event-State Observability — use run_event_state.sh) ---
|
||||
# EXPERIMENT_ID=MM_EDGE_EXP_002
|
||||
# PROBE_VERSION=event_state_v0.1
|
||||
# ENABLE_TRADING=false # hard-enforced; never true for EXP_002
|
||||
# EVENT_STATE_LOG_DIR=logs/event_state
|
||||
# PREFILL_WINDOW_SEC=5.0
|
||||
# PREFILL_MARGIN_SEC=0.25
|
||||
# LARGE_TRADE_QTY=0.1
|
||||
# LOG_EVERY_BOOK_DELTA=true
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
.venv/
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
__pycache__/
|
||||
*.pyc
|
||||
logs/
|
||||
.DS_Store
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
@@ -0,0 +1,531 @@
|
||||
{
|
||||
"experiment_id": "MM_EDGE_EXP_001",
|
||||
"population": {
|
||||
"name": "MATCHED",
|
||||
"fills": 3890,
|
||||
"paths": 3886,
|
||||
"clusters": 3334
|
||||
},
|
||||
"strategy": "v0.1 FROZEN",
|
||||
"execution": "STOPPED",
|
||||
"stage3": "LOCKED",
|
||||
"fee_total_usdt": 42.42214561,
|
||||
"fee_per_fill_usdt": 0.010916661248069994,
|
||||
"fee_per_btc_usdt": 12.74857122550787,
|
||||
"fee_per_cluster_usdt": 0.012724098863227356,
|
||||
"realized_component_usdt": -8.52172967,
|
||||
"gross_markout_30s_usdt": -1.7307799999996094,
|
||||
"net_attr_30s_usdt": -52.67465527999961,
|
||||
"markout_by_horizon": [
|
||||
{
|
||||
"horizon": "1s",
|
||||
"n": 3886,
|
||||
"fill_w": 1.9522065688461466e-05,
|
||||
"cluster_w": 1.9672552540924558e-05,
|
||||
"gross_usdt": 4.140839999999786
|
||||
},
|
||||
{
|
||||
"horizon": "5s",
|
||||
"n": 3886,
|
||||
"fill_w": 1.6056729777556372e-05,
|
||||
"cluster_w": 1.563711857682482e-05,
|
||||
"gross_usdt": 3.4058050000001323
|
||||
},
|
||||
{
|
||||
"horizon": "10s",
|
||||
"n": 3886,
|
||||
"fill_w": 1.1123222170478668e-05,
|
||||
"cluster_w": 1.1253447279750312e-05,
|
||||
"gross_usdt": 2.359355000000128
|
||||
},
|
||||
{
|
||||
"horizon": "30s",
|
||||
"n": 3886,
|
||||
"fill_w": -8.159793870873894e-06,
|
||||
"cluster_w": -8.29897884011306e-06,
|
||||
"gross_usdt": -1.7307799999996094
|
||||
},
|
||||
{
|
||||
"horizon": "300s",
|
||||
"n": 3886,
|
||||
"fill_w": -3.581687863688742e-05,
|
||||
"cluster_w": -3.542059919144397e-05,
|
||||
"gross_usdt": -7.597144999999726
|
||||
}
|
||||
],
|
||||
"inventory_metrics": {
|
||||
"max_net_btc": 0.0059,
|
||||
"min_net_btc": -0.0058000000000000005,
|
||||
"max_abs_net_btc": 0.0059,
|
||||
"avg_abs_net_btc_per_fill": 0.0029282519280205655,
|
||||
"time_weighted_abs_net_btc": 0.0035138619697350987,
|
||||
"time_weighted_signed_net_btc": 0.0009845445677824191,
|
||||
"long_qty": 1.6646,
|
||||
"short_qty": 1.6670000000000003,
|
||||
"turnover_btc": 3.331600000000001
|
||||
},
|
||||
"bucket_rows": [
|
||||
{
|
||||
"dimension": "PathType",
|
||||
"bucket": "A_immediate_edge",
|
||||
"fills": 1481,
|
||||
"clusters": 1279,
|
||||
"btc_qty": 1.2579000000000002,
|
||||
"notional_usdt": 80131.73112,
|
||||
"fee_usdt": 16.02634448,
|
||||
"fee_per_fill": 0.010821299446320053,
|
||||
"fee_per_btc": 12.74055527466412,
|
||||
"markout_1s": 0.00010135972212851294,
|
||||
"markout_5s": 0.00012093929414163527,
|
||||
"markout_10s": 0.00011803045894311667,
|
||||
"markout_30s": 0.00015268377244562442,
|
||||
"markout_300s": 0.00013524292871909752,
|
||||
"gross_markout_30s_usdt": 12.23481500000004,
|
||||
"realized_pnl_usdt": -1.7373983300000004,
|
||||
"net_attr_30s_usdt": -5.52892780999996
|
||||
},
|
||||
{
|
||||
"dimension": "PathType",
|
||||
"bucket": "C_toxic",
|
||||
"fills": 1290,
|
||||
"clusters": 1114,
|
||||
"btc_qty": 1.1028000000000002,
|
||||
"notional_usdt": 70343.84668,
|
||||
"fee_usdt": 14.068767789999999,
|
||||
"fee_per_fill": 0.010906021542635659,
|
||||
"fee_per_btc": 12.757315732680446,
|
||||
"markout_1s": -2.26781740733745e-05,
|
||||
"markout_5s": -5.864727612589716e-05,
|
||||
"markout_10s": -9.311823718992184e-05,
|
||||
"markout_30s": -0.0002122337020878937,
|
||||
"markout_300s": -0.0004798649291039827,
|
||||
"gross_markout_30s_usdt": -14.92933499999959,
|
||||
"realized_pnl_usdt": -2.45849123,
|
||||
"net_attr_30s_usdt": -31.45659401999959
|
||||
},
|
||||
{
|
||||
"dimension": "PathType",
|
||||
"bucket": "B_drawdown_then_recover",
|
||||
"fills": 872,
|
||||
"clusters": 767,
|
||||
"btc_qty": 0.7574000000000001,
|
||||
"notional_usdt": 48282.0527,
|
||||
"fee_usdt": 9.656409660000001,
|
||||
"fee_per_fill": 0.011073864288990827,
|
||||
"fee_per_btc": 12.749418616318987,
|
||||
"markout_1s": -4.145950488969804e-05,
|
||||
"markout_5s": -3.18002221144232e-05,
|
||||
"markout_10s": -1.0215286476420607e-05,
|
||||
"markout_30s": -8.034869652509012e-06,
|
||||
"markout_300s": 0.00040945162631827824,
|
||||
"gross_markout_30s_usdt": -0.3879400000000708,
|
||||
"realized_pnl_usdt": -2.83361474,
|
||||
"net_attr_30s_usdt": -12.87796440000007
|
||||
},
|
||||
{
|
||||
"dimension": "PathType",
|
||||
"bucket": "D_mixed",
|
||||
"fills": 243,
|
||||
"clusters": 215,
|
||||
"btc_qty": 0.20950000000000005,
|
||||
"notional_usdt": 13353.11977,
|
||||
"fee_usdt": 2.6706236800000003,
|
||||
"fee_per_fill": 0.010990220905349795,
|
||||
"fee_per_btc": 12.747607064439139,
|
||||
"markout_1s": -2.877754462019724e-05,
|
||||
"markout_5s": -4.676173139724288e-05,
|
||||
"markout_10s": -4.12787430573779e-06,
|
||||
"markout_30s": 0.00010122578268464162,
|
||||
"markout_300s": -0.00033310680025451153,
|
||||
"gross_markout_30s_usdt": 1.3516800000000115,
|
||||
"realized_pnl_usdt": -1.49222537,
|
||||
"net_attr_30s_usdt": -2.8111690499999886
|
||||
},
|
||||
{
|
||||
"dimension": "Toxicity",
|
||||
"bucket": "non_toxic",
|
||||
"fills": 2596,
|
||||
"clusters": 2236,
|
||||
"btc_qty": 2.2248,
|
||||
"notional_usdt": 141766.90359,
|
||||
"fee_usdt": 28.35337782,
|
||||
"fee_per_fill": 0.010921948312788905,
|
||||
"fee_per_btc": 12.744236704422867,
|
||||
"markout_1s": 4.046155946657924e-05,
|
||||
"markout_5s": 5.3124388057320254e-05,
|
||||
"markout_10s": 6.284717923844347e-05,
|
||||
"markout_30s": 9.310039695986549e-05,
|
||||
"markout_300s": 0.00018451697355013032,
|
||||
"gross_markout_30s_usdt": 13.198554999999981,
|
||||
"realized_pnl_usdt": -6.063238439999999,
|
||||
"net_attr_30s_usdt": -21.21806126000002
|
||||
},
|
||||
{
|
||||
"dimension": "Toxicity",
|
||||
"bucket": "toxic",
|
||||
"fills": 1290,
|
||||
"clusters": 1114,
|
||||
"btc_qty": 1.1028000000000002,
|
||||
"notional_usdt": 70343.84668,
|
||||
"fee_usdt": 14.068767789999999,
|
||||
"fee_per_fill": 0.010906021542635659,
|
||||
"fee_per_btc": 12.757315732680446,
|
||||
"markout_1s": -2.26781740733745e-05,
|
||||
"markout_5s": -5.864727612589716e-05,
|
||||
"markout_10s": -9.311823718992184e-05,
|
||||
"markout_30s": -0.0002122337020878937,
|
||||
"markout_300s": -0.0004798649291039827,
|
||||
"gross_markout_30s_usdt": -14.92933499999959,
|
||||
"realized_pnl_usdt": -2.45849123,
|
||||
"net_attr_30s_usdt": -31.45659401999959
|
||||
},
|
||||
{
|
||||
"dimension": "Volatility",
|
||||
"bucket": "high_vol",
|
||||
"fills": 1944,
|
||||
"clusters": 1704,
|
||||
"btc_qty": 1.7018000000000004,
|
||||
"notional_usdt": 108527.27282000001,
|
||||
"fee_usdt": 21.705452540000003,
|
||||
"fee_per_fill": 0.011165356244855968,
|
||||
"fee_per_btc": 12.754408590903747,
|
||||
"markout_1s": 2.410343439052789e-05,
|
||||
"markout_5s": 2.200234962100767e-05,
|
||||
"markout_10s": 1.3732354654038277e-05,
|
||||
"markout_30s": -1.8503321311044688e-05,
|
||||
"markout_300s": -7.311572283918717e-05,
|
||||
"gross_markout_30s_usdt": -2.0081149999998673,
|
||||
"realized_pnl_usdt": -2.5832574499999996,
|
||||
"net_attr_30s_usdt": -26.296824989999866
|
||||
},
|
||||
{
|
||||
"dimension": "Volatility",
|
||||
"bucket": "low_vol",
|
||||
"fills": 1942,
|
||||
"clusters": 1630,
|
||||
"btc_qty": 1.6258000000000004,
|
||||
"notional_usdt": 103583.47745,
|
||||
"fee_usdt": 20.71669307,
|
||||
"fee_per_fill": 0.010667710128733266,
|
||||
"fee_per_btc": 12.74246098536105,
|
||||
"markout_1s": 1.4722039050446979e-05,
|
||||
"markout_5s": 9.827339504906817e-06,
|
||||
"markout_10s": 8.389561939738874e-06,
|
||||
"markout_30s": 2.677405768059181e-06,
|
||||
"markout_300s": 3.2621515353484367e-06,
|
||||
"gross_markout_30s_usdt": 0.2773350000002581,
|
||||
"realized_pnl_usdt": -5.9384722199999995,
|
||||
"net_attr_30s_usdt": -26.377830289999743
|
||||
},
|
||||
{
|
||||
"dimension": "Trend",
|
||||
"bucket": "range",
|
||||
"fills": 1521,
|
||||
"clusters": 1383,
|
||||
"btc_qty": 1.3480000000000003,
|
||||
"notional_usdt": 85940.04824,
|
||||
"fee_usdt": 17.18800828,
|
||||
"fee_per_fill": 0.011300465667324127,
|
||||
"fee_per_btc": 12.750747982195842,
|
||||
"markout_1s": -1.477018021278579e-06,
|
||||
"markout_5s": -8.501973351929517e-06,
|
||||
"markout_10s": -1.3186808981480409e-05,
|
||||
"markout_30s": -3.577395013107376e-05,
|
||||
"markout_300s": -8.103131360308704e-05,
|
||||
"gross_markout_30s_usdt": -3.074414999999833,
|
||||
"realized_pnl_usdt": -5.27847826,
|
||||
"net_attr_30s_usdt": -25.540901539999833
|
||||
},
|
||||
{
|
||||
"dimension": "Trend",
|
||||
"bucket": "trend_up",
|
||||
"fills": 1255,
|
||||
"clusters": 1097,
|
||||
"btc_qty": 1.0467,
|
||||
"notional_usdt": 66723.59572000001,
|
||||
"fee_usdt": 13.34471747,
|
||||
"fee_per_fill": 0.010633241011952193,
|
||||
"fee_per_btc": 12.749324037451037,
|
||||
"markout_1s": 3.411746587448147e-05,
|
||||
"markout_5s": 3.5157502749763345e-05,
|
||||
"markout_10s": 2.718513563945042e-05,
|
||||
"markout_30s": -2.7502264831481206e-06,
|
||||
"markout_300s": -1.9947965717947442e-06,
|
||||
"gross_markout_30s_usdt": -0.18350500000001263,
|
||||
"realized_pnl_usdt": 0.07479931999999989,
|
||||
"net_attr_30s_usdt": -13.453423150000011
|
||||
},
|
||||
{
|
||||
"dimension": "Trend",
|
||||
"bucket": "trend_down",
|
||||
"fills": 1110,
|
||||
"clusters": 972,
|
||||
"btc_qty": 0.9329000000000001,
|
||||
"notional_usdt": 59447.10631,
|
||||
"fee_usdt": 11.88941986,
|
||||
"fee_per_fill": 0.010711189063063063,
|
||||
"fee_per_btc": 12.744581262729124,
|
||||
"markout_1s": 3.349759346764102e-05,
|
||||
"markout_5s": 3.012139885602704e-05,
|
||||
"markout_10s": 2.823922145589404e-05,
|
||||
"markout_30s": 2.568905527607399e-05,
|
||||
"markout_300s": -8.414370876041091e-06,
|
||||
"gross_markout_30s_usdt": 1.5271400000002369,
|
||||
"realized_pnl_usdt": -3.3180507300000004,
|
||||
"net_attr_30s_usdt": -13.680330589999762
|
||||
},
|
||||
{
|
||||
"dimension": "FillContext",
|
||||
"bucket": "neutral",
|
||||
"fills": 1388,
|
||||
"clusters": 1213,
|
||||
"btc_qty": 1.1838000000000002,
|
||||
"notional_usdt": 75485.50388999999,
|
||||
"fee_usdt": 15.09709914,
|
||||
"fee_per_fill": 0.01087687257925072,
|
||||
"fee_per_btc": 12.7530825646224,
|
||||
"markout_1s": 1.647018216652519e-05,
|
||||
"markout_5s": -3.5886360432144643e-06,
|
||||
"markout_10s": -2.1338136688432127e-05,
|
||||
"markout_30s": -6.403666599409166e-05,
|
||||
"markout_300s": -0.00017397764237140476,
|
||||
"gross_markout_30s_usdt": -4.833839999999636,
|
||||
"realized_pnl_usdt": 0.9367158200000003,
|
||||
"net_attr_30s_usdt": -18.994223319999634
|
||||
},
|
||||
{
|
||||
"dimension": "FillContext",
|
||||
"bucket": "sell_resistance",
|
||||
"fills": 979,
|
||||
"clusters": 861,
|
||||
"btc_qty": 0.8544,
|
||||
"notional_usdt": 54478.163700000005,
|
||||
"fee_usdt": 10.89563175,
|
||||
"fee_per_fill": 0.011129348059244126,
|
||||
"fee_per_btc": 12.752377984550561,
|
||||
"markout_1s": 2.2746912080660278e-05,
|
||||
"markout_5s": 3.396737104044108e-05,
|
||||
"markout_10s": 4.7584661889033e-05,
|
||||
"markout_30s": 6.382970283559833e-05,
|
||||
"markout_300s": 0.00015825056159152353,
|
||||
"gross_markout_30s_usdt": 3.4773250000000804,
|
||||
"realized_pnl_usdt": -6.22020069,
|
||||
"net_attr_30s_usdt": -13.63850743999992
|
||||
},
|
||||
{
|
||||
"dimension": "FillContext",
|
||||
"bucket": "buy_support",
|
||||
"fills": 957,
|
||||
"clusters": 842,
|
||||
"btc_qty": 0.8356000000000002,
|
||||
"notional_usdt": 53220.48314,
|
||||
"fee_usdt": 10.64409564,
|
||||
"fee_per_fill": 0.01112235699059561,
|
||||
"fee_per_btc": 12.73826668262326,
|
||||
"markout_1s": 8.405128507064573e-06,
|
||||
"markout_5s": 1.8714880836011024e-05,
|
||||
"markout_10s": 2.0467871310649115e-05,
|
||||
"markout_30s": 1.984809114230057e-05,
|
||||
"markout_300s": 5.777324478456582e-05,
|
||||
"gross_markout_30s_usdt": 1.0563249999999906,
|
||||
"realized_pnl_usdt": -4.60049982,
|
||||
"net_attr_30s_usdt": -14.188270460000009
|
||||
},
|
||||
{
|
||||
"dimension": "FillContext",
|
||||
"bucket": "buy_pressure_absorbing",
|
||||
"fills": 288,
|
||||
"clusters": 242,
|
||||
"btc_qty": 0.23299999999999998,
|
||||
"notional_usdt": 14849.31926,
|
||||
"fee_usdt": 2.96986344,
|
||||
"fee_per_fill": 0.010312025833333334,
|
||||
"fee_per_btc": 12.74619502145923,
|
||||
"markout_1s": 3.085831693539742e-05,
|
||||
"markout_5s": 9.949951065962983e-07,
|
||||
"markout_10s": -5.396206963900885e-06,
|
||||
"markout_30s": -8.174078412265757e-05,
|
||||
"markout_300s": -0.00018505090717539603,
|
||||
"gross_markout_30s_usdt": -1.2137950000000812,
|
||||
"realized_pnl_usdt": 0.5555894899999999,
|
||||
"net_attr_30s_usdt": -3.628068950000081
|
||||
},
|
||||
{
|
||||
"dimension": "FillContext",
|
||||
"bucket": "sell_pressure_absorbing",
|
||||
"fills": 246,
|
||||
"clusters": 207,
|
||||
"btc_qty": 0.19330000000000003,
|
||||
"notional_usdt": 12320.81768,
|
||||
"fee_usdt": 2.46416312,
|
||||
"fee_per_fill": 0.01001692325203252,
|
||||
"fee_per_btc": 12.74786921883083,
|
||||
"markout_1s": 3.3994902844800725e-05,
|
||||
"markout_5s": 1.3797785537899414e-05,
|
||||
"markout_10s": -3.266666307814853e-05,
|
||||
"markout_30s": -5.2299288629677764e-05,
|
||||
"markout_300s": -0.0003042407652930987,
|
||||
"gross_markout_30s_usdt": -0.6443699999999568,
|
||||
"realized_pnl_usdt": 0.9138141899999999,
|
||||
"net_attr_30s_usdt": -2.194718929999957
|
||||
},
|
||||
{
|
||||
"dimension": "FillContext",
|
||||
"bucket": "sell_pressure_falling",
|
||||
"fills": 10,
|
||||
"clusters": 10,
|
||||
"btc_qty": 0.010000000000000002,
|
||||
"notional_usdt": 638.0611,
|
||||
"fee_usdt": 0.12761222,
|
||||
"fee_per_fill": 0.012761222,
|
||||
"fee_per_btc": 12.761221999999997,
|
||||
"markout_1s": 0.0002923701194132765,
|
||||
"markout_5s": 0.0002596146356515865,
|
||||
"markout_10s": 0.0005288678466685525,
|
||||
"markout_30s": 8.886296312371227e-05,
|
||||
"markout_300s": -2.1392935566876843e-05,
|
||||
"gross_markout_30s_usdt": 0.056699999999975284,
|
||||
"realized_pnl_usdt": 0.02931205,
|
||||
"net_attr_30s_usdt": -0.041600170000024736
|
||||
},
|
||||
{
|
||||
"dimension": "FillContext",
|
||||
"bucket": "buy_pressure_rising",
|
||||
"fills": 8,
|
||||
"clusters": 8,
|
||||
"btc_qty": 0.0075,
|
||||
"notional_usdt": 479.4465,
|
||||
"fee_usdt": 0.0958893,
|
||||
"fee_per_fill": 0.0119861625,
|
||||
"fee_per_btc": 12.78524,
|
||||
"markout_1s": 0.00011800065283618207,
|
||||
"markout_5s": 4.155833862592348e-05,
|
||||
"markout_10s": -0.00018521357440294395,
|
||||
"markout_30s": -0.0002589965720888399,
|
||||
"markout_300s": -0.0008807030607168815,
|
||||
"gross_markout_30s_usdt": -0.12417499999999199,
|
||||
"realized_pnl_usdt": -0.00064907,
|
||||
"net_attr_30s_usdt": -0.22071336999999197
|
||||
},
|
||||
{
|
||||
"dimension": "FillContext",
|
||||
"bucket": "momentum_down_catching_knife",
|
||||
"fills": 5,
|
||||
"clusters": 5,
|
||||
"btc_qty": 0.005,
|
||||
"notional_usdt": 317.4407,
|
||||
"fee_usdt": 0.06348814,
|
||||
"fee_per_fill": 0.012697627999999999,
|
||||
"fee_per_btc": 12.697628,
|
||||
"markout_1s": -8.804794092254599e-05,
|
||||
"markout_5s": 0.00041661324461542815,
|
||||
"markout_10s": 0.0007213945785779604,
|
||||
"markout_30s": 0.000589716441527518,
|
||||
"markout_300s": 0.0009308825238855407,
|
||||
"gross_markout_30s_usdt": 0.18720000000000436,
|
||||
"realized_pnl_usdt": -0.13601287,
|
||||
"net_attr_30s_usdt": -0.012301009999995626
|
||||
},
|
||||
{
|
||||
"dimension": "FillContext",
|
||||
"bucket": "momentum_up_chasing",
|
||||
"fills": 5,
|
||||
"clusters": 5,
|
||||
"btc_qty": 0.005,
|
||||
"notional_usdt": 321.51430000000005,
|
||||
"fee_usdt": 0.06430286,
|
||||
"fee_per_fill": 0.012860572,
|
||||
"fee_per_btc": 12.860572000000001,
|
||||
"markout_1s": 0.00036950144985773765,
|
||||
"markout_5s": 0.001018928240516915,
|
||||
"markout_10s": 0.0009125566110123747,
|
||||
"markout_30s": 0.0009575001796187784,
|
||||
"markout_300s": 0.0014820491654648018,
|
||||
"gross_markout_30s_usdt": 0.30785000000000584,
|
||||
"realized_pnl_usdt": 0.00020122999999998975,
|
||||
"net_attr_30s_usdt": 0.24374837000000582
|
||||
}
|
||||
],
|
||||
"counterfactuals": [
|
||||
{
|
||||
"name": "BASELINE",
|
||||
"fills": 3886,
|
||||
"clusters": 3332,
|
||||
"btc_qty": 3.327600000000001,
|
||||
"fee_usdt": 42.42214561,
|
||||
"gross_markout_30s_usdt": -1.7307799999996094,
|
||||
"realized_pnl_usdt": -8.52172967,
|
||||
"net_attr_30s_usdt": -52.67465527999961,
|
||||
"markout_30s": -8.159793870873894e-06
|
||||
},
|
||||
{
|
||||
"name": "EXCLUDE_PATH_C",
|
||||
"fills": 2596,
|
||||
"clusters": 2236,
|
||||
"btc_qty": 2.2248,
|
||||
"fee_usdt": 28.35337782,
|
||||
"gross_markout_30s_usdt": 13.198554999999981,
|
||||
"realized_pnl_usdt": -6.063238439999999,
|
||||
"net_attr_30s_usdt": -21.21806126000002,
|
||||
"markout_30s": 9.310039695986549e-05
|
||||
},
|
||||
{
|
||||
"name": "EXCLUDE_TOXIC",
|
||||
"fills": 2596,
|
||||
"clusters": 2236,
|
||||
"btc_qty": 2.2248,
|
||||
"fee_usdt": 28.35337782,
|
||||
"gross_markout_30s_usdt": 13.198554999999981,
|
||||
"realized_pnl_usdt": -6.063238439999999,
|
||||
"net_attr_30s_usdt": -21.21806126000002,
|
||||
"markout_30s": 9.310039695986549e-05
|
||||
},
|
||||
{
|
||||
"name": "EXCLUDE_NEGATIVE_STATE",
|
||||
"fills": 1956,
|
||||
"clusters": 1723,
|
||||
"btc_qty": 1.7100000000000002,
|
||||
"fee_usdt": 21.79513061,
|
||||
"gross_markout_30s_usdt": 5.085400000000057,
|
||||
"realized_pnl_usdt": -10.9272001,
|
||||
"net_attr_30s_usdt": -27.636930709999945,
|
||||
"markout_30s": 4.666546513967971e-05
|
||||
}
|
||||
],
|
||||
"recon03_extension": {
|
||||
"venue_confirmed_no_trade_history": 876,
|
||||
"venue_partial_order_canceled": 11
|
||||
},
|
||||
"account_recon_ref": {
|
||||
"experiment_id": "MM_EDGE_EXP_001",
|
||||
"maker_only_status": "PASS",
|
||||
"taker_filled_count": 0,
|
||||
"maker_filled_count": 3890,
|
||||
"exchange_trades": 3890,
|
||||
"jsonl_fills": 4777,
|
||||
"income_by_type": {
|
||||
"COMMISSION": -52.04535895999991,
|
||||
"REALIZED_PNL": -9.671389249999995,
|
||||
"FUNDING_FEE": 0.05826837000000001
|
||||
},
|
||||
"income_sum": -61.658479839999906,
|
||||
"start_wallet_assumed": 5000.0,
|
||||
"end_wallet": 4938.32625704,
|
||||
"end_unrealized": 0.40078863,
|
||||
"end_equity": 4938.72704567,
|
||||
"wallet_residual_gap": -0.015263119999872288,
|
||||
"net_qty": -0.0023999999999990695,
|
||||
"fee_by_asset": {
|
||||
"USDT": 42.47422358999987
|
||||
},
|
||||
"positions": [
|
||||
{
|
||||
"symbol": "BTCUSDT",
|
||||
"amt": 0.0032,
|
||||
"entry": 64233.4,
|
||||
"unrealized": 0.40078863
|
||||
}
|
||||
],
|
||||
"probe": "STOPPED"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
========================================================================
|
||||
Economic Attribution v0.1
|
||||
========================================================================
|
||||
Experiment: MM_EDGE_EXP_001
|
||||
Population: MATCHED=3890
|
||||
Strategy: v0.1 FROZEN
|
||||
Execution: STOPPED
|
||||
Stage3: LOCKED
|
||||
Purpose: Economic Attribution only.
|
||||
No strategy modification. No live execution. No economic simulation.
|
||||
|
||||
Layer 1 — Hard Economic Evidence
|
||||
----------------------------------------
|
||||
Matched fills: 3890
|
||||
Matched paths: 3886
|
||||
Matched clusters: 3334
|
||||
Fee total: 42.422146 USDT
|
||||
Fee / fill: 0.010917 USDT
|
||||
Fee / BTC: 12.748571 USDT
|
||||
Fee / cluster: 0.012724 USDT
|
||||
Realized component: -8.521730 USDT
|
||||
Gross markout @30s: -1.730780 USDT
|
||||
Net attributable @30s: -52.674655 USDT
|
||||
|
||||
Markout by horizon (MATCHED only)
|
||||
----------------------------------------
|
||||
1s n=3886 fill-w=0.0020% cluster-w=0.0020% gross=4.140840 USDT
|
||||
5s n=3886 fill-w=0.0016% cluster-w=0.0016% gross=3.405805 USDT
|
||||
10s n=3886 fill-w=0.0011% cluster-w=0.0011% gross=2.359355 USDT
|
||||
30s n=3886 fill-w=-0.0008% cluster-w=-0.0008% gross=-1.730780 USDT
|
||||
300s n=3886 fill-w=-0.0036% cluster-w=-0.0035% gross=-7.597145 USDT
|
||||
|
||||
Inventory carry / exposure
|
||||
----------------------------------------
|
||||
Max net BTC: 0.005900
|
||||
Min net BTC: -0.005800
|
||||
Max |net BTC|: 0.005900
|
||||
Average |net BTC|: 0.002928
|
||||
TW |net BTC|: 0.003514
|
||||
TW signed net BTC: 0.000985
|
||||
Long qty / Short qty: 1.664600 / 1.667000 BTC
|
||||
Inventory turnover: 3.331600 BTC
|
||||
|
||||
Slices (weighted by notional, MATCHED only)
|
||||
----------------------------------------
|
||||
PathType
|
||||
A_immediate_edge: n=1481 clusters=1279 fee=16.0263 gross30=12.2348 realized=-1.7374 net30=-5.5289 m30=0.0153%
|
||||
C_toxic: n=1290 clusters=1114 fee=14.0688 gross30=-14.9293 realized=-2.4585 net30=-31.4566 m30=-0.0212%
|
||||
B_drawdown_then_recover: n=872 clusters=767 fee=9.6564 gross30=-0.3879 realized=-2.8336 net30=-12.8780 m30=-0.0008%
|
||||
D_mixed: n=243 clusters=215 fee=2.6706 gross30=1.3517 realized=-1.4922 net30=-2.8112 m30=0.0101%
|
||||
|
||||
Toxicity
|
||||
non_toxic: n=2596 clusters=2236 fee=28.3534 gross30=13.1986 realized=-6.0632 net30=-21.2181 m30=0.0093%
|
||||
toxic: n=1290 clusters=1114 fee=14.0688 gross30=-14.9293 realized=-2.4585 net30=-31.4566 m30=-0.0212%
|
||||
|
||||
Volatility
|
||||
high_vol: n=1944 clusters=1704 fee=21.7055 gross30=-2.0081 realized=-2.5833 net30=-26.2968 m30=-0.0019%
|
||||
low_vol: n=1942 clusters=1630 fee=20.7167 gross30=0.2773 realized=-5.9385 net30=-26.3778 m30=0.0003%
|
||||
|
||||
Trend
|
||||
range: n=1521 clusters=1383 fee=17.1880 gross30=-3.0744 realized=-5.2785 net30=-25.5409 m30=-0.0036%
|
||||
trend_up: n=1255 clusters=1097 fee=13.3447 gross30=-0.1835 realized=0.0748 net30=-13.4534 m30=-0.0003%
|
||||
trend_down: n=1110 clusters=972 fee=11.8894 gross30=1.5271 realized=-3.3181 net30=-13.6803 m30=0.0026%
|
||||
|
||||
FillContext
|
||||
neutral: n=1388 clusters=1213 fee=15.0971 gross30=-4.8338 realized=0.9367 net30=-18.9942 m30=-0.0064%
|
||||
sell_resistance: n=979 clusters=861 fee=10.8956 gross30=3.4773 realized=-6.2202 net30=-13.6385 m30=0.0064%
|
||||
buy_support: n=957 clusters=842 fee=10.6441 gross30=1.0563 realized=-4.6005 net30=-14.1883 m30=0.0020%
|
||||
buy_pressure_absorbing: n=288 clusters=242 fee=2.9699 gross30=-1.2138 realized=0.5556 net30=-3.6281 m30=-0.0082%
|
||||
sell_pressure_absorbing: n=246 clusters=207 fee=2.4642 gross30=-0.6444 realized=0.9138 net30=-2.1947 m30=-0.0052%
|
||||
sell_pressure_falling: n=10 clusters=10 fee=0.1276 gross30=0.0567 realized=0.0293 net30=-0.0416 m30=0.0089%
|
||||
buy_pressure_rising: n=8 clusters=8 fee=0.0959 gross30=-0.1242 realized=-0.0006 net30=-0.2207 m30=-0.0259%
|
||||
momentum_down_catching_knife: n=5 clusters=5 fee=0.0635 gross30=0.1872 realized=-0.1360 net30=-0.0123 m30=0.0590%
|
||||
momentum_up_chasing: n=5 clusters=5 fee=0.0643 gross30=0.3079 realized=0.0002 net30=0.2437 m30=0.0958%
|
||||
|
||||
Layer 2 — Evidence Extension (excluded from core conclusion)
|
||||
----------------------------------------
|
||||
VENUE_CONFIRMED_NO_TRADE_HISTORY: 876
|
||||
VENUE_PARTIAL_ORDER_CANCELED: 11
|
||||
These rows are order-confirmed, but not part of the Hard Evidence Population.
|
||||
|
||||
Layer 3 — Counterfactual Attribution (NOT backtest)
|
||||
----------------------------------------
|
||||
Observed vs Exclude-Bucket Attribution. These are contribution decompositions only.
|
||||
BASELINE: fills=3886 clusters=3332 fee=42.4221 gross30=-1.7308 realized=-8.5217 net30=-52.6747 m30=-0.0008%
|
||||
EXCLUDE_PATH_C: fills=2596 clusters=2236 fee=28.3534 gross30=13.1986 realized=-6.0632 net30=-21.2181 m30=0.0093%
|
||||
EXCLUDE_TOXIC: fills=2596 clusters=2236 fee=28.3534 gross30=13.1986 realized=-6.0632 net30=-21.2181 m30=0.0093%
|
||||
EXCLUDE_NEGATIVE_STATE: fills=1956 clusters=1723 fee=21.7951 gross30=5.0854 realized=-10.9272 net30=-27.6369 m30=0.0047%
|
||||
|
||||
Interpretation
|
||||
----------------------------------------
|
||||
Core conclusion is based on 3890 fully matched fills.
|
||||
Economic Attribution asks why MakerAlpha did not convert to money.
|
||||
It does NOT change quote logic, does NOT restart v0.1, and does NOT unlock Stage 3.
|
||||
========================================================================
|
||||
@@ -0,0 +1,342 @@
|
||||
# Research Freeze / Data Collection Phase
|
||||
|
||||
**研究框架已冻结。** 研究对象:可验证的市场现象(不是策略)。
|
||||
|
||||
验证的假设是:
|
||||
|
||||
> 在当前 BTC 永续、当前交易所、当前报价假设、当前执行条件下,被动成交是否产生正向 Maker Alpha。
|
||||
|
||||
不是:「我的策略有没有赚钱」。
|
||||
|
||||
| 报告结论 | 含义 | 下一步 |
|
||||
|----------|------|--------|
|
||||
| **FAIL** | 这个市场假设不成立 | 换假设(Carry / Basis / Funding…) |
|
||||
| **PARTIAL_PASS** | 优势仅局部存在 | Event-driven LP |
|
||||
| **PASS** | 普遍可捕获 | Economic Simulation → Symmetric MM |
|
||||
| **COLLECTING** | 样本不足 | 继续采集 |
|
||||
|
||||
三个答案都推进系统。只有盈利才算成功 —— 错误。
|
||||
|
||||
---
|
||||
|
||||
## 状态机(锁定)
|
||||
|
||||
```
|
||||
Research Freeze
|
||||
|
|
||||
v
|
||||
Data Collection
|
||||
|
|
||||
v
|
||||
Maker Edge Report v0.1
|
||||
|
|
||||
+---- FAIL --------------→ 换假设
|
||||
|
|
||||
+---- PARTIAL_PASS ------→ Event-driven LP
|
||||
|
|
||||
+---- PASS --------------→ Economic Simulation
|
||||
| ↓
|
||||
| Symmetric MM / Quote Engine
|
||||
|
|
||||
+---- COLLECTING --------→ 继续采集
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 观察窗口(不要过早解释)
|
||||
|
||||
| 规模 | 用途 |
|
||||
|------|------|
|
||||
| **~500 fills** | 发现异常 / 管道是否工作 |
|
||||
| **~2000 fills** | 初步判断(Decision 门槛默认) |
|
||||
| **~10000 fills** | 才讨论稳定性 |
|
||||
|
||||
**cluster 数量比 fill 数量更重要。**
|
||||
5000 fills / 80 clusters ≠ 3000 fills / 900 clusters。
|
||||
|
||||
第一份报告最值得看的不是 Decision,而是三个**分布**:Fill Alpha / Cluster Alpha / Toxicity。
|
||||
|
||||
---
|
||||
|
||||
## Unlock Stage3(Economic Simulation)必须同时满足
|
||||
|
||||
```
|
||||
Data Integrity PASS
|
||||
AND cluster-weighted MakerAlpha > 0
|
||||
AND benchmark-adjusted alpha > 0
|
||||
AND edge not concentrated in one event/regime
|
||||
```
|
||||
|
||||
否则:继续 Data Collection,或判定 FAIL / PARTIAL_PASS。
|
||||
**禁止**用改 Quote Logic / 改成本假设 来「凑」解锁。
|
||||
|
||||
---
|
||||
|
||||
## 冻结期间只允许 / 禁止
|
||||
|
||||
### ✅ 允许
|
||||
|
||||
- 数据字段(不改变报价行为)
|
||||
- 数据质量检查
|
||||
- 报告解释能力(分布、归因、可比格式)
|
||||
|
||||
### ❌ 禁止(直到 Gate 解锁)
|
||||
|
||||
- 新交易规则 / 动态 spread / inventory skew
|
||||
- 新过滤条件(Pulse / AI)
|
||||
- 新收益优化参数(为结果改 fee/slip/latency)
|
||||
|
||||
### 三不动
|
||||
|
||||
1. Quote Logic
|
||||
2. 成本模型:`Net = Raw − Fee − Slip − Latency`
|
||||
3. 失败定义:`PASS` / `PARTIAL_PASS` / `COLLECTING` / `FAIL`
|
||||
|
||||
---
|
||||
|
||||
## Report v0.1 固定格式
|
||||
|
||||
```
|
||||
Executive Summary
|
||||
Section 1 — Data Integrity
|
||||
Section 2 — Fill Alpha (+ distribution: mean/median/p25/p75)
|
||||
Section 3 — Toxicity Profile (+ loss concentration)
|
||||
Section 4 — Observed Edge Attribution (事实,非策略建议)
|
||||
Section 5 — Decision + Stage3 unlock checklist
|
||||
```
|
||||
|
||||
第一份报告不期待 PASS。价值在于:市场在哪些情况下愿意付给流动性提供者溢价。
|
||||
|
||||
---
|
||||
|
||||
## Experiment ID(强制绑定)
|
||||
|
||||
每轮 Data Collection 绑定固定身份,写入每条 jsonl + 每份报告:
|
||||
|
||||
```
|
||||
Experiment: MM_EDGE_EXP_001
|
||||
Version: probe_v0.1
|
||||
Quote: frozen
|
||||
Fee: frozen
|
||||
Exchange: frozen
|
||||
```
|
||||
|
||||
环境变量:`EXPERIMENT_ID` / `PROBE_VERSION`(见 `.env.example`)。
|
||||
换实验假设时换新 ID(如 `MM_EDGE_EXP_002`),禁止在同一 ID 下改报价逻辑后重解释旧数据。
|
||||
|
||||
报告阅读顺序:**Integrity → 分布(非均值)→ Cluster → Toxicity → Decision**。
|
||||
Integrity FAIL → `Decision=INVALID`,不解释 Alpha。
|
||||
|
||||
---
|
||||
|
||||
## Post-Report Phase (2026-08-18)
|
||||
|
||||
Maker Edge Report → **PARTIAL_PASS**.
|
||||
Account wallet moved ≈ −62 USDT vs assumed 5000 start — **not** Maker Edge FAIL evidence;
|
||||
also **not** ignorable. Research markout ≠ account equity.
|
||||
|
||||
**Probe volume STOPPED** until:
|
||||
|
||||
1. Maker-only hard check: `TAKER_FILLED_COUNT == 0` (exchange `userTrades.maker`)
|
||||
2. Order→Fill→Fee→Position→Funding→Equity ledger residual ≈ 0
|
||||
|
||||
See `STATUS.md` and `scripts/reconcile_account.py`.
|
||||
|
||||
Stage3 remains **LOCKED**. Economic Edge = **UNKNOWN**.
|
||||
|
||||
---
|
||||
|
||||
## Economic Attribution Phase (2026-08-18)
|
||||
|
||||
`MM_EDGE_EXP_001 / probe_v0.1` is now a **FROZEN BASELINE**.
|
||||
|
||||
Execution state:
|
||||
|
||||
- Probe: **STOPPED**
|
||||
- Strategy modifications: **forbidden**
|
||||
- Purpose: **Economic Attribution only**
|
||||
|
||||
First hard-evidence population:
|
||||
|
||||
```
|
||||
MATCHED = 3890
|
||||
```
|
||||
|
||||
This means:
|
||||
|
||||
- Local Fill
|
||||
- Venue Trade
|
||||
- venue_trade_id
|
||||
- quantity closure
|
||||
- price verification
|
||||
- fee verification
|
||||
- maker-only verification
|
||||
|
||||
Economic Attribution v0.1 must:
|
||||
|
||||
1. Use **MATCHED only** for core conclusions
|
||||
2. Keep `VENUE_CONFIRMED_NO_TRADE_HISTORY` / `VENUE_PARTIAL_ORDER_CANCELED`
|
||||
as extension evidence, not core population
|
||||
3. Treat counterfactuals as **attribution**, not backtest / simulation
|
||||
|
||||
Current baseline conclusion:
|
||||
|
||||
> Maker markout exists, but economic edge is **not established** under v0.1.
|
||||
|
||||
Known explanation path:
|
||||
|
||||
```
|
||||
MakerAlpha
|
||||
↓
|
||||
matched fills
|
||||
↓
|
||||
fee + realized / inventory economics
|
||||
↓
|
||||
account outcome
|
||||
```
|
||||
|
||||
Next allowed work:
|
||||
|
||||
- Fee attribution
|
||||
- Inventory carry / exposure attribution
|
||||
- Counterfactual attribution (`Path C`, toxic, negative states)
|
||||
- Real-time immutable fill ledger design for future runs
|
||||
|
||||
Still forbidden:
|
||||
|
||||
- Resume v0.1 live execution
|
||||
- Change quote offset / TTL / cooldown as a shortcut
|
||||
- Unlock Stage3 from attribution alone
|
||||
|
||||
---
|
||||
|
||||
## Prefill Adverse-Selection Attribution Phase (2026-08-18)
|
||||
|
||||
Purpose: **Pre-fill adverse-selection predictability audit** — NOT strategy / backtest / optimization / model training.
|
||||
|
||||
Hard contract:
|
||||
|
||||
```
|
||||
feature_timestamp <= t_fill - 0.25s
|
||||
```
|
||||
|
||||
Features: sampled `mid_tick` / `inventory_tick` only.
|
||||
Forbidden as features: fill price, post-fill states, Path A/B/C/D, markout, future book/trade/inventory, realized PnL, cancel-after-fill.
|
||||
|
||||
Population: **MATCHED paths = 3886** (100% strict coverage)
|
||||
|
||||
Executability gate (three tiers):
|
||||
|
||||
| Grade | Meaning |
|
||||
|-------|---------|
|
||||
| `NO_PREFILL_SIGNAL` | P(C) / economic almost unchanged |
|
||||
| `STATISTICAL_SIGNAL_ONLY` | probability shift, economic improvement insufficient |
|
||||
| `CANDIDATE_V0_2_SIGNAL` | probability + economic separation — only this enters v0.2 hypothesis |
|
||||
|
||||
Sample-size policy:
|
||||
|
||||
- n < 30 → `LOW_N` (exploratory only)
|
||||
- n < 100 → `WEAK_EVIDENCE`
|
||||
- n >= 100 → `USABLE`
|
||||
|
||||
v0.1 strict conclusion:
|
||||
|
||||
> **0 / 19 features** reach `CANDIDATE_V0_2_SIGNAL`.
|
||||
> Fill 前可观测信号不足以支撑 v0.2 设计。Stage 3 remains **LOCKED**.
|
||||
|
||||
Deferred (requires richer pre-fill event log):
|
||||
|
||||
- `time_since_last_market_event`
|
||||
- intensity / large trades
|
||||
- fill-callback `market_event_before_fill`
|
||||
|
||||
Still forbidden:
|
||||
|
||||
- Treat `STATISTICAL_SIGNAL_ONLY` as v0.2 candidate
|
||||
- Resume probe or design v0.2 strategy without new experiment ID + prefill signal pass
|
||||
|
||||
---
|
||||
|
||||
## MM_EDGE_EXP_002 — Event-State Observability (2026-08-18)
|
||||
|
||||
**Type:** Data Collection / Observability Experiment
|
||||
**NOT:** strategy experiment, backtest, optimization, model training
|
||||
|
||||
```
|
||||
MM_EDGE_EXP_001 → phenomenon PASS, economic FAIL, prefill FAIL (snapshot)
|
||||
MM_EDGE_EXP_002 → close observability gap (immutable event ledger)
|
||||
→ Gate 4 only after Gates 1–3 + fill anchors
|
||||
→ (only then) v0.2 hypothesis allowed
|
||||
```
|
||||
|
||||
Identity:
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| Experiment | MM_EDGE_EXP_002 |
|
||||
| Probe | event_state_v0.1 |
|
||||
| Strategy | NONE |
|
||||
| Trading | NO |
|
||||
| Stage 3 | LOCKED |
|
||||
| Depends on | MM_EDGE_EXP_001 FROZEN |
|
||||
|
||||
Core change: **Immutable Event Ledger** — raw events first, features offline later.
|
||||
|
||||
Pre-fill window schema (frozen):
|
||||
|
||||
```
|
||||
[-5s, fill - 250ms) → all market_event rows
|
||||
fill_anchor → immutable fill metadata (Gate 4; Phase 1 may have none)
|
||||
```
|
||||
|
||||
Success gates:
|
||||
|
||||
1. **Event Completeness** — PASS (smoke)
|
||||
2. **Temporal Integrity** — PASS (smoke)
|
||||
3. **Event Coverage** — PASS (smoke)
|
||||
4. **Predictability** — BLOCKED until frozen sample gates in `MM_EDGE_EXP_002.md`
|
||||
|
||||
Long-run: `EXP-002-RUN-002` / `event-state-probe.service` / trading=NO.
|
||||
|
||||
Frozen before long-run (do not change after seeing more data):
|
||||
|
||||
- Phase 1 dataset: ≥ 7 days AND ≥ 5,000,000 market events
|
||||
- Gate 4: ≥ 2,000 MATCHED fill_anchors AND ≥ 500 clusters (requires later fill-authorized phase)
|
||||
|
||||
No daily Path C analysis during collection.
|
||||
|
||||
Still forbidden:
|
||||
|
||||
- Resume EXP_001 or modify v0.1 quote logic
|
||||
- Enable trading under EXP_002
|
||||
- Unlock Stage 3 from observability data alone
|
||||
- Peek at Gate 4 before the frozen sample threshold
|
||||
|
||||
---
|
||||
|
||||
## MM_EDGE_EXP_002 Phase 1 STOPPED (2026-09-10)
|
||||
|
||||
Human decision: **stop collection**. Do not open a fill-authorized phase. Do not design v0.2.
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| Run | `EXP-002-RUN-002` |
|
||||
| Stopped | **2026-09-10T08:45:24Z** |
|
||||
| systemd | user `event-state-probe` **inactive / disabled** |
|
||||
| Ledger | `logs/event_state/EXP-002-RUN-002/` (24 files, 18.06 GiB, 2026-08-18 → 2026-09-10 partial) |
|
||||
| Trading | **NO** |
|
||||
| Stage 3 | **LOCKED** |
|
||||
| Gate 4 | **BLOCKED** (no `fill_anchor`) |
|
||||
| Hypothesis Closure | **NO** |
|
||||
|
||||
Phase 1 volume thresholds are met as an **observability freeze** only.
|
||||
That does **not** unlock Gate 4 or Stage 3.
|
||||
|
||||
Marker: `logs/event_state/EXP-002-RUN-002/EXP-002-RUN-002.PHASE1_STOPPED.json`
|
||||
|
||||
Still forbidden:
|
||||
|
||||
- Restart `EXP-002-RUN-002`
|
||||
- Enable trading under EXP_002
|
||||
- Path-C / Gate 4 peek on this artifact
|
||||
- Treat this stop as “no pre-fill signal exists”
|
||||
@@ -0,0 +1,214 @@
|
||||
# MM_EDGE_EXP_002 — Event-State Observability Probe
|
||||
|
||||
## Experiment Identity
|
||||
|
||||
```
|
||||
Experiment: MM_EDGE_EXP_002
|
||||
Type: Data Collection / Observability Experiment
|
||||
Strategy: NONE
|
||||
Execution: STOPPED (no trading)
|
||||
Purpose: Capture immutable pre-fill Event State
|
||||
Dependency: MM_EDGE_EXP_001 / v0.1 FROZEN
|
||||
Stage 3: LOCKED
|
||||
```
|
||||
|
||||
**NOT:** strategy experiment, backtest, optimization, model training, v0.2 design.
|
||||
|
||||
---
|
||||
|
||||
## Why EXP_002 Exists
|
||||
|
||||
EXP_001 Prefill Audit conclusion:
|
||||
|
||||
> Under **current snapshot observability** (~1.66s sampled mid/inventory state), no sufficient pre-fill signal was found.
|
||||
|
||||
This must **not** be interpreted as:
|
||||
|
||||
> The market has no pre-fill adverse-selection information.
|
||||
|
||||
EXP_001 strategy is **event-driven**, but observability was **snapshot-driven**. Information between snapshots (e.g. aggressive sweep 100ms before fill) is lost.
|
||||
|
||||
EXP_002 closes the **observability gap**, not the **strategy gap**.
|
||||
|
||||
---
|
||||
|
||||
## Core Design Principle
|
||||
|
||||
```
|
||||
Raw Event > Derived Feature
|
||||
```
|
||||
|
||||
Store immutable events. Features are computed offline later.
|
||||
|
||||
---
|
||||
|
||||
## Immutable Event Ledger
|
||||
|
||||
Each market event records (minimum):
|
||||
|
||||
| Field | Description |
|
||||
|-------|-------------|
|
||||
| `exchange_ts_ns` | Exchange event time |
|
||||
| `local_ts_epoch` / `local_ts` | Local receive time |
|
||||
| `event_type` | `aggressive_trade`, `book_update`, … |
|
||||
| `best_bid` / `best_ask` / `mid` / `spread` | Top-of-book |
|
||||
| `bid_depth_*` / `ask_depth_*` | Depth levels |
|
||||
| `*_delta` | Depth / spread / mid changes |
|
||||
| `time_since_last_*` | Event timing state |
|
||||
|
||||
Fill anchor schema (frozen, for Gate 4 when fills exist):
|
||||
|
||||
```
|
||||
[-5s, fill - 250ms) → all market_event rows
|
||||
fill_anchor → immutable fill metadata
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Event Categories (Priority)
|
||||
|
||||
1. **Aggressive Trade** — side, qty, notional, large_trade_flag, intensity proxy
|
||||
2. **Book Depletion** — depth deltas, level removal velocity
|
||||
3. **Spread / TOB Event** — spread change, bid/ask/mid move
|
||||
4. **Event Timing** — time_since_last_trade / large_trade / depth_change / spread_change
|
||||
|
||||
---
|
||||
|
||||
## Success Gates (frozen before collection)
|
||||
|
||||
### Gate 1 — Event Completeness
|
||||
|
||||
For each `fill_anchor`:
|
||||
|
||||
```
|
||||
100% reconstructable pre-fill event sequence in [-5s, fill - margin)
|
||||
```
|
||||
|
||||
Phase 1 (observability-only): **N/A** until fill anchors exist.
|
||||
|
||||
### Gate 2 — Temporal Integrity
|
||||
|
||||
```
|
||||
all event_ts < fill_ts
|
||||
feature_cutoff = fill_ts - 250ms
|
||||
```
|
||||
|
||||
### Gate 3 — Event Coverage
|
||||
|
||||
| Metric | Threshold |
|
||||
|--------|-----------|
|
||||
| trade events present | ≥ 99% of sessions with trades |
|
||||
| book events present | ≥ 99% of sessions with book updates |
|
||||
| timestamp valid | ≥ 99% rows with exchange_ts_ns or local_ts |
|
||||
|
||||
### Gate 4 — Predictability (BLOCKED until sample freeze + fill anchors)
|
||||
|
||||
Do **not** inspect Path C daily during collection (researcher degrees of freedom).
|
||||
|
||||
Frozen sample thresholds (**set 2026-08-18, before long-run start**):
|
||||
|
||||
**Phase 1 Event-State dataset freeze** (observability-only, no fills):
|
||||
|
||||
| Metric | Minimum |
|
||||
|--------|---------|
|
||||
| Calendar span | **≥ 7 days** |
|
||||
| `market_event` count | **≥ 5,000,000** |
|
||||
| `aggressive_trade` | **≥ 1,500,000** |
|
||||
| `book_update` | **≥ 3,000,000** |
|
||||
| timestamp valid | **≥ 99%** |
|
||||
| parse_fail_lines | **0** |
|
||||
|
||||
Reaching this freeze **does not** unlock Gate 4. It only freezes the Event-State stream as a research artifact.
|
||||
|
||||
**Gate 4 (requires a later fill-authorized phase, not this systemd job):**
|
||||
|
||||
| Metric | Minimum |
|
||||
|--------|---------|
|
||||
| MATCHED `fill_anchor` | **≥ 2,000** |
|
||||
| `event_cluster_id` | **≥ 500** |
|
||||
| reconstructable `[-5s, fill−250ms)` | **100%** of MATCHED anchors |
|
||||
| venue_trade_id + exchange_ts_ns on fill | **100%** |
|
||||
|
||||
Then, **once**:
|
||||
|
||||
```
|
||||
Event State → P(Path C) → Economic separation
|
||||
→ CANDIDATE_V0_2_SIGNAL only if both probability and economic gates pass
|
||||
```
|
||||
|
||||
Future fill collection (if ever authorized) **must** write:
|
||||
|
||||
```
|
||||
Fill → venue_trade_id → exchange_ts_ns → Event Ledger → [-5s, fill_ts)
|
||||
```
|
||||
|
||||
Do not resume snapshot-only fills.
|
||||
|
||||
Only **CANDIDATE_V0_2_SIGNAL** after Gate 4 may enter v0.2 hypothesis design.
|
||||
|
||||
---
|
||||
|
||||
## Research Chain
|
||||
|
||||
```
|
||||
EXP_001 Maker Edge Phenomenon
|
||||
↓
|
||||
conditional markout exists
|
||||
↓
|
||||
Economic FAIL
|
||||
↓
|
||||
Prefill audit (snapshot) → NO SIGNAL
|
||||
↓
|
||||
EXP_002 Event-State Observability
|
||||
↓
|
||||
Gate 1–3 PASS?
|
||||
↓
|
||||
Gate 4 predictability
|
||||
↓
|
||||
(only then) v0.2 hypothesis
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Running
|
||||
|
||||
```bash
|
||||
cd nautilus_mm
|
||||
cp .env.example .env # set EXP_002 block
|
||||
export PYTHONPATH=src
|
||||
./scripts/run_event_state.sh
|
||||
```
|
||||
|
||||
Long-run (systemd, trading=NO):
|
||||
|
||||
```
|
||||
LEDGER_RUN_ID=EXP-002-RUN-002
|
||||
logs/event_state/EXP-002-RUN-002/
|
||||
```
|
||||
|
||||
```bash
|
||||
systemctl --user start event-state-probe
|
||||
./scripts/event_state_status.sh # counts only — not Path C analysis
|
||||
```
|
||||
|
||||
**Phase 1 STOPPED 2026-09-10T08:45:24Z** — user unit `event-state-probe` is **disabled**.
|
||||
Do not start it again on `EXP-002-RUN-002`. Ledger is a frozen observability artifact.
|
||||
|
||||
---
|
||||
|
||||
## Forbidden
|
||||
|
||||
- Resume EXP_001 probe or modify v0.1 quote logic
|
||||
- Restart EXP_002 / `EXP-002-RUN-002` collection
|
||||
- Enable trading under EXP_002
|
||||
- Unlock Stage 3 from observability data alone
|
||||
- Treat weak EXP_001 statistical signals as v0.2 filters
|
||||
- Claim Hypothesis Closure or design v0.2 without a later fill-authorized Gate 4
|
||||
|
||||
---
|
||||
|
||||
## Hypothesis Closure (if Gate 4 also fails)
|
||||
|
||||
> Conditional Maker phenomenon exists, but is not sufficiently predictable pre-fill to be monetizable under this venue/execution model.
|
||||
|
||||
That would be a **strong Research FAIL / Hypothesis Closure** — not "try one more parameter."
|
||||
@@ -0,0 +1,158 @@
|
||||
# nautilus_mm — Research Freeze / Data Collection Phase
|
||||
|
||||
Freqtrade 保留缠论 / 中低频;**Maker / L2 / Fill 事件**迁到 NautilusTrader。
|
||||
|
||||
**研究对象:可验证的市场现象(不是策略)。**
|
||||
|
||||
```
|
||||
Trading OS
|
||||
├─ Market Intelligence (Market Pulse) ← Stage5 才接(Quote Adjustment)
|
||||
├─ Execution Reality Layer (本仓库) ← Fill Alpha Dataset
|
||||
├─ Freqtrade ← Chan / 中低频
|
||||
└─ NautilusTrader ← 事件驱动执行
|
||||
```
|
||||
|
||||
## 冻结研究路径(禁止跳级)
|
||||
|
||||
```
|
||||
Stage 0 Data Integrity
|
||||
↓
|
||||
Stage 1 Fill Alpha ← 当前
|
||||
↓
|
||||
Stage 2 Maker Edge Report ← 当前(决策门)
|
||||
↓
|
||||
Stage 3 Economic Simulation ← LOCKED until Edge PASS
|
||||
↓
|
||||
Stage 4 Quote Engine
|
||||
↓
|
||||
Stage 5 Market Regime Adaptation (Market Pulse → Quote Adjustment)
|
||||
```
|
||||
|
||||
| Stage | 目标 | 状态 |
|
||||
|-------|------|------|
|
||||
| **0** | WS/L2 可信:seq gap / latency / book_age | 探针已记 |
|
||||
| **1** | Fill Alpha Dataset:真实成交 + 路径 | **进行中** |
|
||||
| **2** | Maker Edge 决策门:PASS / FAIL / COLLECTING | **进行中** |
|
||||
| **3** | 经济仿真:quote→fill→inventory→exit(partial/cancel/funding/fee) | **未解锁** |
|
||||
| **4** | Quote Engine | 未开始 |
|
||||
| **5** | Regime Adaptation | 未开始 |
|
||||
|
||||
原则:**先证明成交有优势,再谈账户收益,再设计报价。**
|
||||
禁止现在加:Quote Engine / AI / Market Pulse 交易信号 / 参数优化。
|
||||
|
||||
### 核心问题(交给 ~2000 真实 fills)
|
||||
|
||||
> 个人开发者在 BTC 永续上,通过被动流动性提供,是否能获得统计优势?
|
||||
> 若有:优势来自哪里?
|
||||
|
||||
可能结果:
|
||||
|
||||
| 情况 | 含义 | 下一步 |
|
||||
|------|------|--------|
|
||||
| **A** 全市场 PASS | 稳定被动流动性优势 | Symmetric MM |
|
||||
| **B** 仅特定状态 PASS | 高波动 / 吸收 / 震荡等 | Event-driven LP(更可能) |
|
||||
| **C** 全部 FAIL | 普通 Maker edge 不存在 | Cash Carry / Basis / Funding / 跨所 |
|
||||
|
||||
**最值得等待的不是 PASS,而是 Edge 来自哪里。** 第一份 FAIL 也是高价值结果。
|
||||
|
||||
### 防自我欺骗(已内建)
|
||||
|
||||
1. **Cluster-weighted** — 暴跌 50 笔 Bid ≠ 50 独立样本;同时看 fill-w 与 cluster-w,方向一致才可信
|
||||
2. **Matched Mid / Maker Alpha** — `MakerAlpha = Fill Outcome − Market Move`(剥离方向收益)
|
||||
3. **Adverse Selection** — 成交是否天然站在错误一侧;spread capture 挡不住毒流
|
||||
|
||||
Stage 3(Economic Simulation)只在 Edge PASS 后做:partial fill、cancel latency、inventory limit、position aging、funding、fee tier → 真实账户收益分布。
|
||||
|
||||
## Maker Edge Report v0.1
|
||||
|
||||
```bash
|
||||
./scripts/analyze.sh # 默认 min-fills=2000
|
||||
./scripts/analyze.sh 500 # 早期预览(仍为 COLLECTING)
|
||||
```
|
||||
|
||||
报告结构:Data Integrity → Sample Independence → Fill Quality + Benchmark → Adverse Selection(`POSITIVE_EDGE` / `EDGE_AFTER_COST` / `NO_EDGE`)→ Path Attribution → State Stability → Decision。
|
||||
|
||||
## 快速开始
|
||||
|
||||
```bash
|
||||
cd nautilus_mm
|
||||
python3 -m venv .venv && source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
cp .env.example .env # TESTNET key;代理默认 7897
|
||||
|
||||
./scripts/run_probe.sh # 攒真实 fills
|
||||
./scripts/analyze.sh 2000 # 决策门报告
|
||||
```
|
||||
|
||||
探针只记 quote / fill / outcome。`market_state_snapshot` 预留为 null。
|
||||
|
||||
## 目录
|
||||
|
||||
```
|
||||
nautilus_mm/
|
||||
├── configs/
|
||||
├── logs/maker_edge/ # Fill Alpha Dataset (jsonl)
|
||||
├── scripts/
|
||||
│ ├── run_probe.sh
|
||||
│ ├── analyze.sh
|
||||
│ ├── analyze_maker_edge.py
|
||||
│ └── record_l2_ccxt.py
|
||||
└── src/nautilus_mm/
|
||||
├── recorder.py
|
||||
├── health.py
|
||||
├── book_utils.py
|
||||
├── run_live.py
|
||||
└── strategies/maker_edge_probe.py
|
||||
```
|
||||
|
||||
## 与 Freqtrade
|
||||
|
||||
| | Freqtrade | nautilus_mm |
|
||||
|--|-----------|-------------|
|
||||
| 用途 | 缠论 / 中低频 | Maker / L2 / Edge 验证 |
|
||||
| 驱动 | K 线 | order book / fill 事件 |
|
||||
|
||||
## 安全
|
||||
|
||||
- 默认 `BINANCE_ENVIRONMENT=TESTNET`
|
||||
- `ENABLE_TRADING=false` 可只订数据
|
||||
- **独立 `.venv`**,勿与 freqtrade 混装
|
||||
|
||||
## 实验冻结(见 [FREEZE.md](FREEZE.md))
|
||||
|
||||
**三不动:** Quote Logic / 成本模型 / PASS·COLLECTING·FAIL
|
||||
|
||||
| ✅ 冻结期允许 | ❌ Gate 解锁前禁止 |
|
||||
|-------------|-------------------|
|
||||
| 数据字段 | 新交易规则 |
|
||||
| 数据质量检查 | 新过滤条件 |
|
||||
| 报告解释能力 | 新收益优化参数 |
|
||||
|
||||
Decision:`PASS` / `PARTIAL_PASS` / `COLLECTING` / `FAIL`
|
||||
观察窗:500 异常 · 2000 初步 · 10000 稳定性;**clusters > fills**。
|
||||
第一份报告先看分布(Fill / Cluster / Toxicity),再看 Decision。
|
||||
|
||||
## 服务器 Data Collection(zun_hk)
|
||||
|
||||
默认:
|
||||
|
||||
| | |
|
||||
|--|--|
|
||||
| Host | `jack@jackyu66.com` |
|
||||
| Key | `~/Project/deploy/zun_hk/id_ed25519_hk` |
|
||||
| Dir | `/www/Project/nautilus_mm` |
|
||||
| Experiment | `MM_EDGE_EXP_001` |
|
||||
|
||||
```bash
|
||||
./scripts/deploy_server.sh
|
||||
|
||||
ssh -i ~/Project/deploy/zun_hk/id_ed25519_hk jack@jackyu66.com
|
||||
nano /www/Project/nautilus_mm/.env # BINANCE_API_KEY / SECRET
|
||||
systemctl --user start mm-edge-probe # 用户级 systemd(无需 sudo)
|
||||
journalctl --user -u mm-edge-probe -f
|
||||
|
||||
./scripts/probe_status.sh
|
||||
./scripts/pull_report.sh 2000
|
||||
```
|
||||
|
||||
本地短测:`USE_PROXY=true ./scripts/run_probe.sh`
|
||||
@@ -0,0 +1,205 @@
|
||||
# Project Status
|
||||
|
||||
## MM_EDGE_EXP_001 — CLOSED (FROZEN)
|
||||
|
||||
| Dimension | Status |
|
||||
|-----------|--------|
|
||||
| Research Phenomenon | **PASS** (conditional markout exists) |
|
||||
| Economic Edge | **FAIL** |
|
||||
| Prefill Predictability | **FAIL** (snapshot observability) |
|
||||
| Strategy | **FROZEN** |
|
||||
| Execution | **STOPPED** |
|
||||
| Stage 3 | **LOCKED** |
|
||||
|
||||
EXP_001 Prefill strict conclusion:
|
||||
|
||||
> Under current **snapshot observability** (~1.66s), no CANDIDATE_V0_2_SIGNAL.
|
||||
> This is **NOT** proof that pre-fill signal does not exist in the market.
|
||||
|
||||
Weak signals (research observations only — **not** trading filters):
|
||||
|
||||
- `depth_total_5`: ±3.8pp Path C separation, Economic gate FAIL
|
||||
- `obi_change_5s`: +0.00027 USDT/fill economic Δ
|
||||
- `pre_deteriorated_strict=False`: +0.00116 USDT/fill economic Δ
|
||||
|
||||
Reports: `logs/maker_edge/` — see sections below for detail.
|
||||
|
||||
---
|
||||
|
||||
## MM_EDGE_EXP_002 — Phase 1 STOPPED (artifact FROZEN)
|
||||
|
||||
| Dimension | Status |
|
||||
|-----------|--------|
|
||||
| Type | Data Collection / Observability |
|
||||
| Strategy | **NONE** |
|
||||
| Trading | **NO** |
|
||||
| Execution | **STOPPED** 2026-09-10T08:45:24Z |
|
||||
| Stage 3 | **LOCKED** |
|
||||
| Gate 1 | **PASS** (smoke `EXP-002-RUN-001`) |
|
||||
| Gate 2 | **PASS** |
|
||||
| Gate 3 | **PASS** |
|
||||
| Gate 4 | **BLOCKED** (no `fill_anchor`; not opened) |
|
||||
| Long-run | **EXP-002-RUN-002** STOPPED + systemd **disabled** |
|
||||
| Hypothesis Closure | **NO** — Gate 4 never ran |
|
||||
|
||||
Human decision 2026-09-10: stop further collection. Phase 1 is an observability artifact only.
|
||||
This is **not** “market has no pre-fill signal” and **not** a v0.2 unlock.
|
||||
|
||||
`event-state-probe.service` (user): **inactive / disabled**. Do not restart `EXP-002-RUN-002`.
|
||||
|
||||
Ledger: `logs/event_state/EXP-002-RUN-002/` — 24 files, 2026-08-18 → 2026-09-10 (last day partial), **18.06 GiB**.
|
||||
Event counts not fully censused; size-based estimate ~1.2e7 lines (volume gates clearly exceeded).
|
||||
`fill_anchor` expected **0** (observability-only). Marker: `EXP-002-RUN-002.PHASE1_STOPPED.json`
|
||||
|
||||
Do **not** Path-C snoop this artifact. Frozen Gate 4 sample gates remain in `MM_EDGE_EXP_002.md`.
|
||||
|
||||
Spec: `MM_EDGE_EXP_002.md`
|
||||
Smoke: `./scripts/smoke_test_event_state.sh`
|
||||
Validate: `python scripts/validate_event_ledger.py --dir logs/event_state/EXP-002-RUN-001 --run-id EXP-002-RUN-001`
|
||||
Logs: `logs/event_state/EXP-002-RUN-001/`
|
||||
|
||||
### EXP-002-RUN-001 Smoke (2026-08-18, ~12 min + restart)
|
||||
|
||||
Host: `jADUtR1041803` | Sessions: **2** (restart test) | Schema: `immutable_event_v1`
|
||||
|
||||
| Check | Result |
|
||||
|-------|--------|
|
||||
| Gate 1 Event Completeness | **PASS** (Phase 1 stream; no fill_anchor) |
|
||||
| Gate 2 Temporal Integrity | **PASS** |
|
||||
| Gate 3 Event Coverage | **PASS** |
|
||||
| Restart contract | **PASS** (0 parse fail, 0 dup event_id, seq reset) |
|
||||
| Gate 4 | **BLOCKED** |
|
||||
|
||||
Write rates:
|
||||
- `aggressive_trade`: **3.21 / sec** (2300 events)
|
||||
- `book_update`: **6.09 / sec** (4366 events)
|
||||
- **total**: **9.30 / sec** (6666 market events)
|
||||
|
||||
Timestamp quality (100% exchange + local present):
|
||||
- `local − exchange` lag: p50 **112.5ms**, p95 **237.7ms**, p99 **251.4ms**, max **443.4ms**
|
||||
- `exchange > local + 50ms`: **0** violations
|
||||
|
||||
Event order (recorded, not sorted):
|
||||
- `exchange_ts_ns` regressions: **1053** (max back **276ms**) — multi-source async; explicit in report
|
||||
|
||||
Raw schema sample (n=200 each): **0% missing** on core fields; **0** hollow book_update.
|
||||
|
||||
Manifest: `logs/event_state/EXP-002-RUN-001/EXP-002-RUN-001.manifest.json`
|
||||
Report: `logs/event_state/EXP-002-RUN-001/Event_Ledger_Validation.json`
|
||||
|
||||
Long-run identity: **EXP-002-RUN-002** (separate from smoke) — **STOPPED** 2026-09-10. Status: `./scripts/event_state_status.sh`
|
||||
|
||||
---
|
||||
|
||||
## MM_EDGE_EXP_001 Detail
|
||||
|
||||
**Phase: Prefill Adverse-Selection Attribution v0.1** (probe **STOPPED**)
|
||||
|
||||
| Gate | Status |
|
||||
|------|--------|
|
||||
| Research Freeze | **ACTIVE** |
|
||||
| Probe | **STOPPED** |
|
||||
| Data Integrity | **PASS** |
|
||||
| Maker-only | **PASS** (`TAKER=0`) |
|
||||
| Account Reconciliation (RECON-01) | **PASS** |
|
||||
| RECON-02 classification | **PASS** |
|
||||
| Order-level closure (RECON-03) | **PASS** |
|
||||
| Strict trade-level closure | **FAIL** (Testnet userTrades cutoff — **not** Alpha FAIL) |
|
||||
| Economic Edge | **FAIL** (current execution economics) |
|
||||
| Prefill Predictability (v0.1 strict) | **FAIL** (no CANDIDATE_V0_2_SIGNAL) |
|
||||
| Stage 3 | **LOCKED** |
|
||||
|
||||
## Evidence taxonomy (4777 fills)
|
||||
|
||||
| Class | Count | Grade |
|
||||
|-------|-------|-------|
|
||||
| **MATCHED** | 3890 | Order + Trade (dual) |
|
||||
| **VENUE_CONFIRMED_NO_TRADE_HISTORY** | 876 | Order FILLED, no trade row |
|
||||
| **VENUE_PARTIAL_ORDER_CANCELED** | 11 | Partial fill + TTL cancel |
|
||||
| DUPLICATE / MISMATCH / UNCONFIRMED | 0 | — |
|
||||
|
||||
`userTrades` cutoff: **2026-08-17T03:08 UTC** — see `TESTNET_LIMITATIONS.md`
|
||||
|
||||
Matched trade-level: qty residual **0**, |Δt| p50 **72ms**
|
||||
|
||||
## Separation
|
||||
|
||||
```
|
||||
MakerAlpha (+0.008%) ≠ Account Δ (−61.66 = fee + realized)
|
||||
Strict trade FAIL ≠ Maker Edge FAIL
|
||||
887 orphans = VENUE-HISTORY-CUTOFF (now order-confirmed)
|
||||
```
|
||||
|
||||
## Economic Attribution v0.1 (Hard Evidence Population only)
|
||||
|
||||
Population: **MATCHED=3890**
|
||||
Core report: `logs/maker_edge/Economic_Attribution_v0_1.txt`
|
||||
|
||||
- Matched fills / paths / clusters: **3890 / 3886 / 3334**
|
||||
- Fee total: **42.42 USDT**
|
||||
- Realized component: **−8.52 USDT**
|
||||
- Gross markout @30s: **−1.73 USDT**
|
||||
- Net attributable @30s: **−52.67 USDT**
|
||||
- Inventory carry:
|
||||
- Max `|net BTC|`: **0.0059**
|
||||
- TW `|net BTC|`: **0.0035**
|
||||
- Turnover: **3.3316 BTC**
|
||||
|
||||
Counterfactual Attribution:
|
||||
- Baseline matched net30: **−52.67 USDT**
|
||||
- Exclude `Path C`: **−21.22 USDT**
|
||||
- Exclude toxic: **−21.22 USDT**
|
||||
- Exclude negative states: **−27.64 USDT**
|
||||
|
||||
Interpretation:
|
||||
- v0.1 已完成 baseline 使命:**Maker markout phenomenon exists, economic edge not established**
|
||||
- 当前主要拖累不是“假 alpha”,而是 **fee + realized / inventory economics**
|
||||
|
||||
## Metric Reconciliation (MATCHED=3890)
|
||||
- Return-space MakerAlpha(fill-weighted):-0.000693%
|
||||
- Return-space MakerAlpha(notional-weighted):-0.000816%
|
||||
- Dollar-space gross markout @30s:-1.730780 USDT
|
||||
结论:回报口径一致,但“加权方式”导致返回与美元金额的方向差异。
|
||||
|
||||
## Fee Sensitivity (counterfactual, fee only)
|
||||
net_attr_30s @fee_factor:
|
||||
1.00 → -52.674655 USDT
|
||||
0.50 → -31.463582 USDT
|
||||
0.25 → -20.858046 USDT
|
||||
0.10 → -14.494724 USDT
|
||||
0.00 → -10.252510 USDT
|
||||
结论:即使假设 0 fee,net 仍 < 0,因此“真实拖累”不仅是 fee。
|
||||
|
||||
## Prefill Adverse-Selection Attribution v0.1 (strict contract)
|
||||
|
||||
Population: **MATCHED paths = 3886** (100% strict-prefill coverage)
|
||||
Core report: `logs/maker_edge/Prefill_Adverse_Selection_Attribution_v0_1.txt`
|
||||
|
||||
Time contract: `feature_timestamp <= t_fill - 0.25s`
|
||||
Feature source: sampled `mid_tick` / `inventory_tick` only (no fill-callback leakage)
|
||||
|
||||
Baseline (strict population):
|
||||
- P(Path C): **33.20%**
|
||||
- P(Toxic): **34.61%**
|
||||
- P(Neg30s): **47.43%**
|
||||
- P(Economic<0): **77.61%**
|
||||
- Mean net_attr_30s: **−0.0136 USDT/fill**
|
||||
- Feature age: mean **1681ms**, median **1660ms**
|
||||
|
||||
Conclusion Matrix (auto-grade):
|
||||
- **CANDIDATE_V0_2_SIGNAL**: **0 / 19 features**
|
||||
- **STATISTICAL_SIGNAL_ONLY**: spread_change_5s (Toxic/Neg30s), inventory/inventory_skew (Economic only)
|
||||
- **NO_PREFILL_SIGNAL**: all others under strict gate
|
||||
|
||||
Unavailable under strict contract (deferred to EXP_002):
|
||||
- `time_since_last_market_event`, intensity, large trades, fill-callback `market_event_before_fill`
|
||||
|
||||
## Forbidden (both experiments)
|
||||
|
||||
- Resume EXP_001 probe without new experiment ID
|
||||
- Restart EXP_002 / `EXP-002-RUN-002` collection
|
||||
- Enable trading under EXP_002
|
||||
- Reclassify VENUE_CONFIRMED as MATCHED
|
||||
- Unlock Stage 3 on observability or attribution alone
|
||||
- Treat STATISTICAL_SIGNAL_ONLY as v0.2 candidate
|
||||
- Design v0.2 or claim Hypothesis Closure without a later fill-authorized Gate 4
|
||||
@@ -0,0 +1,55 @@
|
||||
# Testnet Limitations — MM_EDGE_EXP_001
|
||||
|
||||
Observed on **Binance USDT-M Futures Testnet** during Data Collection (Aug 2026).
|
||||
|
||||
## userTrades history cutoff
|
||||
|
||||
```
|
||||
Endpoint: GET /fapi/v1/userTrades
|
||||
Observed max trade time (UTC): 2026-08-17T03:08:23
|
||||
Behavior: Pagination returns 3890 rows; no further trades via time/fromId
|
||||
after cutoff, even while probe continues to produce fills until
|
||||
2026-08-18.
|
||||
```
|
||||
|
||||
**Impact:** Local jsonl fill count can exceed paginated `userTrades` count.
|
||||
This is **not** evidence of duplicate local logging or fake fills.
|
||||
|
||||
## Order API remains available
|
||||
|
||||
```
|
||||
Endpoint: GET /fapi/v1/order?orderId=
|
||||
Behavior: Post-cutoff orders return status=FILLED, executedQty, avgPrice
|
||||
while userTrades?orderId= returns 0 rows for the same orderId.
|
||||
```
|
||||
|
||||
RECON-03 classifies these as:
|
||||
|
||||
```
|
||||
VENUE_CONFIRMED_NO_TRADE_HISTORY
|
||||
```
|
||||
|
||||
Evidence grade: **Order only** (not dual Order+Trade).
|
||||
|
||||
## Income ledger continues
|
||||
|
||||
`GET /fapi/v1/income` continues to record COMMISSION / REALIZED_PNL after
|
||||
the userTrades cutoff. Account reconciliation (RECON-01) uses income, not
|
||||
userTrades alone.
|
||||
|
||||
## Implications for future runs
|
||||
|
||||
1. **Real-time immutable ledger** — persist on every `OrderFilled`:
|
||||
`venue_trade_id`, `venue_order_id`, `liquidity_side`, `commission`,
|
||||
`exchange_ts`, `local_ts`. Do not rely on post-hoc userTrades backfill.
|
||||
|
||||
2. **Reports must use evidence taxonomy** — never compare raw fill count to
|
||||
userTrades count without cutoff annotation.
|
||||
|
||||
3. **Strict trade-level closure** may remain FAIL on Testnet while
|
||||
**order-level closure** can still PASS.
|
||||
|
||||
## Maker-only constraint
|
||||
|
||||
Post-only orders rejected with `-5022` when they would take. Verified:
|
||||
`TAKER_FILLED_COUNT = 0` on all 3890 trades inside userTrades window.
|
||||
@@ -0,0 +1,31 @@
|
||||
# MM_EDGE_EXP_002 — Event-State Observability Probe
|
||||
# Data collection only. NO trading.
|
||||
|
||||
experiment_id: MM_EDGE_EXP_002
|
||||
probe_version: event_state_v0.1
|
||||
experiment_type: Event-State Observability Probe
|
||||
strategy: NONE
|
||||
execution_trading: false
|
||||
depends_on: MM_EDGE_EXP_001
|
||||
|
||||
quote: frozen
|
||||
fee: frozen
|
||||
exchange: frozen
|
||||
|
||||
trader_id: EVENT-STATE-002
|
||||
symbol: BTCUSDT-PERP
|
||||
account_type: USDT_FUTURES
|
||||
environment: TESTNET
|
||||
|
||||
book_depth: 10
|
||||
prefill_window_sec: 5.0
|
||||
prefill_margin_sec: 0.25
|
||||
large_trade_qty: 0.1
|
||||
log_every_book_delta: true
|
||||
|
||||
log_dir: logs/event_state
|
||||
|
||||
# Gate 3 thresholds (frozen)
|
||||
gate3_trade_coverage_min: 0.99
|
||||
gate3_book_coverage_min: 0.99
|
||||
gate3_timestamp_valid_min: 0.99
|
||||
@@ -0,0 +1,24 @@
|
||||
# Reference config (env vars in .env take precedence for secrets)
|
||||
# Experiment identity — do not change mid-run to "make report look better"
|
||||
|
||||
experiment_id: MM_EDGE_EXP_001
|
||||
probe_version: probe_v0.1
|
||||
quote: frozen
|
||||
fee: frozen
|
||||
exchange: frozen
|
||||
|
||||
trader_id: MAKER-EDGE-001
|
||||
symbol: BTCUSDT-PERP
|
||||
account_type: USDT_FUTURES
|
||||
environment: TESTNET # TESTNET | LIVE
|
||||
|
||||
order_qty: "0.001"
|
||||
book_depth: 10
|
||||
quote_offset_ticks: 1
|
||||
max_quotes: 1
|
||||
cooldown_secs: 60
|
||||
obi_enter: 0.25
|
||||
enable_trading: true
|
||||
|
||||
proxy: http://127.0.0.1:7897
|
||||
log_dir: logs/maker_edge
|
||||
@@ -0,0 +1,32 @@
|
||||
[Unit]
|
||||
Description=MM_EDGE_EXP_002 Event-State Observability Probe (no trading)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=jack
|
||||
Group=jack
|
||||
WorkingDirectory=/www/Project/nautilus_mm
|
||||
Environment=PYTHONPATH=/www/Project/nautilus_mm/src
|
||||
EnvironmentFile=-/www/Project/nautilus_mm/.env
|
||||
Environment=ENABLE_TRADING=false
|
||||
Environment=EXPERIMENT_ID=MM_EDGE_EXP_002
|
||||
Environment=PROBE_VERSION=event_state_v0.1
|
||||
Environment=LEDGER_RUN_ID=EXP-002-RUN-002
|
||||
Environment=EVENT_STATE_LOG_DIR=/www/Project/nautilus_mm/logs/event_state/EXP-002-RUN-002
|
||||
Environment=HTTP_PROXY=
|
||||
Environment=HTTPS_PROXY=
|
||||
Environment=http_proxy=
|
||||
Environment=https_proxy=
|
||||
ExecStart=/www/Project/nautilus_mm/scripts/run_event_state.sh
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=event-state-probe
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,30 @@
|
||||
# 用户级 systemd — ~/.config/systemd/user/event-state-probe.service
|
||||
# MM_EDGE_EXP_002 Phase 1 long-term Event-State collection. Trading = NO.
|
||||
[Unit]
|
||||
Description=MM_EDGE_EXP_002 Event-State Observability Probe (no trading)
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/www/Project/nautilus_mm
|
||||
Environment=PYTHONPATH=/www/Project/nautilus_mm/src
|
||||
EnvironmentFile=-/www/Project/nautilus_mm/.env
|
||||
# Layer 0 (systemd): override .env — EXP_002 never trades
|
||||
Environment=ENABLE_TRADING=false
|
||||
Environment=EXPERIMENT_ID=MM_EDGE_EXP_002
|
||||
Environment=PROBE_VERSION=event_state_v0.1
|
||||
Environment=LEDGER_RUN_ID=EXP-002-RUN-002
|
||||
Environment=EVENT_STATE_LOG_DIR=/www/Project/nautilus_mm/logs/event_state/EXP-002-RUN-002
|
||||
Environment=HTTP_PROXY=
|
||||
Environment=HTTPS_PROXY=
|
||||
Environment=http_proxy=
|
||||
Environment=https_proxy=
|
||||
ExecStart=/www/Project/nautilus_mm/scripts/run_event_state.sh
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=event-state-probe
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -0,0 +1,28 @@
|
||||
[Unit]
|
||||
Description=Maker Edge Probe MM_EDGE_EXP_001 (Research Freeze / Data Collection)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=jack
|
||||
Group=jack
|
||||
WorkingDirectory=/www/Project/nautilus_mm
|
||||
Environment=PYTHONPATH=/www/Project/nautilus_mm/src
|
||||
EnvironmentFile=-/www/Project/nautilus_mm/.env
|
||||
# 服务器直连交易所
|
||||
Environment=HTTP_PROXY=
|
||||
Environment=HTTPS_PROXY=
|
||||
Environment=http_proxy=
|
||||
Environment=https_proxy=
|
||||
ExecStart=/www/Project/nautilus_mm/.venv/bin/python -m nautilus_mm.run_live
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=mm-edge-probe
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,21 @@
|
||||
# 用户级 systemd(无需 sudo)— 安装到 ~/.config/systemd/user/
|
||||
[Unit]
|
||||
Description=Maker Edge Probe MM_EDGE_EXP_001
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/www/Project/nautilus_mm
|
||||
Environment=PYTHONPATH=/www/Project/nautilus_mm/src
|
||||
EnvironmentFile=-/www/Project/nautilus_mm/.env
|
||||
Environment=HTTP_PROXY=
|
||||
Environment=HTTPS_PROXY=
|
||||
ExecStart=/www/Project/nautilus_mm/.venv/bin/python -m nautilus_mm.run_live
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=mm-edge-probe
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
Executable
+95
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
# 部署 MM_EDGE_EXP_001 → jack@jackyu66.com:/www/Project/nautilus_mm
|
||||
#
|
||||
# 默认:
|
||||
# SSH_HOST=jack@jackyu66.com
|
||||
# SSH_KEY=~/Project/deploy/zun_hk/id_ed25519_hk
|
||||
# REMOTE_DIR=/www/Project/nautilus_mm
|
||||
#
|
||||
# 覆盖:export SSH_HOST=... SSH_KEY=... REMOTE_DIR=...
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
SSH_HOST="${SSH_HOST:-jack@jackyu66.com}"
|
||||
SSH_KEY="${SSH_KEY:-${HOME}/Project/deploy/zun_hk/id_ed25519_hk}"
|
||||
REMOTE_DIR="${REMOTE_DIR:-/www/Project/nautilus_mm}"
|
||||
|
||||
if [[ ! -f "$SSH_KEY" ]]; then
|
||||
echo "SSH key not found: $SSH_KEY"
|
||||
exit 1
|
||||
fi
|
||||
chmod 400 "$SSH_KEY" 2>/dev/null || true
|
||||
|
||||
SSH_OPTS=(-i "$SSH_KEY" -o StrictHostKeyChecking=accept-new)
|
||||
SSH=(ssh "${SSH_OPTS[@]}" "$SSH_HOST")
|
||||
RSYNC_E="ssh ${SSH_OPTS[*]}"
|
||||
|
||||
echo "==> stop remote probe before sync (if running)"
|
||||
"${SSH[@]}" "systemctl --user stop mm-edge-probe 2>/dev/null || true"
|
||||
|
||||
echo "==> sync $ROOT → $SSH_HOST:$REMOTE_DIR"
|
||||
"${SSH[@]}" "mkdir -p '$REMOTE_DIR' '$REMOTE_DIR/logs/maker_edge'"
|
||||
rsync -avz --delete \
|
||||
-e "$RSYNC_E" \
|
||||
--exclude '.venv' \
|
||||
--exclude '__pycache__' \
|
||||
--exclude '*.pyc' \
|
||||
--exclude 'logs/maker_edge/*.jsonl' \
|
||||
--exclude 'logs/maker_edge/*.txt' \
|
||||
--exclude 'logs/maker_edge_smoke' \
|
||||
--exclude '.env' \
|
||||
"$ROOT/" "$SSH_HOST:$REMOTE_DIR/"
|
||||
|
||||
echo "==> remote setup (uv venv + user systemd)"
|
||||
"${SSH[@]}" "REMOTE_DIR='$REMOTE_DIR' bash -s" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
cd "$REMOTE_DIR"
|
||||
if [[ ! -f .env ]]; then
|
||||
cp .env.example .env
|
||||
{
|
||||
echo ""
|
||||
echo "# Server Data Collection — MM_EDGE_EXP_001"
|
||||
echo "EXPERIMENT_ID=MM_EDGE_EXP_001"
|
||||
echo "PROBE_VERSION=probe_v0.1"
|
||||
echo "EXCHANGE_NAME=binance_usdm"
|
||||
echo "BINANCE_ENVIRONMENT=TESTNET"
|
||||
echo "ENABLE_TRADING=false"
|
||||
echo "QUOTE_TTL_SECS=30"
|
||||
echo "MAX_ABS_INVENTORY=0.005"
|
||||
echo "HTTP_PROXY="
|
||||
echo "HTTPS_PROXY="
|
||||
echo "MAKER_EDGE_LOG_DIR=${REMOTE_DIR}/logs/maker_edge"
|
||||
} >> .env
|
||||
echo "CREATED .env — fill BINANCE_API_KEY / BINANCE_API_SECRET"
|
||||
else
|
||||
echo ".env exists — left untouched"
|
||||
fi
|
||||
|
||||
if [[ ! -x "$HOME/.local/bin/uv" ]]; then
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
fi
|
||||
uv python install 3.12
|
||||
rm -rf .venv
|
||||
uv venv .venv --python 3.12
|
||||
uv pip install -r requirements.txt --python .venv/bin/python
|
||||
|
||||
mkdir -p "$HOME/.config/systemd/user"
|
||||
sed -e "s|/www/Project/nautilus_mm|${REMOTE_DIR}|g" \
|
||||
deploy/mm-edge-probe.user.service > "$HOME/.config/systemd/user/mm-edge-probe.service"
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user enable mm-edge-probe.service
|
||||
loginctl enable-linger "$(whoami)" 2>/dev/null || true
|
||||
echo "User systemd installed (not started — fill keys first)."
|
||||
echo " nano $REMOTE_DIR/.env"
|
||||
echo " systemctl --user start mm-edge-probe"
|
||||
echo " journalctl --user -u mm-edge-probe -f"
|
||||
REMOTE
|
||||
|
||||
echo ""
|
||||
echo "==> done"
|
||||
echo "1) ssh -i $SSH_KEY $SSH_HOST"
|
||||
echo "2) nano $REMOTE_DIR/.env # TESTNET keys"
|
||||
echo "3) systemctl --user start mm-edge-probe"
|
||||
echo "4) ./scripts/probe_status.sh"
|
||||
echo "5) ./scripts/pull_report.sh"
|
||||
@@ -0,0 +1,533 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Economic Attribution v0.1
|
||||
|
||||
Hard Evidence Population only:
|
||||
MATCHED = Local Fill ↔ Venue Trade dual evidence
|
||||
|
||||
Purpose:
|
||||
Economic Attribution only.
|
||||
No strategy modification.
|
||||
No live execution.
|
||||
No economic simulation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
from reconcile_fills import load_local_fills, match, normalize_local, normalize_venue # noqa: E402
|
||||
|
||||
|
||||
def _load_jsonl_df(log_dir: Path) -> pd.DataFrame:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for f in sorted(log_dir.glob("*.jsonl")):
|
||||
if f.name.startswith(("Account_", "Maker_", "RECON")):
|
||||
continue
|
||||
for line in f.open():
|
||||
try:
|
||||
e = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(e, dict):
|
||||
rows.append(e)
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def _parse_fill_context(df: pd.DataFrame) -> pd.DataFrame:
|
||||
if df.empty or "fill_context" not in df.columns:
|
||||
return pd.DataFrame(columns=["fill_id"])
|
||||
rows = []
|
||||
for _, r in df.iterrows():
|
||||
ctx = r.get("fill_context")
|
||||
if not isinstance(ctx, dict):
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"fill_id": r.get("fill_id"),
|
||||
"market_event_before_fill": ctx.get("market_event_before_fill"),
|
||||
"trade_imbalance_5s": ctx.get("trade_imbalance_5s"),
|
||||
"price_velocity_5s": ctx.get("price_velocity_5s"),
|
||||
"fill_type": ctx.get("fill_type"),
|
||||
}
|
||||
)
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def _fav_ret(side: pd.Series, fill: pd.Series, px: pd.Series) -> pd.Series:
|
||||
raw = (pd.to_numeric(px, errors="coerce") - pd.to_numeric(fill, errors="coerce")) / pd.to_numeric(
|
||||
fill, errors="coerce"
|
||||
)
|
||||
return pd.Series(np.where(side == "long", raw, -raw), index=side.index)
|
||||
|
||||
|
||||
def _cluster_weight(frame: pd.DataFrame) -> pd.Series:
|
||||
cnt = frame.groupby("event_cluster_id")["event_cluster_id"].transform("count")
|
||||
return 1.0 / cnt.clip(lower=1)
|
||||
|
||||
|
||||
def _pct(v: float | None) -> str:
|
||||
if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))):
|
||||
return "n/a"
|
||||
return f"{v * 100:.4f}%"
|
||||
|
||||
|
||||
def _num(v: float | None, digits: int = 4) -> str:
|
||||
if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))):
|
||||
return "n/a"
|
||||
return f"{v:.{digits}f}"
|
||||
|
||||
|
||||
def _mean(s: pd.Series) -> float | None:
|
||||
s = pd.to_numeric(s, errors="coerce").dropna()
|
||||
return None if s.empty else float(s.mean())
|
||||
|
||||
|
||||
def _sum(s: pd.Series) -> float:
|
||||
s = pd.to_numeric(s, errors="coerce").fillna(0.0)
|
||||
return float(s.sum())
|
||||
|
||||
|
||||
def _weighted_mean(v: pd.Series, w: pd.Series) -> float | None:
|
||||
vv = pd.to_numeric(v, errors="coerce")
|
||||
ww = pd.to_numeric(w, errors="coerce").fillna(0.0)
|
||||
mask = vv.notna() & ww.notna()
|
||||
vv = vv[mask]
|
||||
ww = ww[mask]
|
||||
if vv.empty or float(ww.sum()) == 0.0:
|
||||
return None
|
||||
return float((vv * ww).sum() / ww.sum())
|
||||
|
||||
|
||||
def _prepare_paths(paths: pd.DataFrame) -> pd.DataFrame:
|
||||
paths = paths.copy()
|
||||
if "max_price" in paths.columns and "min_price" in paths.columns and "fill_price" in paths.columns:
|
||||
rng = (pd.to_numeric(paths["max_price"], errors="coerce") - pd.to_numeric(paths["min_price"], errors="coerce")) / pd.to_numeric(
|
||||
paths["fill_price"], errors="coerce"
|
||||
)
|
||||
med = float(rng.dropna().median()) if rng.notna().any() else 0.0
|
||||
paths["vol_bucket"] = np.where(rng >= med, "high_vol", "low_vol")
|
||||
if "price_velocity_5s" in paths.columns and pd.to_numeric(paths["price_velocity_5s"], errors="coerce").notna().any():
|
||||
v = pd.to_numeric(paths["price_velocity_5s"], errors="coerce")
|
||||
thr = float(v.abs().median()) * 0.5
|
||||
paths["trend_bucket"] = np.where(v > thr, "trend_up", np.where(v < -thr, "trend_down", "range"))
|
||||
if "spread" in paths.columns and "fill_price" in paths.columns and pd.to_numeric(paths["spread"], errors="coerce").notna().any():
|
||||
sp = pd.to_numeric(paths["spread"], errors="coerce") / pd.to_numeric(paths["fill_price"], errors="coerce")
|
||||
med = float(sp.dropna().median()) if sp.notna().any() else 0.0
|
||||
paths["liq_bucket"] = np.where(sp <= med, "tight_spread", "wide_spread")
|
||||
paths["toxicity_bucket"] = np.where(paths["path_type"].astype(str).str.startswith("C"), "toxic", "non_toxic")
|
||||
return paths
|
||||
|
||||
|
||||
def _inventory_metrics(matched: pd.DataFrame) -> dict[str, float | None]:
|
||||
if matched.empty:
|
||||
return {}
|
||||
g = matched.sort_values("venue_time_ms").copy()
|
||||
g["signed_qty"] = np.where(g["side"] == "long", g["qty"], -g["qty"])
|
||||
g["net_btc"] = g["signed_qty"].cumsum()
|
||||
g["abs_net_btc"] = g["net_btc"].abs()
|
||||
times = pd.to_numeric(g["venue_time_ms"], errors="coerce").astype("float64") / 1000.0
|
||||
dt = times.shift(-1) - times
|
||||
dt = dt.fillna(0.0).clip(lower=0.0)
|
||||
total_t = float(dt.sum())
|
||||
tw_abs = float((g["abs_net_btc"] * dt).sum() / total_t) if total_t > 0 else None
|
||||
tw_signed = float((g["net_btc"] * dt).sum() / total_t) if total_t > 0 else None
|
||||
return {
|
||||
"max_net_btc": float(g["net_btc"].max()),
|
||||
"min_net_btc": float(g["net_btc"].min()),
|
||||
"max_abs_net_btc": float(g["abs_net_btc"].max()),
|
||||
"avg_abs_net_btc_per_fill": float(g["abs_net_btc"].mean()),
|
||||
"time_weighted_abs_net_btc": tw_abs,
|
||||
"time_weighted_signed_net_btc": tw_signed,
|
||||
"long_qty": float(g.loc[g["signed_qty"] > 0, "signed_qty"].sum()),
|
||||
"short_qty": float((-g.loc[g["signed_qty"] < 0, "signed_qty"]).sum()),
|
||||
"turnover_btc": float(g["qty"].sum()),
|
||||
}
|
||||
|
||||
|
||||
def _bucket_table(paths: pd.DataFrame, bucket: str, title: str) -> list[dict[str, Any]]:
|
||||
if bucket not in paths.columns or paths.empty:
|
||||
return []
|
||||
rows = []
|
||||
for key, grp in paths.groupby(bucket):
|
||||
notional = grp["notional_usdt"].sum()
|
||||
clusters = grp["event_cluster_id"].nunique()
|
||||
rows.append(
|
||||
{
|
||||
"dimension": title,
|
||||
"bucket": str(key),
|
||||
"fills": int(len(grp)),
|
||||
"clusters": int(clusters),
|
||||
"btc_qty": float(grp["qty"].sum()),
|
||||
"notional_usdt": float(notional),
|
||||
"fee_usdt": float(grp["commission_usdt"].sum()),
|
||||
"fee_per_fill": float(grp["commission_usdt"].mean()) if len(grp) else None,
|
||||
"fee_per_btc": float(grp["commission_usdt"].sum() / grp["qty"].sum()) if grp["qty"].sum() else None,
|
||||
"markout_1s": _weighted_mean(grp["markout_1s"], grp["notional_usdt"]),
|
||||
"markout_5s": _weighted_mean(grp["markout_5s"], grp["notional_usdt"]),
|
||||
"markout_10s": _weighted_mean(grp["markout_10s"], grp["notional_usdt"]),
|
||||
"markout_30s": _weighted_mean(grp["markout_30s"], grp["notional_usdt"]),
|
||||
"markout_300s": _weighted_mean(grp["markout_300s"], grp["notional_usdt"]),
|
||||
"gross_markout_30s_usdt": float(grp["gross_markout_30s_usdt"].sum()),
|
||||
"realized_pnl_usdt": float(grp["realized_pnl_usdt"].sum()),
|
||||
"net_attr_30s_usdt": float(grp["net_attr_30s_usdt"].sum()),
|
||||
}
|
||||
)
|
||||
rows.sort(key=lambda x: (-x["fills"], x["bucket"]))
|
||||
return rows
|
||||
|
||||
|
||||
def _counterfactual(base: pd.DataFrame, exclude_col: str, exclude_values: set[str], label: str) -> dict[str, Any]:
|
||||
kept = base[~base[exclude_col].astype(str).isin(exclude_values)].copy()
|
||||
return {
|
||||
"name": label,
|
||||
"fills": int(len(kept)),
|
||||
"clusters": int(kept["event_cluster_id"].nunique()) if not kept.empty else 0,
|
||||
"btc_qty": float(kept["qty"].sum()) if not kept.empty else 0.0,
|
||||
"fee_usdt": float(kept["commission_usdt"].sum()) if not kept.empty else 0.0,
|
||||
"gross_markout_30s_usdt": float(kept["gross_markout_30s_usdt"].sum()) if not kept.empty else 0.0,
|
||||
"realized_pnl_usdt": float(kept["realized_pnl_usdt"].sum()) if not kept.empty else 0.0,
|
||||
"net_attr_30s_usdt": float(kept["net_attr_30s_usdt"].sum()) if not kept.empty else 0.0,
|
||||
"markout_30s": _weighted_mean(kept["markout_30s"], kept["notional_usdt"]),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="Economic Attribution v0.1 (MATCHED only)")
|
||||
ap.add_argument("--dir", default=str(ROOT / "logs" / "maker_edge"))
|
||||
ap.add_argument("--out", default=str(ROOT / "logs" / "maker_edge" / "Economic_Attribution_v0_1.txt"))
|
||||
ap.add_argument("--recon03", default=str(ROOT / "logs" / "maker_edge" / "RECONCILIATION_03.json"))
|
||||
ap.add_argument("--account", default=str(ROOT / "logs" / "maker_edge" / "Account_Reconciliation.json"))
|
||||
ap.add_argument("--venue-trades", default=str(ROOT / "logs" / "maker_edge" / "venue_trades.json"))
|
||||
args = ap.parse_args()
|
||||
|
||||
log_dir = Path(args.dir)
|
||||
df = _load_jsonl_df(log_dir)
|
||||
fills = df[df["event"] == "fill"].copy()
|
||||
paths = df[df["event"] == "fill_path"].copy()
|
||||
inv = df[df["event"] == "inventory_tick"].copy()
|
||||
|
||||
venue_trades = json.loads(Path(args.venue_trades).read_text())
|
||||
local_fills_raw = load_local_fills(log_dir)
|
||||
locals_norm = [normalize_local(e, i) for i, e in enumerate(local_fills_raw)]
|
||||
venues_norm = [normalize_venue(t, i) for i, t in enumerate(venue_trades)]
|
||||
recon = match(locals_norm, venues_norm)
|
||||
matched_fill_ids = {m["local"]["fill_id"] for m in recon["matched"]}
|
||||
matched_trade_ids = {m["venue"]["venue_trade_id"] for m in recon["matched"]}
|
||||
|
||||
fills = fills[fills["fill_id"].isin(matched_fill_ids)].copy()
|
||||
paths = paths[paths["fill_id"].isin(matched_fill_ids)].copy()
|
||||
|
||||
fc = _parse_fill_context(fills)
|
||||
meta_cols = [
|
||||
c
|
||||
for c in [
|
||||
"fill_id",
|
||||
"side",
|
||||
"fill_price",
|
||||
"spread",
|
||||
"spread_capture_pct",
|
||||
"obi",
|
||||
"trade_imbalance",
|
||||
"bid_depth_5",
|
||||
"ask_depth_5",
|
||||
"book_age_ms",
|
||||
"inventory",
|
||||
"inventory_time",
|
||||
"inventory_skew",
|
||||
"pre_5s_deteriorated",
|
||||
"mid",
|
||||
"event_cluster_id",
|
||||
"pair",
|
||||
]
|
||||
if c in fills.columns
|
||||
]
|
||||
meta = fills.drop_duplicates("fill_id")[meta_cols]
|
||||
paths = paths.merge(meta, on="fill_id", how="left", suffixes=("", "_f"))
|
||||
for col in ["event_cluster_id", "side", "fill_price", "spread_capture_pct", "mid"]:
|
||||
alt = f"{col}_f"
|
||||
if alt in paths.columns:
|
||||
if col not in paths.columns:
|
||||
paths[col] = paths[alt]
|
||||
else:
|
||||
paths[col] = paths[col].fillna(paths[alt])
|
||||
if not fc.empty:
|
||||
paths = paths.merge(fc, on="fill_id", how="left")
|
||||
paths = _prepare_paths(paths)
|
||||
|
||||
venue = pd.DataFrame(venues_norm)
|
||||
venue = venue[venue["venue_trade_id"].isin(matched_trade_ids)].copy()
|
||||
venue = venue.rename(
|
||||
columns={
|
||||
"venue_trade_id": "trade_id_link",
|
||||
"venue_order_id": "venue_order_id",
|
||||
"qty": "qty",
|
||||
"px": "venue_price",
|
||||
"ts": "venue_ts",
|
||||
}
|
||||
)
|
||||
raw_v = pd.DataFrame(venue_trades)
|
||||
raw_v["trade_id_link"] = raw_v["id"].astype(str)
|
||||
raw_v["venue_order_id"] = raw_v["orderId"].astype(str)
|
||||
raw_v["commission_usdt"] = pd.to_numeric(raw_v["commission"], errors="coerce")
|
||||
raw_v["realized_pnl_usdt"] = pd.to_numeric(raw_v["realizedPnl"], errors="coerce").fillna(0.0)
|
||||
raw_v["venue_time_ms"] = pd.to_numeric(raw_v["time"], errors="coerce")
|
||||
raw_v["qty"] = pd.to_numeric(raw_v["qty"], errors="coerce")
|
||||
raw_v["venue_price"] = pd.to_numeric(raw_v["price"], errors="coerce")
|
||||
raw_v["side"] = np.where(raw_v["buyer"].astype(bool), "long", "short")
|
||||
raw_v = raw_v[raw_v["trade_id_link"].isin(matched_trade_ids)].copy()
|
||||
|
||||
matched_map = pd.DataFrame(
|
||||
[
|
||||
{
|
||||
"fill_id": m["local"]["fill_id"],
|
||||
"trade_id_link": m["venue"]["venue_trade_id"],
|
||||
"venue_order_id": m["venue"]["venue_order_id"],
|
||||
}
|
||||
for m in recon["matched"]
|
||||
]
|
||||
)
|
||||
|
||||
paths = paths.merge(
|
||||
matched_map.merge(
|
||||
raw_v[
|
||||
[
|
||||
"trade_id_link",
|
||||
"venue_order_id",
|
||||
"commission_usdt",
|
||||
"realized_pnl_usdt",
|
||||
"venue_time_ms",
|
||||
"qty",
|
||||
"venue_price",
|
||||
"side",
|
||||
]
|
||||
],
|
||||
on=["trade_id_link", "venue_order_id"],
|
||||
how="left",
|
||||
),
|
||||
on="fill_id",
|
||||
how="left",
|
||||
suffixes=("", "_venue"),
|
||||
)
|
||||
|
||||
paths["fill_price"] = pd.to_numeric(paths["fill_price"], errors="coerce")
|
||||
paths["qty"] = pd.to_numeric(paths["qty"], errors="coerce")
|
||||
paths["notional_usdt"] = paths["fill_price"] * paths["qty"]
|
||||
for sec, col in [(1, "after_1s_price"), (5, "after_5s_price"), (10, "after_10s_price"), (30, "after_30s_price"), (300, "after_5m_price")]:
|
||||
paths[f"markout_{sec}s"] = _fav_ret(paths["side"], paths["fill_price"], paths[col])
|
||||
paths["gross_markout_30s_usdt"] = paths["notional_usdt"] * paths["markout_30s"]
|
||||
paths["net_attr_30s_usdt"] = (
|
||||
paths["gross_markout_30s_usdt"]
|
||||
- pd.to_numeric(paths["commission_usdt"], errors="coerce").fillna(0.0)
|
||||
+ pd.to_numeric(paths["realized_pnl_usdt"], errors="coerce").fillna(0.0)
|
||||
)
|
||||
|
||||
inventory_metrics = _inventory_metrics(
|
||||
raw_v[
|
||||
["venue_time_ms", "side", "qty", "commission_usdt", "realized_pnl_usdt", "venue_order_id", "trade_id_link"]
|
||||
].copy()
|
||||
)
|
||||
|
||||
n_matched_paths = len(paths)
|
||||
n_matched_fills = len(fills)
|
||||
n_matched_clusters = int(fills["event_cluster_id"].nunique()) if not fills.empty else 0
|
||||
cluster_w = _cluster_weight(paths) if not paths.empty and "event_cluster_id" in paths.columns else pd.Series(dtype=float)
|
||||
|
||||
horizon_rows = []
|
||||
for sec in (1, 5, 10, 30, 300):
|
||||
col = f"markout_{sec}s"
|
||||
valid = paths[col].notna()
|
||||
sub = paths[valid]
|
||||
w = sub["notional_usdt"]
|
||||
horizon_rows.append(
|
||||
{
|
||||
"horizon": f"{sec}s",
|
||||
"n": int(len(sub)),
|
||||
"fill_w": _weighted_mean(sub[col], w),
|
||||
"cluster_w": _weighted_mean(sub[col], _cluster_weight(sub) if not sub.empty else pd.Series(dtype=float)),
|
||||
"gross_usdt": float((sub["notional_usdt"] * sub[col]).sum()) if not sub.empty else 0.0,
|
||||
}
|
||||
)
|
||||
|
||||
fee_total = float(paths["commission_usdt"].sum())
|
||||
realized_total = float(paths["realized_pnl_usdt"].sum())
|
||||
gross_30_total = float(paths["gross_markout_30s_usdt"].sum())
|
||||
net_attr_30_total = float(paths["net_attr_30s_usdt"].sum())
|
||||
total_qty = float(paths["qty"].sum())
|
||||
total_notional = float(paths["notional_usdt"].sum())
|
||||
|
||||
bucket_rows: list[dict[str, Any]] = []
|
||||
for col, title in [
|
||||
("path_type", "PathType"),
|
||||
("toxicity_bucket", "Toxicity"),
|
||||
("vol_bucket", "Volatility"),
|
||||
("liq_bucket", "Spread"),
|
||||
("trend_bucket", "Trend"),
|
||||
("market_event_before_fill", "FillContext"),
|
||||
]:
|
||||
bucket_rows.extend(_bucket_table(paths, col, title))
|
||||
bucket_df = pd.DataFrame(bucket_rows)
|
||||
|
||||
negative_states: set[str] = set()
|
||||
if not bucket_df.empty:
|
||||
neg = bucket_df[(bucket_df["dimension"] != "PathType") & (bucket_df["markout_30s"] < 0)]
|
||||
negative_states = set(neg["bucket"].astype(str))
|
||||
|
||||
counterfactuals = [
|
||||
{
|
||||
"name": "BASELINE",
|
||||
"fills": int(len(paths)),
|
||||
"clusters": int(paths["event_cluster_id"].nunique()) if not paths.empty else 0,
|
||||
"btc_qty": total_qty,
|
||||
"fee_usdt": fee_total,
|
||||
"gross_markout_30s_usdt": gross_30_total,
|
||||
"realized_pnl_usdt": realized_total,
|
||||
"net_attr_30s_usdt": net_attr_30_total,
|
||||
"markout_30s": _weighted_mean(paths["markout_30s"], paths["notional_usdt"]),
|
||||
},
|
||||
_counterfactual(paths, "path_type", {"C_toxic"}, "EXCLUDE_PATH_C"),
|
||||
_counterfactual(paths, "toxicity_bucket", {"toxic"}, "EXCLUDE_TOXIC"),
|
||||
_counterfactual(paths, "market_event_before_fill", negative_states, "EXCLUDE_NEGATIVE_STATE"),
|
||||
]
|
||||
|
||||
account = json.loads(Path(args.account).read_text()) if Path(args.account).exists() else {}
|
||||
recon03 = json.loads(Path(args.recon03).read_text()) if Path(args.recon03).exists() else {}
|
||||
|
||||
out_txt = Path(args.out)
|
||||
out_json = out_txt.with_suffix(".json")
|
||||
lines: list[str] = []
|
||||
|
||||
def p(s: str = "") -> None:
|
||||
lines.append(s)
|
||||
print(s)
|
||||
|
||||
p("=" * 72)
|
||||
p("Economic Attribution v0.1")
|
||||
p("=" * 72)
|
||||
p("Experiment: MM_EDGE_EXP_001")
|
||||
p("Population: MATCHED=3890")
|
||||
p("Strategy: v0.1 FROZEN")
|
||||
p("Execution: STOPPED")
|
||||
p("Stage3: LOCKED")
|
||||
p("Purpose: Economic Attribution only.")
|
||||
p("No strategy modification. No live execution. No economic simulation.")
|
||||
p()
|
||||
p("Layer 1 — Hard Economic Evidence")
|
||||
p("-" * 40)
|
||||
p(f"Matched fills: {n_matched_fills}")
|
||||
p(f"Matched paths: {n_matched_paths}")
|
||||
p(f"Matched clusters: {n_matched_clusters}")
|
||||
p(f"Fee total: {_num(fee_total, 6)} USDT")
|
||||
p(f"Fee / fill: {_num(fee_total / max(n_matched_paths, 1), 6)} USDT")
|
||||
p(f"Fee / BTC: {_num(fee_total / max(total_qty, 1e-12), 6)} USDT")
|
||||
p(f"Fee / cluster: {_num(fee_total / max(n_matched_clusters, 1), 6)} USDT")
|
||||
p(f"Realized component: {_num(realized_total, 6)} USDT")
|
||||
p(f"Gross markout @30s: {_num(gross_30_total, 6)} USDT")
|
||||
p(f"Net attributable @30s: {_num(net_attr_30_total, 6)} USDT")
|
||||
p()
|
||||
p("Markout by horizon (MATCHED only)")
|
||||
p("-" * 40)
|
||||
for row in horizon_rows:
|
||||
p(
|
||||
f"{row['horizon']:>5} n={row['n']:4d} fill-w={_pct(row['fill_w'])} "
|
||||
f"cluster-w={_pct(row['cluster_w'])} gross={_num(row['gross_usdt'], 6)} USDT"
|
||||
)
|
||||
p()
|
||||
p("Inventory carry / exposure")
|
||||
p("-" * 40)
|
||||
p(f"Max net BTC: {_num(inventory_metrics.get('max_net_btc'), 6)}")
|
||||
p(f"Min net BTC: {_num(inventory_metrics.get('min_net_btc'), 6)}")
|
||||
p(f"Max |net BTC|: {_num(inventory_metrics.get('max_abs_net_btc'), 6)}")
|
||||
p(f"Average |net BTC|: {_num(inventory_metrics.get('avg_abs_net_btc_per_fill'), 6)}")
|
||||
p(f"TW |net BTC|: {_num(inventory_metrics.get('time_weighted_abs_net_btc'), 6)}")
|
||||
p(f"TW signed net BTC: {_num(inventory_metrics.get('time_weighted_signed_net_btc'), 6)}")
|
||||
p(f"Long qty / Short qty: {_num(inventory_metrics.get('long_qty'), 6)} / {_num(inventory_metrics.get('short_qty'), 6)} BTC")
|
||||
p(f"Inventory turnover: {_num(inventory_metrics.get('turnover_btc'), 6)} BTC")
|
||||
p()
|
||||
p("Slices (weighted by notional, MATCHED only)")
|
||||
p("-" * 40)
|
||||
for dim in ["PathType", "Toxicity", "Volatility", "Spread", "Trend", "FillContext"]:
|
||||
sub = bucket_df[bucket_df["dimension"] == dim].copy()
|
||||
if sub.empty:
|
||||
continue
|
||||
p(dim)
|
||||
for _, r in sub.sort_values(["fills", "bucket"], ascending=[False, True]).iterrows():
|
||||
p(
|
||||
f" {r['bucket']}: n={int(r['fills'])} clusters={int(r['clusters'])} "
|
||||
f"fee={_num(r['fee_usdt'], 4)} gross30={_num(r['gross_markout_30s_usdt'], 4)} "
|
||||
f"realized={_num(r['realized_pnl_usdt'], 4)} net30={_num(r['net_attr_30s_usdt'], 4)} "
|
||||
f"m30={_pct(r['markout_30s'])}"
|
||||
)
|
||||
p()
|
||||
p("Layer 2 — Evidence Extension (excluded from core conclusion)")
|
||||
p("-" * 40)
|
||||
p(f"VENUE_CONFIRMED_NO_TRADE_HISTORY: {recon03.get('venue_confirmed_no_trade_history', 'n/a')}")
|
||||
p(f"VENUE_PARTIAL_ORDER_CANCELED: {recon03.get('venue_partial_order_canceled', 'n/a')}")
|
||||
p("These rows are order-confirmed, but not part of the Hard Evidence Population.")
|
||||
p()
|
||||
p("Layer 3 — Counterfactual Attribution (NOT backtest)")
|
||||
p("-" * 40)
|
||||
p("Observed vs Exclude-Bucket Attribution. These are contribution decompositions only.")
|
||||
for row in counterfactuals:
|
||||
p(
|
||||
f"{row['name']}: fills={row['fills']} clusters={row['clusters']} "
|
||||
f"fee={_num(row['fee_usdt'], 4)} gross30={_num(row['gross_markout_30s_usdt'], 4)} "
|
||||
f"realized={_num(row['realized_pnl_usdt'], 4)} net30={_num(row['net_attr_30s_usdt'], 4)} "
|
||||
f"m30={_pct(row['markout_30s'])}"
|
||||
)
|
||||
p()
|
||||
p("Interpretation")
|
||||
p("-" * 40)
|
||||
p("Core conclusion is based on 3890 fully matched fills.")
|
||||
p("Economic Attribution asks why MakerAlpha did not convert to money.")
|
||||
p("It does NOT change quote logic, does NOT restart v0.1, and does NOT unlock Stage 3.")
|
||||
p("=" * 72)
|
||||
|
||||
out_txt.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
sidecar = {
|
||||
"experiment_id": "MM_EDGE_EXP_001",
|
||||
"population": {
|
||||
"name": "MATCHED",
|
||||
"fills": n_matched_fills,
|
||||
"paths": n_matched_paths,
|
||||
"clusters": n_matched_clusters,
|
||||
},
|
||||
"strategy": "v0.1 FROZEN",
|
||||
"execution": "STOPPED",
|
||||
"stage3": "LOCKED",
|
||||
"fee_total_usdt": fee_total,
|
||||
"fee_per_fill_usdt": fee_total / max(n_matched_paths, 1),
|
||||
"fee_per_btc_usdt": fee_total / max(total_qty, 1e-12),
|
||||
"fee_per_cluster_usdt": fee_total / max(n_matched_clusters, 1),
|
||||
"realized_component_usdt": realized_total,
|
||||
"gross_markout_30s_usdt": gross_30_total,
|
||||
"net_attr_30s_usdt": net_attr_30_total,
|
||||
"markout_by_horizon": horizon_rows,
|
||||
"inventory_metrics": inventory_metrics,
|
||||
"bucket_rows": bucket_rows,
|
||||
"counterfactuals": counterfactuals,
|
||||
"recon03_extension": {
|
||||
"venue_confirmed_no_trade_history": recon03.get("venue_confirmed_no_trade_history"),
|
||||
"venue_partial_order_canceled": recon03.get("venue_partial_order_canceled"),
|
||||
},
|
||||
"account_recon_ref": account,
|
||||
}
|
||||
out_json.write_text(json.dumps(sidecar, indent=2) + "\n", encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,6 @@
|
||||
nautilus_trader==1.231.0
|
||||
python-dotenv>=1.0.0
|
||||
PyYAML>=6.0
|
||||
ccxt>=4.0.0
|
||||
pandas>=2.2,<3
|
||||
numpy>=1.26,<2.1
|
||||
@@ -0,0 +1,20 @@
|
||||
========================================================================
|
||||
Economic Fee Sensitivity v0.1 (MATCHED=3890)
|
||||
========================================================================
|
||||
gross_markout_30s_usdt: -1.730780 USDT
|
||||
fee_total_usdt: +42.422146 USDT
|
||||
realized_component_usdt:-8.521730 USDT
|
||||
|
||||
Fee assumption → Net attributable @30s
|
||||
------------------------------------------
|
||||
fee_factor | fee_usdt_assumed | net_attr_30s_usdt
|
||||
1.00 | +42.422146 | -52.674655
|
||||
0.50 | +21.211073 | -31.463582
|
||||
0.25 | +10.605536 | -20.858046
|
||||
0.10 | +4.242215 | -14.494724
|
||||
0.00 | +0.000000 | -10.252510
|
||||
|
||||
Interpretation:
|
||||
- If net remains < 0 at fee_factor=0 → economics not salvageable by fee reduction alone.
|
||||
- If fee reduction flips net > 0 → current venue/fee tier can be the dominant issue.
|
||||
========================================================================
|
||||
@@ -0,0 +1,21 @@
|
||||
========================================================================
|
||||
Economic Metric Reconciliation v0.1 (MATCHED=3890)
|
||||
========================================================================
|
||||
Matched paths: 3886 (expected ~3886)
|
||||
|
||||
Definitions (same math as analyze_maker_edge):
|
||||
- markout_30s_return = _fav_ret(side, fill_price, after_30s_price)
|
||||
- gross_markout_usdt = sum(notional_usdt * markout_30s_return)
|
||||
|
||||
Return-space metrics (sign may differ due to weighting):
|
||||
MakerAlpha fill-weighted mean return: -0.000693%
|
||||
MakerAlpha notional-weighted mean return: -0.000816%
|
||||
MakerAlpha cluster-weighted mean return: -0.000830%
|
||||
|
||||
Dollar-space metrics:
|
||||
gross_markout_usdt (30s): -1.730780 USDT
|
||||
total_notional_usdt: 212110.750 USDT
|
||||
|
||||
If fill-weighted return is + but gross_markout_usdt is negative,
|
||||
it means notional weighting flips sign (alpha is conditionally realized).
|
||||
========================================================================
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
LOG_DIR="${MAKER_EDGE_LOG_DIR:-$ROOT/logs/maker_edge}"
|
||||
# Prefer project venv python if present
|
||||
PY="${ROOT}/.venv/bin/python"
|
||||
if [[ ! -x "$PY" ]]; then
|
||||
PY=python3
|
||||
fi
|
||||
export PYTHONPATH="${ROOT}/src${PYTHONPATH:+:$PYTHONPATH}"
|
||||
exec "$PY" "$ROOT/scripts/analyze_maker_edge.py" --dir "$LOG_DIR" --report --min-fills "${1:-2000}"
|
||||
@@ -0,0 +1,978 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Maker Edge Report v0.1 — Research Freeze / Data Collection Phase
|
||||
|
||||
固定格式(每次运行必须相同、可比较):
|
||||
Executive Summary
|
||||
Section 1 — Data Integrity
|
||||
Section 2 — Fill Alpha
|
||||
Section 3 — Toxicity Profile
|
||||
Section 4 — Observed Edge Attribution
|
||||
Section 5 — Decision
|
||||
|
||||
研究对象:可验证的市场现象(不是策略)。
|
||||
见 FREEZE.md — 只许数据字段/质量检查/报告解释;禁止新交易规则。
|
||||
|
||||
用法:
|
||||
./scripts/analyze.sh 2000
|
||||
python scripts/analyze_maker_edge.py --report --min-fills 2000
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
_ROOT = Path(__file__).resolve().parents[1]
|
||||
_SRC = _ROOT / "src"
|
||||
if str(_SRC) not in sys.path:
|
||||
sys.path.insert(0, str(_SRC))
|
||||
|
||||
|
||||
FEE = 0.00016
|
||||
EXPECTED_SLIPPAGE = 0.00005
|
||||
POSITIVE_EDGE_NET = 0.0002
|
||||
CLUSTER_GAP_SEC = 30.0
|
||||
TOXIC_FAIL_RATIO = 0.60
|
||||
PASS_MIN_FILLS_DEFAULT = 2000
|
||||
|
||||
|
||||
def _load_experiment_from_df(df: pd.DataFrame) -> dict[str, Any]:
|
||||
"""优先用 jsonl 中的 experiment_start / 事件戳;否则回退环境默认。"""
|
||||
try:
|
||||
from nautilus_mm.experiment import load_experiment_meta
|
||||
|
||||
base = load_experiment_meta()
|
||||
except Exception:
|
||||
base = {
|
||||
"experiment_id": "MM_EDGE_EXP_001",
|
||||
"probe_version": "probe_v0.1",
|
||||
"quote_assumption": "frozen",
|
||||
"fee_model": "frozen",
|
||||
"exchange_assumption": "frozen",
|
||||
"exchange": "binance_usdm",
|
||||
"environment": "TESTNET",
|
||||
"symbol": "BTCUSDT-PERP",
|
||||
}
|
||||
if df.empty or "event" not in df.columns:
|
||||
return base
|
||||
starts = df[df["event"] == "experiment_start"]
|
||||
if not starts.empty:
|
||||
row = starts.iloc[-1]
|
||||
for k in ("experiment_id", "probe_version", "exchange", "environment", "symbol"):
|
||||
if k in row and pd.notna(row[k]):
|
||||
base[k] = row[k]
|
||||
return base
|
||||
# 任意带 experiment_id 的事件
|
||||
if "experiment_id" in df.columns and df["experiment_id"].notna().any():
|
||||
base["experiment_id"] = df["experiment_id"].dropna().iloc[-1]
|
||||
if "probe_version" in df.columns and df["probe_version"].notna().any():
|
||||
base["probe_version"] = df["probe_version"].dropna().iloc[-1]
|
||||
return base
|
||||
|
||||
|
||||
def load_events(log_dir: Path) -> pd.DataFrame:
|
||||
rows = []
|
||||
files = sorted(log_dir.glob("*.jsonl"))
|
||||
if not files:
|
||||
raise FileNotFoundError(f"No jsonl in {log_dir}")
|
||||
for f in files:
|
||||
for line in f.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
rows.append(json.loads(line))
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def _fav_ret(side: pd.Series, fill: pd.Series, px: pd.Series) -> pd.Series:
|
||||
raw = (px.astype(float) - fill.astype(float)) / fill.astype(float)
|
||||
return pd.Series(np.where(side == "long", raw, -raw), index=side.index)
|
||||
|
||||
|
||||
def _side_label(side: str) -> str:
|
||||
return "Bid" if side == "long" else "Ask"
|
||||
|
||||
|
||||
def _extract_fill_context(fills: pd.DataFrame) -> pd.DataFrame:
|
||||
if fills.empty or "fill_context" not in fills.columns:
|
||||
return pd.DataFrame()
|
||||
rows = []
|
||||
for _, r in fills.iterrows():
|
||||
ctx = r.get("fill_context")
|
||||
if not isinstance(ctx, dict):
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"fill_id": r.get("fill_id"),
|
||||
"market_event_before_fill": ctx.get("market_event_before_fill"),
|
||||
"trade_imbalance_5s": ctx.get("trade_imbalance_5s"),
|
||||
"price_velocity_5s": ctx.get("price_velocity_5s"),
|
||||
"fill_type": ctx.get("fill_type"),
|
||||
}
|
||||
)
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def _median_safe(s: pd.Series) -> float | None:
|
||||
s = pd.to_numeric(s, errors="coerce").dropna()
|
||||
return float(s.median()) if len(s) else None
|
||||
|
||||
|
||||
def _fmt_pct(x: float | None, digits: int = 4) -> str:
|
||||
if x is None or (isinstance(x, float) and np.isnan(x)):
|
||||
return "n/a"
|
||||
return f"{x*100:+.{digits}f}%"
|
||||
|
||||
|
||||
def _fmt_pp(x: float | None) -> str:
|
||||
if x is None or (isinstance(x, float) and np.isnan(x)):
|
||||
return "n/a"
|
||||
return f"{x*100:+.1f}pp"
|
||||
|
||||
|
||||
def _dist_stats(s: pd.Series) -> dict[str, float | None]:
|
||||
s = pd.to_numeric(s, errors="coerce").dropna()
|
||||
if s.empty:
|
||||
return {"mean": None, "median": None, "p25": None, "p75": None, "n": 0}
|
||||
return {
|
||||
"mean": float(s.mean()),
|
||||
"median": float(s.median()),
|
||||
"p25": float(s.quantile(0.25)),
|
||||
"p75": float(s.quantile(0.75)),
|
||||
"n": int(len(s)),
|
||||
}
|
||||
|
||||
|
||||
def _print_dist(p, title: str, d: dict[str, float | None]) -> None:
|
||||
if not d.get("n"):
|
||||
p(f"{title}: n/a")
|
||||
return
|
||||
p(f"{title} (n={d['n']}):")
|
||||
p(f" mean: {_fmt_pct(d['mean'])}")
|
||||
p(f" median: {_fmt_pct(d['median'])}")
|
||||
p(f" p25: {_fmt_pct(d['p25'])}")
|
||||
p(f" p75: {_fmt_pct(d['p75'])}")
|
||||
|
||||
|
||||
def _observation_window(n_fills: int, n_clusters: int) -> str:
|
||||
if n_fills < 500:
|
||||
return "anomaly-check only (<500 fills)"
|
||||
if n_fills < 2000:
|
||||
return "early look (500+) — do not over-interpret"
|
||||
if n_fills < 10000:
|
||||
return "preliminary judgment (2000+) — clusters still matter more than fills"
|
||||
return "stability discussion eligible (10000+ fills)"
|
||||
|
||||
|
||||
def assign_clusters_offline(fills: pd.DataFrame, gap_sec: float = CLUSTER_GAP_SEC) -> pd.DataFrame:
|
||||
out = fills.copy()
|
||||
if out.empty:
|
||||
return out
|
||||
if "event_cluster_id" in out.columns and out["event_cluster_id"].notna().any():
|
||||
return out
|
||||
if "ts_epoch" not in out.columns:
|
||||
out["event_cluster_id"] = [f"na_{i}" for i in range(len(out))]
|
||||
out["cluster_fill_index"] = 1
|
||||
return out
|
||||
out = out.sort_values("ts_epoch").reset_index(drop=True)
|
||||
cids: list[str] = []
|
||||
idxs: list[int] = []
|
||||
cid = None
|
||||
last_ts = -1e18
|
||||
last_side = None
|
||||
n = 0
|
||||
for _, r in out.iterrows():
|
||||
ts = float(r["ts_epoch"])
|
||||
side = r.get("side")
|
||||
if cid is None or side != last_side or (ts - last_ts) > gap_sec:
|
||||
cid = uuid.uuid4().hex[:12]
|
||||
n = 0
|
||||
n += 1
|
||||
cids.append(cid)
|
||||
idxs.append(n)
|
||||
last_ts = ts
|
||||
last_side = side
|
||||
out["event_cluster_id"] = cids
|
||||
out["cluster_fill_index"] = idxs
|
||||
return out
|
||||
|
||||
|
||||
def classify_space(raw_capture: float, net_edge: float) -> str:
|
||||
if raw_capture <= 0 or net_edge <= 0:
|
||||
return "NO_EDGE"
|
||||
if net_edge < POSITIVE_EDGE_NET:
|
||||
return "EDGE_AFTER_COST"
|
||||
return "POSITIVE_EDGE"
|
||||
|
||||
|
||||
def build_mid_series(df: pd.DataFrame) -> pd.DataFrame:
|
||||
parts = []
|
||||
for ev in ("mid_tick", "inventory_tick"):
|
||||
if "event" not in df.columns:
|
||||
break
|
||||
sub = df[df["event"] == ev]
|
||||
if sub.empty or "mid" not in sub.columns or "ts_epoch" not in sub.columns:
|
||||
continue
|
||||
parts.append(sub[["ts_epoch", "mid"]].dropna())
|
||||
if not parts:
|
||||
return pd.DataFrame(columns=["ts_epoch", "mid"])
|
||||
m = pd.concat(parts, ignore_index=True)
|
||||
m["ts_epoch"] = pd.to_numeric(m["ts_epoch"], errors="coerce")
|
||||
m["mid"] = pd.to_numeric(m["mid"], errors="coerce")
|
||||
return m.dropna().sort_values("ts_epoch").drop_duplicates("ts_epoch").reset_index(drop=True)
|
||||
|
||||
|
||||
def _cluster_weight(frame: pd.DataFrame) -> pd.Series:
|
||||
if "event_cluster_id" not in frame.columns:
|
||||
return pd.Series(1.0, index=frame.index)
|
||||
cnt = frame.groupby("event_cluster_id")["event_cluster_id"].transform("count")
|
||||
return 1.0 / cnt.clip(lower=1)
|
||||
|
||||
|
||||
def _period_str(df: pd.DataFrame) -> str:
|
||||
if df.empty or "ts_epoch" not in df.columns or df["ts_epoch"].isna().all():
|
||||
return "n/a"
|
||||
t0 = float(pd.to_numeric(df["ts_epoch"], errors="coerce").min())
|
||||
t1 = float(pd.to_numeric(df["ts_epoch"], errors="coerce").max())
|
||||
a = datetime.fromtimestamp(t0, tz=timezone.utc).strftime("%Y-%m-%d")
|
||||
b = datetime.fromtimestamp(t1, tz=timezone.utc).strftime("%Y-%m-%d")
|
||||
return f"{a} ~ {b}"
|
||||
|
||||
|
||||
def _instrument(fills: pd.DataFrame, df: pd.DataFrame) -> str:
|
||||
for src in (fills, df):
|
||||
if not src.empty and "pair" in src.columns and src["pair"].notna().any():
|
||||
return str(src["pair"].dropna().iloc[0])
|
||||
return "BTCUSDT Perpetual (assumed)"
|
||||
|
||||
|
||||
def _maker_alpha_frame(g: pd.DataFrame) -> tuple[pd.Series, pd.Series]:
|
||||
"""Return (fill_ret, mkt_signed) for MakerAlpha = fill − market."""
|
||||
mid0 = g["mid"].astype(float)
|
||||
mid1 = g["after_30s_price"].astype(float)
|
||||
mkt_ret = (mid1 - mid0) / mid0
|
||||
mkt_signed = pd.Series(
|
||||
np.where(g["side"] == "long", mkt_ret, -mkt_ret), index=g.index
|
||||
)
|
||||
fill_ret = _fav_ret(g["side"], g["fill_price"], g["after_30s_price"])
|
||||
return fill_ret, mkt_signed
|
||||
|
||||
|
||||
def report(df: pd.DataFrame, min_fills: int = PASS_MIN_FILLS_DEFAULT, out_path: Path | None = None) -> dict[str, Any]:
|
||||
fills = df[df["event"] == "fill"].copy() if "event" in df.columns else pd.DataFrame()
|
||||
paths = df[df["event"] == "fill_path"].copy() if "event" in df.columns else pd.DataFrame()
|
||||
health = df[df["event"] == "health"].copy() if "event" in df.columns else pd.DataFrame()
|
||||
exp = _load_experiment_from_df(df)
|
||||
|
||||
lines: list[str] = []
|
||||
|
||||
def p(s: str = "") -> None:
|
||||
lines.append(s)
|
||||
print(s)
|
||||
|
||||
if not fills.empty:
|
||||
fills = assign_clusters_offline(fills)
|
||||
|
||||
if not paths.empty and not fills.empty and "fill_id" in fills.columns:
|
||||
meta_cols = [
|
||||
c
|
||||
for c in [
|
||||
"side",
|
||||
"fill_price",
|
||||
"fill_reason",
|
||||
"spread",
|
||||
"spread_capture_pct",
|
||||
"obi",
|
||||
"trade_imbalance",
|
||||
"bid_depth_5",
|
||||
"ask_depth_5",
|
||||
"book_age_ms",
|
||||
"inventory",
|
||||
"pre_5s_deteriorated",
|
||||
"mid",
|
||||
"event_cluster_id",
|
||||
"cluster_fill_index",
|
||||
"ts_epoch",
|
||||
"pair",
|
||||
]
|
||||
if c in fills.columns
|
||||
]
|
||||
meta = fills.drop_duplicates("fill_id")[["fill_id"] + meta_cols]
|
||||
paths = paths.merge(meta, on="fill_id", how="left", suffixes=("", "_f"))
|
||||
for col in ("side", "fill_price", "mid", "event_cluster_id", "spread", "spread_capture_pct"):
|
||||
alt = f"{col}_f"
|
||||
if alt in paths.columns:
|
||||
if col not in paths.columns:
|
||||
paths[col] = paths[alt]
|
||||
else:
|
||||
paths[col] = paths[col].fillna(paths[alt])
|
||||
fc = _extract_fill_context(fills)
|
||||
if not fc.empty:
|
||||
paths = paths.merge(fc, on="fill_id", how="left")
|
||||
|
||||
n_fills = len(fills)
|
||||
n_paths = len(paths)
|
||||
n_clusters = int(fills["event_cluster_id"].nunique()) if n_fills and "event_cluster_id" in fills.columns else 0
|
||||
cluster_fill_ratio = n_clusters / max(n_fills, 1)
|
||||
|
||||
# ---------- compute: integrity ----------
|
||||
integrity_ok = True
|
||||
integrity_notes: list[str] = []
|
||||
healthy_ratio = gap_total = gap_win_max = None
|
||||
lat_p50 = lat_p95 = lat_p99 = ba_med = None
|
||||
if health.empty:
|
||||
integrity_ok = False
|
||||
integrity_notes.append("no health telemetry")
|
||||
else:
|
||||
healthy_ratio = float(health["healthy"].astype(bool).mean()) if "healthy" in health.columns else 0.0
|
||||
gap_total = int(health["sequence_gap"].iloc[-1]) if "sequence_gap" in health.columns else 0
|
||||
gap_win_max = (
|
||||
int(pd.to_numeric(health.get("sequence_gap_window"), errors="coerce").fillna(0).max())
|
||||
if "sequence_gap_window" in health.columns
|
||||
else 0
|
||||
)
|
||||
lat_p50 = health["latency_ms_p50"].iloc[-1] if "latency_ms_p50" in health.columns else None
|
||||
lat_p95 = health["latency_ms_p95"].iloc[-1] if "latency_ms_p95" in health.columns else None
|
||||
lat_p99 = health["latency_ms_p99"].iloc[-1] if "latency_ms_p99" in health.columns else None
|
||||
ba_series = (
|
||||
fills["book_age_ms"]
|
||||
if "book_age_ms" in fills.columns and fills["book_age_ms"].notna().any()
|
||||
else health.get("book_age_ms")
|
||||
)
|
||||
ba_med = _median_safe(ba_series) if ba_series is not None else None
|
||||
if healthy_ratio < 0.99:
|
||||
integrity_ok = False
|
||||
integrity_notes.append(f"healthy_ratio={healthy_ratio*100:.2f}% < 99%")
|
||||
# Binance depth update ids are not contiguous — log only, do not INVALID.
|
||||
if gap_win_max and gap_win_max > 0:
|
||||
integrity_notes.append(
|
||||
f"sequence_gap_window_max={gap_win_max} (observe-only; Binance ids skip)"
|
||||
)
|
||||
if ba_med is not None and ba_med > 500:
|
||||
integrity_ok = False
|
||||
integrity_notes.append(f"book_age_median={ba_med:.0f}ms > 500ms")
|
||||
|
||||
decision: dict[str, Any] = {
|
||||
"integrity": integrity_ok,
|
||||
"verdict": "INSUFFICIENT_DATA",
|
||||
"reasons": [],
|
||||
"space_class": None,
|
||||
"benchmark_alpha": None,
|
||||
"maker_alpha_mean": None,
|
||||
"quality": {
|
||||
"fills": n_fills,
|
||||
"clusters": n_clusters,
|
||||
"cluster_fill_ratio": cluster_fill_ratio,
|
||||
"healthy_ratio": healthy_ratio,
|
||||
},
|
||||
}
|
||||
|
||||
if fills.empty:
|
||||
p("=" * 72)
|
||||
p("Maker Edge Report v0.1")
|
||||
p("Phase: Research Freeze / Data Collection")
|
||||
p("=" * 72)
|
||||
p("\nExecutive Summary")
|
||||
p(f" Experiment: {exp.get('experiment_id')}")
|
||||
p(f" Version: {exp.get('probe_version')}")
|
||||
p(" Quote: frozen")
|
||||
p(" Fee: frozen")
|
||||
p(" Exchange: frozen")
|
||||
p(f" Period: {_period_str(df)}")
|
||||
p(f" Instrument: {_instrument(fills, df)}")
|
||||
p(" Samples:")
|
||||
p(" fills: 0")
|
||||
p(" clusters: 0")
|
||||
p(" Decision: INSUFFICIENT_DATA")
|
||||
p(" Reason: no fills yet — run probe")
|
||||
decision["experiment"] = exp
|
||||
_finish(lines, out_path, decision)
|
||||
return decision
|
||||
|
||||
# ---------- compute: fill alpha table + distributions ----------
|
||||
alpha_table: dict[str, dict[str, float | None]] = {
|
||||
"Bid": {"fill_w": None, "cluster_w": None},
|
||||
"Ask": {"fill_w": None, "cluster_w": None},
|
||||
"Overall": {"fill_w": None, "cluster_w": None},
|
||||
}
|
||||
fill_alpha_dist: dict[str, float | None] = {}
|
||||
cluster_alpha_dist: dict[str, float | None] = {}
|
||||
fq_pass = None
|
||||
bench_alpha = None
|
||||
maker_alpha_mean = None
|
||||
agree = None
|
||||
pct_fills_positive_alpha = None
|
||||
|
||||
if not paths.empty and "after_30s_price" in paths.columns and "mid" in paths.columns and paths["mid"].notna().any():
|
||||
for side_name, g in paths.groupby("side"):
|
||||
label = _side_label(str(side_name))
|
||||
fill_ret, mkt_signed = _maker_alpha_frame(g)
|
||||
alpha = fill_ret - mkt_signed
|
||||
w = _cluster_weight(g)
|
||||
alpha_table[label]["fill_w"] = float(alpha.mean())
|
||||
alpha_table[label]["cluster_w"] = float((alpha * w).sum() / w.sum()) if w.sum() else float(alpha.mean())
|
||||
|
||||
fill_ret, mkt_signed = _maker_alpha_frame(paths)
|
||||
alpha = fill_ret - mkt_signed
|
||||
w = _cluster_weight(paths)
|
||||
alpha_table["Overall"]["fill_w"] = float(alpha.mean())
|
||||
alpha_table["Overall"]["cluster_w"] = (
|
||||
float((alpha * w).sum() / w.sum()) if w.sum() else float(alpha.mean())
|
||||
)
|
||||
maker_alpha_mean = alpha_table["Overall"]["cluster_w"]
|
||||
fw, cw = alpha_table["Overall"]["fill_w"], alpha_table["Overall"]["cluster_w"]
|
||||
agree = (fw > 0 and cw > 0) or (fw <= 0 and cw <= 0)
|
||||
fill_alpha_dist = _dist_stats(alpha)
|
||||
pct_fills_positive_alpha = float((alpha > 0).mean())
|
||||
|
||||
# per-cluster mean MakerAlpha(事件级分布)
|
||||
if "event_cluster_id" in paths.columns:
|
||||
tmp = paths.assign(_alpha=alpha)
|
||||
cluster_means = tmp.groupby("event_cluster_id")["_alpha"].mean()
|
||||
cluster_alpha_dist = _dist_stats(cluster_means)
|
||||
|
||||
mkt_fav = mkt_signed > 0
|
||||
fill_fav = fill_ret > 0
|
||||
bench_alpha = float(np.mean(fill_fav) - np.mean(mkt_fav))
|
||||
|
||||
fav30 = fill_ret
|
||||
p30_clu = float((fav30 > 0).astype(float).mul(w).sum() / w.sum()) if w.sum() else float((fav30 > 0).mean())
|
||||
fq_pass = p30_clu > 0.50
|
||||
|
||||
decision["fill_quality"] = fq_pass
|
||||
decision["benchmark_alpha"] = bench_alpha
|
||||
decision["maker_alpha_mean"] = maker_alpha_mean
|
||||
|
||||
# ---------- compute: toxicity + loss concentration ----------
|
||||
toxicity: dict[str, dict[str, float | None]] = {}
|
||||
toxic_bid_ratio = None
|
||||
c_share = None
|
||||
tox_dist: dict[str, Any] = {}
|
||||
if not paths.empty:
|
||||
for side_name, g in paths.groupby("side"):
|
||||
label = _side_label(str(side_name))
|
||||
row: dict[str, float | None] = {}
|
||||
for hz, col in [
|
||||
("1s", "after_1s_price"),
|
||||
("5s", "after_5s_price"),
|
||||
("10s", "after_10s_price"),
|
||||
("30s", "after_30s_price"),
|
||||
("300s", "after_5m_price"),
|
||||
]:
|
||||
if col in g.columns:
|
||||
row[hz] = float(_fav_ret(g["side"], g["fill_price"], g[col]).mean())
|
||||
else:
|
||||
row[hz] = None
|
||||
toxicity[label] = row
|
||||
if "path_type" in paths.columns:
|
||||
c_share = float((paths["path_type"].astype(str).str.startswith("C")).mean())
|
||||
bid = paths[paths["side"] == "long"]
|
||||
if len(bid):
|
||||
toxic_bid_ratio = float((bid["path_type"].astype(str).str.startswith("C")).mean())
|
||||
|
||||
# 毒性分布:多少成交在 10s 不利;最差 20% 占总不利损失比例
|
||||
if "after_10s_price" in paths.columns:
|
||||
fav10 = _fav_ret(paths["side"], paths["fill_price"], paths["after_10s_price"])
|
||||
adverse = fav10[fav10 < 0]
|
||||
tox_dist["pct_adverse_10s"] = float((fav10 < 0).mean())
|
||||
tox_dist["fav10"] = _dist_stats(fav10)
|
||||
if len(adverse) >= 5:
|
||||
worst_n = max(1, int(np.ceil(0.20 * len(fav10))))
|
||||
worst = fav10.nsmallest(worst_n)
|
||||
total_adv = float((-adverse).sum())
|
||||
worst_adv = float((-worst.clip(upper=0)).sum())
|
||||
tox_dist["worst20_share_of_adverse"] = (
|
||||
worst_adv / total_adv if total_adv > 1e-12 else None
|
||||
)
|
||||
else:
|
||||
tox_dist["worst20_share_of_adverse"] = None
|
||||
|
||||
# ---------- compute: cost / adverse ----------
|
||||
space_class = None
|
||||
adv_pass = None
|
||||
raw_capture = net_edge = adv_mag = sc_mean = total_cost = None
|
||||
if not paths.empty and "after_30s_price" in paths.columns:
|
||||
fav30 = _fav_ret(paths["side"], paths["fill_price"], paths["after_30s_price"])
|
||||
w = _cluster_weight(paths)
|
||||
raw_capture = float((fav30 * w).sum() / w.sum()) if w.sum() else float(fav30.mean())
|
||||
adv_mag = (
|
||||
float((-fav30.clip(upper=0) * w).sum() / w.sum())
|
||||
if w.sum()
|
||||
else float((-fav30.clip(upper=0)).mean())
|
||||
)
|
||||
sc_mean = (
|
||||
float(fills["spread_capture_pct"].mean())
|
||||
if "spread_capture_pct" in fills.columns and fills["spread_capture_pct"].notna().any()
|
||||
else 0.0
|
||||
)
|
||||
if "book_age_ms" in fills.columns and fills["book_age_ms"].notna().any():
|
||||
latency_cost = float(fills["book_age_ms"].mean()) / 100.0 * 0.00002
|
||||
else:
|
||||
latency_cost = 0.00002
|
||||
total_cost = 2 * FEE + EXPECTED_SLIPPAGE + latency_cost
|
||||
net_edge = raw_capture - total_cost
|
||||
space_class = classify_space(raw_capture, net_edge)
|
||||
adv_ok = (adv_mag < abs(sc_mean)) if sc_mean != 0 else False
|
||||
adv_pass = bool(adv_ok and space_class in ("POSITIVE_EDGE", "EDGE_AFTER_COST"))
|
||||
|
||||
decision["adverse"] = adv_pass
|
||||
decision["space_class"] = space_class
|
||||
|
||||
# ---------- compute: attribution (facts only) ----------
|
||||
attr_rows: list[tuple[str, str, int, float]] = []
|
||||
stab_pass = None
|
||||
state_coverage_ok = None
|
||||
concentrated = False
|
||||
positive_envs = 0
|
||||
total_envs = 0
|
||||
if not paths.empty and "after_30s_price" in paths.columns:
|
||||
paths = paths.copy()
|
||||
paths["_fav30"] = _fav_ret(paths["side"], paths["fill_price"], paths["after_30s_price"])
|
||||
if "vol_proxy_5m" in paths.columns and paths["vol_proxy_5m"].notna().any():
|
||||
med = paths["vol_proxy_5m"].median()
|
||||
paths["vol_bucket"] = np.where(paths["vol_proxy_5m"] >= med, "high_vol", "low_vol")
|
||||
elif "max_price" in paths.columns:
|
||||
rng = (paths["max_price"] - paths["min_price"]) / paths["fill_price"]
|
||||
paths["vol_bucket"] = np.where(rng >= rng.median(), "high_vol", "low_vol")
|
||||
if "price_velocity_5s" in paths.columns and paths["price_velocity_5s"].notna().any():
|
||||
v = paths["price_velocity_5s"].astype(float)
|
||||
thr = v.abs().median() * 0.5
|
||||
paths["trend_bucket"] = np.where(
|
||||
v > thr, "trend_up", np.where(v < -thr, "trend_down", "range")
|
||||
)
|
||||
if "spread" in paths.columns and paths["spread"].notna().any():
|
||||
sp_pct = paths["spread"] / paths["fill_price"]
|
||||
paths["liq_bucket"] = np.where(sp_pct <= sp_pct.median(), "tight_spread", "wide_spread")
|
||||
if "bid_depth_5" in paths.columns and "ask_depth_5" in paths.columns:
|
||||
depth = paths["bid_depth_5"].fillna(0) + paths["ask_depth_5"].fillna(0)
|
||||
if depth.gt(0).any():
|
||||
paths["depth_bucket"] = np.where(depth >= depth[depth > 0].median(), "deep_book", "thin_book")
|
||||
|
||||
pos_counts: list[int] = []
|
||||
for col, title in [
|
||||
("vol_bucket", "Volatility"),
|
||||
("trend_bucket", "Trend"),
|
||||
("liq_bucket", "Liquidity(spread)"),
|
||||
("depth_bucket", "Liquidity(depth)"),
|
||||
("market_event_before_fill", "FillContext"),
|
||||
("path_type", "PathType"),
|
||||
]:
|
||||
if col not in paths.columns or paths[col].isna().all():
|
||||
continue
|
||||
for idx, row in paths.groupby(col)["_fav30"].agg(["count", "mean"]).iterrows():
|
||||
total_envs += 1
|
||||
mean = float(row["mean"])
|
||||
n = int(row["count"])
|
||||
attr_rows.append((title, str(idx), n, mean))
|
||||
if mean > 0:
|
||||
positive_envs += 1
|
||||
pos_counts.append(n)
|
||||
|
||||
state_coverage_ok = total_envs >= 4
|
||||
if total_envs >= 2:
|
||||
if pos_counts:
|
||||
concentrated = (max(pos_counts) / max(sum(pos_counts), 1)) >= 0.70 and len(pos_counts) == 1
|
||||
stab_pass = positive_envs >= 2 and not concentrated
|
||||
else:
|
||||
state_coverage_ok = False
|
||||
|
||||
decision["stability"] = stab_pass
|
||||
decision["quality"]["state_buckets"] = len(attr_rows)
|
||||
|
||||
# ---------- decision ----------
|
||||
independence_ok = (
|
||||
n_clusters >= max(50, min_fills // 20) if n_fills >= min_fills else None
|
||||
)
|
||||
decision["independence"] = independence_ok
|
||||
reasons: list[str] = []
|
||||
|
||||
min_paths = max(1, min_fills // 10)
|
||||
sample_ok = n_fills >= min_fills and n_paths >= min_paths
|
||||
|
||||
gates = {
|
||||
"integrity": integrity_ok,
|
||||
"fill_quality": fq_pass,
|
||||
"adverse": adv_pass,
|
||||
"stability": stab_pass,
|
||||
}
|
||||
|
||||
hard_fail = False
|
||||
if not integrity_ok:
|
||||
hard_fail = True
|
||||
reasons.append("data integrity failed — stop interpretation")
|
||||
if toxic_bid_ratio is not None and toxic_bid_ratio > TOXIC_FAIL_RATIO:
|
||||
hard_fail = True
|
||||
reasons.append(f"Bid toxic fill ratio {toxic_bid_ratio*100:.0f}% > {TOXIC_FAIL_RATIO*100:.0f}%")
|
||||
if space_class == "NO_EDGE" and sample_ok:
|
||||
hard_fail = True
|
||||
reasons.append("edge disappears after cost / NO_EDGE")
|
||||
if bench_alpha is not None and bench_alpha <= 0 and sample_ok:
|
||||
reasons.append("benchmark-adjusted alpha negative")
|
||||
if maker_alpha_mean is not None and maker_alpha_mean <= 0 and sample_ok:
|
||||
reasons.append("MakerAlpha (fill−market) ≤ 0")
|
||||
if adv_pass is False and sample_ok:
|
||||
reasons.append("adverse selection ≥ spread capture")
|
||||
if concentrated:
|
||||
reasons.append("edge concentrated in single regime")
|
||||
|
||||
pass_extras = True
|
||||
if bench_alpha is not None and bench_alpha <= 0:
|
||||
pass_extras = False
|
||||
if independence_ok is False:
|
||||
pass_extras = False
|
||||
reasons.append(f"insufficient independent clusters ({n_clusters})")
|
||||
if space_class == "NO_EDGE":
|
||||
pass_extras = False
|
||||
|
||||
all_gates = all(v is True for v in gates.values())
|
||||
|
||||
# Stage3 unlock checklist(严格)
|
||||
stage3_unlock = {
|
||||
"data_integrity": integrity_ok is True,
|
||||
"cluster_weighted_alpha_gt_0": bool(maker_alpha_mean is not None and maker_alpha_mean > 0),
|
||||
"benchmark_alpha_gt_0": bool(bench_alpha is not None and bench_alpha > 0),
|
||||
"not_concentrated": not concentrated,
|
||||
}
|
||||
stage3_ready = all(stage3_unlock.values()) and sample_ok and all_gates and pass_extras
|
||||
|
||||
# 局部正 edge:归因桶分化或集中在单一正 regime
|
||||
local_positive = positive_envs >= 1 and total_envs >= 2 and (
|
||||
(positive_envs < total_envs) or concentrated
|
||||
)
|
||||
|
||||
if not integrity_ok:
|
||||
verdict = "INVALID"
|
||||
reasons = ["Data Integrity FAIL — do not interpret Alpha; discard / keep collecting clean data"]
|
||||
reasons.extend(integrity_notes)
|
||||
elif not sample_ok or state_coverage_ok is False:
|
||||
verdict = "COLLECTING"
|
||||
reasons = []
|
||||
if n_fills < min_fills:
|
||||
reasons.append(f"fills {n_fills} < {min_fills}")
|
||||
if n_paths < min_paths:
|
||||
reasons.append(f"fill_paths {n_paths} < {min_paths}")
|
||||
if n_clusters < max(50, min_fills // 20) and n_fills >= 500:
|
||||
reasons.append(f"clusters {n_clusters} insufficient (independent liquidity events)")
|
||||
if state_coverage_ok is False:
|
||||
reasons.append("state coverage incomplete")
|
||||
if not reasons:
|
||||
reasons.append("Insufficient independent liquidity events")
|
||||
elif hard_fail and not local_positive:
|
||||
verdict = "FAIL"
|
||||
if not reasons:
|
||||
reasons.append("market hypothesis does not hold under current quote assumption")
|
||||
elif stage3_ready:
|
||||
verdict = "PASS"
|
||||
reasons = [
|
||||
"Maker alpha survives: cost",
|
||||
"Maker alpha survives: benchmark",
|
||||
"Maker alpha survives: cluster weighting",
|
||||
"Maker alpha survives: multiple states",
|
||||
]
|
||||
elif local_positive and integrity_ok and sample_ok:
|
||||
verdict = "PARTIAL_PASS"
|
||||
reasons = [
|
||||
"edge not universal — observed only in subset of states/events",
|
||||
f"positive attribution buckets: {positive_envs}/{total_envs}",
|
||||
]
|
||||
if concentrated:
|
||||
reasons.append("edge concentrated in one regime/event class")
|
||||
if maker_alpha_mean is not None and maker_alpha_mean <= 0:
|
||||
reasons.append("overall cluster-weighted MakerAlpha ≤ 0")
|
||||
else:
|
||||
verdict = "FAIL"
|
||||
if not reasons:
|
||||
reasons.append("gates failed under current quote assumption")
|
||||
if adv_pass is False:
|
||||
reasons.insert(0, "adverse selection")
|
||||
if sc_mean is not None and abs(sc_mean) < 1e-8:
|
||||
reasons.append("insufficient spread")
|
||||
|
||||
decision["verdict"] = verdict
|
||||
decision["reasons"] = reasons
|
||||
decision["stage3_unlock"] = stage3_unlock
|
||||
decision["stage3_ready"] = stage3_ready
|
||||
decision["experiment"] = exp
|
||||
|
||||
# ==================================================================
|
||||
# PRINT — fixed format
|
||||
# ==================================================================
|
||||
p("=" * 72)
|
||||
p("Maker Edge Report v0.1")
|
||||
p("Phase: Research Freeze / Data Collection")
|
||||
p("Object: verifiable market phenomenon (not a strategy)")
|
||||
p("=" * 72)
|
||||
|
||||
# ----- Executive Summary -----
|
||||
p("\nExecutive Summary")
|
||||
p("-" * 40)
|
||||
p(f"Experiment: {exp.get('experiment_id')}")
|
||||
p(f"Version: {exp.get('probe_version')}")
|
||||
p("Quote: frozen")
|
||||
p("Fee: frozen")
|
||||
p("Exchange: frozen")
|
||||
p(f"Venue: {exp.get('exchange')} / {exp.get('environment')}")
|
||||
p(f"Period: {_period_str(df if not df.empty else fills)}")
|
||||
p(f"Instrument: {_instrument(fills, df)}")
|
||||
p("Samples:")
|
||||
p(f" fills: {n_fills}")
|
||||
p(f" clusters: {n_clusters}")
|
||||
p(f" paths: {n_paths}")
|
||||
p(f" cluster/fill: {cluster_fill_ratio*100:.1f}%")
|
||||
p(f"Observation window: {_observation_window(n_fills, n_clusters)}")
|
||||
p(" (500=anomaly · 2000=preliminary · 10000=stability; clusters > fills)")
|
||||
p(f"Decision: {verdict}")
|
||||
p("Reason:")
|
||||
for r in reasons:
|
||||
p(f" - {r}")
|
||||
p("Hypothesis under test: passive fills produce +MakerAlpha")
|
||||
p(" under current BTC perp / venue / quote / execution — not strategy PnL.")
|
||||
p("Read order: Integrity → distributions (not mean) → Cluster → Toxicity → Decision")
|
||||
|
||||
# ----- Section 1 -----
|
||||
p("\n" + "=" * 72)
|
||||
p("Section 1 — Data Integrity")
|
||||
p("Question: Is the data trustworthy?")
|
||||
p("=" * 72)
|
||||
if health.empty:
|
||||
p("Healthy: n/a (no health events)")
|
||||
p("Sequence gap: n/a")
|
||||
p("Latency: n/a")
|
||||
p("Book freshness:n/a")
|
||||
else:
|
||||
p(f"Healthy: {healthy_ratio*100:.2f}%")
|
||||
p(f"Sequence gap: total={gap_total} window_max={gap_win_max}")
|
||||
p("Latency:")
|
||||
p(f" p50: {lat_p50} ms")
|
||||
p(f" p95: {lat_p95} ms")
|
||||
p(f" p99: {lat_p99} ms")
|
||||
p(f"Book freshness: median={ba_med:.1f} ms" if ba_med is not None else "Book freshness: n/a")
|
||||
p(f"Integrity: [{'PASS' if integrity_ok else 'FAIL'}]")
|
||||
for n in integrity_notes:
|
||||
p(f" · {n}")
|
||||
if not integrity_ok:
|
||||
p("\n★ STOP — Data Integrity FAIL → Decision=INVALID.")
|
||||
p(" Do not interpret Alpha. Bad book/latency/gap fills have no research value.")
|
||||
|
||||
# ----- Section 2 -----
|
||||
p("\n" + "=" * 72)
|
||||
p("Section 2 — Fill Alpha")
|
||||
p("Question: Fill − Matched Market Move (not PnL)")
|
||||
p("Priority: distribution (median/p25/p75) over mean")
|
||||
p("=" * 72)
|
||||
if not integrity_ok:
|
||||
p("(skipped for decision — integrity INVALID; numbers below are not evidence)")
|
||||
if alpha_table["Overall"]["fill_w"] is None:
|
||||
p("(waiting for fill_path with mid + after_30s)")
|
||||
else:
|
||||
p(f"{'':12s} {'Fill weighted':>16s} {'Cluster weighted':>18s}")
|
||||
for lab in ("Bid", "Ask", "Overall"):
|
||||
fw = alpha_table[lab]["fill_w"]
|
||||
cw = alpha_table[lab]["cluster_w"]
|
||||
p(f"{lab+' Alpha':12s} {_fmt_pct(fw):>16s} {_fmt_pct(cw):>18s}")
|
||||
p(f"Direction agree (fill-w vs cluster-w): {'YES' if agree else 'NO ★'}")
|
||||
p(f"Benchmark P(+) Δ (fill − matched mid): {_fmt_pp(bench_alpha)}")
|
||||
p(f"SPACE class: {space_class or 'PENDING'}")
|
||||
if raw_capture is not None and net_edge is not None and total_cost is not None:
|
||||
p(f"Raw capture@30s (cluster-w): {_fmt_pct(raw_capture)}")
|
||||
p(f"Total cost (fee+slip+lat): {_fmt_pct(total_cost)}")
|
||||
p(f"Net edge: {_fmt_pct(net_edge)}")
|
||||
p("")
|
||||
p("Fill Alpha distribution (do not trust mean alone):")
|
||||
_print_dist(p, " per-fill MakerAlpha", fill_alpha_dist)
|
||||
if pct_fills_positive_alpha is not None:
|
||||
p(f" share of fills with +alpha: {pct_fills_positive_alpha*100:.1f}%")
|
||||
if pct_fills_positive_alpha < 0.35 and (fill_alpha_dist.get("mean") or 0) > 0:
|
||||
p(" ★ mean>0 but minority of fills — edge likely event-driven / fat tail")
|
||||
p("")
|
||||
p("Cluster Alpha distribution (independent liquidity events):")
|
||||
_print_dist(p, " per-cluster mean MakerAlpha", cluster_alpha_dist)
|
||||
if (
|
||||
alpha_table["Overall"]["fill_w"] is not None
|
||||
and alpha_table["Overall"]["cluster_w"] is not None
|
||||
):
|
||||
fw, cw = alpha_table["Overall"]["fill_w"], alpha_table["Overall"]["cluster_w"]
|
||||
if fw > 0 >= cw:
|
||||
p(" ★ Fill+ but Cluster≤0 — edge from few burst fills; unstable")
|
||||
elif fw > 0 and cw > 0:
|
||||
p(" Fill+ and Cluster+ — credibility higher")
|
||||
|
||||
# ----- Section 3 -----
|
||||
p("\n" + "=" * 72)
|
||||
p("Section 3 — Toxicity Profile")
|
||||
p("Question: Are fills naturally on the wrong side? (record only — no quote changes)")
|
||||
p("=" * 72)
|
||||
if not toxicity:
|
||||
p("(waiting for fill_path)")
|
||||
else:
|
||||
for label, row in toxicity.items():
|
||||
p(f"\n{label}:")
|
||||
p(" Immediate toxicity:")
|
||||
for hz in ("1s", "5s", "10s"):
|
||||
p(f" {hz}: {_fmt_pct(row.get(hz))}")
|
||||
p(" Recovery:")
|
||||
for hz in ("30s", "300s"):
|
||||
p(f" {hz}: {_fmt_pct(row.get(hz))}")
|
||||
# factual pattern note only
|
||||
t10, t300 = row.get("10s"), row.get("300s")
|
||||
if t10 is not None and t300 is not None:
|
||||
if t10 < 0 < t300:
|
||||
p(" Observed pattern: early toxicity + later recovery (fact; not a rule)")
|
||||
elif t10 < 0 and t300 <= 0:
|
||||
p(" Observed pattern: sustained adverse (fact; not a rule)")
|
||||
elif t10 is not None and t10 > 0:
|
||||
p(" Observed pattern: immediate favorable (fact; not a rule)")
|
||||
if c_share is not None:
|
||||
p(f"\nPath C (toxic) share: {c_share*100:.1f}%")
|
||||
if toxic_bid_ratio is not None:
|
||||
p(f"Bid toxic fill ratio: {toxic_bid_ratio*100:.1f}%")
|
||||
if adv_mag is not None and sc_mean is not None:
|
||||
p(f"mean_adverse vs |spread_capture|: {_fmt_pct(adv_mag)} vs {_fmt_pct(abs(sc_mean))}")
|
||||
if tox_dist:
|
||||
p("\nToxicity distribution:")
|
||||
if tox_dist.get("pct_adverse_10s") is not None:
|
||||
p(f" fills adverse@10s: {tox_dist['pct_adverse_10s']*100:.1f}%")
|
||||
if tox_dist.get("fav10"):
|
||||
_print_dist(p, " fav@10s", tox_dist["fav10"])
|
||||
w20 = tox_dist.get("worst20_share_of_adverse")
|
||||
if w20 is not None:
|
||||
p(f" worst 20% of fills share of adverse loss: {w20*100:.1f}%")
|
||||
if w20 >= 0.70:
|
||||
p(" ★ losses concentrated — future value may be 'which quotes NOT to place'")
|
||||
p(" (record only; no cancel/filter rules in freeze)")
|
||||
|
||||
# ----- Section 4 -----
|
||||
p("\n" + "=" * 72)
|
||||
p("Section 4 — Observed Edge Attribution")
|
||||
p("Facts only. Not strategy recommendations. Not filter rules.")
|
||||
p("=" * 72)
|
||||
if not attr_rows:
|
||||
p("(insufficient state slices)")
|
||||
else:
|
||||
cur_title = None
|
||||
for title, idx, n, mean in attr_rows:
|
||||
if title != cur_title:
|
||||
p(f"\n{title}:")
|
||||
cur_title = title
|
||||
sign = "positive" if mean > 0 else ("negative" if mean < 0 else "flat")
|
||||
p(f" {idx}: n={n} E[fav30]={_fmt_pct(mean)} ({sign})")
|
||||
if concentrated:
|
||||
p("\nObservation: positive mass concentrated in a single bucket (fact).")
|
||||
|
||||
# ----- Section 5 -----
|
||||
p("\n" + "=" * 72)
|
||||
p("Section 5 — Decision")
|
||||
p("=" * 72)
|
||||
p(f"Decision: {verdict}")
|
||||
p("")
|
||||
if verdict == "INVALID":
|
||||
p("Reason:")
|
||||
for r in reasons:
|
||||
p(f" - {r}")
|
||||
p("\nKeep collecting only after Data Integrity is clean.")
|
||||
elif verdict == "COLLECTING":
|
||||
p("Reason:")
|
||||
for r in reasons:
|
||||
p(f" - {r}")
|
||||
p("\nDo not over-interpret before 2000 fills / adequate clusters.")
|
||||
p("500 = anomaly check · 2000 = preliminary · 10000 = stability.")
|
||||
elif verdict == "PASS":
|
||||
p("Maker alpha survives:")
|
||||
for r in reasons:
|
||||
p(f" - {r.replace('Maker alpha survives: ', '')}")
|
||||
p("\n→ Unlock Stage3 Economic Simulation → Symmetric MM")
|
||||
elif verdict == "PARTIAL_PASS":
|
||||
p("Partial: market hypothesis holds only in some states/events.")
|
||||
for r in reasons:
|
||||
p(f" - {r}")
|
||||
p("\n→ Path: Event-driven LP (not all-day Symmetric MM)")
|
||||
p(" Still locked: no new filters yet — attribution is observation only.")
|
||||
else:
|
||||
p("No maker edge under current quote assumption.")
|
||||
p("Dominant reasons:")
|
||||
for r in reasons:
|
||||
p(f" - {r}")
|
||||
p("\nConclusion = hypothesis false (not 'strategy failed'). Avoid futile tuning.")
|
||||
|
||||
p("\nStage3 Unlock Checklist (Economic Simulation):")
|
||||
for k, v in stage3_unlock.items():
|
||||
p(f" [{'OK' if v else '·'}] {k}")
|
||||
p(f" Stage3 ready: {'YES' if stage3_ready else 'NO'}")
|
||||
|
||||
p("")
|
||||
p("State machine:")
|
||||
p(" FAIL → change hypothesis")
|
||||
p(" PARTIAL_PASS → Event-driven LP")
|
||||
p(" PASS → Economic Simulation → Symmetric MM")
|
||||
p(" COLLECTING → keep collecting")
|
||||
p("")
|
||||
p("Action: run probe. Look at distributions first, Decision second.")
|
||||
p("=" * 72)
|
||||
|
||||
_finish(lines, out_path, decision)
|
||||
return decision
|
||||
|
||||
|
||||
def _finish(lines: list[str], out_path: Path | None, decision: dict[str, Any]) -> None:
|
||||
if out_path:
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
footer = {
|
||||
"event": "maker_edge_decision",
|
||||
"report": "Maker Edge Report v0.1",
|
||||
"phase": "Research Freeze / Data Collection",
|
||||
"verdict": decision.get("verdict"),
|
||||
"experiment": decision.get("experiment"),
|
||||
"space_class": decision.get("space_class"),
|
||||
"benchmark_alpha": decision.get("benchmark_alpha"),
|
||||
"maker_alpha_mean": decision.get("maker_alpha_mean"),
|
||||
"stage3_ready": decision.get("stage3_ready"),
|
||||
"stage3_unlock": decision.get("stage3_unlock"),
|
||||
"quality": decision.get("quality"),
|
||||
"gates": {
|
||||
"integrity": decision.get("integrity"),
|
||||
"independence": decision.get("independence"),
|
||||
"fill_quality": decision.get("fill_quality"),
|
||||
"adverse": decision.get("adverse"),
|
||||
"stability": decision.get("stability"),
|
||||
},
|
||||
"reasons": decision.get("reasons"),
|
||||
}
|
||||
text = "\n".join(lines) + "\n\n---\n" + json.dumps(footer, ensure_ascii=False, indent=2) + "\n"
|
||||
out_path.write_text(text, encoding="utf-8")
|
||||
print(f"\nReport saved: {out_path}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description="Maker Edge Report v0.1 — Research Freeze")
|
||||
ap.add_argument(
|
||||
"--dir",
|
||||
type=str,
|
||||
default=str(Path(__file__).resolve().parents[1] / "logs" / "maker_edge"),
|
||||
)
|
||||
ap.add_argument("--min-fills", type=int, default=PASS_MIN_FILLS_DEFAULT)
|
||||
ap.add_argument("--report", action="store_true")
|
||||
args = ap.parse_args()
|
||||
log_dir = Path(args.dir)
|
||||
if not log_dir.exists():
|
||||
print(f"日志目录不存在: {log_dir}")
|
||||
return
|
||||
try:
|
||||
df = load_events(log_dir)
|
||||
except FileNotFoundError as e:
|
||||
print(e)
|
||||
return
|
||||
out = log_dir / "Maker_Edge_Report_v0.1.txt" if args.report else None
|
||||
report(df, min_fills=args.min_fills, out_path=out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+95
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
# 部署 MM_EDGE_EXP_001 → jack@jackyu66.com:/www/Project/nautilus_mm
|
||||
#
|
||||
# 默认:
|
||||
# SSH_HOST=jack@jackyu66.com
|
||||
# SSH_KEY=~/Project/deploy/zun_hk/id_ed25519_hk
|
||||
# REMOTE_DIR=/www/Project/nautilus_mm
|
||||
#
|
||||
# 覆盖:export SSH_HOST=... SSH_KEY=... REMOTE_DIR=...
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
SSH_HOST="${SSH_HOST:-jack@jackyu66.com}"
|
||||
SSH_KEY="${SSH_KEY:-${HOME}/Project/deploy/zun_hk/id_ed25519_hk}"
|
||||
REMOTE_DIR="${REMOTE_DIR:-/www/Project/nautilus_mm}"
|
||||
|
||||
if [[ ! -f "$SSH_KEY" ]]; then
|
||||
echo "SSH key not found: $SSH_KEY"
|
||||
exit 1
|
||||
fi
|
||||
chmod 400 "$SSH_KEY" 2>/dev/null || true
|
||||
|
||||
SSH_OPTS=(-i "$SSH_KEY" -o StrictHostKeyChecking=accept-new)
|
||||
SSH=(ssh "${SSH_OPTS[@]}" "$SSH_HOST")
|
||||
RSYNC_E="ssh ${SSH_OPTS[*]}"
|
||||
|
||||
echo "==> stop remote probe before sync (if running)"
|
||||
"${SSH[@]}" "systemctl --user stop mm-edge-probe 2>/dev/null || true"
|
||||
|
||||
echo "==> sync $ROOT → $SSH_HOST:$REMOTE_DIR"
|
||||
"${SSH[@]}" "mkdir -p '$REMOTE_DIR' '$REMOTE_DIR/logs/maker_edge'"
|
||||
rsync -avz --delete \
|
||||
-e "$RSYNC_E" \
|
||||
--exclude '.venv' \
|
||||
--exclude '__pycache__' \
|
||||
--exclude '*.pyc' \
|
||||
--exclude 'logs/maker_edge/*.jsonl' \
|
||||
--exclude 'logs/maker_edge/*.txt' \
|
||||
--exclude 'logs/maker_edge_smoke' \
|
||||
--exclude '.env' \
|
||||
"$ROOT/" "$SSH_HOST:$REMOTE_DIR/"
|
||||
|
||||
echo "==> remote setup (uv venv + user systemd)"
|
||||
"${SSH[@]}" "REMOTE_DIR='$REMOTE_DIR' bash -s" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
cd "$REMOTE_DIR"
|
||||
if [[ ! -f .env ]]; then
|
||||
cp .env.example .env
|
||||
{
|
||||
echo ""
|
||||
echo "# Server Data Collection — MM_EDGE_EXP_001"
|
||||
echo "EXPERIMENT_ID=MM_EDGE_EXP_001"
|
||||
echo "PROBE_VERSION=probe_v0.1"
|
||||
echo "EXCHANGE_NAME=binance_usdm"
|
||||
echo "BINANCE_ENVIRONMENT=TESTNET"
|
||||
echo "ENABLE_TRADING=false"
|
||||
echo "QUOTE_TTL_SECS=30"
|
||||
echo "MAX_ABS_INVENTORY=0.005"
|
||||
echo "HTTP_PROXY="
|
||||
echo "HTTPS_PROXY="
|
||||
echo "MAKER_EDGE_LOG_DIR=${REMOTE_DIR}/logs/maker_edge"
|
||||
} >> .env
|
||||
echo "CREATED .env — fill BINANCE_API_KEY / BINANCE_API_SECRET"
|
||||
else
|
||||
echo ".env exists — left untouched"
|
||||
fi
|
||||
|
||||
if [[ ! -x "$HOME/.local/bin/uv" ]]; then
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
fi
|
||||
uv python install 3.12
|
||||
rm -rf .venv
|
||||
uv venv .venv --python 3.12
|
||||
uv pip install -r requirements.txt --python .venv/bin/python
|
||||
|
||||
mkdir -p "$HOME/.config/systemd/user"
|
||||
sed -e "s|/www/Project/nautilus_mm|${REMOTE_DIR}|g" \
|
||||
deploy/mm-edge-probe.user.service > "$HOME/.config/systemd/user/mm-edge-probe.service"
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user enable mm-edge-probe.service
|
||||
loginctl enable-linger "$(whoami)" 2>/dev/null || true
|
||||
echo "User systemd installed (not started — fill keys first)."
|
||||
echo " nano $REMOTE_DIR/.env"
|
||||
echo " systemctl --user start mm-edge-probe"
|
||||
echo " journalctl --user -u mm-edge-probe -f"
|
||||
REMOTE
|
||||
|
||||
echo ""
|
||||
echo "==> done"
|
||||
echo "1) ssh -i $SSH_KEY $SSH_HOST"
|
||||
echo "2) nano $REMOTE_DIR/.env # TESTNET keys"
|
||||
echo "3) systemctl --user start mm-edge-probe"
|
||||
echo "4) ./scripts/probe_status.sh"
|
||||
echo "5) ./scripts/pull_report.sh"
|
||||
@@ -0,0 +1,533 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Economic Attribution v0.1
|
||||
|
||||
Hard Evidence Population only:
|
||||
MATCHED = Local Fill ↔ Venue Trade dual evidence
|
||||
|
||||
Purpose:
|
||||
Economic Attribution only.
|
||||
No strategy modification.
|
||||
No live execution.
|
||||
No economic simulation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
from reconcile_fills import load_local_fills, match, normalize_local, normalize_venue # noqa: E402
|
||||
|
||||
|
||||
def _load_jsonl_df(log_dir: Path) -> pd.DataFrame:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for f in sorted(log_dir.glob("*.jsonl")):
|
||||
if f.name.startswith(("Account_", "Maker_", "RECON")):
|
||||
continue
|
||||
for line in f.open():
|
||||
try:
|
||||
e = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(e, dict):
|
||||
rows.append(e)
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def _parse_fill_context(df: pd.DataFrame) -> pd.DataFrame:
|
||||
if df.empty or "fill_context" not in df.columns:
|
||||
return pd.DataFrame(columns=["fill_id"])
|
||||
rows = []
|
||||
for _, r in df.iterrows():
|
||||
ctx = r.get("fill_context")
|
||||
if not isinstance(ctx, dict):
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"fill_id": r.get("fill_id"),
|
||||
"market_event_before_fill": ctx.get("market_event_before_fill"),
|
||||
"trade_imbalance_5s": ctx.get("trade_imbalance_5s"),
|
||||
"price_velocity_5s": ctx.get("price_velocity_5s"),
|
||||
"fill_type": ctx.get("fill_type"),
|
||||
}
|
||||
)
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def _fav_ret(side: pd.Series, fill: pd.Series, px: pd.Series) -> pd.Series:
|
||||
raw = (pd.to_numeric(px, errors="coerce") - pd.to_numeric(fill, errors="coerce")) / pd.to_numeric(
|
||||
fill, errors="coerce"
|
||||
)
|
||||
return pd.Series(np.where(side == "long", raw, -raw), index=side.index)
|
||||
|
||||
|
||||
def _cluster_weight(frame: pd.DataFrame) -> pd.Series:
|
||||
cnt = frame.groupby("event_cluster_id")["event_cluster_id"].transform("count")
|
||||
return 1.0 / cnt.clip(lower=1)
|
||||
|
||||
|
||||
def _pct(v: float | None) -> str:
|
||||
if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))):
|
||||
return "n/a"
|
||||
return f"{v * 100:.4f}%"
|
||||
|
||||
|
||||
def _num(v: float | None, digits: int = 4) -> str:
|
||||
if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))):
|
||||
return "n/a"
|
||||
return f"{v:.{digits}f}"
|
||||
|
||||
|
||||
def _mean(s: pd.Series) -> float | None:
|
||||
s = pd.to_numeric(s, errors="coerce").dropna()
|
||||
return None if s.empty else float(s.mean())
|
||||
|
||||
|
||||
def _sum(s: pd.Series) -> float:
|
||||
s = pd.to_numeric(s, errors="coerce").fillna(0.0)
|
||||
return float(s.sum())
|
||||
|
||||
|
||||
def _weighted_mean(v: pd.Series, w: pd.Series) -> float | None:
|
||||
vv = pd.to_numeric(v, errors="coerce")
|
||||
ww = pd.to_numeric(w, errors="coerce").fillna(0.0)
|
||||
mask = vv.notna() & ww.notna()
|
||||
vv = vv[mask]
|
||||
ww = ww[mask]
|
||||
if vv.empty or float(ww.sum()) == 0.0:
|
||||
return None
|
||||
return float((vv * ww).sum() / ww.sum())
|
||||
|
||||
|
||||
def _prepare_paths(paths: pd.DataFrame) -> pd.DataFrame:
|
||||
paths = paths.copy()
|
||||
if "max_price" in paths.columns and "min_price" in paths.columns and "fill_price" in paths.columns:
|
||||
rng = (pd.to_numeric(paths["max_price"], errors="coerce") - pd.to_numeric(paths["min_price"], errors="coerce")) / pd.to_numeric(
|
||||
paths["fill_price"], errors="coerce"
|
||||
)
|
||||
med = float(rng.dropna().median()) if rng.notna().any() else 0.0
|
||||
paths["vol_bucket"] = np.where(rng >= med, "high_vol", "low_vol")
|
||||
if "price_velocity_5s" in paths.columns and pd.to_numeric(paths["price_velocity_5s"], errors="coerce").notna().any():
|
||||
v = pd.to_numeric(paths["price_velocity_5s"], errors="coerce")
|
||||
thr = float(v.abs().median()) * 0.5
|
||||
paths["trend_bucket"] = np.where(v > thr, "trend_up", np.where(v < -thr, "trend_down", "range"))
|
||||
if "spread" in paths.columns and "fill_price" in paths.columns and pd.to_numeric(paths["spread"], errors="coerce").notna().any():
|
||||
sp = pd.to_numeric(paths["spread"], errors="coerce") / pd.to_numeric(paths["fill_price"], errors="coerce")
|
||||
med = float(sp.dropna().median()) if sp.notna().any() else 0.0
|
||||
paths["liq_bucket"] = np.where(sp <= med, "tight_spread", "wide_spread")
|
||||
paths["toxicity_bucket"] = np.where(paths["path_type"].astype(str).str.startswith("C"), "toxic", "non_toxic")
|
||||
return paths
|
||||
|
||||
|
||||
def _inventory_metrics(matched: pd.DataFrame) -> dict[str, float | None]:
|
||||
if matched.empty:
|
||||
return {}
|
||||
g = matched.sort_values("venue_time_ms").copy()
|
||||
g["signed_qty"] = np.where(g["side"] == "long", g["qty"], -g["qty"])
|
||||
g["net_btc"] = g["signed_qty"].cumsum()
|
||||
g["abs_net_btc"] = g["net_btc"].abs()
|
||||
times = pd.to_numeric(g["venue_time_ms"], errors="coerce").astype("float64") / 1000.0
|
||||
dt = times.shift(-1) - times
|
||||
dt = dt.fillna(0.0).clip(lower=0.0)
|
||||
total_t = float(dt.sum())
|
||||
tw_abs = float((g["abs_net_btc"] * dt).sum() / total_t) if total_t > 0 else None
|
||||
tw_signed = float((g["net_btc"] * dt).sum() / total_t) if total_t > 0 else None
|
||||
return {
|
||||
"max_net_btc": float(g["net_btc"].max()),
|
||||
"min_net_btc": float(g["net_btc"].min()),
|
||||
"max_abs_net_btc": float(g["abs_net_btc"].max()),
|
||||
"avg_abs_net_btc_per_fill": float(g["abs_net_btc"].mean()),
|
||||
"time_weighted_abs_net_btc": tw_abs,
|
||||
"time_weighted_signed_net_btc": tw_signed,
|
||||
"long_qty": float(g.loc[g["signed_qty"] > 0, "signed_qty"].sum()),
|
||||
"short_qty": float((-g.loc[g["signed_qty"] < 0, "signed_qty"]).sum()),
|
||||
"turnover_btc": float(g["qty"].sum()),
|
||||
}
|
||||
|
||||
|
||||
def _bucket_table(paths: pd.DataFrame, bucket: str, title: str) -> list[dict[str, Any]]:
|
||||
if bucket not in paths.columns or paths.empty:
|
||||
return []
|
||||
rows = []
|
||||
for key, grp in paths.groupby(bucket):
|
||||
notional = grp["notional_usdt"].sum()
|
||||
clusters = grp["event_cluster_id"].nunique()
|
||||
rows.append(
|
||||
{
|
||||
"dimension": title,
|
||||
"bucket": str(key),
|
||||
"fills": int(len(grp)),
|
||||
"clusters": int(clusters),
|
||||
"btc_qty": float(grp["qty"].sum()),
|
||||
"notional_usdt": float(notional),
|
||||
"fee_usdt": float(grp["commission_usdt"].sum()),
|
||||
"fee_per_fill": float(grp["commission_usdt"].mean()) if len(grp) else None,
|
||||
"fee_per_btc": float(grp["commission_usdt"].sum() / grp["qty"].sum()) if grp["qty"].sum() else None,
|
||||
"markout_1s": _weighted_mean(grp["markout_1s"], grp["notional_usdt"]),
|
||||
"markout_5s": _weighted_mean(grp["markout_5s"], grp["notional_usdt"]),
|
||||
"markout_10s": _weighted_mean(grp["markout_10s"], grp["notional_usdt"]),
|
||||
"markout_30s": _weighted_mean(grp["markout_30s"], grp["notional_usdt"]),
|
||||
"markout_300s": _weighted_mean(grp["markout_300s"], grp["notional_usdt"]),
|
||||
"gross_markout_30s_usdt": float(grp["gross_markout_30s_usdt"].sum()),
|
||||
"realized_pnl_usdt": float(grp["realized_pnl_usdt"].sum()),
|
||||
"net_attr_30s_usdt": float(grp["net_attr_30s_usdt"].sum()),
|
||||
}
|
||||
)
|
||||
rows.sort(key=lambda x: (-x["fills"], x["bucket"]))
|
||||
return rows
|
||||
|
||||
|
||||
def _counterfactual(base: pd.DataFrame, exclude_col: str, exclude_values: set[str], label: str) -> dict[str, Any]:
|
||||
kept = base[~base[exclude_col].astype(str).isin(exclude_values)].copy()
|
||||
return {
|
||||
"name": label,
|
||||
"fills": int(len(kept)),
|
||||
"clusters": int(kept["event_cluster_id"].nunique()) if not kept.empty else 0,
|
||||
"btc_qty": float(kept["qty"].sum()) if not kept.empty else 0.0,
|
||||
"fee_usdt": float(kept["commission_usdt"].sum()) if not kept.empty else 0.0,
|
||||
"gross_markout_30s_usdt": float(kept["gross_markout_30s_usdt"].sum()) if not kept.empty else 0.0,
|
||||
"realized_pnl_usdt": float(kept["realized_pnl_usdt"].sum()) if not kept.empty else 0.0,
|
||||
"net_attr_30s_usdt": float(kept["net_attr_30s_usdt"].sum()) if not kept.empty else 0.0,
|
||||
"markout_30s": _weighted_mean(kept["markout_30s"], kept["notional_usdt"]),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="Economic Attribution v0.1 (MATCHED only)")
|
||||
ap.add_argument("--dir", default=str(ROOT / "logs" / "maker_edge"))
|
||||
ap.add_argument("--out", default=str(ROOT / "logs" / "maker_edge" / "Economic_Attribution_v0_1.txt"))
|
||||
ap.add_argument("--recon03", default=str(ROOT / "logs" / "maker_edge" / "RECONCILIATION_03.json"))
|
||||
ap.add_argument("--account", default=str(ROOT / "logs" / "maker_edge" / "Account_Reconciliation.json"))
|
||||
ap.add_argument("--venue-trades", default=str(ROOT / "logs" / "maker_edge" / "venue_trades.json"))
|
||||
args = ap.parse_args()
|
||||
|
||||
log_dir = Path(args.dir)
|
||||
df = _load_jsonl_df(log_dir)
|
||||
fills = df[df["event"] == "fill"].copy()
|
||||
paths = df[df["event"] == "fill_path"].copy()
|
||||
inv = df[df["event"] == "inventory_tick"].copy()
|
||||
|
||||
venue_trades = json.loads(Path(args.venue_trades).read_text())
|
||||
local_fills_raw = load_local_fills(log_dir)
|
||||
locals_norm = [normalize_local(e, i) for i, e in enumerate(local_fills_raw)]
|
||||
venues_norm = [normalize_venue(t, i) for i, t in enumerate(venue_trades)]
|
||||
recon = match(locals_norm, venues_norm)
|
||||
matched_fill_ids = {m["local"]["fill_id"] for m in recon["matched"]}
|
||||
matched_trade_ids = {m["venue"]["venue_trade_id"] for m in recon["matched"]}
|
||||
|
||||
fills = fills[fills["fill_id"].isin(matched_fill_ids)].copy()
|
||||
paths = paths[paths["fill_id"].isin(matched_fill_ids)].copy()
|
||||
|
||||
fc = _parse_fill_context(fills)
|
||||
meta_cols = [
|
||||
c
|
||||
for c in [
|
||||
"fill_id",
|
||||
"side",
|
||||
"fill_price",
|
||||
"spread",
|
||||
"spread_capture_pct",
|
||||
"obi",
|
||||
"trade_imbalance",
|
||||
"bid_depth_5",
|
||||
"ask_depth_5",
|
||||
"book_age_ms",
|
||||
"inventory",
|
||||
"inventory_time",
|
||||
"inventory_skew",
|
||||
"pre_5s_deteriorated",
|
||||
"mid",
|
||||
"event_cluster_id",
|
||||
"pair",
|
||||
]
|
||||
if c in fills.columns
|
||||
]
|
||||
meta = fills.drop_duplicates("fill_id")[meta_cols]
|
||||
paths = paths.merge(meta, on="fill_id", how="left", suffixes=("", "_f"))
|
||||
for col in ["event_cluster_id", "side", "fill_price", "spread_capture_pct", "mid"]:
|
||||
alt = f"{col}_f"
|
||||
if alt in paths.columns:
|
||||
if col not in paths.columns:
|
||||
paths[col] = paths[alt]
|
||||
else:
|
||||
paths[col] = paths[col].fillna(paths[alt])
|
||||
if not fc.empty:
|
||||
paths = paths.merge(fc, on="fill_id", how="left")
|
||||
paths = _prepare_paths(paths)
|
||||
|
||||
venue = pd.DataFrame(venues_norm)
|
||||
venue = venue[venue["venue_trade_id"].isin(matched_trade_ids)].copy()
|
||||
venue = venue.rename(
|
||||
columns={
|
||||
"venue_trade_id": "trade_id_link",
|
||||
"venue_order_id": "venue_order_id",
|
||||
"qty": "qty",
|
||||
"px": "venue_price",
|
||||
"ts": "venue_ts",
|
||||
}
|
||||
)
|
||||
raw_v = pd.DataFrame(venue_trades)
|
||||
raw_v["trade_id_link"] = raw_v["id"].astype(str)
|
||||
raw_v["venue_order_id"] = raw_v["orderId"].astype(str)
|
||||
raw_v["commission_usdt"] = pd.to_numeric(raw_v["commission"], errors="coerce")
|
||||
raw_v["realized_pnl_usdt"] = pd.to_numeric(raw_v["realizedPnl"], errors="coerce").fillna(0.0)
|
||||
raw_v["venue_time_ms"] = pd.to_numeric(raw_v["time"], errors="coerce")
|
||||
raw_v["qty"] = pd.to_numeric(raw_v["qty"], errors="coerce")
|
||||
raw_v["venue_price"] = pd.to_numeric(raw_v["price"], errors="coerce")
|
||||
raw_v["side"] = np.where(raw_v["buyer"].astype(bool), "long", "short")
|
||||
raw_v = raw_v[raw_v["trade_id_link"].isin(matched_trade_ids)].copy()
|
||||
|
||||
matched_map = pd.DataFrame(
|
||||
[
|
||||
{
|
||||
"fill_id": m["local"]["fill_id"],
|
||||
"trade_id_link": m["venue"]["venue_trade_id"],
|
||||
"venue_order_id": m["venue"]["venue_order_id"],
|
||||
}
|
||||
for m in recon["matched"]
|
||||
]
|
||||
)
|
||||
|
||||
paths = paths.merge(
|
||||
matched_map.merge(
|
||||
raw_v[
|
||||
[
|
||||
"trade_id_link",
|
||||
"venue_order_id",
|
||||
"commission_usdt",
|
||||
"realized_pnl_usdt",
|
||||
"venue_time_ms",
|
||||
"qty",
|
||||
"venue_price",
|
||||
"side",
|
||||
]
|
||||
],
|
||||
on=["trade_id_link", "venue_order_id"],
|
||||
how="left",
|
||||
),
|
||||
on="fill_id",
|
||||
how="left",
|
||||
suffixes=("", "_venue"),
|
||||
)
|
||||
|
||||
paths["fill_price"] = pd.to_numeric(paths["fill_price"], errors="coerce")
|
||||
paths["qty"] = pd.to_numeric(paths["qty"], errors="coerce")
|
||||
paths["notional_usdt"] = paths["fill_price"] * paths["qty"]
|
||||
for sec, col in [(1, "after_1s_price"), (5, "after_5s_price"), (10, "after_10s_price"), (30, "after_30s_price"), (300, "after_5m_price")]:
|
||||
paths[f"markout_{sec}s"] = _fav_ret(paths["side"], paths["fill_price"], paths[col])
|
||||
paths["gross_markout_30s_usdt"] = paths["notional_usdt"] * paths["markout_30s"]
|
||||
paths["net_attr_30s_usdt"] = (
|
||||
paths["gross_markout_30s_usdt"]
|
||||
- pd.to_numeric(paths["commission_usdt"], errors="coerce").fillna(0.0)
|
||||
+ pd.to_numeric(paths["realized_pnl_usdt"], errors="coerce").fillna(0.0)
|
||||
)
|
||||
|
||||
inventory_metrics = _inventory_metrics(
|
||||
raw_v[
|
||||
["venue_time_ms", "side", "qty", "commission_usdt", "realized_pnl_usdt", "venue_order_id", "trade_id_link"]
|
||||
].copy()
|
||||
)
|
||||
|
||||
n_matched_paths = len(paths)
|
||||
n_matched_fills = len(fills)
|
||||
n_matched_clusters = int(fills["event_cluster_id"].nunique()) if not fills.empty else 0
|
||||
cluster_w = _cluster_weight(paths) if not paths.empty and "event_cluster_id" in paths.columns else pd.Series(dtype=float)
|
||||
|
||||
horizon_rows = []
|
||||
for sec in (1, 5, 10, 30, 300):
|
||||
col = f"markout_{sec}s"
|
||||
valid = paths[col].notna()
|
||||
sub = paths[valid]
|
||||
w = sub["notional_usdt"]
|
||||
horizon_rows.append(
|
||||
{
|
||||
"horizon": f"{sec}s",
|
||||
"n": int(len(sub)),
|
||||
"fill_w": _weighted_mean(sub[col], w),
|
||||
"cluster_w": _weighted_mean(sub[col], _cluster_weight(sub) if not sub.empty else pd.Series(dtype=float)),
|
||||
"gross_usdt": float((sub["notional_usdt"] * sub[col]).sum()) if not sub.empty else 0.0,
|
||||
}
|
||||
)
|
||||
|
||||
fee_total = float(paths["commission_usdt"].sum())
|
||||
realized_total = float(paths["realized_pnl_usdt"].sum())
|
||||
gross_30_total = float(paths["gross_markout_30s_usdt"].sum())
|
||||
net_attr_30_total = float(paths["net_attr_30s_usdt"].sum())
|
||||
total_qty = float(paths["qty"].sum())
|
||||
total_notional = float(paths["notional_usdt"].sum())
|
||||
|
||||
bucket_rows: list[dict[str, Any]] = []
|
||||
for col, title in [
|
||||
("path_type", "PathType"),
|
||||
("toxicity_bucket", "Toxicity"),
|
||||
("vol_bucket", "Volatility"),
|
||||
("liq_bucket", "Spread"),
|
||||
("trend_bucket", "Trend"),
|
||||
("market_event_before_fill", "FillContext"),
|
||||
]:
|
||||
bucket_rows.extend(_bucket_table(paths, col, title))
|
||||
bucket_df = pd.DataFrame(bucket_rows)
|
||||
|
||||
negative_states: set[str] = set()
|
||||
if not bucket_df.empty:
|
||||
neg = bucket_df[(bucket_df["dimension"] != "PathType") & (bucket_df["markout_30s"] < 0)]
|
||||
negative_states = set(neg["bucket"].astype(str))
|
||||
|
||||
counterfactuals = [
|
||||
{
|
||||
"name": "BASELINE",
|
||||
"fills": int(len(paths)),
|
||||
"clusters": int(paths["event_cluster_id"].nunique()) if not paths.empty else 0,
|
||||
"btc_qty": total_qty,
|
||||
"fee_usdt": fee_total,
|
||||
"gross_markout_30s_usdt": gross_30_total,
|
||||
"realized_pnl_usdt": realized_total,
|
||||
"net_attr_30s_usdt": net_attr_30_total,
|
||||
"markout_30s": _weighted_mean(paths["markout_30s"], paths["notional_usdt"]),
|
||||
},
|
||||
_counterfactual(paths, "path_type", {"C_toxic"}, "EXCLUDE_PATH_C"),
|
||||
_counterfactual(paths, "toxicity_bucket", {"toxic"}, "EXCLUDE_TOXIC"),
|
||||
_counterfactual(paths, "market_event_before_fill", negative_states, "EXCLUDE_NEGATIVE_STATE"),
|
||||
]
|
||||
|
||||
account = json.loads(Path(args.account).read_text()) if Path(args.account).exists() else {}
|
||||
recon03 = json.loads(Path(args.recon03).read_text()) if Path(args.recon03).exists() else {}
|
||||
|
||||
out_txt = Path(args.out)
|
||||
out_json = out_txt.with_suffix(".json")
|
||||
lines: list[str] = []
|
||||
|
||||
def p(s: str = "") -> None:
|
||||
lines.append(s)
|
||||
print(s)
|
||||
|
||||
p("=" * 72)
|
||||
p("Economic Attribution v0.1")
|
||||
p("=" * 72)
|
||||
p("Experiment: MM_EDGE_EXP_001")
|
||||
p("Population: MATCHED=3890")
|
||||
p("Strategy: v0.1 FROZEN")
|
||||
p("Execution: STOPPED")
|
||||
p("Stage3: LOCKED")
|
||||
p("Purpose: Economic Attribution only.")
|
||||
p("No strategy modification. No live execution. No economic simulation.")
|
||||
p()
|
||||
p("Layer 1 — Hard Economic Evidence")
|
||||
p("-" * 40)
|
||||
p(f"Matched fills: {n_matched_fills}")
|
||||
p(f"Matched paths: {n_matched_paths}")
|
||||
p(f"Matched clusters: {n_matched_clusters}")
|
||||
p(f"Fee total: {_num(fee_total, 6)} USDT")
|
||||
p(f"Fee / fill: {_num(fee_total / max(n_matched_paths, 1), 6)} USDT")
|
||||
p(f"Fee / BTC: {_num(fee_total / max(total_qty, 1e-12), 6)} USDT")
|
||||
p(f"Fee / cluster: {_num(fee_total / max(n_matched_clusters, 1), 6)} USDT")
|
||||
p(f"Realized component: {_num(realized_total, 6)} USDT")
|
||||
p(f"Gross markout @30s: {_num(gross_30_total, 6)} USDT")
|
||||
p(f"Net attributable @30s: {_num(net_attr_30_total, 6)} USDT")
|
||||
p()
|
||||
p("Markout by horizon (MATCHED only)")
|
||||
p("-" * 40)
|
||||
for row in horizon_rows:
|
||||
p(
|
||||
f"{row['horizon']:>5} n={row['n']:4d} fill-w={_pct(row['fill_w'])} "
|
||||
f"cluster-w={_pct(row['cluster_w'])} gross={_num(row['gross_usdt'], 6)} USDT"
|
||||
)
|
||||
p()
|
||||
p("Inventory carry / exposure")
|
||||
p("-" * 40)
|
||||
p(f"Max net BTC: {_num(inventory_metrics.get('max_net_btc'), 6)}")
|
||||
p(f"Min net BTC: {_num(inventory_metrics.get('min_net_btc'), 6)}")
|
||||
p(f"Max |net BTC|: {_num(inventory_metrics.get('max_abs_net_btc'), 6)}")
|
||||
p(f"Average |net BTC|: {_num(inventory_metrics.get('avg_abs_net_btc_per_fill'), 6)}")
|
||||
p(f"TW |net BTC|: {_num(inventory_metrics.get('time_weighted_abs_net_btc'), 6)}")
|
||||
p(f"TW signed net BTC: {_num(inventory_metrics.get('time_weighted_signed_net_btc'), 6)}")
|
||||
p(f"Long qty / Short qty: {_num(inventory_metrics.get('long_qty'), 6)} / {_num(inventory_metrics.get('short_qty'), 6)} BTC")
|
||||
p(f"Inventory turnover: {_num(inventory_metrics.get('turnover_btc'), 6)} BTC")
|
||||
p()
|
||||
p("Slices (weighted by notional, MATCHED only)")
|
||||
p("-" * 40)
|
||||
for dim in ["PathType", "Toxicity", "Volatility", "Spread", "Trend", "FillContext"]:
|
||||
sub = bucket_df[bucket_df["dimension"] == dim].copy()
|
||||
if sub.empty:
|
||||
continue
|
||||
p(dim)
|
||||
for _, r in sub.sort_values(["fills", "bucket"], ascending=[False, True]).iterrows():
|
||||
p(
|
||||
f" {r['bucket']}: n={int(r['fills'])} clusters={int(r['clusters'])} "
|
||||
f"fee={_num(r['fee_usdt'], 4)} gross30={_num(r['gross_markout_30s_usdt'], 4)} "
|
||||
f"realized={_num(r['realized_pnl_usdt'], 4)} net30={_num(r['net_attr_30s_usdt'], 4)} "
|
||||
f"m30={_pct(r['markout_30s'])}"
|
||||
)
|
||||
p()
|
||||
p("Layer 2 — Evidence Extension (excluded from core conclusion)")
|
||||
p("-" * 40)
|
||||
p(f"VENUE_CONFIRMED_NO_TRADE_HISTORY: {recon03.get('venue_confirmed_no_trade_history', 'n/a')}")
|
||||
p(f"VENUE_PARTIAL_ORDER_CANCELED: {recon03.get('venue_partial_order_canceled', 'n/a')}")
|
||||
p("These rows are order-confirmed, but not part of the Hard Evidence Population.")
|
||||
p()
|
||||
p("Layer 3 — Counterfactual Attribution (NOT backtest)")
|
||||
p("-" * 40)
|
||||
p("Observed vs Exclude-Bucket Attribution. These are contribution decompositions only.")
|
||||
for row in counterfactuals:
|
||||
p(
|
||||
f"{row['name']}: fills={row['fills']} clusters={row['clusters']} "
|
||||
f"fee={_num(row['fee_usdt'], 4)} gross30={_num(row['gross_markout_30s_usdt'], 4)} "
|
||||
f"realized={_num(row['realized_pnl_usdt'], 4)} net30={_num(row['net_attr_30s_usdt'], 4)} "
|
||||
f"m30={_pct(row['markout_30s'])}"
|
||||
)
|
||||
p()
|
||||
p("Interpretation")
|
||||
p("-" * 40)
|
||||
p("Core conclusion is based on 3890 fully matched fills.")
|
||||
p("Economic Attribution asks why MakerAlpha did not convert to money.")
|
||||
p("It does NOT change quote logic, does NOT restart v0.1, and does NOT unlock Stage 3.")
|
||||
p("=" * 72)
|
||||
|
||||
out_txt.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
sidecar = {
|
||||
"experiment_id": "MM_EDGE_EXP_001",
|
||||
"population": {
|
||||
"name": "MATCHED",
|
||||
"fills": n_matched_fills,
|
||||
"paths": n_matched_paths,
|
||||
"clusters": n_matched_clusters,
|
||||
},
|
||||
"strategy": "v0.1 FROZEN",
|
||||
"execution": "STOPPED",
|
||||
"stage3": "LOCKED",
|
||||
"fee_total_usdt": fee_total,
|
||||
"fee_per_fill_usdt": fee_total / max(n_matched_paths, 1),
|
||||
"fee_per_btc_usdt": fee_total / max(total_qty, 1e-12),
|
||||
"fee_per_cluster_usdt": fee_total / max(n_matched_clusters, 1),
|
||||
"realized_component_usdt": realized_total,
|
||||
"gross_markout_30s_usdt": gross_30_total,
|
||||
"net_attr_30s_usdt": net_attr_30_total,
|
||||
"markout_by_horizon": horizon_rows,
|
||||
"inventory_metrics": inventory_metrics,
|
||||
"bucket_rows": bucket_rows,
|
||||
"counterfactuals": counterfactuals,
|
||||
"recon03_extension": {
|
||||
"venue_confirmed_no_trade_history": recon03.get("venue_confirmed_no_trade_history"),
|
||||
"venue_partial_order_canceled": recon03.get("venue_partial_order_canceled"),
|
||||
},
|
||||
"account_recon_ref": account,
|
||||
}
|
||||
out_json.write_text(json.dumps(sidecar, indent=2) + "\n", encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Economic Fee Sensitivity v0.1 (MATCHED=3890)
|
||||
|
||||
Computes:
|
||||
net_attr_30s(fee_factor) = gross_markout_30s_usdt - fee_factor * fee_total_usdt + realized_component_usdt
|
||||
|
||||
Assumption:
|
||||
realized_component_usdt and gross_markout_30s_usdt are fixed (price/path unchanged).
|
||||
Only fee scaling is applied as a counterfactual sensitivity.
|
||||
|
||||
This is NOT a strategy backtest and does NOT modify any execution logic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="Economic Fee Sensitivity v0.1")
|
||||
ap.add_argument(
|
||||
"--attribution",
|
||||
default=str(ROOT / "logs" / "maker_edge" / "Economic_Attribution_v0_1.json"),
|
||||
)
|
||||
ap.add_argument(
|
||||
"--out",
|
||||
default=str(ROOT / "logs" / "maker_edge" / "Economic_Fee_Sensitivity_v0_1.txt"),
|
||||
)
|
||||
args = ap.parse_args()
|
||||
|
||||
data = json.loads(Path(args.attribution).read_text())
|
||||
fee_total = float(data["fee_total_usdt"])
|
||||
realized_total = float(data["realized_component_usdt"])
|
||||
gross_markout = float(data["gross_markout_30s_usdt"])
|
||||
|
||||
factors = [1.0, 0.5, 0.25, 0.1, 0.0]
|
||||
|
||||
lines: list[str] = []
|
||||
|
||||
def p(s: str = "") -> None:
|
||||
lines.append(s)
|
||||
print(s)
|
||||
|
||||
p("=" * 72)
|
||||
p("Economic Fee Sensitivity v0.1 (MATCHED=3890)")
|
||||
p("=" * 72)
|
||||
p(f"gross_markout_30s_usdt: {gross_markout:+.6f} USDT")
|
||||
p(f"fee_total_usdt: {fee_total:+.6f} USDT")
|
||||
p(f"realized_component_usdt:{realized_total:+.6f} USDT")
|
||||
p()
|
||||
p("Fee assumption → Net attributable @30s")
|
||||
p("-" * 42)
|
||||
|
||||
header = ["fee_factor", "fee_usdt_assumed", "net_attr_30s_usdt"]
|
||||
p(" | ".join(header))
|
||||
|
||||
for f in factors:
|
||||
fee_assumed = f * fee_total
|
||||
net = gross_markout - fee_assumed + realized_total
|
||||
row = [f"{f:.2f}", f"{fee_assumed:+.6f}", f"{net:+.6f}"]
|
||||
p(" | ".join(row))
|
||||
|
||||
p()
|
||||
p("Interpretation:")
|
||||
p("- If net remains < 0 at fee_factor=0 → economics not salvageable by fee reduction alone.")
|
||||
p("- If fee reduction flips net > 0 → current venue/fee tier can be the dominant issue.")
|
||||
p("=" * 72)
|
||||
|
||||
Path(args.out).write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Metric Reconciliation v0.1 (MATCHED only)
|
||||
|
||||
Confirms consistency between:
|
||||
- "MakerAlpha" reported in v0.1 research (return space)
|
||||
- "Gross markout @30s" in Economic Attribution (dollar space)
|
||||
- realized component used in Economic Attribution
|
||||
|
||||
Key point:
|
||||
Same definition may flip sign depending on weighting:
|
||||
fill-weighted mean return vs notional-weighted dollar markout
|
||||
|
||||
This script is read-only: it does NOT change any strategy/execution.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
from reconcile_fills import load_local_fills, match, normalize_local, normalize_venue # noqa: E402
|
||||
|
||||
|
||||
def _load_jsonl_df(log_dir: Path) -> pd.DataFrame:
|
||||
rows: list[dict] = []
|
||||
for f in sorted(log_dir.glob("*.jsonl")):
|
||||
if f.name.startswith(("Account_", "Maker_", "RECON")):
|
||||
continue
|
||||
for line in f.open():
|
||||
try:
|
||||
e = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(e, dict):
|
||||
rows.append(e)
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def _fav_ret(side: pd.Series, fill: pd.Series, px: pd.Series) -> pd.Series:
|
||||
# return space, signed by side
|
||||
fill = pd.to_numeric(fill, errors="coerce")
|
||||
px = pd.to_numeric(px, errors="coerce")
|
||||
raw = (px - fill) / fill
|
||||
return pd.Series(np.where(side == "long", raw, -raw), index=side.index)
|
||||
|
||||
|
||||
def _weighted_mean(x: pd.Series, w: pd.Series) -> float | None:
|
||||
xx = pd.to_numeric(x, errors="coerce")
|
||||
ww = pd.to_numeric(w, errors="coerce")
|
||||
mask = xx.notna() & ww.notna()
|
||||
xx = xx[mask]
|
||||
ww = ww[mask]
|
||||
if xx.empty:
|
||||
return None
|
||||
sw = float(ww.sum())
|
||||
if sw == 0:
|
||||
return None
|
||||
return float((xx * ww).sum() / sw)
|
||||
|
||||
|
||||
def _cluster_weight(paths: pd.DataFrame) -> pd.Series:
|
||||
if "event_cluster_id" not in paths.columns:
|
||||
return pd.Series(1.0, index=paths.index)
|
||||
cnt = paths.groupby("event_cluster_id")["event_cluster_id"].transform("count")
|
||||
return 1.0 / cnt.clip(lower=1)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="Economic Metric Reconciliation v0.1")
|
||||
ap.add_argument("--dir", default=str(ROOT / "logs" / "maker_edge"))
|
||||
ap.add_argument("--out", default=str(ROOT / "logs" / "maker_edge" / "Economic_Metric_Reconciliation_v0_1.txt"))
|
||||
ap.add_argument("--venue-trades", default=str(ROOT / "logs" / "maker_edge" / "venue_trades.json"))
|
||||
ap.add_argument("--matched-take", type=int, default=3890)
|
||||
args = ap.parse_args()
|
||||
|
||||
log_dir = Path(args.dir)
|
||||
venue_trades_path = Path(args.venue_trades)
|
||||
|
||||
df = _load_jsonl_df(log_dir)
|
||||
fills = df[df["event"] == "fill"].copy() if "event" in df.columns else pd.DataFrame()
|
||||
paths = df[df["event"] == "fill_path"].copy() if "event" in df.columns else pd.DataFrame()
|
||||
|
||||
# Hard matched population via RECON-02/03 evidence: use existing matcher logic.
|
||||
venue_trades = json.loads(venue_trades_path.read_text())
|
||||
local_fills_raw = load_local_fills(log_dir)
|
||||
locals_norm = [normalize_local(e, i) for i, e in enumerate(local_fills_raw)]
|
||||
venues_norm = [normalize_venue(t, i) for i, t in enumerate(venue_trades)]
|
||||
recon = match(locals_norm, venues_norm)
|
||||
|
||||
matched_fill_ids = {m["local"]["fill_id"] for m in recon["matched"]}
|
||||
matched_trade_ids = {m["venue"]["venue_trade_id"] for m in recon["matched"]}
|
||||
|
||||
fills = fills[fills["fill_id"].isin(matched_fill_ids)].copy()
|
||||
paths = paths[paths["fill_id"].isin(matched_fill_ids)].copy()
|
||||
|
||||
# Build after_30s already present in fill_path fields.
|
||||
# MakerAlpha in analyze_maker_edge uses after_30s_price and _fav_ret definition.
|
||||
# We'll recompute:
|
||||
# return space:
|
||||
# maker_alpha_fill_weighted = mean(markout_30s)
|
||||
# maker_alpha_notional_weighted_return = (gross_markout_usdt / total_notional)
|
||||
# gross_markout_usdt = sum(notional * markout_30s)
|
||||
#
|
||||
if paths.empty:
|
||||
raise SystemExit("No matched paths loaded")
|
||||
|
||||
# Merge meta from fills (side, fill_price, event_cluster_id, notional proxy)
|
||||
meta_cols = [
|
||||
c
|
||||
for c in [
|
||||
"fill_id",
|
||||
"side",
|
||||
"fill_price",
|
||||
"amount",
|
||||
"event_cluster_id",
|
||||
"spread_capture_pct",
|
||||
"pair",
|
||||
]
|
||||
if c in fills.columns
|
||||
]
|
||||
meta = fills.drop_duplicates("fill_id")[meta_cols]
|
||||
paths = paths.merge(meta, on="fill_id", how="left", suffixes=("", "_m"))
|
||||
|
||||
# If fill_path already had these columns, merge created *_m alternates.
|
||||
for col in ["side", "fill_price", "amount", "event_cluster_id"]:
|
||||
alt = f"{col}_m"
|
||||
if alt in paths.columns:
|
||||
if col not in paths.columns:
|
||||
paths[col] = paths[alt]
|
||||
else:
|
||||
paths[col] = paths[col].fillna(paths[alt])
|
||||
|
||||
# Ensure required fields
|
||||
paths["side"] = paths["side"].astype(str)
|
||||
paths["fill_price"] = pd.to_numeric(paths["fill_price"], errors="coerce")
|
||||
paths["qty"] = pd.to_numeric(paths["amount"], errors="coerce")
|
||||
paths["notional_usdt"] = paths["fill_price"] * paths["qty"]
|
||||
paths["after_30s_price"] = pd.to_numeric(paths["after_30s_price"], errors="coerce")
|
||||
|
||||
paths["markout_30s_return"] = _fav_ret(paths["side"], paths["fill_price"], paths["after_30s_price"])
|
||||
|
||||
gross_markout_usdt = float((paths["notional_usdt"] * paths["markout_30s_return"]).sum())
|
||||
total_notional = float(paths["notional_usdt"].sum())
|
||||
maker_alpha_fill_weighted = float(paths["markout_30s_return"].mean())
|
||||
maker_alpha_notional_weighted_return = float(gross_markout_usdt / total_notional) if total_notional else None
|
||||
|
||||
cw = _cluster_weight(paths)
|
||||
maker_alpha_cluster_weighted_return = _weighted_mean(paths["markout_30s_return"], cw)
|
||||
|
||||
# realized component from userTrades is already in Economic Attribution.
|
||||
# Here we only validate return space; realized component sign conventions are asserted elsewhere.
|
||||
out = Path(args.out)
|
||||
lines: list[str] = []
|
||||
|
||||
def p(s: str = "") -> None:
|
||||
lines.append(s)
|
||||
print(s)
|
||||
|
||||
p("=" * 72)
|
||||
p("Economic Metric Reconciliation v0.1 (MATCHED=3890)")
|
||||
p("=" * 72)
|
||||
p(f"Matched paths: {len(paths)} (expected ~3886)")
|
||||
p()
|
||||
p("Definitions (same math as analyze_maker_edge):")
|
||||
p("- markout_30s_return = _fav_ret(side, fill_price, after_30s_price)")
|
||||
p("- gross_markout_usdt = sum(notional_usdt * markout_30s_return)")
|
||||
p()
|
||||
p("Return-space metrics (sign may differ due to weighting):")
|
||||
p(f"MakerAlpha fill-weighted mean return: {_pct(maker_alpha_fill_weighted)}")
|
||||
p(f"MakerAlpha notional-weighted mean return: {_pct(maker_alpha_notional_weighted_return)}")
|
||||
p(f"MakerAlpha cluster-weighted mean return: {_pct(maker_alpha_cluster_weighted_return)}")
|
||||
p()
|
||||
p("Dollar-space metrics:")
|
||||
p(f"gross_markout_usdt (30s): {gross_markout_usdt:+.6f} USDT")
|
||||
p(f"total_notional_usdt: {total_notional:.3f} USDT")
|
||||
p()
|
||||
p("If fill-weighted return is + but gross_markout_usdt is negative,")
|
||||
p("it means notional weighting flips sign (alpha is conditionally realized).")
|
||||
p("=" * 72)
|
||||
|
||||
out.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
def _pct(v: float | None) -> str:
|
||||
if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))):
|
||||
return "n/a"
|
||||
return f"{v*100:.6f}%"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
Executable
+64
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bash
|
||||
# Remote EXP_002 long-run status (read-only). Does not analyze Path C.
|
||||
set -euo pipefail
|
||||
|
||||
SSH_HOST="${SSH_HOST:-jack@jackyu66.com}"
|
||||
SSH_KEY="${SSH_KEY:-${HOME}/Project/deploy/zun_hk/id_ed25519_hk}"
|
||||
REMOTE_DIR="${REMOTE_DIR:-/www/Project/nautilus_mm}"
|
||||
|
||||
SSH_OPTS=(-o StrictHostKeyChecking=accept-new)
|
||||
if [[ -n "$SSH_KEY" ]]; then
|
||||
chmod 400 "$SSH_KEY" 2>/dev/null || true
|
||||
SSH_OPTS+=(-i "$SSH_KEY")
|
||||
fi
|
||||
|
||||
ssh "${SSH_OPTS[@]}" "$SSH_HOST" "REMOTE_DIR='$REMOTE_DIR' bash -s" <<'EOF'
|
||||
set -euo pipefail
|
||||
echo "=== systemd --user event-state-probe ==="
|
||||
systemctl --user is-active event-state-probe || true
|
||||
systemctl --user show event-state-probe -p Environment --no-pager 2>/dev/null | tr ' ' '\n' | grep -E 'ENABLE_TRADING|EXPERIMENT_ID|LEDGER_RUN_ID' || true
|
||||
echo ""
|
||||
echo "=== mm-edge-probe (EXP_001) ==="
|
||||
systemctl --user is-active mm-edge-probe || true
|
||||
echo ""
|
||||
LOG="$REMOTE_DIR/logs/event_state/EXP-002-RUN-002"
|
||||
echo "=== ledger $LOG ==="
|
||||
if [[ ! -d "$LOG" ]]; then
|
||||
echo "no log dir yet"
|
||||
exit 0
|
||||
fi
|
||||
python3 - <<PY
|
||||
import json
|
||||
from pathlib import Path
|
||||
log = Path("$LOG")
|
||||
starts = trades = books = fills = parse_fail = 0
|
||||
run_id = None
|
||||
for f in sorted(log.glob("*.jsonl")):
|
||||
for line in f.open():
|
||||
s = line.strip()
|
||||
if not s:
|
||||
continue
|
||||
try:
|
||||
ev = json.loads(s)
|
||||
except Exception:
|
||||
parse_fail += 1
|
||||
continue
|
||||
run_id = ev.get("run_id") or run_id
|
||||
e = ev.get("event")
|
||||
if e == "experiment_start":
|
||||
starts += 1
|
||||
elif e == "fill_anchor":
|
||||
fills += 1
|
||||
elif e == "market_event":
|
||||
t = ev.get("event_type")
|
||||
if t == "aggressive_trade":
|
||||
trades += 1
|
||||
elif t == "book_update":
|
||||
books += 1
|
||||
print(f"run_id={run_id} starts={starts} trades={trades} books={books} fill_anchors={fills} parse_fail={parse_fail}")
|
||||
print("Gate 4 remains BLOCKED until fill_anchors exist. Do not Path-C snoop.")
|
||||
PY
|
||||
echo ""
|
||||
echo "=== journal (last 15) ==="
|
||||
journalctl --user -u event-state-probe -n 15 --no-pager || true
|
||||
EOF
|
||||
@@ -0,0 +1,595 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Prefill Adverse-Selection Attribution v0.1
|
||||
|
||||
Experiment: MM_EDGE_EXP_001
|
||||
Population: frozen historical fills
|
||||
Strategy: v0.1 FROZEN
|
||||
Execution: STOPPED
|
||||
Purpose:
|
||||
Pre-fill adverse-selection predictability audit
|
||||
NOT:
|
||||
strategy
|
||||
backtest
|
||||
optimization
|
||||
model training
|
||||
|
||||
Hard contract:
|
||||
feature_timestamp <= t_fill - margin_sec
|
||||
|
||||
This script intentionally prefers strict no-leakage over feature richness.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
from reconcile_fills import load_local_fills, match, normalize_local, normalize_venue # noqa: E402
|
||||
|
||||
|
||||
def _load_jsonl_df(log_dir: Path) -> pd.DataFrame:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for f in sorted(log_dir.glob("*.jsonl")):
|
||||
if f.name.startswith(("Account_", "Maker_", "RECON")):
|
||||
continue
|
||||
for line in f.open():
|
||||
try:
|
||||
e = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(e, dict):
|
||||
rows.append(e)
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def _pct(v: float | None) -> str:
|
||||
if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))):
|
||||
return "n/a"
|
||||
return f"{v*100:.2f}%"
|
||||
|
||||
|
||||
def _num(v: float | None, digits: int = 4) -> str:
|
||||
if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))):
|
||||
return "n/a"
|
||||
return f"{v:.{digits}f}"
|
||||
|
||||
|
||||
def _fav_ret(side: pd.Series, fill: pd.Series, px: pd.Series) -> pd.Series:
|
||||
fill = pd.to_numeric(fill, errors="coerce")
|
||||
px = pd.to_numeric(px, errors="coerce")
|
||||
raw = (px - fill) / fill
|
||||
return pd.Series(np.where(side == "long", raw, -raw), index=side.index)
|
||||
|
||||
|
||||
def _weighted_mean(x: pd.Series, w: pd.Series) -> float | None:
|
||||
xx = pd.to_numeric(x, errors="coerce")
|
||||
ww = pd.to_numeric(w, errors="coerce")
|
||||
mask = xx.notna() & ww.notna()
|
||||
xx = xx[mask]
|
||||
ww = ww[mask]
|
||||
if xx.empty:
|
||||
return None
|
||||
sw = float(ww.sum())
|
||||
if sw == 0.0:
|
||||
return None
|
||||
return float((xx * ww).sum() / sw)
|
||||
|
||||
|
||||
def _sample_grade(n: int) -> str:
|
||||
if n < 30:
|
||||
return "LOW_N"
|
||||
if n < 100:
|
||||
return "WEAK_EVIDENCE"
|
||||
return "USABLE"
|
||||
|
||||
|
||||
def _grade_probability(delta_pp: float, n_best: int) -> str:
|
||||
if n_best < 30:
|
||||
return "LOW_N"
|
||||
if delta_pp < 5.0:
|
||||
return "NO_PREFILL_SIGNAL"
|
||||
if n_best < 100 or delta_pp < 10.0:
|
||||
return "STATISTICAL_SIGNAL_ONLY"
|
||||
return "CANDIDATE_V0_2_SIGNAL"
|
||||
|
||||
|
||||
def _grade_economic(delta_usdt_per_fill: float, n_best: int) -> str:
|
||||
if n_best < 30:
|
||||
return "LOW_N"
|
||||
if abs(delta_usdt_per_fill) < 0.003:
|
||||
return "NO_PREFILL_SIGNAL"
|
||||
if n_best < 100 or abs(delta_usdt_per_fill) < 0.008:
|
||||
return "STATISTICAL_SIGNAL_ONLY"
|
||||
return "CANDIDATE_V0_2_SIGNAL"
|
||||
|
||||
|
||||
def _state_table_num(df: pd.DataFrame, feature: str, labels: list[str]) -> tuple[list[dict[str, Any]], dict[str, str]]:
|
||||
s = pd.to_numeric(df[feature], errors="coerce")
|
||||
valid = df[s.notna()].copy()
|
||||
valid[feature] = s[s.notna()]
|
||||
if valid.empty:
|
||||
return [], {k: "NO_DATA" for k in labels + ["Economic"]}
|
||||
q30 = float(valid[feature].quantile(0.30))
|
||||
q70 = float(valid[feature].quantile(0.70))
|
||||
# if no spread, collapse
|
||||
if math.isclose(q30, q70):
|
||||
valid["_state"] = "all"
|
||||
else:
|
||||
valid["_state"] = np.where(
|
||||
valid[feature] <= q30,
|
||||
"low",
|
||||
np.where(valid[feature] >= q70, "high", "mid"),
|
||||
)
|
||||
base = {
|
||||
lab: float(valid[lab].mean()) for lab in labels
|
||||
}
|
||||
base["economic_mean"] = float(valid["net_attr_30s_usdt"].mean())
|
||||
rows = []
|
||||
grades: dict[str, str] = {}
|
||||
for state, g in valid.groupby("_state"):
|
||||
row = {
|
||||
"feature": feature,
|
||||
"state": str(state),
|
||||
"n": int(len(g)),
|
||||
"sample_grade": _sample_grade(int(len(g))),
|
||||
"median": float(g[feature].median()),
|
||||
"p25": float(g[feature].quantile(0.25)),
|
||||
"p75": float(g[feature].quantile(0.75)),
|
||||
"net_attr_mean": float(g["net_attr_30s_usdt"].mean()),
|
||||
}
|
||||
for lab in labels:
|
||||
row[f"p_{lab}"] = float(g[lab].mean())
|
||||
row[f"delta_{lab}_pp"] = (row[f"p_{lab}"] - base[lab]) * 100.0
|
||||
row["delta_economic_per_fill"] = row["net_attr_mean"] - base["economic_mean"]
|
||||
rows.append(row)
|
||||
|
||||
# grade by strongest state-vs-baseline shift
|
||||
for lab in labels:
|
||||
best = max(rows, key=lambda r: abs(r[f"delta_{lab}_pp"]))
|
||||
grades[lab] = _grade_probability(abs(best[f"delta_{lab}_pp"]), int(best["n"]))
|
||||
best_e = max(rows, key=lambda r: abs(r["delta_economic_per_fill"]))
|
||||
grades["Economic"] = _grade_economic(abs(best_e["delta_economic_per_fill"]), int(best_e["n"]))
|
||||
return rows, grades
|
||||
|
||||
|
||||
def _state_table_cat(df: pd.DataFrame, feature: str, labels: list[str]) -> tuple[list[dict[str, Any]], dict[str, str]]:
|
||||
valid = df[df[feature].notna()].copy()
|
||||
if valid.empty:
|
||||
return [], {k: "NO_DATA" for k in labels + ["Economic"]}
|
||||
base = {
|
||||
lab: float(valid[lab].mean()) for lab in labels
|
||||
}
|
||||
base["economic_mean"] = float(valid["net_attr_30s_usdt"].mean())
|
||||
rows = []
|
||||
grades: dict[str, str] = {}
|
||||
for state, g in valid.groupby(feature):
|
||||
n = int(len(g))
|
||||
row = {
|
||||
"feature": feature,
|
||||
"state": str(state),
|
||||
"n": n,
|
||||
"sample_grade": _sample_grade(n),
|
||||
"median": None,
|
||||
"p25": None,
|
||||
"p75": None,
|
||||
"net_attr_mean": float(g["net_attr_30s_usdt"].mean()),
|
||||
}
|
||||
for lab in labels:
|
||||
row[f"p_{lab}"] = float(g[lab].mean())
|
||||
row[f"delta_{lab}_pp"] = (row[f"p_{lab}"] - base[lab]) * 100.0
|
||||
row["delta_economic_per_fill"] = row["net_attr_mean"] - base["economic_mean"]
|
||||
rows.append(row)
|
||||
for lab in labels:
|
||||
best = max(rows, key=lambda r: abs(r[f"delta_{lab}_pp"]))
|
||||
grades[lab] = _grade_probability(abs(best[f"delta_{lab}_pp"]), int(best["n"]))
|
||||
best_e = max(rows, key=lambda r: abs(r["delta_economic_per_fill"]))
|
||||
grades["Economic"] = _grade_economic(abs(best_e["delta_economic_per_fill"]), int(best_e["n"]))
|
||||
return rows, grades
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="Prefill Adverse-Selection Attribution v0.1")
|
||||
ap.add_argument("--dir", default=str(ROOT / "logs" / "maker_edge"))
|
||||
ap.add_argument("--venue-trades", default=str(ROOT / "logs" / "maker_edge" / "venue_trades.json"))
|
||||
ap.add_argument("--out", default=str(ROOT / "logs" / "maker_edge" / "Prefill_Adverse_Selection_Attribution_v0_1.txt"))
|
||||
ap.add_argument("--margin-sec", type=float, default=0.25)
|
||||
args = ap.parse_args()
|
||||
|
||||
log_dir = Path(args.dir)
|
||||
df = _load_jsonl_df(log_dir)
|
||||
fills = df[df["event"] == "fill"].copy()
|
||||
paths = df[df["event"] == "fill_path"].copy()
|
||||
state_ticks = df[df["event"].isin(["mid_tick", "inventory_tick"])].copy()
|
||||
|
||||
venue_trades = json.loads(Path(args.venue_trades).read_text())
|
||||
local_fills_raw = load_local_fills(log_dir)
|
||||
locals_norm = [normalize_local(e, i) for i, e in enumerate(local_fills_raw)]
|
||||
venues_norm = [normalize_venue(t, i) for i, t in enumerate(venue_trades)]
|
||||
recon = match(locals_norm, venues_norm)
|
||||
matched_fill_ids = {m["local"]["fill_id"] for m in recon["matched"]}
|
||||
matched_trade_ids = {m["venue"]["venue_trade_id"] for m in recon["matched"]}
|
||||
|
||||
fills = fills[fills["fill_id"].isin(matched_fill_ids)].copy()
|
||||
paths = paths[paths["fill_id"].isin(matched_fill_ids)].copy()
|
||||
|
||||
# merge labels/path info
|
||||
fill_meta_cols = [
|
||||
c
|
||||
for c in [
|
||||
"fill_id",
|
||||
"side",
|
||||
"fill_price",
|
||||
"amount",
|
||||
"quote_fill_time",
|
||||
"ts_epoch",
|
||||
"event_cluster_id",
|
||||
"pair",
|
||||
]
|
||||
if c in fills.columns
|
||||
]
|
||||
meta = fills.drop_duplicates("fill_id")[fill_meta_cols]
|
||||
paths = paths.merge(meta, on="fill_id", how="left", suffixes=("", "_f"))
|
||||
for col in ["side", "fill_price", "amount", "event_cluster_id", "quote_fill_time", "ts_epoch"]:
|
||||
alt = f"{col}_f"
|
||||
if alt in paths.columns:
|
||||
if col not in paths.columns:
|
||||
paths[col] = paths[alt]
|
||||
else:
|
||||
paths[col] = paths[col].fillna(paths[alt])
|
||||
paths["fill_ts"] = pd.to_datetime(paths["quote_fill_time"], utc=True, errors="coerce")
|
||||
# Prefer fill-event epoch seconds. astype(int64)/1e9 breaks when pandas stores UTC as us.
|
||||
fill_epoch = pd.to_numeric(paths["ts_epoch"], errors="coerce")
|
||||
iso_epoch = paths["fill_ts"].map(lambda ts: ts.timestamp() if pd.notna(ts) else np.nan)
|
||||
paths["fill_ts_epoch"] = fill_epoch.fillna(iso_epoch)
|
||||
paths["qty"] = pd.to_numeric(paths["amount"], errors="coerce")
|
||||
paths["fill_price"] = pd.to_numeric(paths["fill_price"], errors="coerce")
|
||||
paths["after_10s_price"] = pd.to_numeric(paths["after_10s_price"], errors="coerce")
|
||||
paths["after_30s_price"] = pd.to_numeric(paths["after_30s_price"], errors="coerce")
|
||||
paths["notional_usdt"] = paths["qty"] * paths["fill_price"]
|
||||
paths["markout_10s"] = _fav_ret(paths["side"], paths["fill_price"], paths["after_10s_price"])
|
||||
paths["markout_30s"] = _fav_ret(paths["side"], paths["fill_price"], paths["after_30s_price"])
|
||||
paths["path_c"] = paths["path_type"].astype(str).eq("C_toxic")
|
||||
paths["toxic"] = (paths["markout_10s"] < 0) & (paths["markout_30s"] < 0)
|
||||
paths["negative_30s"] = paths["markout_30s"] < 0
|
||||
|
||||
# attach trade economics
|
||||
raw_v = pd.DataFrame(venue_trades)
|
||||
raw_v["trade_id_link"] = raw_v["id"].astype(str)
|
||||
raw_v["commission_usdt"] = pd.to_numeric(raw_v["commission"], errors="coerce")
|
||||
raw_v["realized_pnl_usdt"] = pd.to_numeric(raw_v["realizedPnl"], errors="coerce").fillna(0.0)
|
||||
matched_map = pd.DataFrame(
|
||||
[
|
||||
{
|
||||
"fill_id": m["local"]["fill_id"],
|
||||
"trade_id_link": m["venue"]["venue_trade_id"],
|
||||
}
|
||||
for m in recon["matched"]
|
||||
]
|
||||
)
|
||||
paths = paths.merge(
|
||||
matched_map.merge(raw_v[["trade_id_link", "commission_usdt", "realized_pnl_usdt"]], on="trade_id_link", how="left"),
|
||||
on="fill_id",
|
||||
how="left",
|
||||
)
|
||||
paths["gross_markout_30s_usdt"] = paths["notional_usdt"] * paths["markout_30s"]
|
||||
paths["net_attr_30s_usdt"] = (
|
||||
paths["gross_markout_30s_usdt"]
|
||||
- pd.to_numeric(paths["commission_usdt"], errors="coerce").fillna(0.0)
|
||||
+ pd.to_numeric(paths["realized_pnl_usdt"], errors="coerce").fillna(0.0)
|
||||
)
|
||||
paths["economic_negative"] = paths["net_attr_30s_usdt"] < 0
|
||||
|
||||
# strict prefill state from sampled historical ticks only
|
||||
state_ticks = state_ticks.copy()
|
||||
state_ticks["ts_epoch"] = pd.to_numeric(state_ticks["ts_epoch"], errors="coerce")
|
||||
state_ticks = state_ticks.dropna(subset=["ts_epoch"]).sort_values("ts_epoch").drop_duplicates("ts_epoch")
|
||||
keep_cols = [
|
||||
c
|
||||
for c in [
|
||||
"ts_epoch",
|
||||
"mid",
|
||||
"spread",
|
||||
"bid_depth_1",
|
||||
"ask_depth_1",
|
||||
"bid_depth_5",
|
||||
"ask_depth_5",
|
||||
"obi",
|
||||
"delta",
|
||||
"trade_imbalance",
|
||||
"delta_efficiency",
|
||||
"inventory",
|
||||
"inventory_time",
|
||||
"inventory_skew",
|
||||
]
|
||||
if c in state_ticks.columns
|
||||
]
|
||||
states = state_ticks[keep_cols].copy()
|
||||
num_cols = [c for c in keep_cols if c != "ts_epoch"]
|
||||
for col in num_cols:
|
||||
states[col] = pd.to_numeric(states[col], errors="coerce")
|
||||
|
||||
# 5s lag features using sampled states
|
||||
lag_df = states[["ts_epoch"] + [c for c in ["mid", "spread", "obi", "bid_depth_5", "ask_depth_5", "trade_imbalance", "delta"] if c in states.columns]].copy()
|
||||
lag_df["lag_ts"] = lag_df["ts_epoch"] + 5.0
|
||||
lag_cols = {c: f"{c}_past5s" for c in lag_df.columns if c not in {"ts_epoch", "lag_ts"}}
|
||||
lag_df = lag_df.rename(columns=lag_cols)
|
||||
|
||||
paths = paths[paths["fill_ts_epoch"].notna()].copy()
|
||||
fill_states = paths[["fill_id", "fill_ts_epoch", "side"]].copy().sort_values("fill_ts_epoch")
|
||||
fill_states["feature_cutoff_ts"] = fill_states["fill_ts_epoch"] - float(args.margin_sec)
|
||||
|
||||
# latest sampled tick strictly before fill-margin
|
||||
snap = pd.merge_asof(
|
||||
fill_states.sort_values("feature_cutoff_ts"),
|
||||
states.sort_values("ts_epoch"),
|
||||
left_on="feature_cutoff_ts",
|
||||
right_on="ts_epoch",
|
||||
direction="backward",
|
||||
)
|
||||
snap = snap[snap["ts_epoch"].notna()].copy()
|
||||
snap = pd.merge_asof(
|
||||
snap.sort_values("ts_epoch"),
|
||||
lag_df.sort_values("lag_ts"),
|
||||
left_on="ts_epoch",
|
||||
right_on="lag_ts",
|
||||
direction="backward",
|
||||
)
|
||||
if "ts_epoch_x" in snap.columns:
|
||||
snap = snap.rename(columns={"ts_epoch_x": "ts_epoch"})
|
||||
|
||||
# derived strict-prefill features
|
||||
snap["spread_pct"] = snap["spread"] / snap["mid"]
|
||||
snap["depth_total_5"] = snap["bid_depth_5"] + snap["ask_depth_5"]
|
||||
snap["depth_imbalance_5"] = (snap["bid_depth_5"] - snap["ask_depth_5"]) / snap["depth_total_5"]
|
||||
snap["price_velocity_5s"] = (snap["mid"] - snap["mid_past5s"]) / snap["mid_past5s"]
|
||||
snap["spread_change_5s"] = snap["spread_pct"] - (snap["spread_past5s"] / snap["mid_past5s"])
|
||||
snap["obi_change_5s"] = snap["obi"] - snap["obi_past5s"]
|
||||
snap["depth_total_5_past"] = snap["bid_depth_5_past5s"] + snap["ask_depth_5_past5s"]
|
||||
snap["depth_change_5s"] = snap["depth_total_5"] - snap["depth_total_5_past"]
|
||||
snap["trade_imbalance_change_5s"] = snap["trade_imbalance"] - snap["trade_imbalance_past5s"]
|
||||
snap["delta_change_5s"] = snap["delta"] - snap["delta_past5s"]
|
||||
snap["pre_deteriorated_strict"] = np.where(
|
||||
snap["side"].eq("long"),
|
||||
(snap["price_velocity_5s"] < 0) | (snap["depth_change_5s"] < 0),
|
||||
(snap["price_velocity_5s"] > 0) | (snap["depth_change_5s"] < 0),
|
||||
)
|
||||
snap["feature_age_ms"] = (snap["fill_ts_epoch"] - snap["ts_epoch"]) * 1000.0
|
||||
snap = snap.rename(columns={"ts_epoch": "feature_ts_epoch"})
|
||||
|
||||
snap_feature_cols = [
|
||||
"fill_id",
|
||||
"feature_ts_epoch",
|
||||
"feature_cutoff_ts",
|
||||
"mid",
|
||||
"spread",
|
||||
"bid_depth_1",
|
||||
"ask_depth_1",
|
||||
"bid_depth_5",
|
||||
"ask_depth_5",
|
||||
"obi",
|
||||
"delta",
|
||||
"trade_imbalance",
|
||||
"delta_efficiency",
|
||||
"inventory",
|
||||
"inventory_time",
|
||||
"inventory_skew",
|
||||
"mid_past5s",
|
||||
"spread_past5s",
|
||||
"obi_past5s",
|
||||
"bid_depth_5_past5s",
|
||||
"ask_depth_5_past5s",
|
||||
"trade_imbalance_past5s",
|
||||
"delta_past5s",
|
||||
"spread_pct",
|
||||
"depth_total_5",
|
||||
"depth_imbalance_5",
|
||||
"price_velocity_5s",
|
||||
"spread_change_5s",
|
||||
"depth_total_5_past",
|
||||
"depth_change_5s",
|
||||
"obi_change_5s",
|
||||
"trade_imbalance_change_5s",
|
||||
"delta_change_5s",
|
||||
"pre_deteriorated_strict",
|
||||
"feature_age_ms",
|
||||
]
|
||||
snap_feature_cols = [c for c in snap_feature_cols if c in snap.columns]
|
||||
rename_map = {
|
||||
c: f"strict_{c}"
|
||||
for c in snap_feature_cols
|
||||
if c not in {"fill_id", "feature_ts_epoch", "feature_cutoff_ts", "feature_age_ms", "pre_deteriorated_strict"}
|
||||
}
|
||||
rename_map["feature_ts_epoch"] = "strict_feature_ts_epoch"
|
||||
rename_map["feature_cutoff_ts"] = "strict_feature_cutoff_ts"
|
||||
rename_map["feature_age_ms"] = "strict_feature_age_ms"
|
||||
rename_map["pre_deteriorated_strict"] = "strict_pre_deteriorated"
|
||||
snap_merge = snap[snap_feature_cols].rename(columns=rename_map)
|
||||
pref = paths.merge(snap_merge, on="fill_id", how="left")
|
||||
pref = pref[pref["strict_feature_ts_epoch"].notna()].copy()
|
||||
|
||||
labels = ["path_c", "toxic", "negative_30s"]
|
||||
numeric_features = [
|
||||
"strict_obi",
|
||||
"strict_delta",
|
||||
"strict_trade_imbalance",
|
||||
"strict_spread_pct",
|
||||
"strict_bid_depth_5",
|
||||
"strict_ask_depth_5",
|
||||
"strict_depth_total_5",
|
||||
"strict_depth_imbalance_5",
|
||||
"strict_price_velocity_5s",
|
||||
"strict_spread_change_5s",
|
||||
"strict_depth_change_5s",
|
||||
"strict_obi_change_5s",
|
||||
"strict_trade_imbalance_change_5s",
|
||||
"strict_delta_change_5s",
|
||||
"strict_inventory",
|
||||
"strict_inventory_skew",
|
||||
"strict_inventory_time",
|
||||
"strict_feature_age_ms",
|
||||
]
|
||||
cat_features = ["strict_pre_deteriorated"]
|
||||
|
||||
result_rows: list[dict[str, Any]] = []
|
||||
matrix_rows: list[dict[str, Any]] = []
|
||||
for feat in numeric_features:
|
||||
if feat not in pref.columns:
|
||||
continue
|
||||
rows, grades = _state_table_num(pref, feat, labels)
|
||||
result_rows.extend(rows)
|
||||
matrix_rows.append(
|
||||
{
|
||||
"feature": feat,
|
||||
"Path C": grades["path_c"],
|
||||
"Toxic": grades["toxic"],
|
||||
"Neg30s": grades["negative_30s"],
|
||||
"Economic": grades["Economic"],
|
||||
}
|
||||
)
|
||||
for feat in cat_features:
|
||||
if feat not in pref.columns:
|
||||
continue
|
||||
rows, grades = _state_table_cat(pref, feat, labels)
|
||||
result_rows.extend(rows)
|
||||
matrix_rows.append(
|
||||
{
|
||||
"feature": feat,
|
||||
"Path C": grades["path_c"],
|
||||
"Toxic": grades["toxic"],
|
||||
"Neg30s": grades["negative_30s"],
|
||||
"Economic": grades["Economic"],
|
||||
}
|
||||
)
|
||||
|
||||
baseline = {
|
||||
"path_c": float(pref["path_c"].mean()),
|
||||
"toxic": float(pref["toxic"].mean()),
|
||||
"negative_30s": float(pref["negative_30s"].mean()),
|
||||
"economic_negative": float(pref["economic_negative"].mean()),
|
||||
"net_attr_30s_usdt_mean": float(pref["net_attr_30s_usdt"].mean()),
|
||||
"markout_30s_mean": float(pref["markout_30s"].mean()),
|
||||
}
|
||||
coverage = {
|
||||
"matched_paths": int(len(paths)),
|
||||
"strict_prefill_rows": int(len(pref)),
|
||||
"strict_prefill_coverage_pct": float(len(pref) / max(len(paths), 1) * 100.0),
|
||||
"mean_feature_age_ms": float(pref["strict_feature_age_ms"].mean()),
|
||||
"median_feature_age_ms": float(pref["strict_feature_age_ms"].median()),
|
||||
}
|
||||
|
||||
out_txt = Path(args.out)
|
||||
out_json = out_txt.with_suffix(".json")
|
||||
lines: list[str] = []
|
||||
|
||||
def p(s: str = "") -> None:
|
||||
lines.append(s)
|
||||
print(s)
|
||||
|
||||
p("=" * 72)
|
||||
p("Prefill Adverse-Selection Attribution v0.1")
|
||||
p("=" * 72)
|
||||
p("Experiment: MM_EDGE_EXP_001")
|
||||
p("Population: frozen historical fills (Hard core = MATCHED only)")
|
||||
p("Strategy: v0.1 FROZEN")
|
||||
p("Execution: STOPPED")
|
||||
p("Purpose: Pre-fill adverse-selection predictability audit")
|
||||
p("NOT: strategy / backtest / optimization / model training")
|
||||
p()
|
||||
p("Time Contract")
|
||||
p("-" * 40)
|
||||
p(f"feature_timestamp <= t_fill - {args.margin_sec:.2f}s")
|
||||
p("Only sampled historical mid_tick / inventory_tick states are used.")
|
||||
p("Fill-callback contemporaneous fields are intentionally excluded to avoid leakage.")
|
||||
p()
|
||||
p("Unavailable under strict contract in v0.1")
|
||||
p("-" * 40)
|
||||
p("- event intensity / large trades / time_since_last_market_event")
|
||||
p("- fill-callback market_event_before_fill")
|
||||
p("- any future path / realized / cancel-after-fill info as features")
|
||||
p()
|
||||
p("Baseline labels (MATCHED only)")
|
||||
p("-" * 40)
|
||||
p(f"P(Path C): {_pct(baseline['path_c'])}")
|
||||
p(f"P(Toxic): {_pct(baseline['toxic'])}")
|
||||
p(f"P(Neg30s): {_pct(baseline['negative_30s'])}")
|
||||
p(f"P(Economic<0): {_pct(baseline['economic_negative'])}")
|
||||
p(f"Mean net_attr_30s: {_num(baseline['net_attr_30s_usdt_mean'], 6)} USDT/fill")
|
||||
p(f"Mean markout_30s: {_pct(baseline['markout_30s_mean'])}")
|
||||
p(f"Matched path rows: {coverage['matched_paths']}")
|
||||
p(f"Strict prefill rows: {coverage['strict_prefill_rows']}")
|
||||
p(f"Strict coverage: {coverage['strict_prefill_coverage_pct']:.1f}%")
|
||||
p(f"Feature age ms: mean={coverage['mean_feature_age_ms']:.1f} median={coverage['median_feature_age_ms']:.1f}")
|
||||
p()
|
||||
p("Sample-size policy")
|
||||
p("-" * 40)
|
||||
p("n < 30 exploratory only (LOW_N)")
|
||||
p("n < 100 weak evidence (WEAK_EVIDENCE)")
|
||||
p("n >= 100 usable attribution (USABLE)")
|
||||
p()
|
||||
p("Conclusion Matrix")
|
||||
p("-" * 40)
|
||||
p("feature | Path C | Toxic | Neg30s | Economic")
|
||||
for row in matrix_rows:
|
||||
p(f"{row['feature']} | {row['Path C']} | {row['Toxic']} | {row['Neg30s']} | {row['Economic']}")
|
||||
p()
|
||||
p("State tables")
|
||||
p("-" * 40)
|
||||
for feat in [r["feature"] for r in matrix_rows]:
|
||||
sub = [r for r in result_rows if r["feature"] == feat]
|
||||
if not sub:
|
||||
continue
|
||||
p(feat)
|
||||
for r in sub:
|
||||
med = _num(r["median"], 6) if r["median"] is not None else "n/a"
|
||||
p(
|
||||
f" {r['state']}: n={r['n']} [{r['sample_grade']}] median={med} "
|
||||
f"P(C)={_pct(r['p_path_c'])} Δ={r['delta_path_c_pp']:+.1f}pp "
|
||||
f"P(Toxic)={_pct(r['p_toxic'])} Δ={r['delta_toxic_pp']:+.1f}pp "
|
||||
f"P(Neg30)={_pct(r['p_negative_30s'])} Δ={r['delta_negative_30s_pp']:+.1f}pp "
|
||||
f"E[net30]={_num(r['net_attr_mean'], 5)} Δ={_num(r['delta_economic_per_fill'], 5)}"
|
||||
)
|
||||
p()
|
||||
p("Interpretation")
|
||||
p("-" * 40)
|
||||
p("Only pre-fill observable states count as candidate signals.")
|
||||
p("A feature may separate Path C statistically but still fail Economic relevance.")
|
||||
p("Only rows graded CANDIDATE_V0_2_SIGNAL with usable n should enter v0.2 hypothesis design.")
|
||||
p("=" * 72)
|
||||
|
||||
out_txt.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
out_json.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"experiment_id": "MM_EDGE_EXP_001",
|
||||
"purpose": "prefill adverse-selection predictability audit",
|
||||
"population": {
|
||||
"matched_rows": int(len(pref)),
|
||||
"margin_sec": float(args.margin_sec),
|
||||
},
|
||||
"baseline": baseline,
|
||||
"coverage": coverage,
|
||||
"matrix": matrix_rows,
|
||||
"states": result_rows,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
Executable
+53
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env bash
|
||||
# 远程查看探针状态 + fills/clusters 粗计数
|
||||
# 用法:export SSH_HOST=user@ip [SSH_KEY=...] ./scripts/probe_status.sh
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
SSH_HOST="${SSH_HOST:-jack@jackyu66.com}"
|
||||
SSH_KEY="${SSH_KEY:-${HOME}/Project/deploy/zun_hk/id_ed25519_hk}"
|
||||
REMOTE_DIR="${REMOTE_DIR:-/www/Project/nautilus_mm}"
|
||||
|
||||
SSH_OPTS=(-o StrictHostKeyChecking=accept-new)
|
||||
if [[ -n "$SSH_KEY" ]]; then
|
||||
chmod 400 "$SSH_KEY" 2>/dev/null || true
|
||||
SSH_OPTS+=(-i "$SSH_KEY")
|
||||
fi
|
||||
|
||||
ssh "${SSH_OPTS[@]}" "$SSH_HOST" "REMOTE_DIR='$REMOTE_DIR' bash -s" <<'EOF'
|
||||
set -euo pipefail
|
||||
echo "=== systemd --user ==="
|
||||
systemctl --user is-active mm-edge-probe || true
|
||||
systemctl --user status mm-edge-probe --no-pager -l | head -20 || true
|
||||
echo ""
|
||||
echo "=== experiment (.env) ==="
|
||||
grep -E '^(EXPERIMENT_ID|PROBE_VERSION|BINANCE_ENVIRONMENT|ENABLE_TRADING)=' "$REMOTE_DIR/.env" 2>/dev/null || true
|
||||
grep -E '^BINANCE_API_KEY=.+' "$REMOTE_DIR/.env" >/dev/null && echo "API key: SET" || echo "API key: EMPTY"
|
||||
echo ""
|
||||
echo "=== fills / clusters (jsonl) ==="
|
||||
cd "$REMOTE_DIR/logs/maker_edge" 2>/dev/null || { echo "no log dir"; exit 0; }
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
fills=0
|
||||
cids=set()
|
||||
health=0
|
||||
for f in sorted(Path('.').glob('*.jsonl')):
|
||||
for line in f.read_text().splitlines():
|
||||
try:
|
||||
ev=json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if ev.get('event')=='fill':
|
||||
fills+=1
|
||||
if ev.get('event_cluster_id'):
|
||||
cids.add(ev['event_cluster_id'])
|
||||
elif ev.get('event')=='health':
|
||||
health+=1
|
||||
print(f"fills={fills} clusters={len(cids)} health_ticks={health}")
|
||||
print(f"cluster/fill={len(cids)/fills*100:.1f}%" if fills else "cluster/fill=n/a")
|
||||
PY
|
||||
echo ""
|
||||
echo "=== recent journal (--user) ==="
|
||||
journalctl --user -u mm-edge-probe -n 30 --no-pager || true
|
||||
EOF
|
||||
Executable
+25
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
# 从服务器拉取 jsonl + 本地生成 Maker Edge Report
|
||||
# 用法:export SSH_HOST=user@ip [SSH_KEY=...] ./scripts/pull_report.sh [min_fills]
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
SSH_HOST="${SSH_HOST:-jack@jackyu66.com}"
|
||||
SSH_KEY="${SSH_KEY:-${HOME}/Project/deploy/zun_hk/id_ed25519_hk}"
|
||||
REMOTE_DIR="${REMOTE_DIR:-/www/Project/nautilus_mm}"
|
||||
LOCAL_LOG="${LOCAL_LOG:-$ROOT/logs/maker_edge}"
|
||||
|
||||
SSH_OPTS=(-o StrictHostKeyChecking=accept-new)
|
||||
if [[ -n "$SSH_KEY" ]]; then
|
||||
chmod 400 "$SSH_KEY" 2>/dev/null || true
|
||||
SSH_OPTS+=(-i "$SSH_KEY")
|
||||
fi
|
||||
|
||||
mkdir -p "$LOCAL_LOG"
|
||||
echo "==> pull logs from $SSH_HOST"
|
||||
rsync -avz -e "ssh ${SSH_OPTS[*]}" \
|
||||
"$SSH_HOST:$REMOTE_DIR/logs/maker_edge/" "$LOCAL_LOG/"
|
||||
|
||||
echo "==> analyze"
|
||||
export PYTHONPATH="${ROOT}/src${PYTHONPATH:+:$PYTHONPATH}"
|
||||
exec "$ROOT/scripts/analyze.sh" "${1:-2000}"
|
||||
@@ -0,0 +1,402 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Account Ledger Reconciliation — MM_EDGE_EXP_001
|
||||
|
||||
Separates:
|
||||
MakerAlpha (research markout) ≠ Account Equity (wallet economics)
|
||||
|
||||
Pulls paginated Binance Futures:
|
||||
- /fapi/v1/userTrades (maker flag, commission per fill)
|
||||
- /fapi/v1/income (REALIZED_PNL, COMMISSION, FUNDING_FEE, …)
|
||||
- /fapi/v2/account (wallet + unrealized + position)
|
||||
|
||||
Hard gate:
|
||||
TAKER_FILLED_COUNT == 0 else Maker-only = INVALID
|
||||
|
||||
Equity identity (target error ≈ 0):
|
||||
StartWallet + Σincome_types + (EndUnrealized − StartUnrealized*)
|
||||
+ Transfers/Adjustments ≈ EndMarginBalance
|
||||
|
||||
* StartUnrealized often unknown → report EndUnrealized separately.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
_ROOT = Path(__file__).resolve().parents[1]
|
||||
load_dotenv(_ROOT / ".env")
|
||||
|
||||
|
||||
def _env(name: str, default: str = "") -> str:
|
||||
return os.getenv(name, default).strip()
|
||||
|
||||
|
||||
def _base_url() -> str:
|
||||
env = _env("BINANCE_ENVIRONMENT", "TESTNET").upper()
|
||||
if env == "TESTNET":
|
||||
return "https://testnet.binancefuture.com"
|
||||
if env == "LIVE":
|
||||
return "https://fapi.binance.com"
|
||||
raise SystemExit(f"BINANCE_ENVIRONMENT must be TESTNET|LIVE, got {env!r}")
|
||||
|
||||
|
||||
def _signed_get(path: str, params: dict | None = None) -> object:
|
||||
key = _env("BINANCE_API_KEY")
|
||||
sec = _env("BINANCE_API_SECRET")
|
||||
if not key or not sec:
|
||||
raise SystemExit("BINANCE_API_KEY / BINANCE_API_SECRET required")
|
||||
params = dict(params or {})
|
||||
params["timestamp"] = int(time.time() * 1000)
|
||||
params["recvWindow"] = 60_000
|
||||
qs = urllib.parse.urlencode(params)
|
||||
sig = hmac.new(sec.encode(), qs.encode(), hashlib.sha256).hexdigest()
|
||||
url = f"{_base_url()}{path}?{qs}&signature={sig}"
|
||||
req = urllib.request.Request(url, headers={"X-MBX-APIKEY": key})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
return json.loads(r.read().decode())
|
||||
except urllib.error.HTTPError as e:
|
||||
body = e.read().decode(errors="replace")
|
||||
raise RuntimeError(f"HTTP {e.code} {path} params={params} body={body}") from e
|
||||
|
||||
|
||||
def _fetch_user_trades(symbol: str, start_ms: int, end_ms: int) -> list[dict]:
|
||||
"""Paginate userTrades by time windows (dedupe by trade id).
|
||||
|
||||
Note: Testnet userTrades can stop returning rows after ~7d of dense history
|
||||
even while orders/income continue — RECON-02 must flag that gap separately.
|
||||
"""
|
||||
out: dict[int, dict] = {}
|
||||
cursor = start_ms
|
||||
safety = 0
|
||||
while cursor < end_ms and safety < 2000:
|
||||
safety += 1
|
||||
chunk_end = min(cursor + 7 * 86400_000 - 1, end_ms)
|
||||
batch = _signed_get(
|
||||
"/fapi/v1/userTrades",
|
||||
{
|
||||
"symbol": symbol,
|
||||
"startTime": cursor,
|
||||
"endTime": chunk_end,
|
||||
"limit": 1000,
|
||||
},
|
||||
)
|
||||
assert isinstance(batch, list)
|
||||
if not batch:
|
||||
cursor = chunk_end + 1
|
||||
continue
|
||||
for t in batch:
|
||||
out[int(t["id"])] = t
|
||||
last_t = int(batch[-1]["time"])
|
||||
if len(batch) < 1000:
|
||||
cursor = max(last_t + 1, chunk_end + 1)
|
||||
else:
|
||||
nxt = last_t + 1
|
||||
if nxt <= cursor:
|
||||
nxt = cursor + 1
|
||||
cursor = nxt
|
||||
time.sleep(0.08)
|
||||
return sorted(out.values(), key=lambda x: (int(x["time"]), int(x["id"])))
|
||||
|
||||
|
||||
def _fetch_income(start_ms: int, end_ms: int) -> list[dict]:
|
||||
"""Paginate income by time only."""
|
||||
out: list[dict] = []
|
||||
seen: set[tuple] = set()
|
||||
cursor = start_ms
|
||||
safety = 0
|
||||
while cursor < end_ms and safety < 2000:
|
||||
safety += 1
|
||||
chunk_end = min(cursor + 7 * 86400_000 - 1, end_ms)
|
||||
batch = _signed_get(
|
||||
"/fapi/v1/income",
|
||||
{"startTime": cursor, "endTime": chunk_end, "limit": 1000},
|
||||
)
|
||||
assert isinstance(batch, list)
|
||||
if not batch:
|
||||
cursor = chunk_end + 1
|
||||
continue
|
||||
for row in batch:
|
||||
key = (
|
||||
row.get("tranId"),
|
||||
row.get("time"),
|
||||
row.get("incomeType"),
|
||||
row.get("income"),
|
||||
row.get("asset"),
|
||||
row.get("symbol"),
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(row)
|
||||
last_t = int(batch[-1]["time"])
|
||||
if len(batch) < 1000:
|
||||
cursor = max(last_t + 1, chunk_end + 1)
|
||||
else:
|
||||
cursor = last_t + 1
|
||||
time.sleep(0.08)
|
||||
return out
|
||||
|
||||
|
||||
def _ms_iso(ms: int) -> str:
|
||||
return datetime.fromtimestamp(ms / 1000, tz=timezone.utc).isoformat()
|
||||
|
||||
|
||||
def load_jsonl_fill_count(log_dir: Path) -> int:
|
||||
n = 0
|
||||
if not log_dir.exists():
|
||||
return 0
|
||||
for f in sorted(log_dir.glob("*.jsonl")):
|
||||
for line in f.open():
|
||||
try:
|
||||
e = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(e, dict) and e.get("event") == "fill":
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="Maker Edge account reconciliation")
|
||||
ap.add_argument(
|
||||
"--start-wallet",
|
||||
type=float,
|
||||
default=float(_env("RECON_START_WALLET", "5000")),
|
||||
help="Observed starting USDT wallet (default 5000 testnet grant)",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--symbol",
|
||||
default=_env("RECON_SYMBOL", "BTCUSDT"),
|
||||
help="Futures symbol for userTrades (default BTCUSDT)",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--since-days",
|
||||
type=float,
|
||||
default=float(_env("RECON_SINCE_DAYS", "14")),
|
||||
)
|
||||
ap.add_argument(
|
||||
"--out",
|
||||
default=str(_ROOT / "logs" / "maker_edge" / "Account_Reconciliation.txt"),
|
||||
)
|
||||
args = ap.parse_args()
|
||||
|
||||
end_ms = int(time.time() * 1000)
|
||||
start_ms = end_ms - int(args.since_days * 86400 * 1000)
|
||||
|
||||
print(f"[recon] env={_env('BINANCE_ENVIRONMENT','TESTNET')} base={_base_url()}")
|
||||
print(f"[recon] window {_ms_iso(start_ms)} → {_ms_iso(end_ms)}")
|
||||
|
||||
print("[recon] pulling userTrades (paginated)…")
|
||||
trades = _fetch_user_trades(args.symbol, start_ms, end_ms)
|
||||
|
||||
print(f"[recon] userTrades={len(trades)}")
|
||||
print("[recon] pulling income (paginated)…")
|
||||
income = _fetch_income(start_ms, end_ms)
|
||||
print(f"[recon] income rows={len(income)}")
|
||||
|
||||
acct = _signed_get("/fapi/v2/account")
|
||||
assert isinstance(acct, dict)
|
||||
|
||||
# --- Maker-only hard check ---
|
||||
maker_n = sum(1 for t in trades if t.get("maker") is True)
|
||||
taker_n = sum(1 for t in trades if t.get("maker") is False)
|
||||
unknown_n = len(trades) - maker_n - taker_n
|
||||
maker_only_ok = taker_n == 0 and unknown_n == 0 and len(trades) > 0
|
||||
maker_only_status = "PASS" if maker_only_ok else ("INVALID" if taker_n > 0 else "NEED VERIFY")
|
||||
|
||||
fee_by_asset: dict[str, float] = defaultdict(float)
|
||||
notional = 0.0
|
||||
buy_qty = sell_qty = 0.0
|
||||
for t in trades:
|
||||
fee_by_asset[t.get("commissionAsset") or "?"] += float(t.get("commission") or 0)
|
||||
q = float(t.get("qty") or 0)
|
||||
px = float(t.get("price") or 0)
|
||||
notional += abs(q * px)
|
||||
if t.get("buyer"):
|
||||
buy_qty += q
|
||||
else:
|
||||
sell_qty += q
|
||||
net_qty = buy_qty - sell_qty
|
||||
|
||||
income_by: dict[str, float] = defaultdict(float)
|
||||
for row in income:
|
||||
income_by[str(row.get("incomeType"))] += float(row.get("income") or 0)
|
||||
|
||||
wallet = float(acct.get("totalWalletBalance") or 0)
|
||||
upnl = float(acct.get("totalUnrealizedProfit") or 0)
|
||||
margin = float(acct.get("totalMarginBalance") or 0)
|
||||
avail = float(acct.get("availableBalance") or 0)
|
||||
|
||||
positions = []
|
||||
for p in acct.get("positions") or []:
|
||||
amt = float(p.get("positionAmt") or 0)
|
||||
if abs(amt) > 1e-12:
|
||||
positions.append(
|
||||
{
|
||||
"symbol": p.get("symbol"),
|
||||
"amt": amt,
|
||||
"entry": float(p.get("entryPrice") or 0),
|
||||
"unrealized": float(p.get("unrealizedProfit") or 0),
|
||||
}
|
||||
)
|
||||
|
||||
start_wallet = float(args.start_wallet)
|
||||
income_sum = sum(income_by.values())
|
||||
# Identity without known start upnl:
|
||||
# EndWallet ≈ StartWallet + Σ income (transfers included in income types if any)
|
||||
implied_end_wallet = start_wallet + income_sum
|
||||
wallet_gap = wallet - implied_end_wallet
|
||||
equity_now = margin # wallet + upnl
|
||||
equity_vs_start = equity_now - start_wallet
|
||||
|
||||
jsonl_fills = load_jsonl_fill_count(_ROOT / "logs" / "maker_edge")
|
||||
|
||||
lines: list[str] = []
|
||||
def p(s: str = "") -> None:
|
||||
lines.append(s)
|
||||
print(s)
|
||||
|
||||
p("=" * 72)
|
||||
p("Account Reconciliation — MM_EDGE_EXP_001 / probe_v0.1")
|
||||
p("Research markout (MakerAlpha) ≠ Account equity")
|
||||
p("=" * 72)
|
||||
p()
|
||||
p("Status Snapshot")
|
||||
p("-" * 40)
|
||||
p("Maker Phenomenon PARTIAL_PASS")
|
||||
p("Data Integrity PASS (from Maker Edge Report)")
|
||||
p(f"Maker-only constraint {maker_only_status}")
|
||||
p("Account Reconciliation NOT COMPLETE" if abs(wallet_gap) > 0.5 else "Account Reconciliation CLOSE")
|
||||
p("Economic Edge UNKNOWN")
|
||||
p("Stage 3 LOCKED")
|
||||
p("Probe STOPPED (no further volume until ledger closes)")
|
||||
p()
|
||||
|
||||
p("Section A — Maker-only hard check (exchange userTrades)")
|
||||
p("-" * 40)
|
||||
p(f"Symbol: {args.symbol}")
|
||||
p(f"Exchange trades: {len(trades)}")
|
||||
p(f"Jsonl fills (local): {jsonl_fills}")
|
||||
p(f"MAKER fills: {maker_n}")
|
||||
p(f"TAKER fills: {taker_n}")
|
||||
p(f"Unknown liquidity: {unknown_n}")
|
||||
p(f"TAKER_FILLED_COUNT: {taker_n}")
|
||||
if taker_n > 0:
|
||||
p("→ INVALID: sample contaminated by taker fills")
|
||||
elif maker_only_ok:
|
||||
p("→ PASS: all exchange trades marked maker=true")
|
||||
else:
|
||||
p("→ NEED VERIFY")
|
||||
p(f"Buy qty / Sell qty: {buy_qty:.6f} / {sell_qty:.6f}")
|
||||
p(f"Net inventory (qty): {net_qty:.6f}")
|
||||
p(f"Gross notional: {notional:.4f} USDT")
|
||||
for asset, fee in sorted(fee_by_asset.items()):
|
||||
p(f"Commission ({asset}): {fee}")
|
||||
p()
|
||||
|
||||
p("Section B — Income ledger (paginated, full window)")
|
||||
p("-" * 40)
|
||||
for k, v in sorted(income_by.items(), key=lambda kv: -abs(kv[1])):
|
||||
p(f" {k:24s} {v:+.8f}")
|
||||
p(f" {'Σ income':24s} {income_sum:+.8f}")
|
||||
p()
|
||||
|
||||
p("Section C — Account snapshot (now)")
|
||||
p("-" * 40)
|
||||
p(f"totalWalletBalance: {wallet:.8f}")
|
||||
p(f"totalUnrealizedProfit: {upnl:.8f}")
|
||||
p(f"totalMarginBalance: {margin:.8f} ← equity")
|
||||
p(f"availableBalance: {avail:.8f}")
|
||||
if positions:
|
||||
p("Open positions:")
|
||||
for pos in positions:
|
||||
p(
|
||||
f" {pos['symbol']} amt={pos['amt']} entry={pos['entry']} "
|
||||
f"upnl={pos['unrealized']}"
|
||||
)
|
||||
else:
|
||||
p("Open positions: (none)")
|
||||
p()
|
||||
|
||||
p("Section D — Equity bridge (attempt)")
|
||||
p("-" * 40)
|
||||
p(f"Start wallet (assumed): {start_wallet:.8f}")
|
||||
p(f"+ Σ income: {income_sum:+.8f}")
|
||||
p(f"= Implied end wallet: {implied_end_wallet:.8f}")
|
||||
p(f"Actual end wallet: {wallet:.8f}")
|
||||
p(f"Wallet residual gap: {wallet_gap:+.8f}")
|
||||
p(f"End unrealized: {upnl:+.8f}")
|
||||
p(f"End equity: {equity_now:.8f}")
|
||||
p(f"Equity − start wallet: {equity_vs_start:+.8f}")
|
||||
p()
|
||||
p("Interpretation:")
|
||||
p(" - Do NOT equate EquityΔ with MakerAlpha failure/success.")
|
||||
p(" - Residual gap means incomplete history, wrong start, or missing")
|
||||
p(" transfer/adjustment types — Account Reconciliation stays open.")
|
||||
p(" - Inventory drift (net qty / open position) can dominate economics")
|
||||
p(" even when per-fill markout is slightly positive.")
|
||||
p()
|
||||
|
||||
p("Section E — Next required chain")
|
||||
p("-" * 40)
|
||||
p("QuoteIntent → Submitted → Accepted → Filled")
|
||||
p(" → fill_px/qty → liquidity=MAKER → fee")
|
||||
p(" → position Δ → realized → funding → equity")
|
||||
p("Daily: StartEquity + TradingPnL + Fees + Funding + uPnL + Transfers = EndEquity")
|
||||
p("Target residual ≈ 0 before any Stage3 unlock / further volume.")
|
||||
p("=" * 72)
|
||||
|
||||
out = Path(args.out)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
# machine-readable sidecar
|
||||
sidecar = out.with_suffix(".json")
|
||||
sidecar.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"experiment_id": "MM_EDGE_EXP_001",
|
||||
"maker_only_status": maker_only_status,
|
||||
"taker_filled_count": taker_n,
|
||||
"maker_filled_count": maker_n,
|
||||
"exchange_trades": len(trades),
|
||||
"jsonl_fills": jsonl_fills,
|
||||
"income_by_type": dict(income_by),
|
||||
"income_sum": income_sum,
|
||||
"start_wallet_assumed": start_wallet,
|
||||
"end_wallet": wallet,
|
||||
"end_unrealized": upnl,
|
||||
"end_equity": equity_now,
|
||||
"wallet_residual_gap": wallet_gap,
|
||||
"net_qty": net_qty,
|
||||
"fee_by_asset": dict(fee_by_asset),
|
||||
"positions": positions,
|
||||
"probe": "STOPPED",
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"[recon] saved {out}")
|
||||
print(f"[recon] saved {sidecar}")
|
||||
return 0 if maker_only_ok or taker_n == 0 else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,594 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
RECONCILIATION-02 — Local Fill ↔ Venue Trade 1:1 / quantity-level closure
|
||||
|
||||
Does NOT resume the probe. Does NOT change quote logic.
|
||||
|
||||
Gate: 100% of local fills and venue trades classified into:
|
||||
MATCHED | DUPLICATE | ORPHAN_LOCAL | ORPHAN_VENUE | MISMATCH | MALFORMED
|
||||
|
||||
Primary link: venue_trade_id when present.
|
||||
Fallback (historical jsonl has trade_id=None):
|
||||
venue_order_id + side + qty + price + timestamp window
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
_ROOT = Path(__file__).resolve().parents[1]
|
||||
_SRC = _ROOT / "src"
|
||||
if str(_SRC) not in sys.path:
|
||||
sys.path.insert(0, str(_SRC))
|
||||
|
||||
# reuse pagination from recon-01
|
||||
sys.path.insert(0, str(_ROOT / "scripts"))
|
||||
from reconcile_account import _env, _fetch_user_trades, _signed_get # noqa: E402
|
||||
|
||||
|
||||
PX_TICK = 0.1 # BTCUSDT tick
|
||||
QTY_EPS = 1e-8
|
||||
TIME_MATCH_SEC = 30.0
|
||||
TIME_DUP_SEC = 2.0
|
||||
|
||||
|
||||
def _parse_iso(s: str | None) -> float | None:
|
||||
if not s:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _ms_ts(ms: int | None) -> float | None:
|
||||
if ms is None:
|
||||
return None
|
||||
return int(ms) / 1000.0
|
||||
|
||||
|
||||
def load_local_fills(log_dir: Path) -> list[dict]:
|
||||
fills: list[dict] = []
|
||||
for f in sorted(log_dir.glob("*.jsonl")):
|
||||
if f.name.startswith("Account_") or f.name.startswith("Maker_") or f.name.startswith("RECON"):
|
||||
continue
|
||||
for line in f.open():
|
||||
try:
|
||||
e = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if not isinstance(e, dict) or e.get("event") != "fill":
|
||||
continue
|
||||
fills.append(e)
|
||||
return fills
|
||||
|
||||
|
||||
def normalize_local(e: dict, idx: int) -> dict:
|
||||
px = float(e.get("fill_price") or 0)
|
||||
qty = float(e.get("amount") or 0)
|
||||
side = e.get("side") # long / short
|
||||
venue_oid = e.get("venue_order_id")
|
||||
if venue_oid is not None:
|
||||
venue_oid = str(venue_oid)
|
||||
trade_id = e.get("venue_trade_id") or e.get("trade_id")
|
||||
if trade_id in (None, "None", ""):
|
||||
trade_id = None
|
||||
else:
|
||||
trade_id = str(trade_id)
|
||||
ts = _parse_iso(e.get("quote_fill_time"))
|
||||
malformed = []
|
||||
if px <= 0:
|
||||
malformed.append("bad_price")
|
||||
if qty <= 0:
|
||||
malformed.append("bad_qty")
|
||||
if side not in ("long", "short"):
|
||||
malformed.append("bad_side")
|
||||
if not venue_oid:
|
||||
malformed.append("missing_venue_order_id")
|
||||
return {
|
||||
"idx": idx,
|
||||
"fill_id": e.get("fill_id"),
|
||||
"client_order_id": e.get("client_order_id"),
|
||||
"venue_order_id": venue_oid,
|
||||
"venue_trade_id": trade_id,
|
||||
"side": side,
|
||||
"px": px,
|
||||
"qty": qty,
|
||||
"ts": ts,
|
||||
"ts_iso": e.get("quote_fill_time"),
|
||||
"commission": e.get("commission"),
|
||||
"malformed": malformed,
|
||||
"raw_keys": sorted(e.keys()),
|
||||
}
|
||||
|
||||
|
||||
def normalize_venue(t: dict, idx: int) -> dict:
|
||||
buyer = bool(t.get("buyer"))
|
||||
side = "long" if buyer else "short"
|
||||
return {
|
||||
"idx": idx,
|
||||
"venue_trade_id": str(t.get("id")),
|
||||
"venue_order_id": str(t.get("orderId")),
|
||||
"side": side,
|
||||
"px": float(t.get("price") or 0),
|
||||
"qty": float(t.get("qty") or 0),
|
||||
"ts": _ms_ts(t.get("time")),
|
||||
"ts_iso": datetime.fromtimestamp(int(t["time"]) / 1000, tz=timezone.utc).isoformat()
|
||||
if t.get("time")
|
||||
else None,
|
||||
"commission": float(t.get("commission") or 0),
|
||||
"commission_asset": t.get("commissionAsset"),
|
||||
"maker": t.get("maker"),
|
||||
"symbol": t.get("symbol"),
|
||||
}
|
||||
|
||||
|
||||
def _compatible(loc: dict, ven: dict) -> tuple[bool, str]:
|
||||
if loc["side"] != ven["side"]:
|
||||
return False, "side"
|
||||
if abs(loc["qty"] - ven["qty"]) > QTY_EPS:
|
||||
return False, "qty"
|
||||
if abs(loc["px"] - ven["px"]) > PX_TICK + 1e-9:
|
||||
return False, "price"
|
||||
if loc["ts"] is not None and ven["ts"] is not None:
|
||||
if abs(loc["ts"] - ven["ts"]) > TIME_MATCH_SEC:
|
||||
return False, "time"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def match(locals_: list[dict], venues: list[dict]) -> dict:
|
||||
"""Greedy unique matching. Each venue trade consumed at most once."""
|
||||
used_v: set[int] = set()
|
||||
used_l: set[int] = set()
|
||||
matched: list[dict] = []
|
||||
mismatch: list[dict] = []
|
||||
duplicate: list[dict] = []
|
||||
|
||||
loc_by_tid: dict[str, list[dict]] = defaultdict(list)
|
||||
ven_by_tid: dict[str, dict] = {}
|
||||
for v in venues:
|
||||
ven_by_tid[v["venue_trade_id"]] = v
|
||||
for loc in locals_:
|
||||
if loc["venue_trade_id"]:
|
||||
loc_by_tid[loc["venue_trade_id"]].append(loc)
|
||||
|
||||
# Pass 1: explicit venue_trade_id
|
||||
for tid, locs in loc_by_tid.items():
|
||||
v = ven_by_tid.get(tid)
|
||||
if v is None:
|
||||
continue
|
||||
primary, *rest = locs
|
||||
ok, why = _compatible(primary, v)
|
||||
rec = {"local": primary, "venue": v, "link": "venue_trade_id", "compat": why}
|
||||
if ok:
|
||||
matched.append(rec)
|
||||
else:
|
||||
rec["mismatch_reason"] = why
|
||||
mismatch.append(rec)
|
||||
used_v.add(v["idx"])
|
||||
used_l.add(primary["idx"])
|
||||
for d in rest:
|
||||
duplicate.append(
|
||||
{"local": d, "venue": v, "link": "venue_trade_id_dup", "reason": "same venue_trade_id"}
|
||||
)
|
||||
used_l.add(d["idx"])
|
||||
|
||||
# Pass 2: same venue_order_id, greedy best (qty, px, time)
|
||||
loc_by_oid: dict[str, list[dict]] = defaultdict(list)
|
||||
ven_by_oid: dict[str, list[dict]] = defaultdict(list)
|
||||
for loc in locals_:
|
||||
if loc["idx"] in used_l or loc["malformed"]:
|
||||
continue
|
||||
if loc["venue_order_id"]:
|
||||
loc_by_oid[loc["venue_order_id"]].append(loc)
|
||||
for v in venues:
|
||||
if v["idx"] in used_v:
|
||||
continue
|
||||
ven_by_oid[v["venue_order_id"]].append(v)
|
||||
|
||||
def score(loc: dict, v: dict) -> float:
|
||||
ok, _ = _compatible(loc, v)
|
||||
if not ok:
|
||||
return 1e18
|
||||
dt = 0.0
|
||||
if loc["ts"] is not None and v["ts"] is not None:
|
||||
dt = abs(loc["ts"] - v["ts"])
|
||||
return dt + abs(loc["px"] - v["px"]) * 1e-6
|
||||
|
||||
for oid, locs in loc_by_oid.items():
|
||||
cands = [v for v in ven_by_oid.get(oid, []) if v["idx"] not in used_v]
|
||||
remaining = [x for x in locs if x["idx"] not in used_l]
|
||||
for loc in sorted(remaining, key=lambda x: x["ts"] or 0):
|
||||
best = None
|
||||
best_s = 1e18
|
||||
for v in cands:
|
||||
if v["idx"] in used_v:
|
||||
continue
|
||||
s = score(loc, v)
|
||||
if s < best_s:
|
||||
best_s = s
|
||||
best = v
|
||||
if best is None or best_s >= 1e17:
|
||||
continue
|
||||
matched.append({"local": loc, "venue": best, "link": "order_id+px+qty+time", "compat": "ok"})
|
||||
used_l.add(loc["idx"])
|
||||
used_v.add(best["idx"])
|
||||
|
||||
# Pass 3: remaining locals that share (oid, px, qty) with an already-matched
|
||||
# local → DUPLICATE (restart / double-log of same execution)
|
||||
matched_sig: dict[tuple, dict] = {}
|
||||
for m in matched:
|
||||
loc = m["local"]
|
||||
v = m["venue"]
|
||||
matched_sig[(loc["venue_order_id"], round(loc["px"], 2), round(loc["qty"], 8), loc["side"])] = v
|
||||
|
||||
for loc in locals_:
|
||||
if loc["idx"] in used_l or loc["malformed"]:
|
||||
continue
|
||||
key = (loc["venue_order_id"], round(loc["px"], 2), round(loc["qty"], 8), loc["side"])
|
||||
v = matched_sig.get(key)
|
||||
if v is None:
|
||||
continue
|
||||
dt_ok = True
|
||||
if loc["ts"] is not None and v["ts"] is not None:
|
||||
dt_ok = abs(loc["ts"] - v["ts"]) <= TIME_MATCH_SEC
|
||||
if not dt_ok:
|
||||
continue
|
||||
duplicate.append(
|
||||
{
|
||||
"local": loc,
|
||||
"venue": v,
|
||||
"link": "dup_of_matched",
|
||||
"reason": "same order/px/qty/side as a matched fill",
|
||||
}
|
||||
)
|
||||
used_l.add(loc["idx"])
|
||||
|
||||
# Pass 4: global leftover by px+qty+side+time (order id mismatch)
|
||||
leftover_v = [v for v in venues if v["idx"] not in used_v]
|
||||
leftover_l = [x for x in locals_ if x["idx"] not in used_l and not x["malformed"]]
|
||||
for loc in leftover_l:
|
||||
best = None
|
||||
best_s = 1e18
|
||||
for v in leftover_v:
|
||||
if v["idx"] in used_v:
|
||||
continue
|
||||
s = score(loc, v)
|
||||
if s < best_s:
|
||||
best_s = s
|
||||
best = v
|
||||
if best is None or best_s >= 1e17:
|
||||
continue
|
||||
matched.append({"local": loc, "venue": best, "link": "global_px_qty_time", "compat": "ok"})
|
||||
used_l.add(loc["idx"])
|
||||
used_v.add(best["idx"])
|
||||
|
||||
malformed = [x for x in locals_ if x["malformed"]]
|
||||
for x in malformed:
|
||||
used_l.add(x["idx"])
|
||||
|
||||
orphan_local = [x for x in locals_ if x["idx"] not in used_l]
|
||||
orphan_venue = [v for v in venues if v["idx"] not in used_v]
|
||||
|
||||
return {
|
||||
"matched": matched,
|
||||
"duplicate": duplicate,
|
||||
"mismatch": mismatch,
|
||||
"malformed": malformed,
|
||||
"orphan_local": orphan_local,
|
||||
"orphan_venue": orphan_venue,
|
||||
}
|
||||
|
||||
|
||||
def _qty(xs, key="qty") -> float:
|
||||
return sum(float(x[key]) for x in xs)
|
||||
|
||||
|
||||
def audit_orphan_orders(orphans: list[dict], symbol: str, max_checks: int = 40) -> dict:
|
||||
"""Cross-check orphan locals against /fapi/v1/order and /userTrades?orderId=."""
|
||||
stats = {
|
||||
"checked": 0,
|
||||
"order_filled_no_trades": 0,
|
||||
"order_missing": 0,
|
||||
"order_other": 0,
|
||||
"trades_found": 0,
|
||||
}
|
||||
samples: list[dict] = []
|
||||
for loc in orphans[:max_checks]:
|
||||
oid = loc["venue_order_id"]
|
||||
if not oid:
|
||||
continue
|
||||
stats["checked"] += 1
|
||||
try:
|
||||
order = _signed_get("/fapi/v1/order", {"symbol": symbol, "orderId": oid})
|
||||
except Exception as exc:
|
||||
stats["order_missing"] += 1
|
||||
samples.append({"oid": oid, "fill_id": loc["fill_id"], "order": "ERR", "detail": str(exc)})
|
||||
continue
|
||||
st = order.get("status")
|
||||
try:
|
||||
tr = _signed_get("/fapi/v1/userTrades", {"symbol": symbol, "orderId": oid})
|
||||
except Exception:
|
||||
tr = []
|
||||
ntr = len(tr) if isinstance(tr, list) else 0
|
||||
if st == "FILLED" and ntr == 0:
|
||||
stats["order_filled_no_trades"] += 1
|
||||
elif ntr > 0:
|
||||
stats["trades_found"] += 1
|
||||
else:
|
||||
stats["order_other"] += 1
|
||||
if len(samples) < 8:
|
||||
samples.append(
|
||||
{
|
||||
"oid": oid,
|
||||
"fill_id": loc["fill_id"],
|
||||
"status": st,
|
||||
"execQty": order.get("executedQty"),
|
||||
"avgPrice": order.get("avgPrice"),
|
||||
"userTrades_n": ntr,
|
||||
}
|
||||
)
|
||||
stats["samples"] = samples
|
||||
return stats
|
||||
|
||||
|
||||
def write_report(
|
||||
out: Path,
|
||||
result: dict,
|
||||
n_local: int,
|
||||
n_venue: int,
|
||||
*,
|
||||
venue_t_max: str | None = None,
|
||||
orphan_audit: dict | None = None,
|
||||
) -> None:
|
||||
m = result["matched"]
|
||||
d = result["duplicate"]
|
||||
mm = result["mismatch"]
|
||||
mal = result["malformed"]
|
||||
ol = result["orphan_local"]
|
||||
ov = result["orphan_venue"]
|
||||
|
||||
loc_explained = len(m) + len(d) + len(mm) + len(mal) + len(ol)
|
||||
ven_explained = len(m) + len(mm) + len(ov) # dups share venue; orphans leftover
|
||||
# every local in exactly one bucket
|
||||
# every venue in matched, mismatch, or orphan_venue (dups don't extra-count venue)
|
||||
|
||||
m_qty_l = sum(x["local"]["qty"] for x in m)
|
||||
m_qty_v = sum(x["venue"]["qty"] for x in m)
|
||||
m_fee_v = sum(x["venue"]["commission"] for x in m)
|
||||
dt = [
|
||||
abs(x["local"]["ts"] - x["venue"]["ts"])
|
||||
for x in m
|
||||
if x["local"]["ts"] is not None and x["venue"]["ts"] is not None
|
||||
]
|
||||
dt.sort()
|
||||
|
||||
def pctile(a, q):
|
||||
if not a:
|
||||
return None
|
||||
i = min(len(a) - 1, max(0, int(round(q * (len(a) - 1)))))
|
||||
return a[i]
|
||||
|
||||
unexplained_local = n_local - (len(m) + len(d) + len(mm) + len(mal))
|
||||
# orphan_local IS unexplained in the sense of no venue link, but classified
|
||||
classified_local = len(m) + len(d) + len(mm) + len(mal) + len(ol)
|
||||
classified_venue = len({x["venue"]["idx"] for x in m + mm} | {x["idx"] for x in ov})
|
||||
|
||||
gate = (
|
||||
classified_local == n_local
|
||||
and classified_venue == n_venue
|
||||
and len(ol) == 0
|
||||
and len(ov) == 0
|
||||
and len(mm) == 0
|
||||
and len(mal) == 0
|
||||
)
|
||||
# 100% explainable ≠ zero orphans. User asked 100% explainable.
|
||||
# We treat orphans as classified. Gate PASS if all rows classified (always if logic sound)
|
||||
# Strict gate: no orphans/mismatch/malformed
|
||||
explainable = classified_local == n_local and classified_venue == n_venue
|
||||
|
||||
lines = []
|
||||
|
||||
def p(s: str = "") -> None:
|
||||
lines.append(s)
|
||||
p("=" * 72)
|
||||
p("RECONCILIATION-02 — Local Fill ↔ Venue Trade")
|
||||
p("MM_EDGE_EXP_001 / probe_v0.1 / TESTNET BTCUSDT")
|
||||
p("Probe remains STOPPED")
|
||||
p("=" * 72)
|
||||
p()
|
||||
p("Counts")
|
||||
p("-" * 40)
|
||||
p(f"Local JSONL fills: {n_local}")
|
||||
p(f"Venue userTrades: {n_venue}")
|
||||
p(f" MATCHED: {len(m)}")
|
||||
p(f" DUPLICATE (local): {len(d)}")
|
||||
p(f" MISMATCH: {len(mm)}")
|
||||
p(f" MALFORMED (local): {len(mal)}")
|
||||
p(f" ORPHAN_LOCAL: {len(ol)}")
|
||||
p(f" ORPHAN_VENUE: {len(ov)}")
|
||||
p(f"Local classified: {classified_local}/{n_local}")
|
||||
p(f"Venue classified: {classified_venue}/{n_venue}")
|
||||
venue_t_max_ts = None
|
||||
if venue_t_max:
|
||||
p(f"Venue history max (UTC): {venue_t_max}")
|
||||
try:
|
||||
venue_t_max_ts = datetime.fromisoformat(venue_t_max).timestamp()
|
||||
except Exception:
|
||||
venue_t_max_ts = None
|
||||
if ol and venue_t_max_ts:
|
||||
orphan_after = sum(1 for x in ol if x["ts"] is not None and x["ts"] > venue_t_max_ts)
|
||||
orphan_before = len(ol) - orphan_after
|
||||
p(f"Orphan after venue cutoff: {orphan_after} (userTrades history gap on testnet)")
|
||||
p(f"Orphan before cutoff: {orphan_before}")
|
||||
if orphan_audit:
|
||||
p()
|
||||
p("Orphan order audit (sample)")
|
||||
p("-" * 40)
|
||||
p(f" checked: {orphan_audit.get('checked')}")
|
||||
p(f" order FILLED, 0 trades: {orphan_audit.get('order_filled_no_trades')}")
|
||||
p(f" userTrades found: {orphan_audit.get('trades_found')}")
|
||||
for s in orphan_audit.get("samples") or []:
|
||||
p(f" oid={s.get('oid')} status={s.get('status')} exec={s.get('execQty')} trades={s.get('userTrades_n')}")
|
||||
p()
|
||||
p("Quantity (BTC)")
|
||||
p("-" * 40)
|
||||
p(f"Matched local qty: {m_qty_l:.6f}")
|
||||
p(f"Matched venue qty: {m_qty_v:.6f}")
|
||||
p(f"Qty residual: {m_qty_l - m_qty_v:+.8f}")
|
||||
p(f"Orphan local qty: {sum(x['qty'] for x in ol):.6f}")
|
||||
p(f"Orphan venue qty: {sum(x['qty'] for x in ov):.6f}")
|
||||
p(f"Duplicate local qty: {sum(x['local']['qty'] for x in d):.6f}")
|
||||
p()
|
||||
p("Fee / time (matched only)")
|
||||
p("-" * 40)
|
||||
p(f"Venue commission sum: {m_fee_v:.8f} USDT")
|
||||
if dt:
|
||||
p(f"|Δt| n={len(dt)} p50={pctile(dt,0.5):.3f}s p95={pctile(dt,0.95):.3f}s max={dt[-1]:.3f}s")
|
||||
p()
|
||||
p("Link methods (matched)")
|
||||
p("-" * 40)
|
||||
by = defaultdict(int)
|
||||
for x in m:
|
||||
by[x["link"]] += 1
|
||||
for k, v in sorted(by.items(), key=lambda kv: -kv[1]):
|
||||
p(f" {k:28s} {v}")
|
||||
p()
|
||||
p("Gate")
|
||||
p("-" * 40)
|
||||
p(f"100% classified: {'PASS' if explainable else 'FAIL'}")
|
||||
p(f"Strict (no orphan/mismatch/malformed): {'PASS' if gate else 'FAIL'}")
|
||||
p("Do not resume probe until strict gate PASS or leftovers 100% attributed.")
|
||||
p()
|
||||
|
||||
def dump_sample(title: str, rows: list, kind: str, n: int = 8) -> None:
|
||||
if not rows:
|
||||
return
|
||||
p(f"Samples — {title} (showing {min(n, len(rows))}/{len(rows)})")
|
||||
p("-" * 40)
|
||||
for row in rows[:n]:
|
||||
if kind == "match":
|
||||
loc, v = row["local"], row["venue"]
|
||||
p(
|
||||
f" fill={loc['fill_id']} oid={loc['venue_order_id']} "
|
||||
f"tid={v['venue_trade_id']} px={loc['px']}/{v['px']} "
|
||||
f"qty={loc['qty']}/{v['qty']} link={row['link']}"
|
||||
)
|
||||
elif kind == "dup":
|
||||
loc, v = row["local"], row["venue"]
|
||||
p(
|
||||
f" fill={loc['fill_id']} oid={loc['venue_order_id']} "
|
||||
f"tid={v['venue_trade_id']} reason={row.get('reason')}"
|
||||
)
|
||||
elif kind == "local":
|
||||
p(
|
||||
f" fill={row['fill_id']} oid={row['venue_order_id']} "
|
||||
f"px={row['px']} qty={row['qty']} side={row['side']} ts={row['ts_iso']}"
|
||||
)
|
||||
elif kind == "venue":
|
||||
p(
|
||||
f" tid={row['venue_trade_id']} oid={row['venue_order_id']} "
|
||||
f"px={row['px']} qty={row['qty']} side={row['side']} ts={row['ts_iso']}"
|
||||
)
|
||||
p()
|
||||
|
||||
dump_sample("ORPHAN_LOCAL", ol, "local")
|
||||
dump_sample("ORPHAN_VENUE", ov, "venue")
|
||||
dump_sample("DUPLICATE", d, "dup")
|
||||
dump_sample("MISMATCH", mm, "match")
|
||||
p("=" * 72)
|
||||
|
||||
out.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
print("\n".join(lines))
|
||||
|
||||
sidecar = {
|
||||
"experiment_id": "MM_EDGE_EXP_001",
|
||||
"recon": "RECONCILIATION-02",
|
||||
"n_local": n_local,
|
||||
"n_venue": n_venue,
|
||||
"matched": len(m),
|
||||
"duplicate": len(d),
|
||||
"mismatch": len(mm),
|
||||
"malformed": len(mal),
|
||||
"orphan_local": len(ol),
|
||||
"orphan_venue": len(ov),
|
||||
"classified_local": classified_local,
|
||||
"classified_venue": classified_venue,
|
||||
"qty_matched_local": m_qty_l,
|
||||
"qty_matched_venue": m_qty_v,
|
||||
"qty_orphan_local": sum(x["qty"] for x in ol),
|
||||
"qty_orphan_venue": sum(x["qty"] for x in ov),
|
||||
"qty_duplicate_local": sum(x["local"]["qty"] for x in d),
|
||||
"fee_matched_venue": m_fee_v,
|
||||
"strict_gate": gate,
|
||||
"classified_gate": explainable,
|
||||
"dt_p50_sec": pctile(dt, 0.5),
|
||||
"dt_p95_sec": pctile(dt, 0.95),
|
||||
"orphan_local_oids": [x["venue_order_id"] for x in ol[:50]],
|
||||
"orphan_venue_tids": [x["venue_trade_id"] for x in ov[:50]],
|
||||
"venue_history_max": venue_t_max,
|
||||
"orphan_audit": orphan_audit,
|
||||
"probe": "STOPPED",
|
||||
}
|
||||
out.with_suffix(".json").write_text(json.dumps(sidecar, indent=2) + "\n")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--dir", default=str(_ROOT / "logs" / "maker_edge"))
|
||||
ap.add_argument("--symbol", default=_env("RECON_SYMBOL", "BTCUSDT"))
|
||||
ap.add_argument("--since-days", type=float, default=20.0)
|
||||
ap.add_argument("--trades-cache", default="")
|
||||
ap.add_argument("--fetch", action="store_true", help="Fetch userTrades from exchange")
|
||||
ap.add_argument("--out", default="")
|
||||
args = ap.parse_args()
|
||||
|
||||
log_dir = Path(args.dir)
|
||||
cache = Path(args.trades_cache) if args.trades_cache else log_dir / "venue_trades.json"
|
||||
|
||||
if args.fetch or not cache.exists():
|
||||
import time
|
||||
|
||||
end_ms = int(time.time() * 1000)
|
||||
start_ms = end_ms - int(args.since_days * 86400 * 1000)
|
||||
print(f"[recon-02] fetching userTrades {args.symbol} …")
|
||||
trades = _fetch_user_trades(args.symbol, start_ms, end_ms)
|
||||
cache.write_text(json.dumps(trades))
|
||||
print(f"[recon-02] cached {len(trades)} trades → {cache}")
|
||||
else:
|
||||
trades = json.loads(cache.read_text())
|
||||
print(f"[recon-02] loaded {len(trades)} trades from {cache}")
|
||||
|
||||
raw_fills = load_local_fills(log_dir)
|
||||
locals_ = [normalize_local(e, i) for i, e in enumerate(raw_fills)]
|
||||
venues = [normalize_venue(t, i) for i, t in enumerate(trades)]
|
||||
print(f"[recon-02] local fills={len(locals_)} venue={len(venues)}")
|
||||
|
||||
result = match(locals_, venues)
|
||||
venue_t_max = None
|
||||
if venues:
|
||||
venue_t_max = datetime.fromtimestamp(
|
||||
max(int(t["time"]) for t in trades) / 1000, tz=timezone.utc
|
||||
).isoformat()
|
||||
orphan_audit = audit_orphan_orders(result["orphan_local"], args.symbol)
|
||||
out = Path(args.out) if args.out else log_dir / "RECONCILIATION_02.txt"
|
||||
write_report(
|
||||
out,
|
||||
result,
|
||||
len(locals_),
|
||||
len(venues),
|
||||
venue_t_max=venue_t_max,
|
||||
orphan_audit=orphan_audit,
|
||||
)
|
||||
print(f"[recon-02] saved {out}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
RECONCILIATION-03 — Order-level evidence for ORPHAN_LOCAL (post userTrades cutoff)
|
||||
|
||||
Does NOT resume probe. Does NOT reclassify as MATCHED.
|
||||
|
||||
For each ORPHAN_LOCAL from RECON-02, query /fapi/v1/order and validate:
|
||||
status == FILLED
|
||||
executedQty ~= sum(local qty per order)
|
||||
avgPrice ~= local weighted avg
|
||||
side consistent
|
||||
|
||||
Reclassify passing rows as:
|
||||
VENUE_CONFIRMED_NO_TRADE_HISTORY
|
||||
(Order evidence only — no userTrades row on Testnet after cutoff)
|
||||
|
||||
See TESTNET_LIMITATIONS.md
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(_ROOT / "scripts"))
|
||||
sys.path.insert(0, str(_ROOT / "src"))
|
||||
|
||||
from reconcile_fills import ( # noqa: E402
|
||||
load_local_fills,
|
||||
match,
|
||||
normalize_local,
|
||||
normalize_venue,
|
||||
)
|
||||
from reconcile_account import _env, _fetch_user_trades, _signed_get # noqa: E402
|
||||
|
||||
PX_TICK = 0.1
|
||||
QTY_EPS = 1e-8
|
||||
|
||||
|
||||
def _order_side_to_local(side: str) -> str:
|
||||
return "long" if side.upper() == "BUY" else "short"
|
||||
|
||||
|
||||
def fetch_order(symbol: str, order_id: str, cache: dict) -> dict | None:
|
||||
if order_id in cache:
|
||||
return cache[order_id]
|
||||
try:
|
||||
o = _signed_get("/fapi/v1/order", {"symbol": symbol, "orderId": order_id})
|
||||
except Exception as exc:
|
||||
cache[order_id] = {"_error": str(exc)}
|
||||
return cache[order_id]
|
||||
cache[order_id] = o if isinstance(o, dict) else {"_error": "bad_response"}
|
||||
time.sleep(0.05)
|
||||
return cache[order_id]
|
||||
|
||||
|
||||
def validate_order_group(fills: list[dict], order: dict) -> tuple[str, list[str]]:
|
||||
"""Return (classification, reasons)."""
|
||||
reasons: list[str] = []
|
||||
if order.get("_error"):
|
||||
return "ORPHAN_LOCAL_UNCONFIRMED", [f"order_api_error:{order['_error']}"]
|
||||
st = order.get("status")
|
||||
exec_qty = float(order.get("executedQty") or 0)
|
||||
avg_px = float(order.get("avgPrice") or 0)
|
||||
local_qty = sum(f["qty"] for f in fills)
|
||||
if exec_qty <= 0:
|
||||
return "ORPHAN_LOCAL_UNCONFIRMED", [f"status={st} executedQty=0"]
|
||||
# Partial fill then TTL cancel: status=CANCELED but executedQty>0
|
||||
if st not in ("FILLED", "CANCELED"):
|
||||
return "ORPHAN_LOCAL_UNCONFIRMED", [f"status={st}"]
|
||||
if abs(local_qty - exec_qty) > QTY_EPS:
|
||||
reasons.append(f"qty local={local_qty} order={exec_qty}")
|
||||
wavg = sum(f["px"] * f["qty"] for f in fills) / local_qty if local_qty else 0
|
||||
if avg_px > 0 and abs(wavg - avg_px) > PX_TICK + 1e-6:
|
||||
reasons.append(f"px local_wavg={wavg:.2f} order_avg={avg_px:.2f}")
|
||||
order_side = _order_side_to_local(str(order.get("side", "")))
|
||||
for f in fills:
|
||||
if f["side"] != order_side:
|
||||
reasons.append(f"side local={f['side']} order={order_side}")
|
||||
break
|
||||
if reasons:
|
||||
return "ORDER_MISMATCH", reasons
|
||||
if st == "CANCELED":
|
||||
return "VENUE_PARTIAL_ORDER_CANCELED", []
|
||||
return "VENUE_CONFIRMED_NO_TRADE_HISTORY", []
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--dir", default=str(_ROOT / "logs" / "maker_edge"))
|
||||
ap.add_argument("--symbol", default=_env("RECON_SYMBOL", "BTCUSDT"))
|
||||
ap.add_argument("--trades-cache", default="")
|
||||
ap.add_argument("--out", default="")
|
||||
args = ap.parse_args()
|
||||
|
||||
log_dir = Path(args.dir)
|
||||
cache_path = Path(args.trades_cache) if args.trades_cache else log_dir / "venue_trades.json"
|
||||
trades = json.loads(cache_path.read_text()) if cache_path.exists() else []
|
||||
|
||||
raw = load_local_fills(log_dir)
|
||||
locals_ = [normalize_local(e, i) for i, e in enumerate(raw)]
|
||||
venues = [normalize_venue(t, i) for i, t in enumerate(trades)]
|
||||
r02 = match(locals_, venues)
|
||||
orphans = r02["orphan_local"]
|
||||
|
||||
by_oid: dict[str, list[dict]] = defaultdict(list)
|
||||
for f in orphans:
|
||||
if f.get("venue_order_id"):
|
||||
by_oid[f["venue_order_id"]].append(f)
|
||||
|
||||
order_cache: dict[str, dict] = {}
|
||||
fill_class: dict[str, tuple[str, list[str], dict | None]] = {}
|
||||
counts = defaultdict(int)
|
||||
order_rows: list[dict] = []
|
||||
|
||||
for oid, fills in sorted(by_oid.items()):
|
||||
order = fetch_order(args.symbol, oid, order_cache)
|
||||
cls, reasons = validate_order_group(fills, order or {})
|
||||
counts[cls] += len(fills)
|
||||
order_rows.append(
|
||||
{
|
||||
"venue_order_id": oid,
|
||||
"classification": cls,
|
||||
"n_local_fills": len(fills),
|
||||
"local_qty": sum(f["qty"] for f in fills),
|
||||
"order_executedQty": order.get("executedQty") if order else None,
|
||||
"order_avgPrice": order.get("avgPrice") if order else None,
|
||||
"order_status": order.get("status") if order else None,
|
||||
"order_updateTime": order.get("updateTime") if order else None,
|
||||
"reasons": reasons,
|
||||
}
|
||||
)
|
||||
for f in fills:
|
||||
fill_class[f["fill_id"]] = (cls, reasons, order)
|
||||
|
||||
# Summary from RECON-02 matched
|
||||
n_matched = len(r02["matched"])
|
||||
n_dup = len(r02["duplicate"])
|
||||
n_mismatch = len(r02["mismatch"])
|
||||
n_mal = len(r02["malformed"])
|
||||
n_confirmed = counts["VENUE_CONFIRMED_NO_TRADE_HISTORY"]
|
||||
n_partial_canceled = counts["VENUE_PARTIAL_ORDER_CANCELED"]
|
||||
n_order_mismatch = counts["ORDER_MISMATCH"]
|
||||
n_unconfirmed = counts["ORPHAN_LOCAL_UNCONFIRMED"]
|
||||
n_local = len(locals_)
|
||||
|
||||
venue_t_max = None
|
||||
if trades:
|
||||
venue_t_max = datetime.fromtimestamp(
|
||||
max(int(t["time"]) for t in trades) / 1000, tz=timezone.utc
|
||||
).isoformat()
|
||||
|
||||
order_evidence_ok = (
|
||||
n_unconfirmed == 0
|
||||
and n_order_mismatch == 0
|
||||
and (n_confirmed + n_partial_canceled) == len(orphans)
|
||||
)
|
||||
classified = (
|
||||
n_matched + n_dup + n_mismatch + n_mal
|
||||
+ n_confirmed + n_partial_canceled + n_order_mismatch + n_unconfirmed
|
||||
)
|
||||
|
||||
lines: list[str] = []
|
||||
|
||||
def p(s: str = "") -> None:
|
||||
lines.append(s)
|
||||
print(s)
|
||||
|
||||
p("=" * 72)
|
||||
p("RECONCILIATION-03 — Order-level evidence (ORPHAN_LOCAL backfill)")
|
||||
p("MM_EDGE_EXP_001 / probe_v0.1 / TESTNET BTCUSDT")
|
||||
p("Probe remains STOPPED")
|
||||
p("=" * 72)
|
||||
p()
|
||||
p("Prior RECON-02 (trade-level)")
|
||||
p("-" * 40)
|
||||
p(f"MATCHED (Order+Trade): {n_matched}")
|
||||
p(f"DUPLICATE: {n_dup}")
|
||||
p(f"MISMATCH: {n_mismatch}")
|
||||
p(f"MALFORMED: {n_mal}")
|
||||
p(f"ORPHAN_LOCAL (pre-03): {len(orphans)}")
|
||||
if venue_t_max:
|
||||
p(f"userTrades history max (UTC): {venue_t_max}")
|
||||
p()
|
||||
p("RECON-03 order-level reclassification")
|
||||
p("-" * 40)
|
||||
p(f"VENUE_CONFIRMED_NO_TRADE_HISTORY: {n_confirmed}")
|
||||
p(f"VENUE_PARTIAL_ORDER_CANCELED: {n_partial_canceled}")
|
||||
p(f"ORDER_MISMATCH: {n_order_mismatch}")
|
||||
p(f"ORPHAN_LOCAL_UNCONFIRMED: {n_unconfirmed}")
|
||||
p(f"Unique orders checked: {len(by_oid)}")
|
||||
p()
|
||||
p("Evidence grades (permanent taxonomy)")
|
||||
p("-" * 40)
|
||||
p("MATCHED = Order + Trade row (dual evidence)")
|
||||
p("VENUE_CONFIRMED_NO_TRADE_HISTORY = Order FILLED, no userTrades row")
|
||||
p("VENUE_PARTIAL_ORDER_CANCELED = Partial fill, order later CANCELED (TTL)")
|
||||
p("ORDER_MISMATCH = Order exists but qty/px/side disagree")
|
||||
p("ORPHAN_LOCAL_UNCONFIRMED = No reliable order evidence")
|
||||
p()
|
||||
p("Gates")
|
||||
p("-" * 40)
|
||||
p(f"RECON-02 classification (all buckets): {'PASS' if classified == n_local else 'FAIL'}")
|
||||
p(f"Order-level closure (887 backfill): {'PASS' if order_evidence_ok else 'FAIL'}")
|
||||
p(f"Strict trade-level closure: FAIL (by design until live trade_id ledger)")
|
||||
p()
|
||||
p("Testnet limitation")
|
||||
p("-" * 40)
|
||||
p("userTrades history is NOT guaranteed complete after observed cutoff.")
|
||||
p("Order-level FILLED status remains queryable via /fapi/v1/order.")
|
||||
p("Do NOT treat VENUE_CONFIRMED fills as fake or duplicate.")
|
||||
p()
|
||||
|
||||
fails = [r for r in order_rows if r["classification"] in ("ORDER_MISMATCH", "ORPHAN_LOCAL_UNCONFIRMED")]
|
||||
if fails:
|
||||
p(f"Non-confirmed orders (showing {min(8, len(fails))}/{len(fails)})")
|
||||
p("-" * 40)
|
||||
for r in fails[:8]:
|
||||
p(
|
||||
f" oid={r['venue_order_id']} cls={r['classification']} "
|
||||
f"local_qty={r['local_qty']} exec={r['order_executedQty']} reasons={r['reasons']}"
|
||||
)
|
||||
p()
|
||||
|
||||
p("=" * 72)
|
||||
|
||||
out = Path(args.out) if args.out else log_dir / "RECONCILIATION_03.txt"
|
||||
out.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
sidecar = {
|
||||
"experiment_id": "MM_EDGE_EXP_001",
|
||||
"recon": "RECONCILIATION-03",
|
||||
"n_local": n_local,
|
||||
"matched_trade_level": n_matched,
|
||||
"orphan_local_pre03": len(orphans),
|
||||
"venue_confirmed_no_trade_history": n_confirmed,
|
||||
"venue_partial_order_canceled": n_partial_canceled,
|
||||
"order_mismatch": n_order_mismatch,
|
||||
"orphan_local_unconfirmed": n_unconfirmed,
|
||||
"unique_orders_checked": len(by_oid),
|
||||
"userTrades_cutoff_utc": venue_t_max,
|
||||
"recon02_classification_pass": classified == n_local,
|
||||
"order_level_closure_pass": order_evidence_ok,
|
||||
"strict_trade_level_pass": False,
|
||||
"probe": "STOPPED",
|
||||
"order_rows": order_rows,
|
||||
}
|
||||
out.with_suffix(".json").write_text(json.dumps(sidecar, indent=2) + "\n")
|
||||
print(f"[recon-03] saved {out}")
|
||||
return 0 if order_evidence_ok else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
CCXT 轻量 L2 录音机(不依赖 Nautilus)
|
||||
|
||||
用途:在 Nautilus 节点未就绪时,先用代理拉 Binance USDT-M 盘口 + trades,
|
||||
写入与 Maker Edge 相同的 jsonl schema(book history + 模拟 quote 心跳)。
|
||||
|
||||
用法:
|
||||
cd nautilus_mm
|
||||
source .venv/bin/activate
|
||||
export PYTHONPATH=src
|
||||
export HTTPS_PROXY=http://127.0.0.1:7897
|
||||
python scripts/record_l2_ccxt.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(_ROOT / "src"))
|
||||
|
||||
import ccxt # type: ignore
|
||||
|
||||
from nautilus_mm.recorder import MakerEdgeLogger
|
||||
|
||||
|
||||
def main() -> None:
|
||||
proxy = os.getenv("HTTPS_PROXY") or os.getenv("HTTP_PROXY") or "http://127.0.0.1:7897"
|
||||
symbol = os.getenv("CCXT_SYMBOL", "BTC/USDT:USDT")
|
||||
poll = float(os.getenv("POLL_SECS", "2"))
|
||||
log_dir = os.getenv("MAKER_EDGE_LOG_DIR", str(_ROOT / "logs" / "maker_edge"))
|
||||
|
||||
ex = ccxt.binanceusdm(
|
||||
{
|
||||
"enableRateLimit": True,
|
||||
"proxies": {"http": proxy, "https": proxy},
|
||||
"options": {"defaultType": "future"},
|
||||
}
|
||||
)
|
||||
lg = MakerEdgeLogger(log_dir=log_dir, levels=10)
|
||||
last_mid = None
|
||||
print(f"[ccxt-recorder] {symbol} proxy={proxy} log={log_dir}")
|
||||
print("Ctrl+C to stop. This mode records book only (no live orders).")
|
||||
|
||||
while True:
|
||||
try:
|
||||
ob = ex.fetch_order_book(symbol, limit=10)
|
||||
trades = ex.fetch_trades(symbol, limit=100)
|
||||
snap = MakerEdgeLogger.snapshot_from_orderbook(
|
||||
ob, levels=10, recent_trades=trades, last_mid=last_mid
|
||||
)
|
||||
if snap.mid:
|
||||
last_mid = snap.mid
|
||||
now = time.time()
|
||||
lg.record_book(snap, now=now)
|
||||
# 心跳 quote(不挂单,仅记录可报价位置)
|
||||
if snap.best_bid:
|
||||
lg.write(
|
||||
{
|
||||
"event": "book_tick",
|
||||
"pair": symbol,
|
||||
"inventory": 0,
|
||||
**snap.to_book_fields(),
|
||||
}
|
||||
)
|
||||
lg.update_paths(symbol, snap.mid or 0, now=now)
|
||||
print(
|
||||
f"\r mid={snap.mid:.1f} spread={snap.spread:.2f} obi={snap.obi:+.3f} "
|
||||
f"timb={snap.trade_imbalance:+.3f} pending_fills={lg.pending_count}",
|
||||
end="",
|
||||
flush=True,
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"\nerror: {e}")
|
||||
time.sleep(poll)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# Preserve systemd/caller identity before .env (which still belongs to EXP_001)
|
||||
PRESERVE_RUN_ID="${LEDGER_RUN_ID:-}"
|
||||
PRESERVE_LOG_DIR="${EVENT_STATE_LOG_DIR:-}"
|
||||
|
||||
if [[ -f .env ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
source .env
|
||||
set +a
|
||||
fi
|
||||
|
||||
# Layer 1 (script): force EXP_002 contract after .env
|
||||
export EXPERIMENT_ID=MM_EDGE_EXP_002
|
||||
export PROBE_VERSION=event_state_v0.1
|
||||
export ENABLE_TRADING=false
|
||||
export LEDGER_RUN_ID="${PRESERVE_RUN_ID:-${LEDGER_RUN_ID:-EXP-002-RUN-002}}"
|
||||
export EVENT_STATE_LOG_DIR="${PRESERVE_LOG_DIR:-$ROOT/logs/event_state/$LEDGER_RUN_ID}"
|
||||
export PYTHONPATH="${PYTHONPATH:-$ROOT/src}"
|
||||
mkdir -p "$EVENT_STATE_LOG_DIR"
|
||||
|
||||
echo "[run_event_state] EXP_002 observability | trading=NO | run=$LEDGER_RUN_ID | log=$EVENT_STATE_LOG_DIR"
|
||||
exec "$ROOT/.venv/bin/python" -m nautilus_mm.run_event_state
|
||||
Executable
+34
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
if [[ ! -d .venv ]]; then
|
||||
python3 -m venv .venv
|
||||
.venv/bin/pip install -U pip
|
||||
.venv/bin/pip install -r requirements.txt
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source .venv/bin/activate
|
||||
export PYTHONPATH="${ROOT}/src:${PYTHONPATH:-}"
|
||||
|
||||
if [[ -f .env ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
source .env
|
||||
set +a
|
||||
fi
|
||||
|
||||
# 本地可开代理;服务器 systemd 直连,勿强制 7897
|
||||
if [[ "${USE_PROXY:-}" == "1" || "${USE_PROXY:-}" == "true" ]]; then
|
||||
export HTTP_PROXY="${HTTP_PROXY:-http://127.0.0.1:7897}"
|
||||
export HTTPS_PROXY="${HTTPS_PROXY:-http://127.0.0.1:7897}"
|
||||
echo "[run_probe] proxy=$HTTPS_PROXY"
|
||||
elif [[ -n "${HTTPS_PROXY:-}${HTTP_PROXY:-}" ]]; then
|
||||
echo "[run_probe] proxy=${HTTPS_PROXY:-$HTTP_PROXY}"
|
||||
else
|
||||
echo "[run_probe] direct (no proxy)"
|
||||
fi
|
||||
|
||||
exec python -m nautilus_mm.run_live
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
# MM_EDGE_EXP_002 smoke test — 10–15 min, restart in the middle, NO trading.
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
RUN_ID="${LEDGER_RUN_ID:-EXP-002-RUN-001}"
|
||||
SESSION_SECS="${SESSION_SECS:-360}" # 6 min × 2 = 12 min collect
|
||||
LOG_DIR="${EVENT_STATE_LOG_DIR:-$ROOT/logs/event_state/$RUN_ID}"
|
||||
PYTHON="${ROOT}/.venv/bin/python"
|
||||
|
||||
if [[ -f .env ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
source .env
|
||||
set +a
|
||||
fi
|
||||
|
||||
export EXPERIMENT_ID=MM_EDGE_EXP_002
|
||||
export PROBE_VERSION=event_state_v0.1
|
||||
export ENABLE_TRADING=false
|
||||
export LEDGER_RUN_ID="$RUN_ID"
|
||||
export EVENT_STATE_LOG_DIR="$LOG_DIR"
|
||||
export PYTHONPATH="$ROOT/src"
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
run_session() {
|
||||
local label="$1"
|
||||
export LEDGER_SESSION_ID="$(python3 -c 'import uuid; print(uuid.uuid4().hex[:12])')"
|
||||
echo "[smoke] session ${label} start session_id=${LEDGER_SESSION_ID} secs=${SESSION_SECS}"
|
||||
"$PYTHON" -m nautilus_mm.run_event_state &
|
||||
local pid=$!
|
||||
echo "[smoke] pid=${pid}"
|
||||
sleep "$SESSION_SECS"
|
||||
echo "[smoke] session ${label} stopping pid=${pid}"
|
||||
kill -INT "$pid" 2>/dev/null || true
|
||||
# allow experiment_stop flush
|
||||
local i=0
|
||||
while kill -0 "$pid" 2>/dev/null && [[ $i -lt 30 ]]; do
|
||||
sleep 1
|
||||
i=$((i + 1))
|
||||
done
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
echo "[smoke] SIGINT timeout — SIGTERM"
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sleep 3
|
||||
fi
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
echo "[smoke] SIGTERM timeout — SIGKILL"
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
fi
|
||||
wait "$pid" 2>/dev/null || true
|
||||
echo "[smoke] session ${label} stopped"
|
||||
}
|
||||
|
||||
echo "[smoke] RUN_ID=${RUN_ID} log=${LOG_DIR} trading=NO"
|
||||
run_session A
|
||||
echo "[smoke] restart gap 5s"
|
||||
sleep 5
|
||||
run_session B
|
||||
|
||||
echo "[smoke] validating ledger"
|
||||
"$PYTHON" "$ROOT/scripts/validate_event_ledger.py" \
|
||||
--dir "$LOG_DIR" \
|
||||
--run-id "$RUN_ID" \
|
||||
--out "$LOG_DIR/Event_Ledger_Validation.json"
|
||||
echo "[smoke] done"
|
||||
@@ -0,0 +1,453 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Validate MM_EDGE_EXP_002 Immutable Event Ledger.
|
||||
|
||||
Phase 1 smoke: Gates 1–3 plus ledger engineering contract.
|
||||
Gate 4 (predictability) is blocked until fill anchors exist.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import random
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
TRADE_REQUIRED = [
|
||||
"event_type",
|
||||
"exchange_ts_ns",
|
||||
"local_ts_epoch",
|
||||
"local_ts_ns",
|
||||
"trade_side",
|
||||
"trade_qty",
|
||||
"trade_price",
|
||||
"best_bid", # optional on trade; counted separately
|
||||
]
|
||||
TRADE_CORE = [
|
||||
"event_type",
|
||||
"exchange_ts_ns",
|
||||
"local_ts_epoch",
|
||||
"local_ts_ns",
|
||||
"trade_side",
|
||||
"trade_qty",
|
||||
"trade_price",
|
||||
"price",
|
||||
"quantity",
|
||||
"best_bid",
|
||||
"best_ask",
|
||||
"mid",
|
||||
"spread",
|
||||
]
|
||||
BOOK_CORE = [
|
||||
"event_type",
|
||||
"exchange_ts_ns",
|
||||
"local_ts_epoch",
|
||||
"local_ts_ns",
|
||||
"best_bid",
|
||||
"best_ask",
|
||||
"mid",
|
||||
"spread",
|
||||
"bid_depth_1",
|
||||
"ask_depth_1",
|
||||
"bid_depth_5",
|
||||
"ask_depth_5",
|
||||
]
|
||||
BOOK_DELTA_KEYS = [
|
||||
"bid_depth_delta_1",
|
||||
"ask_depth_delta_1",
|
||||
"bid_move",
|
||||
"ask_move",
|
||||
"spread_change",
|
||||
]
|
||||
|
||||
|
||||
def _pctile(xs: list[float], q: float) -> float | None:
|
||||
if not xs:
|
||||
return None
|
||||
ys = sorted(xs)
|
||||
if len(ys) == 1:
|
||||
return ys[0]
|
||||
i = (len(ys) - 1) * q
|
||||
lo = math.floor(i)
|
||||
hi = math.ceil(i)
|
||||
if lo == hi:
|
||||
return ys[lo]
|
||||
return ys[lo] * (hi - i) + ys[hi] * (i - lo)
|
||||
|
||||
|
||||
def _num(v: float | None, digits: int = 3) -> str:
|
||||
if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))):
|
||||
return "n/a"
|
||||
return f"{v:.{digits}f}"
|
||||
|
||||
|
||||
def load_jsonl(log_dir: Path) -> tuple[list[dict[str, Any]], int, int]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
parse_fail = 0
|
||||
empty = 0
|
||||
for f in sorted(log_dir.glob("*.jsonl")):
|
||||
for line in f.open():
|
||||
s = line.strip()
|
||||
if not s:
|
||||
empty += 1
|
||||
continue
|
||||
try:
|
||||
e = json.loads(s)
|
||||
except Exception:
|
||||
parse_fail += 1
|
||||
continue
|
||||
if isinstance(e, dict):
|
||||
rows.append(e)
|
||||
else:
|
||||
parse_fail += 1
|
||||
return rows, parse_fail, empty
|
||||
|
||||
|
||||
def _present(ev: dict[str, Any], key: str) -> bool:
|
||||
v = ev.get(key)
|
||||
return v is not None and v != ""
|
||||
|
||||
|
||||
def _hollow_book(ev: dict[str, Any]) -> bool:
|
||||
depths = [
|
||||
ev.get("bid_depth_1"),
|
||||
ev.get("ask_depth_1"),
|
||||
ev.get("bid_depth_5"),
|
||||
ev.get("ask_depth_5"),
|
||||
ev.get("mid"),
|
||||
]
|
||||
nums = []
|
||||
for d in depths:
|
||||
try:
|
||||
nums.append(float(d))
|
||||
except (TypeError, ValueError):
|
||||
nums.append(0.0)
|
||||
return all(abs(x) < 1e-12 for x in nums)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="Validate EXP_002 event ledger / smoke contract")
|
||||
ap.add_argument("--dir", default="logs/event_state")
|
||||
ap.add_argument("--out", default="")
|
||||
ap.add_argument("--run-id", default="")
|
||||
ap.add_argument("--sample", type=int, default=200)
|
||||
ap.add_argument("--latency-tolerance-ms", type=float, default=50.0)
|
||||
ap.add_argument("--seed", type=int, default=42)
|
||||
args = ap.parse_args()
|
||||
|
||||
log_dir = Path(args.dir)
|
||||
rows, parse_fail, empty_lines = load_jsonl(log_dir)
|
||||
if args.run_id:
|
||||
rows = [r for r in rows if r.get("run_id") == args.run_id]
|
||||
|
||||
market = [r for r in rows if r.get("event") == "market_event"]
|
||||
trades = [r for r in market if r.get("event_type") == "aggressive_trade"]
|
||||
books = [r for r in market if r.get("event_type") == "book_update"]
|
||||
starts = [r for r in rows if r.get("event") == "experiment_start"]
|
||||
stops = [r for r in rows if r.get("event") == "experiment_stop"]
|
||||
anchors = [r for r in rows if r.get("event") == "fill_anchor"]
|
||||
|
||||
run_ids = sorted({r.get("run_id") for r in rows if r.get("run_id")})
|
||||
sessions = [r.get("session_id") for r in starts]
|
||||
|
||||
# Duration from first/last local_ts
|
||||
local_epochs = [float(r["local_ts_epoch"]) for r in market if r.get("local_ts_epoch") is not None]
|
||||
duration_s = (max(local_epochs) - min(local_epochs)) if len(local_epochs) >= 2 else 0.0
|
||||
if duration_s <= 0:
|
||||
duration_s = 1.0
|
||||
|
||||
rates = {
|
||||
"aggressive_trade_per_sec": len(trades) / duration_s,
|
||||
"book_update_per_sec": len(books) / duration_s,
|
||||
"total_market_events_per_sec": len(market) / duration_s,
|
||||
"duration_sec": duration_s,
|
||||
}
|
||||
|
||||
# Timestamp quality
|
||||
ex_ok = sum(1 for r in market if r.get("exchange_ts_ns") is not None)
|
||||
loc_ok = sum(1 for r in market if r.get("local_ts_epoch") is not None and r.get("local_ts_ns") is not None)
|
||||
latencies_ms: list[float] = []
|
||||
skew_violations = 0
|
||||
for r in market:
|
||||
ex = r.get("exchange_ts_ns")
|
||||
loc = r.get("local_ts_ns")
|
||||
if ex is None or loc is None:
|
||||
continue
|
||||
lag_ms = (float(loc) - float(ex)) / 1e6
|
||||
latencies_ms.append(lag_ms)
|
||||
if float(ex) > float(loc) + args.latency_tolerance_ms * 1e6:
|
||||
skew_violations += 1
|
||||
|
||||
ts_quality = {
|
||||
"exchange_ts_ns_pct": (ex_ok / len(market)) if market else 0.0,
|
||||
"local_ts_pct": (loc_ok / len(market)) if market else 0.0,
|
||||
"latency_n": len(latencies_ms),
|
||||
"latency_ms_p50": _pctile(latencies_ms, 0.50),
|
||||
"latency_ms_p95": _pctile(latencies_ms, 0.95),
|
||||
"latency_ms_p99": _pctile(latencies_ms, 0.99),
|
||||
"latency_ms_max": max(latencies_ms) if latencies_ms else None,
|
||||
"latency_ms_min": min(latencies_ms) if latencies_ms else None,
|
||||
"exchange_after_local_violations": skew_violations,
|
||||
"tolerance_ms": args.latency_tolerance_ms,
|
||||
}
|
||||
|
||||
# Event order: exchange_ts regression (do not silently sort)
|
||||
regressions = 0
|
||||
max_back_ns = 0
|
||||
prev_ex = None
|
||||
for r in market:
|
||||
ex = r.get("exchange_ts_ns")
|
||||
if ex is None:
|
||||
continue
|
||||
ex = int(ex)
|
||||
if prev_ex is not None and ex < prev_ex:
|
||||
regressions += 1
|
||||
max_back_ns = max(max_back_ns, prev_ex - ex)
|
||||
prev_ex = ex
|
||||
|
||||
# Schema completeness (sample)
|
||||
rng = random.Random(args.seed)
|
||||
n_trade_s = min(args.sample, len(trades))
|
||||
n_book_s = min(args.sample, len(books))
|
||||
trade_sample = rng.sample(trades, n_trade_s) if n_trade_s else []
|
||||
book_sample = rng.sample(books, n_book_s) if n_book_s else []
|
||||
|
||||
def missing_rate(sample: list[dict], keys: list[str]) -> dict[str, float]:
|
||||
if not sample:
|
||||
return {k: 1.0 for k in keys}
|
||||
out = {}
|
||||
for k in keys:
|
||||
miss = sum(1 for e in sample if not _present(e, k))
|
||||
out[k] = miss / len(sample)
|
||||
return out
|
||||
|
||||
trade_missing = missing_rate(trade_sample, TRADE_CORE)
|
||||
book_missing = missing_rate(book_sample, BOOK_CORE)
|
||||
book_delta_key_miss = 0.0
|
||||
if book_sample:
|
||||
book_delta_key_miss = sum(
|
||||
1 for e in book_sample if any(k not in e for k in BOOK_DELTA_KEYS)
|
||||
) / len(book_sample)
|
||||
hollow = sum(1 for e in book_sample if _hollow_book(e))
|
||||
|
||||
# Restart / integrity
|
||||
event_ids = [r.get("event_id") for r in market if r.get("event_id")]
|
||||
dup_ids = [k for k, v in Counter(event_ids).items() if v > 1]
|
||||
|
||||
seq_ok = True
|
||||
seq_notes = []
|
||||
by_session: dict[str, list[int]] = {}
|
||||
for r in rows:
|
||||
sid = r.get("session_id")
|
||||
seq = r.get("event_seq")
|
||||
if sid is None or seq is None:
|
||||
continue
|
||||
by_session.setdefault(str(sid), []).append(int(seq))
|
||||
for sid, seqs in by_session.items():
|
||||
if seqs != list(range(1, len(seqs) + 1)) and seqs != sorted(seqs):
|
||||
# allow gaps only if we filtered; within session expect 1..n
|
||||
expected = list(range(min(seqs), max(seqs) + 1))
|
||||
if seqs != expected:
|
||||
seq_ok = False
|
||||
seq_notes.append(f"{sid}: not contiguous {seqs[:5]}...{seqs[-3:]}")
|
||||
if seqs and seqs[0] != 1:
|
||||
seq_notes.append(f"{sid}: seq starts at {seqs[0]} (expected 1 after restart)")
|
||||
|
||||
seq_reset_expected = len(sessions) >= 2 and all(
|
||||
(by_session.get(str(s), [None])[0] == 1) for s in sessions if s
|
||||
)
|
||||
|
||||
# Gates
|
||||
gate1_pass: bool | None
|
||||
if anchors:
|
||||
reconstruct_fail = 0
|
||||
for anc in anchors:
|
||||
fill_ts = float(anc["fill_ts_epoch"])
|
||||
start = float(anc.get("window_start_epoch", fill_ts - 5.0))
|
||||
cutoff = float(anc.get("feature_cutoff_epoch", fill_ts - 0.25))
|
||||
window = []
|
||||
for r in market:
|
||||
ex = r.get("exchange_ts_ns")
|
||||
ts = float(ex) / 1e9 if ex is not None else r.get("local_ts_epoch")
|
||||
if ts is None:
|
||||
continue
|
||||
if start <= float(ts) < cutoff:
|
||||
window.append(r)
|
||||
if not window:
|
||||
reconstruct_fail += 1
|
||||
gate1_pass = reconstruct_fail == 0
|
||||
gate1_status = "PASS" if gate1_pass else "FAIL"
|
||||
else:
|
||||
# Phase 1: stream completeness stands in for fill reconstruction
|
||||
stream_ok = parse_fail == 0 and len(market) > 0 and loc_ok == len(market)
|
||||
gate1_pass = stream_ok
|
||||
gate1_status = (
|
||||
"PASS (Phase 1 stream completeness; no fill_anchor — expected)"
|
||||
if stream_ok
|
||||
else "FAIL (stream incomplete)"
|
||||
)
|
||||
|
||||
gate2_ok = (
|
||||
ts_quality["exchange_ts_ns_pct"] >= 0.99
|
||||
and ts_quality["local_ts_pct"] >= 0.99
|
||||
and skew_violations == 0
|
||||
)
|
||||
gate2_status = "PASS" if gate2_ok else "FAIL"
|
||||
|
||||
schema_ok = (
|
||||
all(v == 0.0 for v in trade_missing.values())
|
||||
and all(v == 0.0 for v in book_missing.values())
|
||||
and book_delta_key_miss == 0.0
|
||||
and hollow == 0
|
||||
and len(trades) > 0
|
||||
and len(books) > 0
|
||||
)
|
||||
gate3_ok = schema_ok and ts_quality["exchange_ts_ns_pct"] >= 0.99
|
||||
gate3_status = "PASS" if gate3_ok else "FAIL"
|
||||
|
||||
restart_ok = (
|
||||
parse_fail == 0
|
||||
and len(dup_ids) == 0
|
||||
and len(starts) >= 1
|
||||
and (len(starts) == 1 or (len(stops) >= len(starts) - 1 and seq_reset_expected))
|
||||
)
|
||||
|
||||
integrity = {
|
||||
"parse_fail_lines": parse_fail,
|
||||
"empty_lines": empty_lines,
|
||||
"duplicate_event_ids": len(dup_ids),
|
||||
"experiment_start_count": len(starts),
|
||||
"experiment_stop_count": len(stops),
|
||||
"sessions": sessions,
|
||||
"seq_contiguous_ok": seq_ok,
|
||||
"seq_reset_expected": seq_reset_expected,
|
||||
"seq_notes": seq_notes[:8],
|
||||
"restart_contract": "PASS" if restart_ok else "FAIL",
|
||||
}
|
||||
|
||||
run_id = args.run_id or (run_ids[0] if len(run_ids) == 1 else ",".join(run_ids) or "UNSET")
|
||||
start0 = starts[0] if starts else {}
|
||||
manifest = {
|
||||
"run_id": run_id,
|
||||
"start_ts": start0.get("local_ts"),
|
||||
"end_ts": stops[-1].get("local_ts") if stops else (rows[-1].get("local_ts") if rows else None),
|
||||
"host": start0.get("host"),
|
||||
"commit": start0.get("commit"),
|
||||
"config_hash": start0.get("config_hash"),
|
||||
"schema_version": start0.get("schema_version"),
|
||||
"event_count": len(rows),
|
||||
"trade_event_count": len(trades),
|
||||
"book_event_count": len(books),
|
||||
"session_count": len(sessions),
|
||||
}
|
||||
|
||||
report = {
|
||||
"experiment_id": start0.get("experiment_id", "MM_EDGE_EXP_002"),
|
||||
"run_id": run_id,
|
||||
"purpose": "ledger smoke / Gates 1-3",
|
||||
"gate4_predictability": "BLOCKED",
|
||||
"gates": {
|
||||
"gate1_event_completeness": gate1_status,
|
||||
"gate2_temporal_integrity": gate2_status,
|
||||
"gate3_event_coverage": gate3_status,
|
||||
},
|
||||
"manifest": manifest,
|
||||
"rates": rates,
|
||||
"timestamp_quality": ts_quality,
|
||||
"order": {
|
||||
"exchange_ts_regressions": regressions,
|
||||
"max_regression_ns": max_back_ns,
|
||||
"max_regression_ms": max_back_ns / 1e6 if regressions else 0.0,
|
||||
"note": "regressions recorded, not silently sorted",
|
||||
},
|
||||
"schema": {
|
||||
"trade_sample_n": n_trade_s,
|
||||
"book_sample_n": n_book_s,
|
||||
"trade_missing_rate": trade_missing,
|
||||
"book_missing_rate": book_missing,
|
||||
"hollow_book_in_sample": hollow,
|
||||
},
|
||||
"integrity": integrity,
|
||||
"counts": {
|
||||
"total_rows": len(rows),
|
||||
"market_events": len(market),
|
||||
"aggressive_trades": len(trades),
|
||||
"book_updates": len(books),
|
||||
"fill_anchors": len(anchors),
|
||||
},
|
||||
}
|
||||
|
||||
lines = [
|
||||
"=" * 68,
|
||||
"MM_EDGE_EXP_002 Ledger Smoke / Gates 1–3",
|
||||
"=" * 68,
|
||||
f"run_id: {run_id}",
|
||||
f"sessions: {len(sessions)} {sessions}",
|
||||
f"host/commit:{start0.get('host')} / {str(start0.get('commit') or '')[:12]}",
|
||||
f"config_hash:{start0.get('config_hash')}",
|
||||
f"schema: {start0.get('schema_version')}",
|
||||
"",
|
||||
"Gate 1 Event Completeness: " + gate1_status,
|
||||
"Gate 2 Temporal Integrity: " + gate2_status,
|
||||
"Gate 3 Event Coverage: " + gate3_status,
|
||||
"Gate 4 Predictability: BLOCKED",
|
||||
"",
|
||||
"1. Event write rates",
|
||||
"-" * 40,
|
||||
f"duration_sec: {_num(duration_s, 1)}",
|
||||
f"aggressive_trade / sec: {_num(rates['aggressive_trade_per_sec'], 3)}",
|
||||
f"book_update / sec: {_num(rates['book_update_per_sec'], 3)}",
|
||||
f"total market events / sec: {_num(rates['total_market_events_per_sec'], 3)}",
|
||||
f"counts: trades={len(trades)} books={len(books)} total={len(market)}",
|
||||
"",
|
||||
"2. Timestamp quality",
|
||||
"-" * 40,
|
||||
f"exchange_ts_ns != null: {ts_quality['exchange_ts_ns_pct']*100:.2f}%",
|
||||
f"local_ts_ns != null: {ts_quality['local_ts_pct']*100:.2f}%",
|
||||
f"exchange > local+tol: {skew_violations} (tol={args.latency_tolerance_ms}ms)",
|
||||
f"local-exchange lag ms: p50={_num(ts_quality['latency_ms_p50'])} "
|
||||
f"p95={_num(ts_quality['latency_ms_p95'])} p99={_num(ts_quality['latency_ms_p99'])} "
|
||||
f"max={_num(ts_quality['latency_ms_max'])}",
|
||||
"",
|
||||
"3. Event order (exchange_ts_ns regression, not sorted)",
|
||||
"-" * 40,
|
||||
f"regressions: {regressions} max_back_ms={_num(max_back_ns/1e6 if regressions else 0.0)}",
|
||||
"",
|
||||
"4. Raw event completeness (sample)",
|
||||
"-" * 40,
|
||||
f"trade sample={n_trade_s} missing={trade_missing}",
|
||||
f"book sample={n_book_s} missing={book_missing}",
|
||||
f"hollow book_update (all depth/mid empty): {hollow}",
|
||||
"",
|
||||
"5. Restart / immutable integrity",
|
||||
"-" * 40,
|
||||
f"parse_fail_lines={parse_fail} empty_lines={empty_lines}",
|
||||
f"duplicate_event_ids={len(dup_ids)}",
|
||||
f"start={len(starts)} stop={len(stops)} seq_ok={seq_ok} seq_reset_expected={seq_reset_expected}",
|
||||
f"restart_contract={integrity['restart_contract']}",
|
||||
"",
|
||||
"Gate 4 remains BLOCKED until fill_anchor exists. Do not resume trading.",
|
||||
"=" * 68,
|
||||
]
|
||||
text = "\n".join(lines) + "\n"
|
||||
print(text)
|
||||
|
||||
out_json = Path(args.out) if args.out else log_dir / "Event_Ledger_Validation.json"
|
||||
out_txt = out_json.with_suffix(".txt")
|
||||
out_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_json.write_text(json.dumps(report, indent=2, default=str) + "\n", encoding="utf-8")
|
||||
out_txt.write_text(text, encoding="utf-8")
|
||||
(log_dir / f"{run_id.replace('/', '_')}.manifest.json").write_text(
|
||||
json.dumps(manifest, indent=2, default=str) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
ok = gate1_pass is not False and gate2_ok and gate3_ok and restart_ok and parse_fail == 0
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,3 @@
|
||||
"""Nautilus MM — Trading OS Execution Reality Layer (Maker Edge)."""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Order book → MicroSnapshot helpers (Nautilus OrderBook / dict)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Optional
|
||||
|
||||
from nautilus_mm.recorder import MicroSnapshot
|
||||
|
||||
|
||||
def snapshot_from_nautilus_book(
|
||||
book,
|
||||
levels: int = 10,
|
||||
recent_buy_qty: float = 0.0,
|
||||
recent_sell_qty: float = 0.0,
|
||||
last_mid: float | None = None,
|
||||
liq_low: float | None = None,
|
||||
liq_high: float | None = None,
|
||||
) -> MicroSnapshot:
|
||||
"""Convert nautilus_trader OrderBook to MicroSnapshot."""
|
||||
try:
|
||||
bids = list(book.bids())[:levels] if callable(getattr(book, "bids", None)) else []
|
||||
asks = list(book.asks())[:levels] if callable(getattr(book, "asks", None)) else []
|
||||
except Exception:
|
||||
# Some versions expose bid/ask sequences differently
|
||||
bids = getattr(book, "bids", [])[:levels] or []
|
||||
asks = getattr(book, "asks", [])[:levels] or []
|
||||
|
||||
def _px_qty(level) -> tuple[float, float]:
|
||||
# Nautilus BookLevel: price is attribute, size() is method
|
||||
if hasattr(level, "price") and hasattr(level, "size"):
|
||||
size = level.size() if callable(level.size) else level.size
|
||||
return float(level.price), float(size)
|
||||
if isinstance(level, (list, tuple)) and len(level) >= 2:
|
||||
return float(level[0]), float(level[1])
|
||||
return 0.0, 0.0
|
||||
|
||||
if not bids or not asks:
|
||||
# try best bid/ask API
|
||||
try:
|
||||
bb = float(book.best_bid_price()) if book.best_bid_price() is not None else 0.0
|
||||
ba = float(book.best_ask_price()) if book.best_ask_price() is not None else 0.0
|
||||
bs = float(book.best_bid_size() or 0)
|
||||
az = float(book.best_ask_size() or 0)
|
||||
if bb and ba:
|
||||
mid = (bb + ba) / 2
|
||||
return MicroSnapshot(
|
||||
best_bid=bb,
|
||||
best_ask=ba,
|
||||
mid=mid,
|
||||
spread=ba - bb,
|
||||
bid_depth_1=bs,
|
||||
ask_depth_1=az,
|
||||
bid_depth_5=bs,
|
||||
ask_depth_5=az,
|
||||
bid_depth=bs,
|
||||
ask_depth=az,
|
||||
obi=((bs - az) / (bs + az)) if (bs + az) else 0.0,
|
||||
)
|
||||
except Exception:
|
||||
return MicroSnapshot()
|
||||
return MicroSnapshot()
|
||||
|
||||
bid_levels = [_px_qty(x) for x in bids]
|
||||
ask_levels = [_px_qty(x) for x in asks]
|
||||
best_bid, bid1 = bid_levels[0]
|
||||
best_ask, ask1 = ask_levels[0]
|
||||
mid = (best_bid + best_ask) / 2.0
|
||||
spread = best_ask - best_bid
|
||||
|
||||
def depth(lvls, n):
|
||||
return sum(q for _, q in lvls[:n])
|
||||
|
||||
bid_depth_5 = depth(bid_levels, 5)
|
||||
ask_depth_5 = depth(ask_levels, 5)
|
||||
bid_depth = depth(bid_levels, levels)
|
||||
ask_depth = depth(ask_levels, levels)
|
||||
tot = bid_depth + ask_depth
|
||||
obi = ((bid_depth - ask_depth) / tot) if tot else 0.0
|
||||
|
||||
delta = recent_buy_qty - recent_sell_qty
|
||||
timb_den = recent_buy_qty + recent_sell_qty
|
||||
trade_imbalance = (delta / timb_den) if timb_den else 0.0
|
||||
|
||||
de = 0.0
|
||||
if last_mid and mid and abs(delta) > 1e-12:
|
||||
de = ((mid - last_mid) / last_mid) / delta
|
||||
|
||||
liq_dist = 0.0
|
||||
if liq_low and liq_high and mid and (liq_high - liq_low) > 0:
|
||||
liq_dist = ((mid - liq_low) / (liq_high - liq_low)) * 2 - 1
|
||||
|
||||
return MicroSnapshot(
|
||||
best_bid=best_bid,
|
||||
best_ask=best_ask,
|
||||
mid=mid,
|
||||
spread=spread,
|
||||
bid_depth_1=bid1,
|
||||
ask_depth_1=ask1,
|
||||
bid_depth_5=bid_depth_5,
|
||||
ask_depth_5=ask_depth_5,
|
||||
bid_depth=bid_depth,
|
||||
ask_depth=ask_depth,
|
||||
obi=obi,
|
||||
delta=delta,
|
||||
trade_imbalance=trade_imbalance,
|
||||
delta_efficiency=de,
|
||||
liquidation_distance=liq_dist,
|
||||
)
|
||||
|
||||
|
||||
def snapshot_from_ccxt_ob(
|
||||
ob: dict[str, Any],
|
||||
levels: int = 10,
|
||||
recent_trades: list | None = None,
|
||||
last_mid: float | None = None,
|
||||
) -> MicroSnapshot:
|
||||
from nautilus_mm.recorder import MakerEdgeLogger
|
||||
|
||||
return MakerEdgeLogger.snapshot_from_orderbook(
|
||||
ob,
|
||||
levels=levels,
|
||||
recent_trades=recent_trades,
|
||||
last_mid=last_mid,
|
||||
)
|
||||
@@ -0,0 +1,403 @@
|
||||
"""
|
||||
Immutable Event Ledger — MM_EDGE_EXP_002
|
||||
|
||||
Raw Event > Derived Feature
|
||||
|
||||
Stores immutable market events for later reconstruction of pre-fill windows.
|
||||
Features are computed offline; this module only persists observability data.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from nautilus_mm.experiment import load_experiment_meta, stamp_event
|
||||
from nautilus_mm.recorder import MicroSnapshot
|
||||
|
||||
LEDGER_SCHEMA_VERSION = "immutable_event_v1"
|
||||
|
||||
|
||||
def _git_commit(root: Path) -> str | None:
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["git", "rev-parse", "HEAD"],
|
||||
cwd=root,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=3,
|
||||
check=False,
|
||||
)
|
||||
if r.returncode == 0:
|
||||
return r.stdout.strip() or None
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def load_run_identity(*, extra_config: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
"""Freeze one Ledger Integrity ID per collection run (sessions share the same run_id)."""
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
cfg = {
|
||||
"prefill_window_sec": os.getenv("PREFILL_WINDOW_SEC", "5.0"),
|
||||
"prefill_margin_sec": os.getenv("PREFILL_MARGIN_SEC", "0.25"),
|
||||
"large_trade_qty": os.getenv("LARGE_TRADE_QTY", "0.1"),
|
||||
"book_depth": os.getenv("BOOK_DEPTH", "10"),
|
||||
"symbol": os.getenv("SYMBOL", "BTCUSDT-PERP"),
|
||||
"environment": os.getenv("BINANCE_ENVIRONMENT", "TESTNET"),
|
||||
"schema_version": LEDGER_SCHEMA_VERSION,
|
||||
}
|
||||
if extra_config:
|
||||
cfg.update({k: str(v) for k, v in extra_config.items()})
|
||||
payload = json.dumps(cfg, sort_keys=True, default=str)
|
||||
return {
|
||||
"run_id": os.getenv("LEDGER_RUN_ID", "EXP-002-RUN-UNSET"),
|
||||
"session_id": os.getenv("LEDGER_SESSION_ID") or uuid.uuid4().hex[:12],
|
||||
"schema_version": LEDGER_SCHEMA_VERSION,
|
||||
"host": socket.gethostname(),
|
||||
"commit": _git_commit(root),
|
||||
"config_hash": hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16],
|
||||
"config_snapshot": cfg,
|
||||
}
|
||||
|
||||
|
||||
def _utc_iso(ts: float | None = None) -> str:
|
||||
t = datetime.fromtimestamp(ts or time.time(), tz=timezone.utc)
|
||||
return t.isoformat()
|
||||
|
||||
|
||||
@dataclass
|
||||
class EventTimingState:
|
||||
last_trade_ts: float | None = None
|
||||
last_large_trade_ts: float | None = None
|
||||
last_book_event_ts: float | None = None
|
||||
last_tob_change_ts: float | None = None
|
||||
last_spread_change_ts: float | None = None
|
||||
last_mid_change_ts: float | None = None
|
||||
last_depth_change_ts: float | None = None
|
||||
|
||||
|
||||
class ImmutableEventLedger:
|
||||
"""Append-only JSONL ledger for market events."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
log_dir: str | Path | None = None,
|
||||
*,
|
||||
prefill_window_sec: float = 5.0,
|
||||
prefill_margin_sec: float = 0.25,
|
||||
large_trade_qty: float = 0.1,
|
||||
book_levels: int = 10,
|
||||
) -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
self.log_dir = Path(log_dir) if log_dir else root / "logs" / "event_state"
|
||||
self.log_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.prefill_window_sec = float(prefill_window_sec)
|
||||
self.prefill_margin_sec = float(prefill_margin_sec)
|
||||
self.large_trade_qty = float(large_trade_qty)
|
||||
self.book_levels = int(book_levels)
|
||||
self.experiment = load_experiment_meta()
|
||||
self.run_identity = load_run_identity(
|
||||
extra_config={
|
||||
"prefill_window_sec": self.prefill_window_sec,
|
||||
"prefill_margin_sec": self.prefill_margin_sec,
|
||||
"large_trade_qty": self.large_trade_qty,
|
||||
"book_levels": self.book_levels,
|
||||
}
|
||||
)
|
||||
self._timing = EventTimingState()
|
||||
self._prev_snap: MicroSnapshot | None = None
|
||||
self._event_seq = 0
|
||||
self._session_event_count = 0
|
||||
|
||||
def _file(self) -> Path:
|
||||
return self.log_dir / f"{datetime.now(timezone.utc).strftime('%Y%m%d')}.jsonl"
|
||||
|
||||
def _next_seq(self) -> int:
|
||||
self._event_seq += 1
|
||||
return self._event_seq
|
||||
|
||||
def _timing_fields(self, now: float, *, is_trade: bool = False, is_large_trade: bool = False) -> dict[str, Any]:
|
||||
def _since(last: float | None) -> float | None:
|
||||
if last is None:
|
||||
return None
|
||||
return (now - last) * 1000.0
|
||||
|
||||
fields = {
|
||||
"time_since_last_trade_ms": _since(self._timing.last_trade_ts),
|
||||
"time_since_last_large_trade_ms": _since(self._timing.last_large_trade_ts),
|
||||
"time_since_last_book_event_ms": _since(self._timing.last_book_event_ts),
|
||||
"time_since_last_tob_change_ms": _since(self._timing.last_tob_change_ts),
|
||||
"time_since_last_spread_change_ms": _since(self._timing.last_spread_change_ts),
|
||||
"time_since_last_mid_change_ms": _since(self._timing.last_mid_change_ts),
|
||||
"time_since_last_depth_change_ms": _since(self._timing.last_depth_change_ts),
|
||||
}
|
||||
self._timing.last_book_event_ts = now
|
||||
if is_trade:
|
||||
self._timing.last_trade_ts = now
|
||||
if is_large_trade:
|
||||
self._timing.last_large_trade_ts = now
|
||||
return fields
|
||||
|
||||
def write(self, event: dict[str, Any]) -> None:
|
||||
# Receive time is always local wall clock. Never copy exchange_ts into local_ts.
|
||||
now = time.time()
|
||||
now_ns = time.time_ns()
|
||||
event["ledger"] = LEDGER_SCHEMA_VERSION
|
||||
event["schema_version"] = LEDGER_SCHEMA_VERSION
|
||||
event["run_id"] = self.run_identity["run_id"]
|
||||
event["session_id"] = self.run_identity["session_id"]
|
||||
event["local_ts_epoch"] = now
|
||||
event["local_ts_ns"] = now_ns
|
||||
event["local_ts"] = _utc_iso(now)
|
||||
event["event_seq"] = self._next_seq()
|
||||
self._session_event_count += 1
|
||||
stamp_event(event, self.experiment)
|
||||
with self._file().open("a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(event, ensure_ascii=False, default=str) + "\n")
|
||||
f.flush()
|
||||
|
||||
def write_experiment_start(self, extra: dict | None = None) -> None:
|
||||
ev = {
|
||||
"event": "experiment_start",
|
||||
"experiment_type": "Event-State Observability Probe",
|
||||
"strategy": "NONE",
|
||||
"execution_trading": False,
|
||||
"prefill_window_sec": self.prefill_window_sec,
|
||||
"prefill_margin_sec": self.prefill_margin_sec,
|
||||
"large_trade_qty": self.large_trade_qty,
|
||||
**self.experiment,
|
||||
**{k: v for k, v in self.run_identity.items() if k != "config_snapshot"},
|
||||
"config_snapshot": self.run_identity["config_snapshot"],
|
||||
}
|
||||
if extra:
|
||||
ev.update(extra)
|
||||
self.write(ev)
|
||||
|
||||
def write_experiment_stop(self, extra: dict | None = None) -> None:
|
||||
ev = {
|
||||
"event": "experiment_stop",
|
||||
"session_event_count": self._session_event_count,
|
||||
"run_id": self.run_identity["run_id"],
|
||||
"session_id": self.run_identity["session_id"],
|
||||
"execution_trading": False,
|
||||
}
|
||||
if extra:
|
||||
ev.update(extra)
|
||||
self.write(ev)
|
||||
|
||||
def log_aggressive_trade(
|
||||
self,
|
||||
*,
|
||||
pair: str,
|
||||
price: float,
|
||||
qty: float,
|
||||
trade_side: str,
|
||||
exchange_ts_ns: int | None,
|
||||
local_ts_epoch: float | None = None,
|
||||
aggressor_side: str | None = None,
|
||||
trade_id: str | None = None,
|
||||
snap: MicroSnapshot | None = None,
|
||||
) -> str:
|
||||
now = local_ts_epoch or time.time()
|
||||
notional = price * qty
|
||||
is_large = qty >= self.large_trade_qty
|
||||
event_id = uuid.uuid4().hex[:16]
|
||||
ev = {
|
||||
"event": "market_event",
|
||||
"event_id": event_id,
|
||||
"event_type": "aggressive_trade",
|
||||
"pair": pair,
|
||||
"price": price,
|
||||
"quantity": qty,
|
||||
"trade_qty": qty,
|
||||
"trade_price": price,
|
||||
"trade_notional": notional,
|
||||
"trade_side": trade_side,
|
||||
"aggressor_side": aggressor_side or trade_side,
|
||||
"large_trade_flag": is_large,
|
||||
"trade_id": trade_id,
|
||||
"exchange_ts_ns": exchange_ts_ns,
|
||||
}
|
||||
if snap is not None:
|
||||
ev.update(self._snap_book_fields(snap))
|
||||
ev.update(self._depth_deltas(self._prev_snap, snap))
|
||||
ev.update(self._timing_fields(now, is_trade=True, is_large_trade=is_large))
|
||||
self.write(ev)
|
||||
return event_id
|
||||
|
||||
def _snap_book_fields(self, snap: MicroSnapshot) -> dict[str, Any]:
|
||||
return {
|
||||
"best_bid": snap.best_bid,
|
||||
"best_ask": snap.best_ask,
|
||||
"mid": snap.mid,
|
||||
"spread": snap.spread,
|
||||
"bid_depth_1": snap.bid_depth_1,
|
||||
"ask_depth_1": snap.ask_depth_1,
|
||||
"bid_depth_5": snap.bid_depth_5,
|
||||
"ask_depth_5": snap.ask_depth_5,
|
||||
"bid_depth": snap.bid_depth,
|
||||
"ask_depth": snap.ask_depth,
|
||||
"obi": snap.obi,
|
||||
}
|
||||
|
||||
def _depth_deltas(self, prev: MicroSnapshot | None, cur: MicroSnapshot) -> dict[str, Any]:
|
||||
if prev is None:
|
||||
return {
|
||||
"bid_depth_delta_1": None,
|
||||
"ask_depth_delta_1": None,
|
||||
"bid_depth_delta_5": None,
|
||||
"ask_depth_delta_5": None,
|
||||
"bid_depth_delta": None,
|
||||
"ask_depth_delta": None,
|
||||
}
|
||||
return {
|
||||
"bid_depth_delta_1": cur.bid_depth_1 - prev.bid_depth_1,
|
||||
"ask_depth_delta_1": cur.ask_depth_1 - prev.ask_depth_1,
|
||||
"bid_depth_delta_5": cur.bid_depth_5 - prev.bid_depth_5,
|
||||
"ask_depth_delta_5": cur.ask_depth_5 - prev.ask_depth_5,
|
||||
"bid_depth_delta": cur.bid_depth - prev.bid_depth,
|
||||
"ask_depth_delta": cur.ask_depth - prev.ask_depth,
|
||||
}
|
||||
|
||||
def log_book_state(
|
||||
self,
|
||||
*,
|
||||
pair: str,
|
||||
snap: MicroSnapshot,
|
||||
exchange_ts_ns: int | None,
|
||||
local_ts_epoch: float | None = None,
|
||||
sequence: int | None = None,
|
||||
delta_count: int | None = None,
|
||||
event_type: str = "book_update",
|
||||
) -> str:
|
||||
now = local_ts_epoch or time.time()
|
||||
prev = self._prev_snap
|
||||
event_id = uuid.uuid4().hex[:16]
|
||||
|
||||
bid_move = None
|
||||
ask_move = None
|
||||
mid_move = None
|
||||
spread_change = None
|
||||
if prev and prev.mid > 0:
|
||||
bid_move = snap.best_bid - prev.best_bid
|
||||
ask_move = snap.best_ask - prev.best_ask
|
||||
mid_move = snap.mid - prev.mid
|
||||
spread_change = snap.spread - prev.spread
|
||||
|
||||
depth_deltas = self._depth_deltas(prev, snap)
|
||||
ev = {
|
||||
"event": "market_event",
|
||||
"event_id": event_id,
|
||||
"event_type": event_type,
|
||||
"pair": pair,
|
||||
"exchange_ts_ns": exchange_ts_ns,
|
||||
"sequence": sequence,
|
||||
"delta_count": delta_count,
|
||||
**self._snap_book_fields(snap),
|
||||
**depth_deltas,
|
||||
"bid_move": bid_move,
|
||||
"ask_move": ask_move,
|
||||
"mid_move": mid_move,
|
||||
"spread_change": spread_change,
|
||||
}
|
||||
ev.update(self._timing_fields(now))
|
||||
|
||||
if prev is not None:
|
||||
if bid_move not in (None, 0.0) or ask_move not in (None, 0.0):
|
||||
self._timing.last_tob_change_ts = now
|
||||
if spread_change not in (None, 0.0):
|
||||
self._timing.last_spread_change_ts = now
|
||||
if mid_move not in (None, 0.0):
|
||||
self._timing.last_mid_change_ts = now
|
||||
if any(
|
||||
depth_deltas[k] not in (None, 0.0)
|
||||
for k in (
|
||||
"bid_depth_delta_1",
|
||||
"ask_depth_delta_1",
|
||||
"bid_depth_delta_5",
|
||||
"ask_depth_delta_5",
|
||||
)
|
||||
):
|
||||
self._timing.last_depth_change_ts = now
|
||||
|
||||
self._prev_snap = snap
|
||||
self.write(ev)
|
||||
return event_id
|
||||
|
||||
def log_fill_anchor(
|
||||
self,
|
||||
*,
|
||||
fill_id: str,
|
||||
fill_ts_epoch: float,
|
||||
exchange_ts_ns: int | None,
|
||||
side: str,
|
||||
fill_price: float,
|
||||
fill_qty: float,
|
||||
order_id: str | None = None,
|
||||
venue_order_id: str | None = None,
|
||||
venue_trade_id: str | None = None,
|
||||
pair: str | None = None,
|
||||
snap: MicroSnapshot | None = None,
|
||||
extra: dict | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Anchor for offline [-prefill_window_sec, fill) reconstruction.
|
||||
|
||||
EXP_002 Phase 1 may not emit these (no trading). Schema is frozen for
|
||||
future fill-anchored analysis (Gate 4).
|
||||
"""
|
||||
window_id = uuid.uuid4().hex[:16]
|
||||
window_start = fill_ts_epoch - self.prefill_window_sec
|
||||
feature_cutoff = fill_ts_epoch - self.prefill_margin_sec
|
||||
ev = {
|
||||
"event": "fill_anchor",
|
||||
"window_id": window_id,
|
||||
"fill_id": fill_id,
|
||||
"fill_ts_epoch": fill_ts_epoch,
|
||||
"fill_ts": _utc_iso(fill_ts_epoch),
|
||||
"exchange_ts_ns": exchange_ts_ns,
|
||||
"window_start_epoch": window_start,
|
||||
"feature_cutoff_epoch": feature_cutoff,
|
||||
"prefill_window_sec": self.prefill_window_sec,
|
||||
"prefill_margin_sec": self.prefill_margin_sec,
|
||||
"side": side,
|
||||
"fill_price": fill_price,
|
||||
"fill_qty": fill_qty,
|
||||
"order_id": order_id,
|
||||
"venue_order_id": venue_order_id,
|
||||
"venue_trade_id": venue_trade_id,
|
||||
"pair": pair,
|
||||
}
|
||||
if snap is not None:
|
||||
ev.update(self._snap_book_fields(snap))
|
||||
if extra:
|
||||
ev.update(extra)
|
||||
self.write(ev)
|
||||
return window_id
|
||||
|
||||
def events_in_window(self, events: list[dict], fill_ts_epoch: float) -> list[dict]:
|
||||
"""Offline helper: filter market_event rows in [-window, fill-margin)."""
|
||||
start = fill_ts_epoch - self.prefill_window_sec
|
||||
cutoff = fill_ts_epoch - self.prefill_margin_sec
|
||||
out = []
|
||||
for ev in events:
|
||||
if ev.get("event") != "market_event":
|
||||
continue
|
||||
ts = ev.get("exchange_ts_ns")
|
||||
if ts is not None:
|
||||
ts_epoch = float(ts) / 1e9
|
||||
else:
|
||||
ts_epoch = float(ev.get("local_ts_epoch", 0.0))
|
||||
if start <= ts_epoch < cutoff:
|
||||
out.append(ev)
|
||||
return out
|
||||
@@ -0,0 +1,63 @@
|
||||
"""
|
||||
实验身份绑定 — Research Freeze / Data Collection
|
||||
|
||||
每条 jsonl 与每份 Maker Edge Report 必须绑定同一 Experiment ID,
|
||||
避免 v2/v3 混淆「哪个实验验证出了什么」。
|
||||
|
||||
冻结字段(运行期不可为「结果好看」而改):
|
||||
quote / fee / exchange / probe version
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
|
||||
# 默认实验身份(可用环境变量覆盖 ID,其余保持 frozen 语义)
|
||||
DEFAULT_EXPERIMENT_ID = "MM_EDGE_EXP_001"
|
||||
DEFAULT_PROBE_VERSION = "probe_v0.1"
|
||||
|
||||
EXP_002_ID = "MM_EDGE_EXP_002"
|
||||
EXP_002_PROBE_VERSION = "event_state_v0.1"
|
||||
|
||||
|
||||
def load_experiment_meta() -> dict[str, Any]:
|
||||
"""从环境变量加载实验元数据;冻结维度固定为 frozen。"""
|
||||
exp_id = os.getenv("EXPERIMENT_ID", DEFAULT_EXPERIMENT_ID)
|
||||
probe_version = os.getenv("PROBE_VERSION", DEFAULT_PROBE_VERSION)
|
||||
if exp_id == EXP_002_ID:
|
||||
phase = "Event-State Observability / Data Collection"
|
||||
experiment_type = "Event-State Observability Probe"
|
||||
else:
|
||||
phase = "Research Freeze / Data Collection"
|
||||
experiment_type = "Maker Edge Phenomenon Probe"
|
||||
return {
|
||||
"experiment_id": exp_id,
|
||||
"probe_version": probe_version,
|
||||
"experiment_type": experiment_type,
|
||||
"quote_assumption": "frozen",
|
||||
"fee_model": "frozen",
|
||||
"exchange_assumption": "frozen",
|
||||
"exchange": os.getenv("EXCHANGE_NAME", "binance_usdm"),
|
||||
"environment": os.getenv("BINANCE_ENVIRONMENT", "TESTNET").upper(),
|
||||
"symbol": os.getenv("SYMBOL", "BTCUSDT-PERP"),
|
||||
"phase": phase,
|
||||
"depends_on": "MM_EDGE_EXP_001" if exp_id == EXP_002_ID else None,
|
||||
}
|
||||
|
||||
|
||||
def stamp_event(event: dict[str, Any], meta: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
"""给单条事件打上实验身份(不覆盖已有显式字段)。"""
|
||||
m = meta or load_experiment_meta()
|
||||
event.setdefault("experiment_id", m["experiment_id"])
|
||||
event.setdefault("probe_version", m["probe_version"])
|
||||
event.setdefault("experiment", {
|
||||
"quote": m["quote_assumption"],
|
||||
"fee": m["fee_model"],
|
||||
"exchange": m["exchange_assumption"],
|
||||
"venue": m["exchange"],
|
||||
"environment": m["environment"],
|
||||
"symbol": m["symbol"],
|
||||
})
|
||||
return event
|
||||
@@ -0,0 +1,147 @@
|
||||
"""
|
||||
Phase 0 — 连接 / 数据健康度
|
||||
|
||||
关键指标:
|
||||
sequence_gap — order book 失真信号(>0 需警惕)
|
||||
latency_ms — p50 / p95 / p99 / max(做市看尾部)
|
||||
book_age_ms — quote/fill 使用盘口时的新鲜度
|
||||
book/trade update rate
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def empty_market_state_snapshot() -> dict:
|
||||
"""预留给 Market Pulse;探针阶段全部为 null,不做预测/下单决策。"""
|
||||
return {
|
||||
"regime": None,
|
||||
"trend_state": None,
|
||||
"liquidity_state": None,
|
||||
"volatility_state": None,
|
||||
}
|
||||
|
||||
|
||||
def _percentile(sorted_vals: list[float], q: float) -> Optional[float]:
|
||||
if not sorted_vals:
|
||||
return None
|
||||
if len(sorted_vals) == 1:
|
||||
return sorted_vals[0]
|
||||
idx = min(len(sorted_vals) - 1, max(0, int(round(q * (len(sorted_vals) - 1)))))
|
||||
return sorted_vals[idx]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConnectionHealth:
|
||||
window_sec: float = 60.0
|
||||
report_every_sec: float = 30.0
|
||||
_book_ts: deque = field(default_factory=lambda: deque(maxlen=50_000))
|
||||
_trade_ts: deque = field(default_factory=lambda: deque(maxlen=50_000))
|
||||
_latencies_ms: deque = field(default_factory=lambda: deque(maxlen=5_000))
|
||||
_seq_gaps: int = 0
|
||||
_seq_gaps_window: deque = field(default_factory=lambda: deque(maxlen=10_000))
|
||||
_last_seq: Optional[int] = None
|
||||
_last_report: float = 0.0
|
||||
_book_count: int = 0
|
||||
_trade_count: int = 0
|
||||
_last_book_wall: float = 0.0 # 本地收到最新 book 的时间
|
||||
|
||||
def on_book(self, seq: int | None = None, event_ts_ns: int | None = None) -> None:
|
||||
now = time.time()
|
||||
self._book_ts.append(now)
|
||||
self._last_book_wall = now
|
||||
self._book_count += 1
|
||||
if event_ts_ns is not None and event_ts_ns > 0:
|
||||
lat = (now * 1e9 - event_ts_ns) / 1e6
|
||||
if -1000 < lat < 60_000:
|
||||
self._latencies_ms.append(lat)
|
||||
if seq is not None:
|
||||
if self._last_seq is not None and seq > self._last_seq + 1:
|
||||
gap = seq - self._last_seq - 1
|
||||
# Binance L2 update ids often jump across snapshot/reconnect;
|
||||
# only count modest gaps as packet loss. Huge jumps → reset.
|
||||
if gap <= 1000:
|
||||
self._seq_gaps += gap
|
||||
self._seq_gaps_window.append((now, gap))
|
||||
self._last_seq = seq
|
||||
self._trim(now)
|
||||
|
||||
def on_trade(self, event_ts_ns: int | None = None) -> None:
|
||||
now = time.time()
|
||||
self._trade_ts.append(now)
|
||||
self._trade_count += 1
|
||||
if event_ts_ns is not None and event_ts_ns > 0:
|
||||
lat = (now * 1e9 - event_ts_ns) / 1e6
|
||||
if -1000 < lat < 60_000:
|
||||
self._latencies_ms.append(lat)
|
||||
self._trim(now)
|
||||
|
||||
def book_age_ms(self, now: float | None = None) -> Optional[float]:
|
||||
"""当前时刻距离最近一次 book 更新的年龄(ms)。"""
|
||||
if self._last_book_wall <= 0:
|
||||
return None
|
||||
now = now or time.time()
|
||||
return max(0.0, (now - self._last_book_wall) * 1000.0)
|
||||
|
||||
def _trim(self, now: float) -> None:
|
||||
cut = now - self.window_sec
|
||||
while self._book_ts and self._book_ts[0] < cut:
|
||||
self._book_ts.popleft()
|
||||
while self._trade_ts and self._trade_ts[0] < cut:
|
||||
self._trade_ts.popleft()
|
||||
while self._seq_gaps_window and self._seq_gaps_window[0][0] < cut:
|
||||
self._seq_gaps_window.popleft()
|
||||
|
||||
def snapshot(self) -> dict:
|
||||
now = time.time()
|
||||
self._trim(now)
|
||||
w = max(self.window_sec, 1e-6)
|
||||
lat = sorted(self._latencies_ms)
|
||||
gaps_in_window = sum(g for _, g in self._seq_gaps_window)
|
||||
book_age = self.book_age_ms(now)
|
||||
# Binance depth update ids are not contiguous; gap is observe-only.
|
||||
# Healthy = sufficient book rate + fresh book.
|
||||
healthy = (
|
||||
len(self._book_ts) / w >= 0.5
|
||||
and (book_age is None or book_age < 500.0)
|
||||
)
|
||||
return {
|
||||
"event": "health",
|
||||
"window_sec": self.window_sec,
|
||||
"book_update_rate": len(self._book_ts) / w,
|
||||
"trade_update_rate": len(self._trade_ts) / w,
|
||||
"latency_ms_mean": (sum(lat) / len(lat)) if lat else None,
|
||||
"latency_ms_p50": _percentile(lat, 0.50),
|
||||
"latency_ms_p95": _percentile(lat, 0.95),
|
||||
"latency_ms_p99": _percentile(lat, 0.99),
|
||||
"latency_ms_max": lat[-1] if lat else None,
|
||||
"sequence_gap": self._seq_gaps, # 累计
|
||||
"sequence_gap_window": gaps_in_window, # 近窗
|
||||
"book_age_ms": book_age,
|
||||
"book_total": self._book_count,
|
||||
"trade_total": self._trade_count,
|
||||
"healthy": healthy,
|
||||
}
|
||||
|
||||
def maybe_report(self) -> Optional[dict]:
|
||||
now = time.time()
|
||||
if now - self._last_report < self.report_every_sec:
|
||||
return None
|
||||
self._last_report = now
|
||||
return self.snapshot()
|
||||
|
||||
def allow_quoting(self, max_book_age_ms: float = 500.0) -> bool:
|
||||
"""Gate new quotes on freshness + update rate (not Binance seq jumps)."""
|
||||
age = self.book_age_ms()
|
||||
if age is None or age >= max_book_age_ms:
|
||||
return False
|
||||
now = time.time()
|
||||
self._trim(now)
|
||||
w = max(self.window_sec, 1e-6)
|
||||
if len(self._book_ts) / w < 0.5:
|
||||
return False
|
||||
return True
|
||||
@@ -0,0 +1,861 @@
|
||||
"""
|
||||
Maker Edge 事件记录器 — Execution Reality Layer(Nautilus / CCXT 共用)
|
||||
|
||||
事件:
|
||||
- quote_created / quote_canceled / quote_filled
|
||||
- fill / fill_path / fill_exit
|
||||
|
||||
默认输出:nautilus_mm/logs/maker_edge/YYYYMMDD.jsonl
|
||||
(与 Freqtrade MakerEdgeProbe schema 对齐,可用同一 analyze 脚本)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
from nautilus_mm.experiment import load_experiment_meta, stamp_event
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def empty_market_state_snapshot() -> dict:
|
||||
"""Market Pulse 预留位;探针阶段保持 null,不做交易决策。"""
|
||||
return {
|
||||
"regime": None,
|
||||
"trend_state": None,
|
||||
"liquidity_state": None,
|
||||
"volatility_state": None,
|
||||
}
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _iso(ts: datetime | float | None = None) -> str:
|
||||
if ts is None:
|
||||
t = _utc_now()
|
||||
elif isinstance(ts, (int, float)):
|
||||
t = datetime.fromtimestamp(ts, tz=timezone.utc)
|
||||
else:
|
||||
t = ts if ts.tzinfo else ts.replace(tzinfo=timezone.utc)
|
||||
return t.isoformat()
|
||||
|
||||
|
||||
@dataclass
|
||||
class MicroSnapshot:
|
||||
best_bid: float = 0.0
|
||||
best_ask: float = 0.0
|
||||
mid: float = 0.0
|
||||
spread: float = 0.0
|
||||
bid_depth_1: float = 0.0
|
||||
ask_depth_1: float = 0.0
|
||||
bid_depth_5: float = 0.0
|
||||
ask_depth_5: float = 0.0
|
||||
bid_depth: float = 0.0 # top-N
|
||||
ask_depth: float = 0.0
|
||||
obi: float = 0.0
|
||||
delta: float = 0.0
|
||||
trade_imbalance: float = 0.0 # (buy-sell)/(buy+sell) on recent trades
|
||||
delta_efficiency: float = 0.0
|
||||
liquidation_distance: float = 0.0
|
||||
|
||||
def to_book_fields(self) -> dict[str, float]:
|
||||
return {
|
||||
"bid_price": self.best_bid,
|
||||
"ask_price": self.best_ask,
|
||||
"mid": self.mid,
|
||||
"spread": self.spread,
|
||||
"bid_depth_1": self.bid_depth_1,
|
||||
"ask_depth_1": self.ask_depth_1,
|
||||
"bid_depth_5": self.bid_depth_5,
|
||||
"ask_depth_5": self.ask_depth_5,
|
||||
"bid_depth": self.bid_depth,
|
||||
"ask_depth": self.ask_depth,
|
||||
"obi": self.obi,
|
||||
"delta": self.delta,
|
||||
"trade_imbalance": self.trade_imbalance,
|
||||
"delta_efficiency": self.delta_efficiency,
|
||||
"liquidation_distance": self.liquidation_distance,
|
||||
# 兼容旧字段
|
||||
"buy1_depth": self.bid_depth_1,
|
||||
"sell1_depth": self.ask_depth_1,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ActiveQuote:
|
||||
quote_id: str
|
||||
pair: str
|
||||
side: str # bid / ask
|
||||
quote_price: float
|
||||
created_ts: float
|
||||
reason: str = ""
|
||||
trade_id: Optional[int] = None
|
||||
status: str = "open" # open / filled / canceled
|
||||
|
||||
|
||||
@dataclass
|
||||
class PendingFillPath:
|
||||
fill_id: str
|
||||
pair: str
|
||||
side: str
|
||||
fill_price: float
|
||||
fill_ts: float
|
||||
quote_id: Optional[str] = None
|
||||
exit_reason: Optional[str] = None
|
||||
# horizon prices: +1s +5s +10s +30s +60s +300s
|
||||
after_1s_price: Optional[float] = None
|
||||
after_5s_price: Optional[float] = None
|
||||
after_10s_price: Optional[float] = None
|
||||
after_30s_price: Optional[float] = None
|
||||
after_1m_price: Optional[float] = None
|
||||
after_5m_price: Optional[float] = None
|
||||
# running extrema
|
||||
min_price: float = 0.0
|
||||
max_price: float = 0.0
|
||||
# time-MAE / MFE at horizons
|
||||
mae_1s: Optional[float] = None
|
||||
mae_5s: Optional[float] = None
|
||||
mae_10s: Optional[float] = None
|
||||
mae_30s: Optional[float] = None
|
||||
mae_1m: Optional[float] = None
|
||||
mae_5m: Optional[float] = None
|
||||
mfe_1s: Optional[float] = None
|
||||
mfe_5s: Optional[float] = None
|
||||
mfe_10s: Optional[float] = None
|
||||
mfe_30s: Optional[float] = None
|
||||
mfe_1m: Optional[float] = None
|
||||
mfe_5m: Optional[float] = None
|
||||
done: bool = False
|
||||
|
||||
def __post_init__(self):
|
||||
self.min_price = self.fill_price
|
||||
self.max_price = self.fill_price
|
||||
|
||||
def signed_excursions(self) -> tuple[float, float]:
|
||||
"""Return (mae, mfe) at current min/max. mae<=0 adverse, mfe>=0 favorable."""
|
||||
if self.side == "long":
|
||||
mae = (self.min_price - self.fill_price) / self.fill_price
|
||||
mfe = (self.max_price - self.fill_price) / self.fill_price
|
||||
else:
|
||||
mae = (self.fill_price - self.max_price) / self.fill_price
|
||||
mfe = (self.fill_price - self.min_price) / self.fill_price
|
||||
return mae, mfe
|
||||
|
||||
def fav_ret_at(self, px: Optional[float]) -> Optional[float]:
|
||||
if px is None or self.fill_price <= 0:
|
||||
return None
|
||||
raw = (px - self.fill_price) / self.fill_price
|
||||
return raw if self.side == "long" else -raw
|
||||
|
||||
|
||||
def classify_path_type(p: "PendingFillPath") -> str:
|
||||
"""
|
||||
成交后路径形态(决定未来 Quote Logic):
|
||||
A_immediate_edge — 立即有利(1s/5s 已正,30s 仍正)
|
||||
B_drawdown_then_recover — 先亏后赚(早期 MAE,末期有利)
|
||||
C_toxic — 成交即错误(持续不利)
|
||||
D_mixed — 其它
|
||||
"""
|
||||
r1 = p.fav_ret_at(p.after_1s_price)
|
||||
r5 = p.fav_ret_at(p.after_5s_price)
|
||||
r30 = p.fav_ret_at(p.after_30s_price)
|
||||
r300 = p.fav_ret_at(p.after_5m_price)
|
||||
mae30 = p.mae_30s or 0.0
|
||||
|
||||
if r30 is not None and r30 > 0 and (r1 or 0) >= 0 and (r5 or 0) >= 0:
|
||||
return "A_immediate_edge"
|
||||
if mae30 < -1e-6 and r300 is not None and r300 > 0:
|
||||
return "B_drawdown_then_recover"
|
||||
if r30 is not None and r30 < 0 and (r300 is None or r300 <= 0):
|
||||
return "C_toxic"
|
||||
return "D_mixed"
|
||||
|
||||
|
||||
class MakerEdgeLogger:
|
||||
def __init__(
|
||||
self,
|
||||
log_dir: str | Path | None = None,
|
||||
levels: int = 10,
|
||||
book_history_sec: float = 30.0,
|
||||
cluster_gap_sec: float = 30.0,
|
||||
mid_tick_every_sec: float = 1.0,
|
||||
):
|
||||
# .../nautilus_mm/src/nautilus_mm/recorder.py → parents[2] = nautilus_mm
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
self.log_dir = Path(log_dir) if log_dir else root / "logs" / "maker_edge"
|
||||
self.log_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.levels = levels
|
||||
self.book_history_sec = book_history_sec
|
||||
self.cluster_gap_sec = cluster_gap_sec
|
||||
self.mid_tick_every_sec = mid_tick_every_sec
|
||||
self._pending: dict[str, PendingFillPath] = {}
|
||||
self._quotes: dict[str, ActiveQuote] = {} # quote_id -> ActiveQuote
|
||||
self._quotes_by_trade: dict[int, str] = {} # trade_id -> quote_id
|
||||
self._book_hist: deque[tuple[float, MicroSnapshot]] = deque(maxlen=2000)
|
||||
# inventory tracking for future quote engine
|
||||
self._inv: float = 0.0
|
||||
self._inv_nonzero_since: Optional[float] = None
|
||||
self._inv_target: float = 0.0 # flat target; skew = inv - target
|
||||
# liquidity-event clustering (样本独立性)
|
||||
self._cluster_id: Optional[str] = None
|
||||
self._cluster_side: Optional[str] = None
|
||||
self._cluster_last_ts: float = 0.0
|
||||
self._cluster_start_mid: Optional[float] = None
|
||||
self._cluster_n: int = 0
|
||||
self._last_mid_tick_ts: float = 0.0
|
||||
self.experiment = load_experiment_meta()
|
||||
|
||||
def _file(self) -> Path:
|
||||
return self.log_dir / f"{_utc_now().strftime('%Y%m%d')}.jsonl"
|
||||
|
||||
def update_inventory(self, inventory: float, now: float | None = None) -> dict:
|
||||
"""更新库存并返回 inventory / inventory_time / inventory_skew。"""
|
||||
now = now or time.time()
|
||||
self._inv = float(inventory)
|
||||
if abs(self._inv) < 1e-12:
|
||||
self._inv_nonzero_since = None
|
||||
inv_time = 0.0
|
||||
else:
|
||||
if self._inv_nonzero_since is None:
|
||||
self._inv_nonzero_since = now
|
||||
inv_time = now - self._inv_nonzero_since
|
||||
skew = self._inv - self._inv_target
|
||||
return {
|
||||
"inventory": self._inv,
|
||||
"inventory_time": inv_time,
|
||||
"inventory_skew": skew,
|
||||
}
|
||||
|
||||
def _attach_common(
|
||||
self,
|
||||
ev: dict[str, Any],
|
||||
inventory: float | None = None,
|
||||
state: dict | None = None,
|
||||
now: float | None = None,
|
||||
) -> dict[str, Any]:
|
||||
now = now or time.time()
|
||||
if inventory is not None:
|
||||
ev.update(self.update_inventory(inventory, now=now))
|
||||
# 始终带 market_state_snapshot(可被 state 覆盖内部字段)
|
||||
mss = empty_market_state_snapshot()
|
||||
if state:
|
||||
for k in mss:
|
||||
if k in state and state[k] is not None:
|
||||
mss[k] = state[k]
|
||||
# 兼容旧扁平字段
|
||||
for k, v in state.items():
|
||||
if k not in mss and k != "market_state_snapshot":
|
||||
ev.setdefault(k, v)
|
||||
ev["market_state_snapshot"] = mss
|
||||
return ev
|
||||
|
||||
def write(self, event: dict[str, Any]) -> None:
|
||||
event.setdefault("ts", _iso())
|
||||
event.setdefault("ts_epoch", time.time())
|
||||
if "market_state_snapshot" not in event:
|
||||
event["market_state_snapshot"] = empty_market_state_snapshot()
|
||||
stamp_event(event, self.experiment)
|
||||
with self._file().open("a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(event, ensure_ascii=False, default=str) + "\n")
|
||||
|
||||
def write_experiment_start(self, extra: dict | None = None) -> None:
|
||||
"""探针启动时写入一次,绑定本轮 Data Collection。"""
|
||||
ev = {
|
||||
"event": "experiment_start",
|
||||
**self.experiment,
|
||||
}
|
||||
if extra:
|
||||
ev.update(extra)
|
||||
self.write(ev)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Snapshot
|
||||
# ------------------------------------------------------------------ #
|
||||
@staticmethod
|
||||
def snapshot_from_orderbook(
|
||||
ob: dict,
|
||||
levels: int = 10,
|
||||
recent_trades: list | None = None,
|
||||
last_mid: float | None = None,
|
||||
liq_proxy_low: float | None = None,
|
||||
liq_proxy_high: float | None = None,
|
||||
) -> MicroSnapshot:
|
||||
bids = (ob.get("bids") or [])[:levels]
|
||||
asks = (ob.get("asks") or [])[:levels]
|
||||
if not bids or not asks:
|
||||
return MicroSnapshot()
|
||||
|
||||
best_bid = float(bids[0][0])
|
||||
best_ask = float(asks[0][0])
|
||||
mid = (best_bid + best_ask) / 2.0
|
||||
spread = best_ask - best_bid
|
||||
|
||||
def depth(levels_side, n):
|
||||
return sum(float(x[1]) for x in levels_side[:n])
|
||||
|
||||
bid_depth_1 = depth(bids, 1)
|
||||
ask_depth_1 = depth(asks, 1)
|
||||
bid_depth_5 = depth(bids, 5)
|
||||
ask_depth_5 = depth(asks, 5)
|
||||
bid_depth = depth(bids, levels)
|
||||
ask_depth = depth(asks, levels)
|
||||
tot = bid_depth + ask_depth
|
||||
obi = ((bid_depth - ask_depth) / tot) if tot > 0 else 0.0
|
||||
|
||||
buy_v = sell_v = 0.0
|
||||
if recent_trades:
|
||||
for t in recent_trades:
|
||||
amt = float(t.get("amount") or t.get("qty") or 0.0)
|
||||
side = (t.get("side") or "").lower()
|
||||
if side in ("buy", "b"):
|
||||
buy_v += amt
|
||||
elif side in ("sell", "s"):
|
||||
sell_v += amt
|
||||
delta = buy_v - sell_v
|
||||
timb_den = buy_v + sell_v
|
||||
trade_imbalance = ((buy_v - sell_v) / timb_den) if timb_den > 0 else 0.0
|
||||
|
||||
de = 0.0
|
||||
if last_mid and mid and abs(delta) > 1e-12:
|
||||
de = ((mid - last_mid) / last_mid) / delta
|
||||
|
||||
liq_dist = 0.0
|
||||
if liq_proxy_low and liq_proxy_high and mid:
|
||||
rng = liq_proxy_high - liq_proxy_low
|
||||
if rng > 0:
|
||||
liq_dist = ((mid - liq_proxy_low) / rng) * 2 - 1
|
||||
|
||||
return MicroSnapshot(
|
||||
best_bid=best_bid,
|
||||
best_ask=best_ask,
|
||||
mid=mid,
|
||||
spread=spread,
|
||||
bid_depth_1=bid_depth_1,
|
||||
ask_depth_1=ask_depth_1,
|
||||
bid_depth_5=bid_depth_5,
|
||||
ask_depth_5=ask_depth_5,
|
||||
bid_depth=bid_depth,
|
||||
ask_depth=ask_depth,
|
||||
obi=obi,
|
||||
delta=delta,
|
||||
trade_imbalance=trade_imbalance,
|
||||
delta_efficiency=de,
|
||||
liquidation_distance=liq_dist,
|
||||
)
|
||||
|
||||
def record_book(
|
||||
self,
|
||||
snap: MicroSnapshot,
|
||||
now: float | None = None,
|
||||
*,
|
||||
emit_mid_tick: bool = True,
|
||||
pair: str | None = None,
|
||||
) -> None:
|
||||
now = now or time.time()
|
||||
self._book_hist.append((now, snap))
|
||||
# trim old
|
||||
cutoff = now - self.book_history_sec
|
||||
while self._book_hist and self._book_hist[0][0] < cutoff:
|
||||
self._book_hist.popleft()
|
||||
# mid 时间序列:供 Fill vs Random Benchmark(研究保护栏)
|
||||
if (
|
||||
emit_mid_tick
|
||||
and snap.mid > 0
|
||||
and (now - self._last_mid_tick_ts) >= self.mid_tick_every_sec
|
||||
):
|
||||
self._last_mid_tick_ts = now
|
||||
self.write(
|
||||
{
|
||||
"event": "mid_tick",
|
||||
"pair": pair,
|
||||
"mid": snap.mid,
|
||||
"best_bid": snap.best_bid,
|
||||
"best_ask": snap.best_ask,
|
||||
"spread": snap.spread,
|
||||
"ts_epoch": now,
|
||||
}
|
||||
)
|
||||
|
||||
def assign_event_cluster(
|
||||
self,
|
||||
side: str,
|
||||
mid: float,
|
||||
now: float | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
将连续同侧成交归入同一流动性事件(event_cluster_id)。
|
||||
|
||||
规则(研究保护栏,非策略):
|
||||
- 同 side
|
||||
- 与上一笔间隔 < cluster_gap_sec
|
||||
→ 同一 cluster;否则新开 cluster。
|
||||
|
||||
统计时应用 cluster 加权,避免「暴跌连续 50 笔 Bid」当成 50 个独立样本。
|
||||
"""
|
||||
now = now or time.time()
|
||||
new_cluster = (
|
||||
self._cluster_id is None
|
||||
or self._cluster_side != side
|
||||
or (now - self._cluster_last_ts) > self.cluster_gap_sec
|
||||
)
|
||||
if new_cluster:
|
||||
self._cluster_id = uuid.uuid4().hex[:12]
|
||||
self._cluster_side = side
|
||||
self._cluster_start_mid = mid if mid > 0 else None
|
||||
self._cluster_n = 0
|
||||
self._cluster_n += 1
|
||||
self._cluster_last_ts = now
|
||||
mid_move = None
|
||||
if self._cluster_start_mid and mid > 0:
|
||||
mid_move = (mid - self._cluster_start_mid) / self._cluster_start_mid
|
||||
return {
|
||||
"event_cluster_id": self._cluster_id,
|
||||
"cluster_fill_index": self._cluster_n,
|
||||
"cluster_mid_move_from_start": mid_move,
|
||||
}
|
||||
|
||||
def book_at(self, target_ts: float) -> Optional[MicroSnapshot]:
|
||||
"""取最接近 target_ts 的历史盘口(用于成交前5s)。"""
|
||||
if not self._book_hist:
|
||||
return None
|
||||
best = min(self._book_hist, key=lambda x: abs(x[0] - target_ts))
|
||||
return best[1]
|
||||
|
||||
def build_fill_context(self, side: str, now: float | None = None) -> dict:
|
||||
"""
|
||||
成交主动性上下文:区分「砸盘后吸收」vs「下跌接刀」。
|
||||
不接 Market Pulse,仅用本地 book history + trade imbalance。
|
||||
"""
|
||||
now = now or time.time()
|
||||
cur = self.book_at(now)
|
||||
past = self.book_at(now - 5.0)
|
||||
fill_type = "bid" if side == "long" else "ask"
|
||||
ctx: dict[str, Any] = {
|
||||
"fill_type": fill_type,
|
||||
"market_event_before_fill": "unknown",
|
||||
"trade_imbalance_5s": None,
|
||||
"price_velocity_5s": None,
|
||||
}
|
||||
if not cur or not past or past.mid <= 0:
|
||||
return {"fill_context": ctx}
|
||||
|
||||
vel = (cur.mid - past.mid) / past.mid
|
||||
# 用当前与 5s 前 imbalance 的平均作代理
|
||||
timb = (cur.trade_imbalance + past.trade_imbalance) / 2.0
|
||||
ctx["trade_imbalance_5s"] = timb
|
||||
ctx["price_velocity_5s"] = vel
|
||||
|
||||
if fill_type == "bid":
|
||||
# 卖压后吸收:价格下行/企稳 + 卖向 imbalance,但盘口未继续恶化太狠
|
||||
if timb < -0.2 and vel < 0:
|
||||
if abs(vel) < 0.0003:
|
||||
ctx["market_event_before_fill"] = "sell_pressure_absorbing"
|
||||
else:
|
||||
ctx["market_event_before_fill"] = "sell_pressure_falling"
|
||||
elif vel < -0.0005:
|
||||
ctx["market_event_before_fill"] = "momentum_down_catching_knife"
|
||||
elif timb > 0.15:
|
||||
ctx["market_event_before_fill"] = "buy_support"
|
||||
else:
|
||||
ctx["market_event_before_fill"] = "neutral"
|
||||
else:
|
||||
if timb > 0.2 and vel > 0:
|
||||
if abs(vel) < 0.0003:
|
||||
ctx["market_event_before_fill"] = "buy_pressure_absorbing"
|
||||
else:
|
||||
ctx["market_event_before_fill"] = "buy_pressure_rising"
|
||||
elif vel > 0.0005:
|
||||
ctx["market_event_before_fill"] = "momentum_up_chasing"
|
||||
elif timb < -0.15:
|
||||
ctx["market_event_before_fill"] = "sell_resistance"
|
||||
else:
|
||||
ctx["market_event_before_fill"] = "neutral"
|
||||
return {"fill_context": ctx}
|
||||
|
||||
def book_deterioration(self, side: str, now: float | None = None, lookback: float = 5.0) -> dict:
|
||||
"""
|
||||
成交前 lookback 秒盘口是否恶化。
|
||||
long: bid_depth 下降 / ask_depth 上升 / mid 下跌 → 恶化
|
||||
"""
|
||||
now = now or time.time()
|
||||
cur = self.book_at(now)
|
||||
past = self.book_at(now - lookback)
|
||||
if not cur or not past or past.mid <= 0:
|
||||
return {"book_ok": False}
|
||||
mid_chg = (cur.mid - past.mid) / past.mid
|
||||
bid5_chg = (cur.bid_depth_5 - past.bid_depth_5) / past.bid_depth_5 if past.bid_depth_5 else 0.0
|
||||
ask5_chg = (cur.ask_depth_5 - past.ask_depth_5) / past.ask_depth_5 if past.ask_depth_5 else 0.0
|
||||
obi_chg = cur.obi - past.obi
|
||||
if side == "long":
|
||||
deteriorated = (mid_chg < -0.00005) or (bid5_chg < -0.15) or (obi_chg < -0.1)
|
||||
else:
|
||||
deteriorated = (mid_chg > 0.00005) or (ask5_chg < -0.15) or (obi_chg > 0.1)
|
||||
return {
|
||||
"book_ok": True,
|
||||
"pre_5s_mid_chg": mid_chg,
|
||||
"pre_5s_bid_depth_5_chg": bid5_chg,
|
||||
"pre_5s_ask_depth_5_chg": ask5_chg,
|
||||
"pre_5s_obi_chg": obi_chg,
|
||||
"pre_5s_deteriorated": bool(deteriorated),
|
||||
"pre_5s_bid_depth_1": past.bid_depth_1,
|
||||
"pre_5s_ask_depth_1": past.ask_depth_1,
|
||||
"pre_5s_bid_depth_5": past.bid_depth_5,
|
||||
"pre_5s_ask_depth_5": past.ask_depth_5,
|
||||
"pre_5s_obi": past.obi,
|
||||
"pre_5s_spread": past.spread,
|
||||
"pre_5s_trade_imbalance": past.trade_imbalance,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Quote lifecycle
|
||||
# ------------------------------------------------------------------ #
|
||||
def create_quote(
|
||||
self,
|
||||
pair: str,
|
||||
side: str,
|
||||
quote_price: float,
|
||||
inventory: float,
|
||||
snap: MicroSnapshot,
|
||||
reason: str = "",
|
||||
trade_id: Optional[int] = None,
|
||||
state: dict | None = None,
|
||||
extra: dict | None = None,
|
||||
) -> str:
|
||||
qid = uuid.uuid4().hex[:16]
|
||||
now = time.time()
|
||||
q = ActiveQuote(
|
||||
quote_id=qid,
|
||||
pair=pair,
|
||||
side=side,
|
||||
quote_price=quote_price,
|
||||
created_ts=now,
|
||||
reason=reason,
|
||||
trade_id=trade_id,
|
||||
status="open",
|
||||
)
|
||||
self._quotes[qid] = q
|
||||
if trade_id is not None:
|
||||
self._quotes_by_trade[trade_id] = qid
|
||||
|
||||
ev = {
|
||||
"event": "quote_created",
|
||||
"quote_id": qid,
|
||||
"pair": pair,
|
||||
"side": side,
|
||||
"quote_price": quote_price,
|
||||
"quote_created_time": _iso(now),
|
||||
"quote_created_epoch": now,
|
||||
"reason": reason,
|
||||
"trade_id": trade_id,
|
||||
"status": "open",
|
||||
"filled": False,
|
||||
}
|
||||
ev.update(snap.to_book_fields())
|
||||
self._attach_common(ev, inventory=inventory, state=state, now=now)
|
||||
if extra:
|
||||
ev.update(extra)
|
||||
self.write(ev)
|
||||
return qid
|
||||
|
||||
def cancel_quote(
|
||||
self,
|
||||
quote_id: str | None = None,
|
||||
trade_id: Optional[int] = None,
|
||||
reason: str = "timeout",
|
||||
snap: MicroSnapshot | None = None,
|
||||
) -> None:
|
||||
q = None
|
||||
if quote_id and quote_id in self._quotes:
|
||||
q = self._quotes[quote_id]
|
||||
elif trade_id is not None and trade_id in self._quotes_by_trade:
|
||||
q = self._quotes.get(self._quotes_by_trade[trade_id])
|
||||
if q is None or q.status != "open":
|
||||
return
|
||||
|
||||
now = time.time()
|
||||
q.status = "canceled"
|
||||
ev = {
|
||||
"event": "quote_canceled",
|
||||
"quote_id": q.quote_id,
|
||||
"pair": q.pair,
|
||||
"side": q.side,
|
||||
"quote_price": q.quote_price,
|
||||
"quote_created_time": _iso(q.created_ts),
|
||||
"quote_cancel_time": _iso(now),
|
||||
"quote_cancel_epoch": now,
|
||||
"time_alive_sec": now - q.created_ts,
|
||||
"cancel_reason": reason,
|
||||
"filled": False,
|
||||
"status": "canceled",
|
||||
"trade_id": q.trade_id,
|
||||
}
|
||||
if snap:
|
||||
ev.update(snap.to_book_fields())
|
||||
self.write(ev)
|
||||
|
||||
def bind_trade(self, quote_id: str, trade_id: int) -> None:
|
||||
if quote_id in self._quotes:
|
||||
self._quotes[quote_id].trade_id = trade_id
|
||||
self._quotes_by_trade[trade_id] = quote_id
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Fill + path
|
||||
# ------------------------------------------------------------------ #
|
||||
def log_fill(
|
||||
self,
|
||||
pair: str,
|
||||
side: str,
|
||||
fill_price: float,
|
||||
amount: float,
|
||||
inventory: float,
|
||||
snap: MicroSnapshot | None,
|
||||
order_type: str = "limit",
|
||||
quote_id: str | None = None,
|
||||
trade_id: Optional[int] = None,
|
||||
fill_reason: str = "maker_hit",
|
||||
state: dict | None = None,
|
||||
extra: dict | None = None,
|
||||
quote_terminal: bool = True,
|
||||
) -> str:
|
||||
"""Record a fill. Always writes even if snap is None (book unavailable).
|
||||
|
||||
quote_terminal=False keeps quote open for partial fills so later slices
|
||||
retain quote_id linkage until the order closes.
|
||||
"""
|
||||
now = time.time()
|
||||
fill_id = uuid.uuid4().hex[:16]
|
||||
snap = snap or MicroSnapshot()
|
||||
|
||||
# resolve quote lifecycle
|
||||
q: Optional[ActiveQuote] = None
|
||||
if quote_id and quote_id in self._quotes:
|
||||
q = self._quotes[quote_id]
|
||||
elif trade_id is not None and trade_id in self._quotes_by_trade:
|
||||
q = self._quotes.get(self._quotes_by_trade[trade_id])
|
||||
|
||||
time_to_fill = None
|
||||
quote_created_time = None
|
||||
quote_price = fill_price
|
||||
if q is not None:
|
||||
if quote_terminal:
|
||||
q.status = "filled"
|
||||
time_to_fill = now - q.created_ts
|
||||
quote_created_time = _iso(q.created_ts)
|
||||
quote_price = q.quote_price
|
||||
quote_id = q.quote_id
|
||||
|
||||
det = self.book_deterioration(side, now=now, lookback=5.0)
|
||||
fctx = self.build_fill_context(side, now=now)
|
||||
mid_for_cluster = snap.mid if snap.mid > 0 else fill_price
|
||||
cluster = self.assign_event_cluster(side, mid_for_cluster, now=now)
|
||||
|
||||
ev = {
|
||||
"event": "fill",
|
||||
"fill_id": fill_id,
|
||||
"quote_id": quote_id,
|
||||
"pair": pair,
|
||||
"side": side,
|
||||
"fill_price": fill_price,
|
||||
"quote_price": quote_price,
|
||||
"amount": amount,
|
||||
"order_type": order_type,
|
||||
"fill_reason": fill_reason,
|
||||
"quote_created_time": quote_created_time,
|
||||
"quote_fill_time": _iso(now),
|
||||
"time_to_fill": time_to_fill,
|
||||
"trade_id": trade_id,
|
||||
"filled": True,
|
||||
"quote_terminal": quote_terminal,
|
||||
"book_available": bool(snap.mid > 0),
|
||||
}
|
||||
ev.update(snap.to_book_fields())
|
||||
ev.update(det)
|
||||
ev.update(fctx)
|
||||
ev.update(cluster)
|
||||
# Effective spread capture proxy: 相对 mid 的被动成交优势
|
||||
if snap.mid > 0:
|
||||
if side == "long":
|
||||
ev["spread_capture_pct"] = (snap.mid - fill_price) / snap.mid
|
||||
else:
|
||||
ev["spread_capture_pct"] = (fill_price - snap.mid) / snap.mid
|
||||
self._attach_common(ev, inventory=inventory, state=state, now=now)
|
||||
if extra:
|
||||
ev.update(extra)
|
||||
self.write(ev)
|
||||
|
||||
# also emit quote_filled lifecycle event (only when order fully done)
|
||||
if q is not None and quote_terminal:
|
||||
self.write(
|
||||
{
|
||||
"event": "quote_filled",
|
||||
"quote_id": q.quote_id,
|
||||
"fill_id": fill_id,
|
||||
"pair": pair,
|
||||
"side": q.side,
|
||||
"quote_price": q.quote_price,
|
||||
"quote_created_time": _iso(q.created_ts),
|
||||
"quote_fill_time": _iso(now),
|
||||
"time_to_fill": time_to_fill,
|
||||
"fill_reason": fill_reason,
|
||||
"filled": True,
|
||||
"status": "filled",
|
||||
"trade_id": trade_id,
|
||||
**snap.to_book_fields(),
|
||||
**det,
|
||||
}
|
||||
)
|
||||
|
||||
self._pending[fill_id] = PendingFillPath(
|
||||
fill_id=fill_id,
|
||||
pair=pair,
|
||||
side=side,
|
||||
fill_price=fill_price,
|
||||
fill_ts=now,
|
||||
quote_id=quote_id,
|
||||
)
|
||||
return fill_id
|
||||
|
||||
def attach_exit_reason(self, fill_id: str, exit_reason: str) -> None:
|
||||
if fill_id in self._pending:
|
||||
self._pending[fill_id].exit_reason = exit_reason
|
||||
# also write lightweight annotation
|
||||
self.write(
|
||||
{
|
||||
"event": "fill_exit",
|
||||
"fill_id": fill_id,
|
||||
"exit_reason": exit_reason,
|
||||
}
|
||||
)
|
||||
|
||||
def update_paths(self, pair: str, last_price: float, now: float | None = None) -> None:
|
||||
now = now or time.time()
|
||||
finished = []
|
||||
for fid, p in self._pending.items():
|
||||
if p.pair != pair or p.done:
|
||||
continue
|
||||
p.min_price = min(p.min_price, last_price)
|
||||
p.max_price = max(p.max_price, last_price)
|
||||
mae, mfe = p.signed_excursions()
|
||||
age = now - p.fill_ts
|
||||
|
||||
def mark(horizon_attr_price, horizon_mae, horizon_mfe, sec, price_val):
|
||||
if getattr(p, horizon_attr_price) is None and age >= sec:
|
||||
setattr(p, horizon_attr_price, price_val)
|
||||
setattr(p, horizon_mae, mae)
|
||||
setattr(p, horizon_mfe, mfe)
|
||||
|
||||
mark("after_1s_price", "mae_1s", "mfe_1s", 1, last_price)
|
||||
mark("after_5s_price", "mae_5s", "mfe_5s", 5, last_price)
|
||||
mark("after_10s_price", "mae_10s", "mfe_10s", 10, last_price)
|
||||
mark("after_30s_price", "mae_30s", "mfe_30s", 30, last_price)
|
||||
mark("after_1m_price", "mae_1m", "mfe_1m", 60, last_price)
|
||||
|
||||
if p.after_5m_price is None and age >= 300:
|
||||
p.after_5m_price = last_price
|
||||
p.mae_5m = mae
|
||||
p.mfe_5m = mfe
|
||||
p.done = True
|
||||
# Price MAE absolute
|
||||
if p.side == "long":
|
||||
price_mae = p.min_price - p.fill_price
|
||||
price_mfe = p.max_price - p.fill_price
|
||||
else:
|
||||
price_mae = p.fill_price - p.max_price
|
||||
price_mfe = p.fill_price - p.min_price
|
||||
|
||||
fav_30 = p.fav_ret_at(p.after_30s_price) or 0.0
|
||||
fav_1 = p.fav_ret_at(p.after_1s_price)
|
||||
fav_5 = p.fav_ret_at(p.after_5s_price)
|
||||
fav_10 = p.fav_ret_at(p.after_10s_price)
|
||||
fav_60 = p.fav_ret_at(p.after_1m_price)
|
||||
fav_300 = p.fav_ret_at(p.after_5m_price) or 0.0
|
||||
|
||||
vol_proxy = abs(p.max_price - p.min_price) / p.fill_price if p.fill_price else 0.0
|
||||
toxicity_score = max(0.0, -fav_30) / max(vol_proxy, 1e-8)
|
||||
mfe_gt_mae_30 = (p.mfe_30s or 0.0) > abs(p.mae_30s or 0.0)
|
||||
path_type = classify_path_type(p)
|
||||
|
||||
# 路径点(供形态分析 / 复现)
|
||||
price_path = {
|
||||
"t0": p.fill_price,
|
||||
"t1s": p.after_1s_price,
|
||||
"t5s": p.after_5s_price,
|
||||
"t10s": p.after_10s_price,
|
||||
"t30s": p.after_30s_price,
|
||||
"t60s": p.after_1m_price,
|
||||
"t300s": p.after_5m_price,
|
||||
}
|
||||
ret_path = {
|
||||
"t1s": fav_1,
|
||||
"t5s": fav_5,
|
||||
"t10s": fav_10,
|
||||
"t30s": fav_30,
|
||||
"t60s": fav_60,
|
||||
"t300s": fav_300,
|
||||
}
|
||||
|
||||
self.write(
|
||||
{
|
||||
"event": "fill_path",
|
||||
"fill_id": p.fill_id,
|
||||
"quote_id": p.quote_id,
|
||||
"pair": p.pair,
|
||||
"side": p.side,
|
||||
"fill_price": p.fill_price,
|
||||
"exit_reason": p.exit_reason,
|
||||
"after_1s_price": p.after_1s_price,
|
||||
"after_5s_price": p.after_5s_price,
|
||||
"after_10s_price": p.after_10s_price,
|
||||
"after_30s_price": p.after_30s_price,
|
||||
"after_1m_price": p.after_1m_price,
|
||||
"after_5m_price": p.after_5m_price,
|
||||
"price_path": price_path,
|
||||
"ret_path": ret_path,
|
||||
"path_type": path_type,
|
||||
"min_price": p.min_price,
|
||||
"max_price": p.max_price,
|
||||
"mae_1s": p.mae_1s,
|
||||
"mae_5s": p.mae_5s,
|
||||
"mae_10s": p.mae_10s,
|
||||
"mae_30s": p.mae_30s,
|
||||
"mae_1m": p.mae_1m,
|
||||
"mae_5m": p.mae_5m,
|
||||
"mfe_1s": p.mfe_1s,
|
||||
"mfe_5s": p.mfe_5s,
|
||||
"mfe_10s": p.mfe_10s,
|
||||
"mfe_30s": p.mfe_30s,
|
||||
"mfe_1m": p.mfe_1m,
|
||||
"mfe_5m": p.mfe_5m,
|
||||
"price_mae": price_mae,
|
||||
"price_mfe": price_mfe,
|
||||
"price_mae_pct": mae,
|
||||
"price_mfe_pct": mfe,
|
||||
"fav_ret_30s": fav_30,
|
||||
"vol_proxy_5m": vol_proxy,
|
||||
"toxicity_score": toxicity_score,
|
||||
"mfe_gt_mae_30s": mfe_gt_mae_30,
|
||||
}
|
||||
)
|
||||
finished.append(fid)
|
||||
|
||||
for fid in finished:
|
||||
self._pending.pop(fid, None)
|
||||
|
||||
@property
|
||||
def pending_count(self) -> int:
|
||||
return len(self._pending)
|
||||
|
||||
# 兼容旧 API
|
||||
def log_quote(self, *args, **kwargs):
|
||||
"""Deprecated wrapper → create_quote for live quotes; heartbeat uses book only."""
|
||||
return self.create_quote(*args, **kwargs)
|
||||
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Launch Event-State Observability Probe (MM_EDGE_EXP_002)
|
||||
|
||||
Data collection only — NO trading, NO strategy, NO Stage 3 unlock.
|
||||
|
||||
Environment:
|
||||
EXPERIMENT_ID=MM_EDGE_EXP_002
|
||||
PROBE_VERSION=event_state_v0.1
|
||||
ENABLE_TRADING=false (hard-enforced; any true value is ignored)
|
||||
|
||||
Usage:
|
||||
cd nautilus_mm
|
||||
source .venv/bin/activate
|
||||
export PYTHONPATH=src
|
||||
python -m nautilus_mm.run_event_state
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
_ROOT = Path(__file__).resolve().parents[2]
|
||||
_SRC = _ROOT / "src"
|
||||
if str(_SRC) not in sys.path:
|
||||
sys.path.insert(0, str(_SRC))
|
||||
|
||||
load_dotenv(_ROOT / ".env")
|
||||
|
||||
from nautilus_trader.adapters.binance import BINANCE
|
||||
from nautilus_trader.adapters.binance import BinanceAccountType
|
||||
from nautilus_trader.adapters.binance import BinanceDataClientConfig
|
||||
from nautilus_trader.adapters.binance import BinanceExecClientConfig
|
||||
from nautilus_trader.adapters.binance import BinanceInstrumentProviderConfig
|
||||
from nautilus_trader.adapters.binance import BinanceLiveDataClientFactory
|
||||
from nautilus_trader.adapters.binance import BinanceLiveExecClientFactory
|
||||
from nautilus_trader.adapters.binance.common.enums import BinanceEnvironment
|
||||
from nautilus_trader.config import LiveDataEngineConfig
|
||||
from nautilus_trader.config import LiveExecEngineConfig
|
||||
from nautilus_trader.config import LoggingConfig
|
||||
from nautilus_trader.config import TradingNodeConfig
|
||||
from nautilus_trader.live.node import TradingNode
|
||||
from nautilus_trader.model.identifiers import ClientId
|
||||
from nautilus_trader.model.identifiers import InstrumentId
|
||||
from nautilus_trader.model.identifiers import TraderId
|
||||
|
||||
from nautilus_mm.experiment import load_experiment_meta
|
||||
from nautilus_mm.strategies.event_state_probe import EventStateProbe
|
||||
from nautilus_mm.strategies.event_state_probe import EventStateProbeConfig
|
||||
|
||||
|
||||
def _env_bool(name: str, default: bool = False) -> bool:
|
||||
v = os.getenv(name)
|
||||
if v is None:
|
||||
return default
|
||||
return v.strip().lower() in ("1", "true", "yes", "y")
|
||||
|
||||
|
||||
def _resolve_environment() -> BinanceEnvironment:
|
||||
raw = os.getenv("BINANCE_ENVIRONMENT", "TESTNET")
|
||||
env_name = raw.strip().upper()
|
||||
if env_name not in ("TESTNET", "LIVE"):
|
||||
print(f"ERROR: BINANCE_ENVIRONMENT must be TESTNET or LIVE, got {raw!r}")
|
||||
sys.exit(1)
|
||||
if env_name == "LIVE" and not _env_bool("I_UNDERSTAND_LIVE", False):
|
||||
print("ERROR: LIVE blocked for EXP_002 unless I_UNDERSTAND_LIVE=yes")
|
||||
sys.exit(1)
|
||||
return BinanceEnvironment.LIVE if env_name == "LIVE" else BinanceEnvironment.TESTNET
|
||||
|
||||
|
||||
def main() -> None:
|
||||
# Layer 2: runner hard-forces trading off even if .env / systemd is wrong
|
||||
if _env_bool("ENABLE_TRADING", False):
|
||||
print(
|
||||
"WARNING: ENABLE_TRADING=true ignored — MM_EDGE_EXP_002 is observability-only"
|
||||
)
|
||||
os.environ["ENABLE_TRADING"] = "false"
|
||||
os.environ["EXPERIMENT_ID"] = "MM_EDGE_EXP_002"
|
||||
os.environ.setdefault("PROBE_VERSION", "event_state_v0.1")
|
||||
|
||||
exp_id = os.getenv("EXPERIMENT_ID", "MM_EDGE_EXP_002")
|
||||
if exp_id != "MM_EDGE_EXP_002":
|
||||
print(
|
||||
f"WARNING: EXPERIMENT_ID={exp_id!r} — expected MM_EDGE_EXP_002 for this runner"
|
||||
)
|
||||
|
||||
api_key = os.getenv("BINANCE_API_KEY", "")
|
||||
api_secret = os.getenv("BINANCE_API_SECRET", "")
|
||||
environment = _resolve_environment()
|
||||
symbol = os.getenv("SYMBOL", "BTCUSDT-PERP")
|
||||
instrument_id = InstrumentId.from_str(f"{symbol}.{BINANCE}")
|
||||
log_dir = os.getenv("EVENT_STATE_LOG_DIR", str(_ROOT / "logs" / "event_state"))
|
||||
|
||||
if not api_key or not api_secret:
|
||||
print("ERROR: set BINANCE_API_KEY / BINANCE_API_SECRET in nautilus_mm/.env")
|
||||
sys.exit(1)
|
||||
|
||||
config_node = TradingNodeConfig(
|
||||
trader_id=TraderId("EVENT-STATE-002"),
|
||||
logging=LoggingConfig(log_level="INFO", log_colors=True, use_pyo3=True),
|
||||
data_engine=LiveDataEngineConfig(external_clients=[ClientId(BINANCE)]),
|
||||
exec_engine=LiveExecEngineConfig(
|
||||
reconciliation=False,
|
||||
open_check_interval_secs=0.0,
|
||||
graceful_shutdown_on_exception=True,
|
||||
),
|
||||
data_clients={
|
||||
BINANCE: BinanceDataClientConfig(
|
||||
api_key=api_key,
|
||||
api_secret=api_secret,
|
||||
account_type=BinanceAccountType.USDT_FUTURES,
|
||||
environment=environment,
|
||||
instrument_provider=BinanceInstrumentProviderConfig(
|
||||
load_ids=frozenset([instrument_id]),
|
||||
),
|
||||
),
|
||||
},
|
||||
exec_clients={
|
||||
BINANCE: BinanceExecClientConfig(
|
||||
api_key=api_key,
|
||||
api_secret=api_secret,
|
||||
account_type=BinanceAccountType.USDT_FUTURES,
|
||||
environment=environment,
|
||||
instrument_provider=BinanceInstrumentProviderConfig(
|
||||
load_ids=frozenset([instrument_id]),
|
||||
),
|
||||
max_retries=3,
|
||||
),
|
||||
},
|
||||
timeout_connection=30.0,
|
||||
timeout_reconciliation=10.0,
|
||||
timeout_portfolio=10.0,
|
||||
timeout_disconnection=10.0,
|
||||
timeout_post_stop=5.0,
|
||||
)
|
||||
|
||||
node = TradingNode(config=config_node)
|
||||
strat_config = EventStateProbeConfig(
|
||||
instrument_id=instrument_id,
|
||||
book_depth=int(os.getenv("BOOK_DEPTH", "10")),
|
||||
log_dir=log_dir,
|
||||
prefill_window_sec=float(os.getenv("PREFILL_WINDOW_SEC", "5.0")),
|
||||
prefill_margin_sec=float(os.getenv("PREFILL_MARGIN_SEC", "0.25")),
|
||||
large_trade_qty=float(os.getenv("LARGE_TRADE_QTY", "0.1")),
|
||||
log_every_book_delta=_env_bool("LOG_EVERY_BOOK_DELTA", True),
|
||||
)
|
||||
node.trader.add_strategy(EventStateProbe(config=strat_config))
|
||||
node.add_data_client_factory(BINANCE, BinanceLiveDataClientFactory)
|
||||
node.add_exec_client_factory(BINANCE, BinanceLiveExecClientFactory)
|
||||
node.build()
|
||||
|
||||
exp = load_experiment_meta()
|
||||
print(
|
||||
f"[event_state] Experiment={exp['experiment_id']} {exp['probe_version']} | "
|
||||
f"type=Event-State Observability | trading=NO | {symbol} env={environment} | "
|
||||
f"run={os.getenv('LEDGER_RUN_ID', 'EXP-002-RUN-UNSET')} | log={log_dir}"
|
||||
)
|
||||
try:
|
||||
node.run()
|
||||
finally:
|
||||
node.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
启动 Nautilus TradingNode + MakerEdgeProbe(Binance USDT-M)
|
||||
|
||||
环境变量(或 .env):
|
||||
BINANCE_API_KEY
|
||||
BINANCE_API_SECRET
|
||||
BINANCE_ENVIRONMENT=TESTNET|LIVE (仅允许这两个值;默认 TESTNET)
|
||||
I_UNDERSTAND_LIVE=yes (LIVE 必填)
|
||||
ENABLE_TRADING=false (默认关闭;显式 true 才挂单)
|
||||
HTTP_PROXY / HTTPS_PROXY (可选)
|
||||
|
||||
用法:
|
||||
cd nautilus_mm
|
||||
source .venv/bin/activate
|
||||
export PYTHONPATH=src
|
||||
python -m nautilus_mm.run_live
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# ensure src on path when run as script
|
||||
_ROOT = Path(__file__).resolve().parents[2]
|
||||
_SRC = _ROOT / "src"
|
||||
if str(_SRC) not in sys.path:
|
||||
sys.path.insert(0, str(_SRC))
|
||||
|
||||
load_dotenv(_ROOT / ".env")
|
||||
|
||||
from nautilus_trader.adapters.binance import BINANCE
|
||||
from nautilus_trader.adapters.binance import BinanceAccountType
|
||||
from nautilus_trader.adapters.binance import BinanceDataClientConfig
|
||||
from nautilus_trader.adapters.binance import BinanceExecClientConfig
|
||||
from nautilus_trader.adapters.binance import BinanceInstrumentProviderConfig
|
||||
from nautilus_trader.adapters.binance import BinanceLiveDataClientFactory
|
||||
from nautilus_trader.adapters.binance import BinanceLiveExecClientFactory
|
||||
from nautilus_trader.adapters.binance.common.enums import BinanceEnvironment
|
||||
from nautilus_trader.config import LiveDataEngineConfig
|
||||
from nautilus_trader.config import LiveExecEngineConfig
|
||||
from nautilus_trader.config import LoggingConfig
|
||||
from nautilus_trader.config import TradingNodeConfig
|
||||
from nautilus_trader.live.node import TradingNode
|
||||
from nautilus_trader.model.identifiers import ClientId
|
||||
from nautilus_trader.model.identifiers import InstrumentId
|
||||
from nautilus_trader.model.identifiers import TraderId
|
||||
|
||||
from nautilus_mm.experiment import load_experiment_meta
|
||||
from nautilus_mm.strategies.maker_edge_probe import MakerEdgeProbe
|
||||
from nautilus_mm.strategies.maker_edge_probe import MakerEdgeProbeConfig
|
||||
|
||||
|
||||
def _env_bool(name: str, default: bool = False) -> bool:
|
||||
v = os.getenv(name)
|
||||
if v is None:
|
||||
return default
|
||||
return v.strip().lower() in ("1", "true", "yes", "y")
|
||||
|
||||
|
||||
def _resolve_environment() -> BinanceEnvironment:
|
||||
raw = os.getenv("BINANCE_ENVIRONMENT", "TESTNET")
|
||||
env_name = raw.strip().upper()
|
||||
if env_name not in ("TESTNET", "LIVE"):
|
||||
print(
|
||||
f"ERROR: BINANCE_ENVIRONMENT must be exactly TESTNET or LIVE, got {raw!r}"
|
||||
)
|
||||
sys.exit(1)
|
||||
if env_name == "LIVE":
|
||||
if not _env_bool("I_UNDERSTAND_LIVE", False):
|
||||
print(
|
||||
"ERROR: LIVE trading blocked. Set I_UNDERSTAND_LIVE=yes "
|
||||
"only after you accept real-money risk."
|
||||
)
|
||||
sys.exit(1)
|
||||
return BinanceEnvironment.LIVE
|
||||
return BinanceEnvironment.TESTNET
|
||||
|
||||
|
||||
def main() -> None:
|
||||
api_key = os.getenv("BINANCE_API_KEY", "")
|
||||
api_secret = os.getenv("BINANCE_API_SECRET", "")
|
||||
environment = _resolve_environment()
|
||||
|
||||
symbol = os.getenv("SYMBOL", "BTCUSDT-PERP")
|
||||
instrument_id = InstrumentId.from_str(f"{symbol}.{BINANCE}")
|
||||
order_qty = Decimal(os.getenv("ORDER_QTY", "0.001"))
|
||||
enable_trading = _env_bool("ENABLE_TRADING", False)
|
||||
max_abs_inventory = Decimal(os.getenv("MAX_ABS_INVENTORY", "0.005"))
|
||||
quote_ttl_secs = float(os.getenv("QUOTE_TTL_SECS", "30"))
|
||||
log_dir = os.getenv("MAKER_EDGE_LOG_DIR", str(_ROOT / "logs" / "maker_edge"))
|
||||
|
||||
# 代理:Nautilus/httpx 会读 HTTP(S)_PROXY;这里仅提示
|
||||
proxy = os.getenv("HTTPS_PROXY") or os.getenv("HTTP_PROXY") or ""
|
||||
if proxy:
|
||||
print(f"[nautilus_mm] using proxy: {proxy}")
|
||||
|
||||
if not api_key or not api_secret:
|
||||
print("ERROR: set BINANCE_API_KEY / BINANCE_API_SECRET in nautilus_mm/.env")
|
||||
print("For TESTNET keys: https://testnet.binancefuture.com/")
|
||||
sys.exit(1)
|
||||
|
||||
config_node = TradingNodeConfig(
|
||||
trader_id=TraderId("MAKER-EDGE-001"),
|
||||
logging=LoggingConfig(log_level="INFO", log_colors=True, use_pyo3=True),
|
||||
data_engine=LiveDataEngineConfig(external_clients=[ClientId(BINANCE)]),
|
||||
exec_engine=LiveExecEngineConfig(
|
||||
reconciliation=True,
|
||||
open_check_interval_secs=5.0,
|
||||
graceful_shutdown_on_exception=True,
|
||||
),
|
||||
data_clients={
|
||||
BINANCE: BinanceDataClientConfig(
|
||||
api_key=api_key,
|
||||
api_secret=api_secret,
|
||||
account_type=BinanceAccountType.USDT_FUTURES,
|
||||
environment=environment,
|
||||
instrument_provider=BinanceInstrumentProviderConfig(
|
||||
load_ids=frozenset([instrument_id]),
|
||||
),
|
||||
),
|
||||
},
|
||||
exec_clients={
|
||||
BINANCE: BinanceExecClientConfig(
|
||||
api_key=api_key,
|
||||
api_secret=api_secret,
|
||||
account_type=BinanceAccountType.USDT_FUTURES,
|
||||
environment=environment,
|
||||
instrument_provider=BinanceInstrumentProviderConfig(
|
||||
load_ids=frozenset([instrument_id]),
|
||||
),
|
||||
max_retries=3,
|
||||
),
|
||||
},
|
||||
timeout_connection=30.0,
|
||||
timeout_reconciliation=10.0,
|
||||
timeout_portfolio=10.0,
|
||||
timeout_disconnection=10.0,
|
||||
timeout_post_stop=5.0,
|
||||
)
|
||||
|
||||
node = TradingNode(config=config_node)
|
||||
|
||||
strat_config = MakerEdgeProbeConfig(
|
||||
instrument_id=instrument_id,
|
||||
order_qty=order_qty,
|
||||
book_depth=10,
|
||||
quote_offset_ticks=1,
|
||||
max_quotes=1,
|
||||
quote_ttl_secs=quote_ttl_secs,
|
||||
cooldown_secs=float(os.getenv("COOLDOWN_SECS", "60")),
|
||||
book_sample_secs=2.0,
|
||||
log_dir=log_dir,
|
||||
obi_enter=float(os.getenv("OBI_ENTER", "0.25")),
|
||||
enable_trading=enable_trading,
|
||||
max_abs_inventory=max_abs_inventory,
|
||||
)
|
||||
strategy = MakerEdgeProbe(config=strat_config)
|
||||
node.trader.add_strategy(strategy)
|
||||
|
||||
node.add_data_client_factory(BINANCE, BinanceLiveDataClientFactory)
|
||||
node.add_exec_client_factory(BINANCE, BinanceLiveExecClientFactory)
|
||||
node.build()
|
||||
|
||||
exp = load_experiment_meta()
|
||||
print(
|
||||
f"[nautilus_mm] Experiment={exp['experiment_id']} {exp['probe_version']} "
|
||||
f"quote/fee/exchange=frozen | {symbol} env={environment} "
|
||||
f"trading={enable_trading} ttl={quote_ttl_secs}s max_inv={max_abs_inventory} "
|
||||
f"log={log_dir}"
|
||||
)
|
||||
try:
|
||||
node.run()
|
||||
finally:
|
||||
node.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,9 @@
|
||||
from nautilus_mm.strategies.event_state_probe import EventStateProbe, EventStateProbeConfig
|
||||
from nautilus_mm.strategies.maker_edge_probe import MakerEdgeProbe, MakerEdgeProbeConfig
|
||||
|
||||
__all__ = [
|
||||
"MakerEdgeProbe",
|
||||
"MakerEdgeProbeConfig",
|
||||
"EventStateProbe",
|
||||
"EventStateProbeConfig",
|
||||
]
|
||||
@@ -0,0 +1,187 @@
|
||||
"""
|
||||
Event-State Observability Probe — MM_EDGE_EXP_002
|
||||
|
||||
Type: Data Collection / Observability Experiment
|
||||
Strategy: NONE (no quotes, no orders, no trading)
|
||||
Purpose: Capture immutable pre-fill Event State
|
||||
|
||||
EXP_001 remains FROZEN. This probe never submits orders.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections import deque
|
||||
|
||||
from nautilus_trader.common.enums import LogColor
|
||||
from nautilus_trader.config import PositiveInt, StrategyConfig
|
||||
from nautilus_trader.model.data import OrderBookDeltas, TradeTick
|
||||
from nautilus_trader.model.enums import AggressorSide, BookType
|
||||
from nautilus_trader.model.identifiers import InstrumentId
|
||||
from nautilus_trader.model.instruments import Instrument
|
||||
from nautilus_trader.trading.strategy import Strategy
|
||||
|
||||
from nautilus_mm.book_utils import snapshot_from_nautilus_book
|
||||
from nautilus_mm.event_ledger import ImmutableEventLedger
|
||||
from nautilus_mm.health import ConnectionHealth
|
||||
|
||||
|
||||
class EventStateProbeConfig(StrategyConfig, frozen=True):
|
||||
instrument_id: InstrumentId
|
||||
book_depth: PositiveInt = 10
|
||||
log_dir: str = ""
|
||||
prefill_window_sec: float = 5.0
|
||||
prefill_margin_sec: float = 0.25
|
||||
large_trade_qty: float = 0.1
|
||||
# Log every book delta batch (raw). Do not downsample.
|
||||
log_every_book_delta: bool = True
|
||||
|
||||
|
||||
class EventStateProbe(Strategy):
|
||||
"""Read-only market observability — immutable event ledger only."""
|
||||
|
||||
def __init__(self, config: EventStateProbeConfig) -> None:
|
||||
super().__init__(config)
|
||||
self.instrument: Instrument | None = None
|
||||
self._ledger = ImmutableEventLedger(
|
||||
log_dir=config.log_dir or None,
|
||||
prefill_window_sec=float(config.prefill_window_sec),
|
||||
prefill_margin_sec=float(config.prefill_margin_sec),
|
||||
large_trade_qty=float(config.large_trade_qty),
|
||||
book_levels=int(config.book_depth),
|
||||
)
|
||||
self._health = ConnectionHealth(window_sec=60.0, report_every_sec=30.0)
|
||||
self._last_mid: float | None = None
|
||||
self._recent_buys: deque[tuple[float, float]] = deque(maxlen=500)
|
||||
self._recent_sells: deque[tuple[float, float]] = deque(maxlen=500)
|
||||
|
||||
def on_start(self) -> None:
|
||||
self.instrument = self.cache.instrument(self.config.instrument_id)
|
||||
if self.instrument is None:
|
||||
self.log.error(f"Instrument not found: {self.config.instrument_id}")
|
||||
self.stop()
|
||||
return
|
||||
|
||||
self.subscribe_order_book_deltas(
|
||||
instrument_id=self.config.instrument_id,
|
||||
book_type=BookType.L2_MBP,
|
||||
depth=int(self.config.book_depth),
|
||||
)
|
||||
self.subscribe_trade_ticks(self.config.instrument_id)
|
||||
|
||||
exp = self._ledger.experiment
|
||||
ident = self._ledger.run_identity
|
||||
self._ledger.write_experiment_start(
|
||||
extra={
|
||||
"instrument_id": str(self.config.instrument_id),
|
||||
"log_dir": str(self._ledger.log_dir),
|
||||
"log_every_book_delta": bool(self.config.log_every_book_delta),
|
||||
"depends_on": "MM_EDGE_EXP_001 / v0.1 FROZEN",
|
||||
}
|
||||
)
|
||||
self.log.info(
|
||||
f"EXP_002 Event-State Observability | {exp['experiment_id']} | "
|
||||
f"{exp['probe_version']} | run={ident['run_id']} session={ident['session_id']} | "
|
||||
f"trading=NO | log={self._ledger.log_dir}",
|
||||
LogColor.GREEN,
|
||||
)
|
||||
|
||||
def submit_order(self, *args, **kwargs): # noqa: ANN002
|
||||
raise RuntimeError(
|
||||
"MM_EDGE_EXP_002 forbids submit_order — observability probe, trading=NO"
|
||||
)
|
||||
|
||||
def submit_order_list(self, *args, **kwargs): # noqa: ANN002
|
||||
raise RuntimeError(
|
||||
"MM_EDGE_EXP_002 forbids submit_order_list — observability probe, trading=NO"
|
||||
)
|
||||
|
||||
def on_stop(self) -> None:
|
||||
try:
|
||||
self._ledger.write_experiment_stop()
|
||||
except Exception as exc:
|
||||
self.log.warning(f"experiment_stop write failed: {exc}")
|
||||
self.log.info("EventStateProbe stopped (no orders were submitted)")
|
||||
|
||||
def _trade_qty_window(self, window_sec: float = 20.0) -> tuple[float, float]:
|
||||
now = time.time()
|
||||
buy = sum(q for t, q in self._recent_buys if now - t <= window_sec)
|
||||
sell = sum(q for t, q in self._recent_sells if now - t <= window_sec)
|
||||
return buy, sell
|
||||
|
||||
def _current_snap(self):
|
||||
book = self.cache.order_book(self.config.instrument_id)
|
||||
if book is None:
|
||||
return None
|
||||
buy, sell = self._trade_qty_window()
|
||||
snap = snapshot_from_nautilus_book(
|
||||
book,
|
||||
levels=int(self.config.book_depth),
|
||||
recent_buy_qty=buy,
|
||||
recent_sell_qty=sell,
|
||||
last_mid=self._last_mid,
|
||||
)
|
||||
if snap.mid:
|
||||
self._last_mid = snap.mid
|
||||
return snap
|
||||
|
||||
def on_order_book_deltas(self, deltas: OrderBookDeltas) -> None:
|
||||
seq = getattr(deltas, "sequence", None)
|
||||
ts_event = getattr(deltas, "ts_event", None)
|
||||
self._health.on_book(seq=int(seq) if seq is not None else None, event_ts_ns=ts_event)
|
||||
|
||||
report = self._health.maybe_report()
|
||||
if report:
|
||||
self._ledger.write({**report, "event": "phase0_health"})
|
||||
|
||||
if not self.config.log_every_book_delta:
|
||||
return
|
||||
|
||||
snap = self._current_snap()
|
||||
if snap is None or snap.mid <= 0:
|
||||
return
|
||||
|
||||
delta_count = len(getattr(deltas, "deltas", []) or [])
|
||||
exchange_ts = int(ts_event) if ts_event is not None else None
|
||||
self._ledger.log_book_state(
|
||||
pair=str(self.config.instrument_id),
|
||||
snap=snap,
|
||||
exchange_ts_ns=exchange_ts,
|
||||
local_ts_epoch=time.time(),
|
||||
sequence=int(seq) if seq is not None else None,
|
||||
delta_count=delta_count,
|
||||
event_type="book_update",
|
||||
)
|
||||
|
||||
def on_trade_tick(self, tick: TradeTick) -> None:
|
||||
ts_event = getattr(tick, "ts_event", None)
|
||||
self._health.on_trade(event_ts_ns=ts_event)
|
||||
|
||||
qty = float(tick.size)
|
||||
price = float(tick.price)
|
||||
now = time.time()
|
||||
trade_side = "unknown"
|
||||
aggressor = str(getattr(tick, "aggressor_side", ""))
|
||||
try:
|
||||
if tick.aggressor_side == AggressorSide.BUYER:
|
||||
trade_side = "buy"
|
||||
self._recent_buys.append((now, qty))
|
||||
elif tick.aggressor_side == AggressorSide.SELLER:
|
||||
trade_side = "sell"
|
||||
self._recent_sells.append((now, qty))
|
||||
except Exception:
|
||||
trade_side = "unknown"
|
||||
|
||||
exchange_ts = int(ts_event) if ts_event is not None else None
|
||||
trade_id = str(getattr(tick, "trade_id", "") or getattr(tick, "id", "") or "")
|
||||
self._ledger.log_aggressive_trade(
|
||||
pair=str(self.config.instrument_id),
|
||||
price=price,
|
||||
qty=qty,
|
||||
trade_side=trade_side,
|
||||
exchange_ts_ns=exchange_ts,
|
||||
local_ts_epoch=now,
|
||||
aggressor_side=aggressor,
|
||||
trade_id=trade_id or None,
|
||||
snap=self._current_snap(),
|
||||
)
|
||||
@@ -0,0 +1,430 @@
|
||||
"""
|
||||
MakerEdgeProbe — Nautilus 事件驱动探针(v0)
|
||||
|
||||
实验冻结见 nautilus_mm/FREEZE.md — 三不动:
|
||||
1. 不动 Quote Logic(无动态 spread / inv skew / AI / Pulse)
|
||||
2. 不动成本模型
|
||||
3. 不动 PASS/COLLECTING/FAIL 定义
|
||||
|
||||
只记录:quote / fill / outcome + Phase0 健康度。
|
||||
market_state_snapshot 必须保持 null,禁止注入交易决策。
|
||||
Stage3+ 未解锁前禁止进化为本文件的「聪明报价」。
|
||||
|
||||
安全闸(非报价进化):TTL 撤单、健康度 gate、库存上限、fill 必记。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections import deque
|
||||
from decimal import Decimal
|
||||
from typing import Optional
|
||||
|
||||
from nautilus_trader.common.enums import LogColor
|
||||
from nautilus_trader.config import PositiveInt, StrategyConfig
|
||||
from nautilus_trader.model.data import OrderBookDeltas, TradeTick
|
||||
from nautilus_trader.model.enums import BookType, OrderSide, TimeInForce
|
||||
from nautilus_trader.model.events import OrderCanceled, OrderDenied, OrderFilled, OrderRejected
|
||||
from nautilus_trader.model.identifiers import InstrumentId
|
||||
from nautilus_trader.model.instruments import Instrument
|
||||
from nautilus_trader.model.objects import Price, Quantity
|
||||
from nautilus_trader.trading.strategy import Strategy
|
||||
|
||||
from nautilus_mm.book_utils import snapshot_from_nautilus_book
|
||||
from nautilus_mm.health import ConnectionHealth, empty_market_state_snapshot
|
||||
from nautilus_mm.recorder import MakerEdgeLogger
|
||||
|
||||
|
||||
class MakerEdgeProbeConfig(StrategyConfig, frozen=True):
|
||||
instrument_id: InstrumentId
|
||||
order_qty: Decimal = Decimal("0.001")
|
||||
book_depth: PositiveInt = 10
|
||||
quote_offset_ticks: PositiveInt = 1
|
||||
max_quotes: PositiveInt = 1
|
||||
quote_ttl_secs: float = 30.0
|
||||
cooldown_secs: float = 60.0
|
||||
book_sample_secs: float = 2.0
|
||||
log_dir: str = ""
|
||||
# 探针:仅在 OBI 极端时挂一侧(吸收叙事),避免噪音
|
||||
obi_enter: float = 0.25
|
||||
enable_trading: bool = False # False = 只录盘口不挂单
|
||||
# 风险熔断:|inventory| 达上限后只允许减仓方向挂单
|
||||
max_abs_inventory: Decimal = Decimal("0.005")
|
||||
|
||||
|
||||
class MakerEdgeProbe(Strategy):
|
||||
def __init__(self, config: MakerEdgeProbeConfig) -> None:
|
||||
super().__init__(config)
|
||||
self.instrument: Instrument | None = None
|
||||
self._logger = MakerEdgeLogger(
|
||||
log_dir=config.log_dir or None,
|
||||
levels=int(config.book_depth),
|
||||
)
|
||||
self._last_mid: float | None = None
|
||||
self._last_book_sample = 0.0
|
||||
self._last_quote_ts = 0.0
|
||||
self._recent_buys = deque(maxlen=200)
|
||||
self._recent_sells = deque(maxlen=200)
|
||||
self._quote_id_by_client: dict[str, str] = {}
|
||||
self._quote_submit_ts: dict[str, float] = {}
|
||||
self._fill_id_by_client: dict[str, str] = {}
|
||||
self._liq_high = 0.0
|
||||
self._liq_low = 0.0
|
||||
self._health = ConnectionHealth(window_sec=60.0, report_every_sec=30.0)
|
||||
self._quoting_halted = False
|
||||
|
||||
def on_start(self) -> None:
|
||||
self.instrument = self.cache.instrument(self.config.instrument_id)
|
||||
if self.instrument is None:
|
||||
self.log.error(f"Instrument not found: {self.config.instrument_id}")
|
||||
self.stop()
|
||||
return
|
||||
|
||||
# 启动清场:避免上次硬杀残留挂单污染实验
|
||||
try:
|
||||
self.cancel_all_orders(self.config.instrument_id)
|
||||
self.log.info("Startup cancel_all_orders issued", LogColor.BLUE)
|
||||
except Exception as exc:
|
||||
self.log.warning(f"Startup cancel_all failed: {exc}")
|
||||
|
||||
self.subscribe_order_book_deltas(
|
||||
instrument_id=self.config.instrument_id,
|
||||
book_type=BookType.L2_MBP,
|
||||
depth=int(self.config.book_depth),
|
||||
)
|
||||
self.subscribe_trade_ticks(self.config.instrument_id)
|
||||
exp = self._logger.experiment
|
||||
self._logger.write_experiment_start(
|
||||
extra={
|
||||
"instrument_id": str(self.config.instrument_id),
|
||||
"enable_trading": bool(self.config.enable_trading),
|
||||
"quote_ttl_secs": float(self.config.quote_ttl_secs),
|
||||
"max_abs_inventory": str(self.config.max_abs_inventory),
|
||||
"log_dir": str(self._logger.log_dir),
|
||||
}
|
||||
)
|
||||
self.log.info(
|
||||
f"Experiment {exp['experiment_id']} | {exp['probe_version']} | "
|
||||
f"quote/fee/exchange=frozen | log={self._logger.log_dir} | "
|
||||
f"trading={self.config.enable_trading} ttl={self.config.quote_ttl_secs}s "
|
||||
f"max_inv={self.config.max_abs_inventory}",
|
||||
LogColor.GREEN,
|
||||
)
|
||||
self.log.info(
|
||||
"Research Freeze: Data Collection only — no Pulse / no quote evolution",
|
||||
LogColor.BLUE,
|
||||
)
|
||||
|
||||
def on_stop(self) -> None:
|
||||
try:
|
||||
self.cancel_all_orders(self.config.instrument_id)
|
||||
except Exception as exc:
|
||||
self.log.warning(f"Stop cancel_all failed: {exc}")
|
||||
self.log.info("MakerEdgeProbe stopped")
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
def _trade_qty_window(self, window_sec: float = 20.0) -> tuple[float, float]:
|
||||
now = time.time()
|
||||
buy = sum(q for t, q in self._recent_buys if now - t <= window_sec)
|
||||
sell = sum(q for t, q in self._recent_sells if now - t <= window_sec)
|
||||
return buy, sell
|
||||
|
||||
def _current_snap(self):
|
||||
book = self.cache.order_book(self.config.instrument_id)
|
||||
if book is None:
|
||||
return None
|
||||
buy, sell = self._trade_qty_window()
|
||||
snap = snapshot_from_nautilus_book(
|
||||
book,
|
||||
levels=int(self.config.book_depth),
|
||||
recent_buy_qty=buy,
|
||||
recent_sell_qty=sell,
|
||||
last_mid=self._last_mid,
|
||||
liq_low=self._liq_low or None,
|
||||
liq_high=self._liq_high or None,
|
||||
)
|
||||
if snap.mid:
|
||||
self._last_mid = snap.mid
|
||||
self._liq_high = max(self._liq_high or snap.mid, snap.mid)
|
||||
self._liq_low = min(self._liq_low or snap.mid, snap.mid) if self._liq_low else snap.mid
|
||||
return snap
|
||||
|
||||
def _inventory(self) -> float:
|
||||
try:
|
||||
positions = self.cache.positions_open(instrument_id=self.config.instrument_id)
|
||||
except TypeError:
|
||||
positions = [
|
||||
p
|
||||
for p in self.cache.positions_open()
|
||||
if p.instrument_id == self.config.instrument_id
|
||||
]
|
||||
if not positions:
|
||||
return 0.0
|
||||
inv = 0.0
|
||||
for pos in positions:
|
||||
qty = float(pos.quantity)
|
||||
inv += -qty if pos.is_short else qty
|
||||
return inv
|
||||
|
||||
def _open_orders(self):
|
||||
try:
|
||||
return list(self.cache.orders_open(instrument_id=self.config.instrument_id))
|
||||
except TypeError:
|
||||
return [
|
||||
o
|
||||
for o in self.cache.orders_open()
|
||||
if o.instrument_id == self.config.instrument_id
|
||||
]
|
||||
|
||||
def _expire_stale_quotes(self, now: float) -> None:
|
||||
"""Cancel GTC quotes older than quote_ttl_secs."""
|
||||
ttl = float(self.config.quote_ttl_secs)
|
||||
if ttl <= 0:
|
||||
return
|
||||
for order in self._open_orders():
|
||||
cid = order.client_order_id.value
|
||||
submitted = self._quote_submit_ts.get(cid)
|
||||
if submitted is None:
|
||||
# 非本进程跟踪的单(启动残留等)— 一并撤掉
|
||||
self.log.warning(f"TTL cancel untracked open order {cid}")
|
||||
self.cancel_order(order)
|
||||
continue
|
||||
if now - submitted >= ttl:
|
||||
self.log.info(f"TTL cancel {cid} age={now - submitted:.1f}s", LogColor.YELLOW)
|
||||
self.cancel_order(order)
|
||||
|
||||
def _inventory_allows(self, side: OrderSide, inv: float) -> bool:
|
||||
max_abs = float(self.config.max_abs_inventory)
|
||||
if max_abs <= 0:
|
||||
return True
|
||||
if abs(inv) < max_abs:
|
||||
return True
|
||||
# 超限:只允许减仓方向
|
||||
if inv >= max_abs and side == OrderSide.SELL:
|
||||
return True
|
||||
if inv <= -max_abs and side == OrderSide.BUY:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _release_quote_client(self, cid: str, reason: str, snap=None) -> None:
|
||||
qid = self._quote_id_by_client.pop(cid, None)
|
||||
self._quote_submit_ts.pop(cid, None)
|
||||
if qid is not None:
|
||||
self._logger.cancel_quote(quote_id=qid, reason=reason, snap=snap)
|
||||
|
||||
def on_order_book_deltas(self, deltas: OrderBookDeltas) -> None:
|
||||
now = time.time()
|
||||
# Phase 0 health
|
||||
seq = getattr(deltas, "sequence", None)
|
||||
ts_event = getattr(deltas, "ts_event", None)
|
||||
self._health.on_book(seq=int(seq) if seq is not None else None, event_ts_ns=ts_event)
|
||||
report = self._health.maybe_report()
|
||||
if report:
|
||||
self._logger.write(report)
|
||||
gap_w = report.get("sequence_gap_window", 0)
|
||||
color = LogColor.RED if gap_w or not report.get("healthy") else LogColor.CYAN
|
||||
self.log.info(
|
||||
f"Phase0 book/s={report['book_update_rate']:.1f} "
|
||||
f"trade/s={report['trade_update_rate']:.1f} "
|
||||
f"lat_p50/p99/max={report['latency_ms_p50']}/"
|
||||
f"{report['latency_ms_p99']}/{report['latency_ms_max']} "
|
||||
f"gap_win={gap_w} book_age_ms={report.get('book_age_ms')}",
|
||||
color,
|
||||
)
|
||||
|
||||
# TTL 撤单:与报价逻辑无关的生命周期闭环
|
||||
self._expire_stale_quotes(now)
|
||||
|
||||
snap = self._current_snap()
|
||||
if snap is None or snap.mid <= 0:
|
||||
return
|
||||
|
||||
if now - self._last_book_sample >= float(self.config.book_sample_secs):
|
||||
self._last_book_sample = now
|
||||
self._logger.record_book(
|
||||
snap,
|
||||
now=now,
|
||||
emit_mid_tick=True,
|
||||
pair=str(self.config.instrument_id),
|
||||
)
|
||||
inv_fields = self._logger.update_inventory(self._inventory(), now=now)
|
||||
self._logger.write(
|
||||
{
|
||||
"event": "inventory_tick",
|
||||
"pair": str(self.config.instrument_id),
|
||||
**inv_fields,
|
||||
"market_state_snapshot": empty_market_state_snapshot(),
|
||||
**snap.to_book_fields(),
|
||||
}
|
||||
)
|
||||
|
||||
# 推进 fill path
|
||||
self._logger.update_paths(str(self.config.instrument_id), snap.mid, now=now)
|
||||
|
||||
if not self.config.enable_trading:
|
||||
return
|
||||
if not self._health.allow_quoting():
|
||||
if not self._quoting_halted:
|
||||
self._quoting_halted = True
|
||||
self.log.warning("Quoting halted: health gate (stale book / low update rate)")
|
||||
return
|
||||
if self._quoting_halted:
|
||||
self._quoting_halted = False
|
||||
self.log.info("Quoting resumed: health OK", LogColor.GREEN)
|
||||
|
||||
if now - self._last_quote_ts < float(self.config.cooldown_secs):
|
||||
return
|
||||
if len(self._open_orders()) >= int(self.config.max_quotes):
|
||||
return
|
||||
|
||||
if self.instrument is None:
|
||||
return
|
||||
|
||||
# 简单吸收探针:OBI 极端 → 挂被动单
|
||||
tick = float(self.instrument.price_increment)
|
||||
offset = int(self.config.quote_offset_ticks) * tick
|
||||
qty = self.instrument.make_qty(self.config.order_qty)
|
||||
inv = self._inventory()
|
||||
|
||||
if snap.obi >= float(self.config.obi_enter):
|
||||
side = OrderSide.SELL
|
||||
if not self._inventory_allows(side, inv):
|
||||
return
|
||||
price = self.instrument.make_price(snap.best_ask + offset)
|
||||
self._submit_quote(side, price, qty, snap, reason="obi_bid_thick")
|
||||
elif snap.obi <= -float(self.config.obi_enter):
|
||||
side = OrderSide.BUY
|
||||
if not self._inventory_allows(side, inv):
|
||||
return
|
||||
price = self.instrument.make_price(snap.best_bid - offset)
|
||||
self._submit_quote(side, price, qty, snap, reason="obi_ask_thick")
|
||||
|
||||
def on_trade_tick(self, tick: TradeTick) -> None:
|
||||
now = time.time()
|
||||
self._health.on_trade(event_ts_ns=getattr(tick, "ts_event", None))
|
||||
qty = float(tick.size)
|
||||
# Aggressor side
|
||||
try:
|
||||
from nautilus_trader.model.enums import AggressorSide
|
||||
|
||||
if tick.aggressor_side == AggressorSide.BUYER:
|
||||
self._recent_buys.append((now, qty))
|
||||
elif tick.aggressor_side == AggressorSide.SELLER:
|
||||
self._recent_sells.append((now, qty))
|
||||
except Exception:
|
||||
self._recent_buys.append((now, qty * 0.5))
|
||||
self._recent_sells.append((now, qty * 0.5))
|
||||
|
||||
snap_mid = self._last_mid or float(tick.price)
|
||||
self._logger.update_paths(str(self.config.instrument_id), snap_mid, now=now)
|
||||
|
||||
def _submit_quote(self, side: OrderSide, price: Price, qty: Quantity, snap, reason: str) -> None:
|
||||
assert self.instrument is not None
|
||||
order = self.order_factory.limit(
|
||||
instrument_id=self.config.instrument_id,
|
||||
order_side=side,
|
||||
quantity=qty,
|
||||
price=price,
|
||||
time_in_force=TimeInForce.GTC,
|
||||
post_only=True,
|
||||
)
|
||||
qside = "bid" if side == OrderSide.BUY else "ask"
|
||||
cid = order.client_order_id.value
|
||||
# 先 submit,成功后再记 quote(避免幽灵 quote_created)
|
||||
try:
|
||||
self.submit_order(order)
|
||||
except Exception as exc:
|
||||
self.log.error(f"submit_order failed: {exc}")
|
||||
return
|
||||
|
||||
qid = self._logger.create_quote(
|
||||
pair=str(self.config.instrument_id),
|
||||
side=qside,
|
||||
quote_price=float(price),
|
||||
inventory=self._inventory(),
|
||||
snap=snap,
|
||||
reason=reason,
|
||||
state=empty_market_state_snapshot(), # 故意不接 Market Pulse
|
||||
extra={"book_age_ms": self._health.book_age_ms()},
|
||||
)
|
||||
self._quote_id_by_client[cid] = qid
|
||||
self._quote_submit_ts[cid] = time.time()
|
||||
self._last_quote_ts = time.time()
|
||||
self.log.info(f"QUOTE {qside} {price} qty={qty} reason={reason}", LogColor.BLUE)
|
||||
|
||||
def on_order_canceled(self, event: OrderCanceled) -> None:
|
||||
cid = event.client_order_id.value
|
||||
snap = self._current_snap()
|
||||
self._release_quote_client(cid, reason="canceled", snap=snap)
|
||||
|
||||
def on_order_rejected(self, event: OrderRejected) -> None:
|
||||
cid = event.client_order_id.value
|
||||
reason = getattr(event, "reason", None) or "rejected"
|
||||
self.log.warning(f"OrderRejected {cid}: {reason}")
|
||||
snap = self._current_snap()
|
||||
self._release_quote_client(cid, reason=f"rejected:{reason}", snap=snap)
|
||||
|
||||
def on_order_denied(self, event: OrderDenied) -> None:
|
||||
cid = event.client_order_id.value
|
||||
reason = getattr(event, "reason", None) or "denied"
|
||||
self.log.warning(f"OrderDenied {cid}: {reason}")
|
||||
snap = self._current_snap()
|
||||
self._release_quote_client(cid, reason=f"denied:{reason}", snap=snap)
|
||||
|
||||
def on_order_filled(self, event: OrderFilled) -> None:
|
||||
cid = event.client_order_id.value
|
||||
qid = self._quote_id_by_client.get(cid)
|
||||
snap = self._current_snap()
|
||||
# snap 缺失仍必须记 fill(book 字段可空)
|
||||
side = "long" if event.order_side == OrderSide.BUY else "short"
|
||||
det = self._logger.book_deterioration(side)
|
||||
fill_reason = "toxic_passive" if det.get("pre_5s_deteriorated") else "maker_hit"
|
||||
|
||||
order = self.cache.order(event.client_order_id)
|
||||
terminal = True
|
||||
if order is not None:
|
||||
terminal = bool(order.is_closed) or float(order.leaves_qty) <= 0
|
||||
|
||||
fill_id = self._logger.log_fill(
|
||||
pair=str(self.config.instrument_id),
|
||||
side=side,
|
||||
fill_price=float(event.last_px),
|
||||
amount=float(event.last_qty),
|
||||
inventory=self._inventory(),
|
||||
snap=snap,
|
||||
order_type="limit",
|
||||
quote_id=qid,
|
||||
fill_reason=fill_reason,
|
||||
state=empty_market_state_snapshot(),
|
||||
quote_terminal=terminal,
|
||||
extra={
|
||||
"client_order_id": cid,
|
||||
"venue_order_id": str(event.venue_order_id),
|
||||
"trade_id": str(event.trade_id),
|
||||
"venue_trade_id": str(event.trade_id),
|
||||
"exchange_ts_ns": int(event.ts_event) if getattr(event, "ts_event", None) else None,
|
||||
"local_ts": time.time(),
|
||||
"book_age_ms": self._health.book_age_ms(),
|
||||
"leaves_qty": float(order.leaves_qty) if order is not None else None,
|
||||
# Maker-only hard evidence (do not trust post_only param alone)
|
||||
"liquidity_side": str(event.liquidity_side),
|
||||
"is_maker": event.liquidity_side.name == "MAKER"
|
||||
if hasattr(event.liquidity_side, "name")
|
||||
else str(event.liquidity_side) == "MAKER",
|
||||
"commission": float(event.commission) if event.commission is not None else None,
|
||||
"commission_currency": (
|
||||
str(event.commission.currency) if event.commission is not None else None
|
||||
),
|
||||
"post_only": True,
|
||||
"execution_type": "TRADE",
|
||||
},
|
||||
)
|
||||
self._fill_id_by_client[cid] = fill_id
|
||||
if terminal:
|
||||
self._quote_id_by_client.pop(cid, None)
|
||||
self._quote_submit_ts.pop(cid, None)
|
||||
self.log.info(
|
||||
f"FILL {side} px={event.last_px} qty={event.last_qty} "
|
||||
f"reason={fill_reason} terminal={terminal} book={'ok' if snap else 'none'}",
|
||||
LogColor.YELLOW,
|
||||
)
|
||||
Reference in New Issue
Block a user