jack/nautilus_mm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Snapshot nautilus_mm after EXP_002 Phase 1 stop.

Browse Source 
Keep frozen research conclusions and code; raw ledgers and secrets stay out of git.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
jackyu66git
2026-09-10 16:53:22 +08:00
co-authored by Cursor
commit e2fbe1c2b3
49 changed files with 9633 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+41
View File
@@ -0,0 +1,41 @@
# Binance Futures API (prefer TESTNET first)
BINANCE_API_KEY=
BINANCE_API_SECRET=
# Must be exactly TESTNET or LIVE (case-insensitive). Anything else exits.
BINANCE_ENVIRONMENT=TESTNET
# Required only for LIVE:
# I_UNDERSTAND_LIVE=yes
# Proxy: 本地可开；服务器请留空（直连）
# 本地: USE_PROXY=true ./scripts/run_probe.sh
HTTP_PROXY=
HTTPS_PROXY=
# Instrument
SYMBOL=BTCUSDT-PERP
ORDER_QTY=0.001
# Default false — set true explicitly to place post-only quotes
ENABLE_TRADING=false
COOLDOWN_SECS=60
OBI_ENTER=0.25
QUOTE_TTL_SECS=30
# Absolute inventory circuit breaker (BTC). Beyond: reduce-only side.
MAX_ABS_INVENTORY=0.005
# Logs (shared schema with Freqtrade MakerEdgeProbe)
MAKER_EDGE_LOG_DIR=
# Experiment identity (bind every fill + every report)
EXPERIMENT_ID=MM_EDGE_EXP_001
PROBE_VERSION=probe_v0.1
EXCHANGE_NAME=binance_usdm
# --- MM_EDGE_EXP_002 (Event-State Observability — use run_event_state.sh) ---
# EXPERIMENT_ID=MM_EDGE_EXP_002
# PROBE_VERSION=event_state_v0.1
# ENABLE_TRADING=false # hard-enforced; never true for EXP_002
# EVENT_STATE_LOG_DIR=logs/event_state
# PREFILL_WINDOW_SEC=5.0
# PREFILL_MARGIN_SEC=0.25
# LARGE_TRADE_QTY=0.1
# LOG_EVERY_BOOK_DELTA=true
.gitignore
+11
View File
@@ -0,0 +1,11 @@
.venv/
.env
.env.*
!.env.example
__pycache__/
*.pyc
logs/
.DS_Store
*.egg-info/
dist/
build/
Economic_Attribution_v0_1.json
+531
View File
@@ -0,0 +1,531 @@
{
"experiment_id": "MM_EDGE_EXP_001",
"population": {
"name": "MATCHED",
"fills": 3890,
"paths": 3886,
"clusters": 3334
},
"strategy": "v0.1 FROZEN",
"execution": "STOPPED",
"stage3": "LOCKED",
"fee_total_usdt": 42.42214561,
"fee_per_fill_usdt": 0.010916661248069994,
"fee_per_btc_usdt": 12.74857122550787,
"fee_per_cluster_usdt": 0.012724098863227356,
"realized_component_usdt": -8.52172967,
"gross_markout_30s_usdt": -1.7307799999996094,
"net_attr_30s_usdt": -52.67465527999961,
"markout_by_horizon": [
{
"horizon": "1s",
"n": 3886,
"fill_w": 1.9522065688461466e-05,
"cluster_w": 1.9672552540924558e-05,
"gross_usdt": 4.140839999999786
},
{
"horizon": "5s",
"n": 3886,
"fill_w": 1.6056729777556372e-05,
"cluster_w": 1.563711857682482e-05,
"gross_usdt": 3.4058050000001323
},
{
"horizon": "10s",
"n": 3886,
"fill_w": 1.1123222170478668e-05,
"cluster_w": 1.1253447279750312e-05,
"gross_usdt": 2.359355000000128
},
{
"horizon": "30s",
"n": 3886,
"fill_w": -8.159793870873894e-06,
"cluster_w": -8.29897884011306e-06,
"gross_usdt": -1.7307799999996094
},
{
"horizon": "300s",
"n": 3886,
"fill_w": -3.581687863688742e-05,
"cluster_w": -3.542059919144397e-05,
"gross_usdt": -7.597144999999726
}
],
"inventory_metrics": {
"max_net_btc": 0.0059,
"min_net_btc": -0.0058000000000000005,
"max_abs_net_btc": 0.0059,
"avg_abs_net_btc_per_fill": 0.0029282519280205655,
"time_weighted_abs_net_btc": 0.0035138619697350987,
"time_weighted_signed_net_btc": 0.0009845445677824191,
"long_qty": 1.6646,
"short_qty": 1.6670000000000003,
"turnover_btc": 3.331600000000001
},
"bucket_rows": [
{
"dimension": "PathType",
"bucket": "A_immediate_edge",
"fills": 1481,
"clusters": 1279,
"btc_qty": 1.2579000000000002,
"notional_usdt": 80131.73112,
"fee_usdt": 16.02634448,
"fee_per_fill": 0.010821299446320053,
"fee_per_btc": 12.74055527466412,
"markout_1s": 0.00010135972212851294,
"markout_5s": 0.00012093929414163527,
"markout_10s": 0.00011803045894311667,
"markout_30s": 0.00015268377244562442,
"markout_300s": 0.00013524292871909752,
"gross_markout_30s_usdt": 12.23481500000004,
"realized_pnl_usdt": -1.7373983300000004,
"net_attr_30s_usdt": -5.52892780999996
},
{
"dimension": "PathType",
"bucket": "C_toxic",
"fills": 1290,
"clusters": 1114,
"btc_qty": 1.1028000000000002,
"notional_usdt": 70343.84668,
"fee_usdt": 14.068767789999999,
"fee_per_fill": 0.010906021542635659,
"fee_per_btc": 12.757315732680446,
"markout_1s": -2.26781740733745e-05,
"markout_5s": -5.864727612589716e-05,
"markout_10s": -9.311823718992184e-05,
"markout_30s": -0.0002122337020878937,
"markout_300s": -0.0004798649291039827,
"gross_markout_30s_usdt": -14.92933499999959,
"realized_pnl_usdt": -2.45849123,
"net_attr_30s_usdt": -31.45659401999959
},
{
"dimension": "PathType",
"bucket": "B_drawdown_then_recover",
"fills": 872,
"clusters": 767,
"btc_qty": 0.7574000000000001,
"notional_usdt": 48282.0527,
"fee_usdt": 9.656409660000001,
"fee_per_fill": 0.011073864288990827,
"fee_per_btc": 12.749418616318987,
"markout_1s": -4.145950488969804e-05,
"markout_5s": -3.18002221144232e-05,
"markout_10s": -1.0215286476420607e-05,
"markout_30s": -8.034869652509012e-06,
"markout_300s": 0.00040945162631827824,
"gross_markout_30s_usdt": -0.3879400000000708,
"realized_pnl_usdt": -2.83361474,
"net_attr_30s_usdt": -12.87796440000007
},
{
"dimension": "PathType",
"bucket": "D_mixed",
"fills": 243,
"clusters": 215,
"btc_qty": 0.20950000000000005,
"notional_usdt": 13353.11977,
"fee_usdt": 2.6706236800000003,
"fee_per_fill": 0.010990220905349795,
"fee_per_btc": 12.747607064439139,
"markout_1s": -2.877754462019724e-05,
"markout_5s": -4.676173139724288e-05,
"markout_10s": -4.12787430573779e-06,
"markout_30s": 0.00010122578268464162,
"markout_300s": -0.00033310680025451153,
"gross_markout_30s_usdt": 1.3516800000000115,
"realized_pnl_usdt": -1.49222537,
"net_attr_30s_usdt": -2.8111690499999886
},
{
"dimension": "Toxicity",
"bucket": "non_toxic",
"fills": 2596,
"clusters": 2236,
"btc_qty": 2.2248,
"notional_usdt": 141766.90359,
"fee_usdt": 28.35337782,
"fee_per_fill": 0.010921948312788905,
"fee_per_btc": 12.744236704422867,
"markout_1s": 4.046155946657924e-05,
"markout_5s": 5.3124388057320254e-05,
"markout_10s": 6.284717923844347e-05,
"markout_30s": 9.310039695986549e-05,
"markout_300s": 0.00018451697355013032,
"gross_markout_30s_usdt": 13.198554999999981,
"realized_pnl_usdt": -6.063238439999999,
"net_attr_30s_usdt": -21.21806126000002
},
{
"dimension": "Toxicity",
"bucket": "toxic",
"fills": 1290,
"clusters": 1114,
"btc_qty": 1.1028000000000002,
"notional_usdt": 70343.84668,
"fee_usdt": 14.068767789999999,
"fee_per_fill": 0.010906021542635659,
"fee_per_btc": 12.757315732680446,
"markout_1s": -2.26781740733745e-05,
"markout_5s": -5.864727612589716e-05,
"markout_10s": -9.311823718992184e-05,
"markout_30s": -0.0002122337020878937,
"markout_300s": -0.0004798649291039827,
"gross_markout_30s_usdt": -14.92933499999959,
"realized_pnl_usdt": -2.45849123,
"net_attr_30s_usdt": -31.45659401999959
},
{
"dimension": "Volatility",
"bucket": "high_vol",
"fills": 1944,
"clusters": 1704,
"btc_qty": 1.7018000000000004,
"notional_usdt": 108527.27282000001,
"fee_usdt": 21.705452540000003,
"fee_per_fill": 0.011165356244855968,
"fee_per_btc": 12.754408590903747,
"markout_1s": 2.410343439052789e-05,
"markout_5s": 2.200234962100767e-05,
"markout_10s": 1.3732354654038277e-05,
"markout_30s": -1.8503321311044688e-05,
"markout_300s": -7.311572283918717e-05,
"gross_markout_30s_usdt": -2.0081149999998673,
"realized_pnl_usdt": -2.5832574499999996,
"net_attr_30s_usdt": -26.296824989999866
},
{
"dimension": "Volatility",
"bucket": "low_vol",
"fills": 1942,
"clusters": 1630,
"btc_qty": 1.6258000000000004,
"notional_usdt": 103583.47745,
"fee_usdt": 20.71669307,
"fee_per_fill": 0.010667710128733266,
"fee_per_btc": 12.74246098536105,
"markout_1s": 1.4722039050446979e-05,
"markout_5s": 9.827339504906817e-06,
"markout_10s": 8.389561939738874e-06,
"markout_30s": 2.677405768059181e-06,
"markout_300s": 3.2621515353484367e-06,
"gross_markout_30s_usdt": 0.2773350000002581,
"realized_pnl_usdt": -5.9384722199999995,
"net_attr_30s_usdt": -26.377830289999743
},
{
"dimension": "Trend",
"bucket": "range",
"fills": 1521,
"clusters": 1383,
"btc_qty": 1.3480000000000003,
"notional_usdt": 85940.04824,
"fee_usdt": 17.18800828,
"fee_per_fill": 0.011300465667324127,
"fee_per_btc": 12.750747982195842,
"markout_1s": -1.477018021278579e-06,
"markout_5s": -8.501973351929517e-06,
"markout_10s": -1.3186808981480409e-05,
"markout_30s": -3.577395013107376e-05,
"markout_300s": -8.103131360308704e-05,
"gross_markout_30s_usdt": -3.074414999999833,
"realized_pnl_usdt": -5.27847826,
"net_attr_30s_usdt": -25.540901539999833
},
{
"dimension": "Trend",
"bucket": "trend_up",
"fills": 1255,
"clusters": 1097,
"btc_qty": 1.0467,
"notional_usdt": 66723.59572000001,
"fee_usdt": 13.34471747,
"fee_per_fill": 0.010633241011952193,
"fee_per_btc": 12.749324037451037,
"markout_1s": 3.411746587448147e-05,
"markout_5s": 3.5157502749763345e-05,
"markout_10s": 2.718513563945042e-05,
"markout_30s": -2.7502264831481206e-06,
"markout_300s": -1.9947965717947442e-06,
"gross_markout_30s_usdt": -0.18350500000001263,
"realized_pnl_usdt": 0.07479931999999989,
"net_attr_30s_usdt": -13.453423150000011
},
{
"dimension": "Trend",
"bucket": "trend_down",
"fills": 1110,
"clusters": 972,
"btc_qty": 0.9329000000000001,
"notional_usdt": 59447.10631,
"fee_usdt": 11.88941986,
"fee_per_fill": 0.010711189063063063,
"fee_per_btc": 12.744581262729124,
"markout_1s": 3.349759346764102e-05,
"markout_5s": 3.012139885602704e-05,
"markout_10s": 2.823922145589404e-05,
"markout_30s": 2.568905527607399e-05,
"markout_300s": -8.414370876041091e-06,
"gross_markout_30s_usdt": 1.5271400000002369,
"realized_pnl_usdt": -3.3180507300000004,
"net_attr_30s_usdt": -13.680330589999762
},
{
"dimension": "FillContext",
"bucket": "neutral",
"fills": 1388,
"clusters": 1213,
"btc_qty": 1.1838000000000002,
"notional_usdt": 75485.50388999999,
"fee_usdt": 15.09709914,
"fee_per_fill": 0.01087687257925072,
"fee_per_btc": 12.7530825646224,
"markout_1s": 1.647018216652519e-05,
"markout_5s": -3.5886360432144643e-06,
"markout_10s": -2.1338136688432127e-05,
"markout_30s": -6.403666599409166e-05,
"markout_300s": -0.00017397764237140476,
"gross_markout_30s_usdt": -4.833839999999636,
"realized_pnl_usdt": 0.9367158200000003,
"net_attr_30s_usdt": -18.994223319999634
},
{
"dimension": "FillContext",
"bucket": "sell_resistance",
"fills": 979,
"clusters": 861,
"btc_qty": 0.8544,
"notional_usdt": 54478.163700000005,
"fee_usdt": 10.89563175,
"fee_per_fill": 0.011129348059244126,
"fee_per_btc": 12.752377984550561,
"markout_1s": 2.2746912080660278e-05,
"markout_5s": 3.396737104044108e-05,
"markout_10s": 4.7584661889033e-05,
"markout_30s": 6.382970283559833e-05,
"markout_300s": 0.00015825056159152353,
"gross_markout_30s_usdt": 3.4773250000000804,
"realized_pnl_usdt": -6.22020069,
"net_attr_30s_usdt": -13.63850743999992
},
{
"dimension": "FillContext",
"bucket": "buy_support",
"fills": 957,
"clusters": 842,
"btc_qty": 0.8356000000000002,
"notional_usdt": 53220.48314,
"fee_usdt": 10.64409564,
"fee_per_fill": 0.01112235699059561,
"fee_per_btc": 12.73826668262326,
"markout_1s": 8.405128507064573e-06,
"markout_5s": 1.8714880836011024e-05,
"markout_10s": 2.0467871310649115e-05,
"markout_30s": 1.984809114230057e-05,
"markout_300s": 5.777324478456582e-05,
"gross_markout_30s_usdt": 1.0563249999999906,
"realized_pnl_usdt": -4.60049982,
"net_attr_30s_usdt": -14.188270460000009
},
{
"dimension": "FillContext",
"bucket": "buy_pressure_absorbing",
"fills": 288,
"clusters": 242,
"btc_qty": 0.23299999999999998,
"notional_usdt": 14849.31926,
"fee_usdt": 2.96986344,
"fee_per_fill": 0.010312025833333334,
"fee_per_btc": 12.74619502145923,
"markout_1s": 3.085831693539742e-05,
"markout_5s": 9.949951065962983e-07,
"markout_10s": -5.396206963900885e-06,
"markout_30s": -8.174078412265757e-05,
"markout_300s": -0.00018505090717539603,
"gross_markout_30s_usdt": -1.2137950000000812,
"realized_pnl_usdt": 0.5555894899999999,
"net_attr_30s_usdt": -3.628068950000081
},
{
"dimension": "FillContext",
"bucket": "sell_pressure_absorbing",
"fills": 246,
"clusters": 207,
"btc_qty": 0.19330000000000003,
"notional_usdt": 12320.81768,
"fee_usdt": 2.46416312,
"fee_per_fill": 0.01001692325203252,
"fee_per_btc": 12.74786921883083,
"markout_1s": 3.3994902844800725e-05,
"markout_5s": 1.3797785537899414e-05,
"markout_10s": -3.266666307814853e-05,
"markout_30s": -5.2299288629677764e-05,
"markout_300s": -0.0003042407652930987,
"gross_markout_30s_usdt": -0.6443699999999568,
"realized_pnl_usdt": 0.9138141899999999,
"net_attr_30s_usdt": -2.194718929999957
},
{
"dimension": "FillContext",
"bucket": "sell_pressure_falling",
"fills": 10,
"clusters": 10,
"btc_qty": 0.010000000000000002,
"notional_usdt": 638.0611,
"fee_usdt": 0.12761222,
"fee_per_fill": 0.012761222,
"fee_per_btc": 12.761221999999997,
"markout_1s": 0.0002923701194132765,
"markout_5s": 0.0002596146356515865,
"markout_10s": 0.0005288678466685525,
"markout_30s": 8.886296312371227e-05,
"markout_300s": -2.1392935566876843e-05,
"gross_markout_30s_usdt": 0.056699999999975284,
"realized_pnl_usdt": 0.02931205,
"net_attr_30s_usdt": -0.041600170000024736
},
{
"dimension": "FillContext",
"bucket": "buy_pressure_rising",
"fills": 8,
"clusters": 8,
"btc_qty": 0.0075,
"notional_usdt": 479.4465,
"fee_usdt": 0.0958893,
"fee_per_fill": 0.0119861625,
"fee_per_btc": 12.78524,
"markout_1s": 0.00011800065283618207,
"markout_5s": 4.155833862592348e-05,
"markout_10s": -0.00018521357440294395,
"markout_30s": -0.0002589965720888399,
"markout_300s": -0.0008807030607168815,
"gross_markout_30s_usdt": -0.12417499999999199,
"realized_pnl_usdt": -0.00064907,
"net_attr_30s_usdt": -0.22071336999999197
},
{
"dimension": "FillContext",
"bucket": "momentum_down_catching_knife",
"fills": 5,
"clusters": 5,
"btc_qty": 0.005,
"notional_usdt": 317.4407,
"fee_usdt": 0.06348814,
"fee_per_fill": 0.012697627999999999,
"fee_per_btc": 12.697628,
"markout_1s": -8.804794092254599e-05,
"markout_5s": 0.00041661324461542815,
"markout_10s": 0.0007213945785779604,
"markout_30s": 0.000589716441527518,
"markout_300s": 0.0009308825238855407,
"gross_markout_30s_usdt": 0.18720000000000436,
"realized_pnl_usdt": -0.13601287,
"net_attr_30s_usdt": -0.012301009999995626
},
{
"dimension": "FillContext",
"bucket": "momentum_up_chasing",
"fills": 5,
"clusters": 5,
"btc_qty": 0.005,
"notional_usdt": 321.51430000000005,
"fee_usdt": 0.06430286,
"fee_per_fill": 0.012860572,
"fee_per_btc": 12.860572000000001,
"markout_1s": 0.00036950144985773765,
"markout_5s": 0.001018928240516915,
"markout_10s": 0.0009125566110123747,
"markout_30s": 0.0009575001796187784,
"markout_300s": 0.0014820491654648018,
"gross_markout_30s_usdt": 0.30785000000000584,
"realized_pnl_usdt": 0.00020122999999998975,
"net_attr_30s_usdt": 0.24374837000000582
}
],
"counterfactuals": [
{
"name": "BASELINE",
"fills": 3886,
"clusters": 3332,
"btc_qty": 3.327600000000001,
"fee_usdt": 42.42214561,
"gross_markout_30s_usdt": -1.7307799999996094,
"realized_pnl_usdt": -8.52172967,
"net_attr_30s_usdt": -52.67465527999961,
"markout_30s": -8.159793870873894e-06
},
{
"name": "EXCLUDE_PATH_C",
"fills": 2596,
"clusters": 2236,
"btc_qty": 2.2248,
"fee_usdt": 28.35337782,
"gross_markout_30s_usdt": 13.198554999999981,
"realized_pnl_usdt": -6.063238439999999,
"net_attr_30s_usdt": -21.21806126000002,
"markout_30s": 9.310039695986549e-05
},
{
"name": "EXCLUDE_TOXIC",
"fills": 2596,
"clusters": 2236,
"btc_qty": 2.2248,
"fee_usdt": 28.35337782,
"gross_markout_30s_usdt": 13.198554999999981,
"realized_pnl_usdt": -6.063238439999999,
"net_attr_30s_usdt": -21.21806126000002,
"markout_30s": 9.310039695986549e-05
},
{
"name": "EXCLUDE_NEGATIVE_STATE",
"fills": 1956,
"clusters": 1723,
"btc_qty": 1.7100000000000002,
"fee_usdt": 21.79513061,
"gross_markout_30s_usdt": 5.085400000000057,
"realized_pnl_usdt": -10.9272001,
"net_attr_30s_usdt": -27.636930709999945,
"markout_30s": 4.666546513967971e-05
}
],
"recon03_extension": {
"venue_confirmed_no_trade_history": 876,
"venue_partial_order_canceled": 11
},
"account_recon_ref": {
"experiment_id": "MM_EDGE_EXP_001",
"maker_only_status": "PASS",
"taker_filled_count": 0,
"maker_filled_count": 3890,
"exchange_trades": 3890,
"jsonl_fills": 4777,
"income_by_type": {
"COMMISSION": -52.04535895999991,
"REALIZED_PNL": -9.671389249999995,
"FUNDING_FEE": 0.05826837000000001
},
"income_sum": -61.658479839999906,
"start_wallet_assumed": 5000.0,
"end_wallet": 4938.32625704,
"end_unrealized": 0.40078863,
"end_equity": 4938.72704567,
"wallet_residual_gap": -0.015263119999872288,
"net_qty": -0.0023999999999990695,
"fee_by_asset": {
"USDT": 42.47422358999987
},
"positions": [
{
"symbol": "BTCUSDT",
"amt": 0.0032,
"entry": 64233.4,
"unrealized": 0.40078863
}
],
"probe": "STOPPED"
}
}
Economic_Attribution_v0_1.txt
+95
View File
@@ -0,0 +1,95 @@
========================================================================
Economic Attribution v0.1
========================================================================
Experiment: MM_EDGE_EXP_001
Population: MATCHED=3890
Strategy: v0.1 FROZEN
Execution: STOPPED
Stage3: LOCKED
Purpose: Economic Attribution only.
No strategy modification. No live execution. No economic simulation.
Layer 1 — Hard Economic Evidence
----------------------------------------
Matched fills: 3890
Matched paths: 3886
Matched clusters: 3334
Fee total: 42.422146 USDT
Fee / fill: 0.010917 USDT
Fee / BTC: 12.748571 USDT
Fee / cluster: 0.012724 USDT
Realized component: -8.521730 USDT
Gross markout @30s: -1.730780 USDT
Net attributable @30s: -52.674655 USDT
Markout by horizon (MATCHED only)
----------------------------------------
1s n=3886 fill-w=0.0020% cluster-w=0.0020% gross=4.140840 USDT
5s n=3886 fill-w=0.0016% cluster-w=0.0016% gross=3.405805 USDT
10s n=3886 fill-w=0.0011% cluster-w=0.0011% gross=2.359355 USDT
30s n=3886 fill-w=-0.0008% cluster-w=-0.0008% gross=-1.730780 USDT
300s n=3886 fill-w=-0.0036% cluster-w=-0.0035% gross=-7.597145 USDT
Inventory carry / exposure
----------------------------------------
Max net BTC: 0.005900
Min net BTC: -0.005800
Max |net BTC|: 0.005900
Average |net BTC|: 0.002928
TW |net BTC|: 0.003514
TW signed net BTC: 0.000985
Long qty / Short qty: 1.664600 / 1.667000 BTC
Inventory turnover: 3.331600 BTC
Slices (weighted by notional, MATCHED only)
----------------------------------------
PathType
A_immediate_edge: n=1481 clusters=1279 fee=16.0263 gross30=12.2348 realized=-1.7374 net30=-5.5289 m30=0.0153%
C_toxic: n=1290 clusters=1114 fee=14.0688 gross30=-14.9293 realized=-2.4585 net30=-31.4566 m30=-0.0212%
B_drawdown_then_recover: n=872 clusters=767 fee=9.6564 gross30=-0.3879 realized=-2.8336 net30=-12.8780 m30=-0.0008%
D_mixed: n=243 clusters=215 fee=2.6706 gross30=1.3517 realized=-1.4922 net30=-2.8112 m30=0.0101%
Toxicity
non_toxic: n=2596 clusters=2236 fee=28.3534 gross30=13.1986 realized=-6.0632 net30=-21.2181 m30=0.0093%
toxic: n=1290 clusters=1114 fee=14.0688 gross30=-14.9293 realized=-2.4585 net30=-31.4566 m30=-0.0212%
Volatility
high_vol: n=1944 clusters=1704 fee=21.7055 gross30=-2.0081 realized=-2.5833 net30=-26.2968 m30=-0.0019%
low_vol: n=1942 clusters=1630 fee=20.7167 gross30=0.2773 realized=-5.9385 net30=-26.3778 m30=0.0003%
Trend
range: n=1521 clusters=1383 fee=17.1880 gross30=-3.0744 realized=-5.2785 net30=-25.5409 m30=-0.0036%
trend_up: n=1255 clusters=1097 fee=13.3447 gross30=-0.1835 realized=0.0748 net30=-13.4534 m30=-0.0003%
trend_down: n=1110 clusters=972 fee=11.8894 gross30=1.5271 realized=-3.3181 net30=-13.6803 m30=0.0026%
FillContext
neutral: n=1388 clusters=1213 fee=15.0971 gross30=-4.8338 realized=0.9367 net30=-18.9942 m30=-0.0064%
sell_resistance: n=979 clusters=861 fee=10.8956 gross30=3.4773 realized=-6.2202 net30=-13.6385 m30=0.0064%
buy_support: n=957 clusters=842 fee=10.6441 gross30=1.0563 realized=-4.6005 net30=-14.1883 m30=0.0020%
buy_pressure_absorbing: n=288 clusters=242 fee=2.9699 gross30=-1.2138 realized=0.5556 net30=-3.6281 m30=-0.0082%
sell_pressure_absorbing: n=246 clusters=207 fee=2.4642 gross30=-0.6444 realized=0.9138 net30=-2.1947 m30=-0.0052%
sell_pressure_falling: n=10 clusters=10 fee=0.1276 gross30=0.0567 realized=0.0293 net30=-0.0416 m30=0.0089%
buy_pressure_rising: n=8 clusters=8 fee=0.0959 gross30=-0.1242 realized=-0.0006 net30=-0.2207 m30=-0.0259%
momentum_down_catching_knife: n=5 clusters=5 fee=0.0635 gross30=0.1872 realized=-0.1360 net30=-0.0123 m30=0.0590%
momentum_up_chasing: n=5 clusters=5 fee=0.0643 gross30=0.3079 realized=0.0002 net30=0.2437 m30=0.0958%
Layer 2 — Evidence Extension (excluded from core conclusion)
----------------------------------------
VENUE_CONFIRMED_NO_TRADE_HISTORY: 876
VENUE_PARTIAL_ORDER_CANCELED: 11
These rows are order-confirmed, but not part of the Hard Evidence Population.
Layer 3 — Counterfactual Attribution (NOT backtest)
----------------------------------------
Observed vs Exclude-Bucket Attribution. These are contribution decompositions only.
BASELINE: fills=3886 clusters=3332 fee=42.4221 gross30=-1.7308 realized=-8.5217 net30=-52.6747 m30=-0.0008%
EXCLUDE_PATH_C: fills=2596 clusters=2236 fee=28.3534 gross30=13.1986 realized=-6.0632 net30=-21.2181 m30=0.0093%
EXCLUDE_TOXIC: fills=2596 clusters=2236 fee=28.3534 gross30=13.1986 realized=-6.0632 net30=-21.2181 m30=0.0093%
EXCLUDE_NEGATIVE_STATE: fills=1956 clusters=1723 fee=21.7951 gross30=5.0854 realized=-10.9272 net30=-27.6369 m30=0.0047%
Interpretation
----------------------------------------
Core conclusion is based on 3890 fully matched fills.
Economic Attribution asks why MakerAlpha did not convert to money.
It does NOT change quote logic, does NOT restart v0.1, and does NOT unlock Stage 3.
========================================================================
FREEZE.md
+342
View File
@@ -0,0 +1,342 @@
# Research Freeze / Data Collection Phase
**研究框架已冻结。** 研究对象：可验证的市场现象（不是策略）。
验证的假设是：
> 在当前 BTC 永续、当前交易所、当前报价假设、当前执行条件下，被动成交是否产生正向 Maker Alpha。
不是：「我的策略有没有赚钱」。
| 报告结论 | 含义 | 下一步 |
|----------|------|--------|
| **FAIL** | 这个市场假设不成立 | 换假设（Carry / Basis / Funding…） |
| **PARTIAL_PASS** | 优势仅局部存在 | Event-driven LP |
| **PASS** | 普遍可捕获 | Economic Simulation → Symmetric MM |
| **COLLECTING** | 样本不足 | 继续采集 |
三个答案都推进系统。只有盈利才算成功 —— 错误。
---
## 状态机（锁定）
```
Research Freeze
|
v
Data Collection
|
v
Maker Edge Report v0.1
|
+---- FAIL --------------→ 换假设
|
+---- PARTIAL_PASS ------→ Event-driven LP
|
+---- PASS --------------→ Economic Simulation
| ↓
| Symmetric MM / Quote Engine
|
+---- COLLECTING --------→ 继续采集
```
---
## 观察窗口（不要过早解释）
| 规模 | 用途 |
|------|------|
| **~500 fills** | 发现异常 / 管道是否工作 |
| **~2000 fills** | 初步判断（Decision 门槛默认） |
| **~10000 fills** | 才讨论稳定性 |
**cluster 数量比 fill 数量更重要。**
5000 fills / 80 clusters ≠ 3000 fills / 900 clusters。
第一份报告最值得看的不是 Decision，而是三个**分布**：Fill Alpha / Cluster Alpha / Toxicity。
---
## Unlock Stage3（Economic Simulation）必须同时满足
```
Data Integrity PASS
AND cluster-weighted MakerAlpha > 0
AND benchmark-adjusted alpha > 0
AND edge not concentrated in one event/regime
```
否则：继续 Data Collection，或判定 FAIL / PARTIAL_PASS。
**禁止**用改 Quote Logic / 改成本假设 来「凑」解锁。
---
## 冻结期间只允许 / 禁止
### ✅ 允许
- 数据字段（不改变报价行为）
- 数据质量检查
- 报告解释能力（分布、归因、可比格式）
### ❌ 禁止（直到 Gate 解锁）
- 新交易规则 / 动态 spread / inventory skew
- 新过滤条件（Pulse / AI）
- 新收益优化参数（为结果改 fee/slip/latency）
### 三不动
1. Quote Logic
2. 成本模型：`Net = Raw − Fee − Slip − Latency`
3. 失败定义：`PASS` / `PARTIAL_PASS` / `COLLECTING` / `FAIL`
---
## Report v0.1 固定格式
```
Executive Summary
Section 1 — Data Integrity
Section 2 — Fill Alpha (+ distribution: mean/median/p25/p75)
Section 3 — Toxicity Profile (+ loss concentration)
Section 4 — Observed Edge Attribution (事实，非策略建议)
Section 5 — Decision + Stage3 unlock checklist
```
第一份报告不期待 PASS。价值在于：市场在哪些情况下愿意付给流动性提供者溢价。
---
## Experiment ID（强制绑定）
每轮 Data Collection 绑定固定身份，写入每条 jsonl + 每份报告：
```
Experiment: MM_EDGE_EXP_001
Version: probe_v0.1
Quote: frozen
Fee: frozen
Exchange: frozen
```
环境变量：`EXPERIMENT_ID` / `PROBE_VERSION`（见 `.env.example`）。
换实验假设时换新 ID（如 `MM_EDGE_EXP_002`），禁止在同一 ID 下改报价逻辑后重解释旧数据。
报告阅读顺序：**Integrity → 分布（非均值）→ Cluster → Toxicity → Decision**。
Integrity FAIL → `Decision=INVALID`，不解释 Alpha。
---
## Post-Report Phase (2026-08-18)
Maker Edge Report → **PARTIAL_PASS**.
Account wallet moved ≈ −62 USDT vs assumed 5000 start — **not** Maker Edge FAIL evidence;
also **not** ignorable. Research markout ≠ account equity.
**Probe volume STOPPED** until:
1. Maker-only hard check: `TAKER_FILLED_COUNT == 0` (exchange `userTrades.maker`)
2. Order→Fill→Fee→Position→Funding→Equity ledger residual ≈ 0
See `STATUS.md` and `scripts/reconcile_account.py`.
Stage3 remains **LOCKED**. Economic Edge = **UNKNOWN**.
---
## Economic Attribution Phase (2026-08-18)
`MM_EDGE_EXP_001 / probe_v0.1` is now a **FROZEN BASELINE**.
Execution state:
- Probe: **STOPPED**
- Strategy modifications: **forbidden**
- Purpose: **Economic Attribution only**
First hard-evidence population:
```
MATCHED = 3890
```
This means:
- Local Fill
- Venue Trade
- venue_trade_id
- quantity closure
- price verification
- fee verification
- maker-only verification
Economic Attribution v0.1 must:
1. Use **MATCHED only** for core conclusions
2. Keep `VENUE_CONFIRMED_NO_TRADE_HISTORY` / `VENUE_PARTIAL_ORDER_CANCELED`
as extension evidence, not core population
3. Treat counterfactuals as **attribution**, not backtest / simulation
Current baseline conclusion:
> Maker markout exists, but economic edge is **not established** under v0.1.
Known explanation path:
```
MakerAlpha
↓
matched fills
↓
fee + realized / inventory economics
↓
account outcome
```
Next allowed work:
- Fee attribution
- Inventory carry / exposure attribution
- Counterfactual attribution (`Path C`, toxic, negative states)
- Real-time immutable fill ledger design for future runs
Still forbidden:
- Resume v0.1 live execution
- Change quote offset / TTL / cooldown as a shortcut
- Unlock Stage3 from attribution alone
---
## Prefill Adverse-Selection Attribution Phase (2026-08-18)
Purpose: **Pre-fill adverse-selection predictability audit** — NOT strategy / backtest / optimization / model training.
Hard contract:
```
feature_timestamp <= t_fill - 0.25s
```
Features: sampled `mid_tick` / `inventory_tick` only.
Forbidden as features: fill price, post-fill states, Path A/B/C/D, markout, future book/trade/inventory, realized PnL, cancel-after-fill.
Population: **MATCHED paths = 3886** (100% strict coverage)
Executability gate (three tiers):
| Grade | Meaning |
|-------|---------|
| `NO_PREFILL_SIGNAL` | P(C) / economic almost unchanged |
| `STATISTICAL_SIGNAL_ONLY` | probability shift, economic improvement insufficient |
| `CANDIDATE_V0_2_SIGNAL` | probability + economic separation — only this enters v0.2 hypothesis |
Sample-size policy:
- n < 30 → `LOW_N` (exploratory only)
- n < 100 → `WEAK_EVIDENCE`
- n >= 100 → `USABLE`
v0.1 strict conclusion:
> **0 / 19 features** reach `CANDIDATE_V0_2_SIGNAL`.
> Fill 前可观测信号不足以支撑 v0.2 设计。Stage 3 remains **LOCKED**.
Deferred (requires richer pre-fill event log):
- `time_since_last_market_event`
- intensity / large trades
- fill-callback `market_event_before_fill`
Still forbidden:
- Treat `STATISTICAL_SIGNAL_ONLY` as v0.2 candidate
- Resume probe or design v0.2 strategy without new experiment ID + prefill signal pass
---
## MM_EDGE_EXP_002 — Event-State Observability (2026-08-18)
**Type:** Data Collection / Observability Experiment
**NOT:** strategy experiment, backtest, optimization, model training
```
MM_EDGE_EXP_001 → phenomenon PASS, economic FAIL, prefill FAIL (snapshot)
MM_EDGE_EXP_002 → close observability gap (immutable event ledger)
→ Gate 4 only after Gates 1–3 + fill anchors
→ (only then) v0.2 hypothesis allowed
```
Identity:
| Field | Value |
|-------|-------|
| Experiment | MM_EDGE_EXP_002 |
| Probe | event_state_v0.1 |
| Strategy | NONE |
| Trading | NO |
| Stage 3 | LOCKED |
| Depends on | MM_EDGE_EXP_001 FROZEN |
Core change: **Immutable Event Ledger** — raw events first, features offline later.
Pre-fill window schema (frozen):
```
[-5s, fill - 250ms) → all market_event rows
fill_anchor → immutable fill metadata (Gate 4; Phase 1 may have none)
```
Success gates:
1. **Event Completeness** — PASS (smoke)
2. **Temporal Integrity** — PASS (smoke)
3. **Event Coverage** — PASS (smoke)
4. **Predictability** — BLOCKED until frozen sample gates in `MM_EDGE_EXP_002.md`
Long-run: `EXP-002-RUN-002` / `event-state-probe.service` / trading=NO.
Frozen before long-run (do not change after seeing more data):
- Phase 1 dataset: ≥ 7 days AND ≥ 5,000,000 market events
- Gate 4: ≥ 2,000 MATCHED fill_anchors AND ≥ 500 clusters (requires later fill-authorized phase)
No daily Path C analysis during collection.
Still forbidden:
- Resume EXP_001 or modify v0.1 quote logic
- Enable trading under EXP_002
- Unlock Stage 3 from observability data alone
- Peek at Gate 4 before the frozen sample threshold
---
## MM_EDGE_EXP_002 Phase 1 STOPPED (2026-09-10)
Human decision: **stop collection**. Do not open a fill-authorized phase. Do not design v0.2.
| Field | Value |
|-------|-------|
| Run | `EXP-002-RUN-002` |
| Stopped | **2026-09-10T08:45:24Z** |
| systemd | user `event-state-probe` **inactive / disabled** |
| Ledger | `logs/event_state/EXP-002-RUN-002/` (24 files, 18.06 GiB, 2026-08-18 → 2026-09-10 partial) |
| Trading | **NO** |
| Stage 3 | **LOCKED** |
| Gate 4 | **BLOCKED** (no `fill_anchor`) |
| Hypothesis Closure | **NO** |
Phase 1 volume thresholds are met as an **observability freeze** only.
That does **not** unlock Gate 4 or Stage 3.
Marker: `logs/event_state/EXP-002-RUN-002/EXP-002-RUN-002.PHASE1_STOPPED.json`
Still forbidden:
- Restart `EXP-002-RUN-002`
- Enable trading under EXP_002
- Path-C / Gate 4 peek on this artifact
- Treat this stop as “no pre-fill signal exists”
MM_EDGE_EXP_002.md
+214
View File
@@ -0,0 +1,214 @@
# MM_EDGE_EXP_002 — Event-State Observability Probe
## Experiment Identity
```
Experiment: MM_EDGE_EXP_002
Type: Data Collection / Observability Experiment
Strategy: NONE
Execution: STOPPED (no trading)
Purpose: Capture immutable pre-fill Event State
Dependency: MM_EDGE_EXP_001 / v0.1 FROZEN
Stage 3: LOCKED
```
**NOT:** strategy experiment, backtest, optimization, model training, v0.2 design.
---
## Why EXP_002 Exists
EXP_001 Prefill Audit conclusion:
> Under **current snapshot observability** (~1.66s sampled mid/inventory state), no sufficient pre-fill signal was found.
This must **not** be interpreted as:
> The market has no pre-fill adverse-selection information.
EXP_001 strategy is **event-driven**, but observability was **snapshot-driven**. Information between snapshots (e.g. aggressive sweep 100ms before fill) is lost.
EXP_002 closes the **observability gap**, not the **strategy gap**.
---
## Core Design Principle
```
Raw Event > Derived Feature
```
Store immutable events. Features are computed offline later.
---
## Immutable Event Ledger
Each market event records (minimum):
| Field | Description |
|-------|-------------|
| `exchange_ts_ns` | Exchange event time |
| `local_ts_epoch` / `local_ts` | Local receive time |
| `event_type` | `aggressive_trade`, `book_update`, … |
| `best_bid` / `best_ask` / `mid` / `spread` | Top-of-book |
| `bid_depth_*` / `ask_depth_*` | Depth levels |
| `*_delta` | Depth / spread / mid changes |
| `time_since_last_*` | Event timing state |
Fill anchor schema (frozen, for Gate 4 when fills exist):
```
[-5s, fill - 250ms) → all market_event rows
fill_anchor → immutable fill metadata
```
---
## Event Categories (Priority)
1. **Aggressive Trade** — side, qty, notional, large_trade_flag, intensity proxy
2. **Book Depletion** — depth deltas, level removal velocity
3. **Spread / TOB Event** — spread change, bid/ask/mid move
4. **Event Timing** — time_since_last_trade / large_trade / depth_change / spread_change
---
## Success Gates (frozen before collection)
### Gate 1 — Event Completeness
For each `fill_anchor`:
```
100% reconstructable pre-fill event sequence in [-5s, fill - margin)
```
Phase 1 (observability-only): **N/A** until fill anchors exist.
### Gate 2 — Temporal Integrity
```
all event_ts < fill_ts
feature_cutoff = fill_ts - 250ms
```
### Gate 3 — Event Coverage
| Metric | Threshold |
|--------|-----------|
| trade events present | ≥ 99% of sessions with trades |
| book events present | ≥ 99% of sessions with book updates |
| timestamp valid | ≥ 99% rows with exchange_ts_ns or local_ts |
### Gate 4 — Predictability (BLOCKED until sample freeze + fill anchors)
Do **not** inspect Path C daily during collection (researcher degrees of freedom).
Frozen sample thresholds (**set 2026-08-18, before long-run start**):
**Phase 1 Event-State dataset freeze** (observability-only, no fills):
| Metric | Minimum |
|--------|---------|
| Calendar span | **≥ 7 days** |
| `market_event` count | **≥ 5,000,000** |
| `aggressive_trade` | **≥ 1,500,000** |
| `book_update` | **≥ 3,000,000** |
| timestamp valid | **≥ 99%** |
| parse_fail_lines | **0** |
Reaching this freeze **does not** unlock Gate 4. It only freezes the Event-State stream as a research artifact.
**Gate 4 (requires a later fill-authorized phase, not this systemd job):**
| Metric | Minimum |
|--------|---------|
| MATCHED `fill_anchor` | **≥ 2,000** |
| `event_cluster_id` | **≥ 500** |
| reconstructable `[-5s, fill−250ms)` | **100%** of MATCHED anchors |
| venue_trade_id + exchange_ts_ns on fill | **100%** |
Then, **once**:
```
Event State → P(Path C) → Economic separation
→ CANDIDATE_V0_2_SIGNAL only if both probability and economic gates pass
```
Future fill collection (if ever authorized) **must** write:
```
Fill → venue_trade_id → exchange_ts_ns → Event Ledger → [-5s, fill_ts)
```
Do not resume snapshot-only fills.
Only **CANDIDATE_V0_2_SIGNAL** after Gate 4 may enter v0.2 hypothesis design.
---
## Research Chain
```
EXP_001 Maker Edge Phenomenon
↓
conditional markout exists
↓
Economic FAIL
↓
Prefill audit (snapshot) → NO SIGNAL
↓
EXP_002 Event-State Observability
↓
Gate 1–3 PASS?
↓
Gate 4 predictability
↓
(only then) v0.2 hypothesis
```
---
## Running
```bash
cd nautilus_mm
cp .env.example .env # set EXP_002 block
export PYTHONPATH=src
./scripts/run_event_state.sh
```
Long-run (systemd, trading=NO):
```
LEDGER_RUN_ID=EXP-002-RUN-002
logs/event_state/EXP-002-RUN-002/
```
```bash
systemctl --user start event-state-probe
./scripts/event_state_status.sh # counts only — not Path C analysis
```
**Phase 1 STOPPED 2026-09-10T08:45:24Z** — user unit `event-state-probe` is **disabled**.
Do not start it again on `EXP-002-RUN-002`. Ledger is a frozen observability artifact.
---
## Forbidden
- Resume EXP_001 probe or modify v0.1 quote logic
- Restart EXP_002 / `EXP-002-RUN-002` collection
- Enable trading under EXP_002
- Unlock Stage 3 from observability data alone
- Treat weak EXP_001 statistical signals as v0.2 filters
- Claim Hypothesis Closure or design v0.2 without a later fill-authorized Gate 4
---
## Hypothesis Closure (if Gate 4 also fails)
> Conditional Maker phenomenon exists, but is not sufficiently predictable pre-fill to be monetizable under this venue/execution model.
That would be a **strong Research FAIL / Hypothesis Closure** — not "try one more parameter."
README.md
+158
View File
@@ -0,0 +1,158 @@
# nautilus_mm — Research Freeze / Data Collection Phase
Freqtrade 保留缠论 / 中低频；**Maker / L2 / Fill 事件**迁到 NautilusTrader。
**研究对象：可验证的市场现象（不是策略）。**
```
Trading OS
├─ Market Intelligence (Market Pulse) ← Stage5 才接（Quote Adjustment）
├─ Execution Reality Layer (本仓库) ← Fill Alpha Dataset
├─ Freqtrade ← Chan / 中低频
└─ NautilusTrader ← 事件驱动执行
```
## 冻结研究路径（禁止跳级）
```
Stage 0 Data Integrity
↓
Stage 1 Fill Alpha ← 当前
↓
Stage 2 Maker Edge Report ← 当前（决策门）
↓
Stage 3 Economic Simulation ← LOCKED until Edge PASS
↓
Stage 4 Quote Engine
↓
Stage 5 Market Regime Adaptation (Market Pulse → Quote Adjustment)
```
| Stage | 目标 | 状态 |
|-------|------|------|
| **0** | WS/L2 可信：seq gap / latency / book_age | 探针已记 |
| **1** | Fill Alpha Dataset：真实成交 + 路径 | **进行中** |
| **2** | Maker Edge 决策门：PASS / FAIL / COLLECTING | **进行中** |
| **3** | 经济仿真：quote→fill→inventory→exit（partial/cancel/funding/fee） | **未解锁** |
| **4** | Quote Engine | 未开始 |
| **5** | Regime Adaptation | 未开始 |
原则：**先证明成交有优势，再谈账户收益，再设计报价。**
禁止现在加：Quote Engine / AI / Market Pulse 交易信号 / 参数优化。
### 核心问题（交给 ~2000 真实 fills）
> 个人开发者在 BTC 永续上，通过被动流动性提供，是否能获得统计优势？
> 若有：优势来自哪里？
可能结果：
| 情况 | 含义 | 下一步 |
|------|------|--------|
| **A** 全市场 PASS | 稳定被动流动性优势 | Symmetric MM |
| **B** 仅特定状态 PASS | 高波动 / 吸收 / 震荡等 | Event-driven LP（更可能） |
| **C** 全部 FAIL | 普通 Maker edge 不存在 | Cash Carry / Basis / Funding / 跨所 |
**最值得等待的不是 PASS，而是 Edge 来自哪里。** 第一份 FAIL 也是高价值结果。
### 防自我欺骗（已内建）
1. **Cluster-weighted** — 暴跌 50 笔 Bid ≠ 50 独立样本；同时看 fill-w 与 cluster-w，方向一致才可信
2. **Matched Mid / Maker Alpha** — `MakerAlpha = Fill Outcome − Market Move`（剥离方向收益）
3. **Adverse Selection** — 成交是否天然站在错误一侧；spread capture 挡不住毒流
Stage 3（Economic Simulation）只在 Edge PASS 后做：partial fill、cancel latency、inventory limit、position aging、funding、fee tier → 真实账户收益分布。
## Maker Edge Report v0.1
```bash
./scripts/analyze.sh # 默认 min-fills=2000
./scripts/analyze.sh 500 # 早期预览（仍为 COLLECTING）
```
报告结构：Data Integrity → Sample Independence → Fill Quality + Benchmark → Adverse Selection（`POSITIVE_EDGE` / `EDGE_AFTER_COST` / `NO_EDGE`）→ Path Attribution → State Stability → Decision。
## 快速开始
```bash
cd nautilus_mm
python3 -m venv .venv && source .venv/bin/activate
pip install -r requirements.txt
cp .env.example .env # TESTNET key；代理默认 7897
./scripts/run_probe.sh # 攒真实 fills
./scripts/analyze.sh 2000 # 决策门报告
```
探针只记 quote / fill / outcome。`market_state_snapshot` 预留为 null。
## 目录
```
nautilus_mm/
├── configs/
├── logs/maker_edge/ # Fill Alpha Dataset (jsonl)
├── scripts/
│ ├── run_probe.sh
│ ├── analyze.sh
│ ├── analyze_maker_edge.py
│ └── record_l2_ccxt.py
└── src/nautilus_mm/
├── recorder.py
├── health.py
├── book_utils.py
├── run_live.py
└── strategies/maker_edge_probe.py
```
## 与 Freqtrade
| | Freqtrade | nautilus_mm |
|--|-----------|-------------|
| 用途 | 缠论 / 中低频 | Maker / L2 / Edge 验证 |
| 驱动 | K 线 | order book / fill 事件 |
## 安全
- 默认 `BINANCE_ENVIRONMENT=TESTNET`
- `ENABLE_TRADING=false` 可只订数据
- **独立 `.venv`**，勿与 freqtrade 混装
## 实验冻结（见 [FREEZE.md](FREEZE.md)）
**三不动：** Quote Logic / 成本模型 / PASS·COLLECTING·FAIL
| ✅ 冻结期允许 | ❌ Gate 解锁前禁止 |
|-------------|-------------------|
| 数据字段 | 新交易规则 |
| 数据质量检查 | 新过滤条件 |
| 报告解释能力 | 新收益优化参数 |
Decision：`PASS` / `PARTIAL_PASS` / `COLLECTING` / `FAIL`
观察窗：500 异常 · 2000 初步 · 10000 稳定性；**clusters > fills**。
第一份报告先看分布（Fill / Cluster / Toxicity），再看 Decision。
## 服务器 Data Collection（zun_hk）
默认：
| | |
|--|--|
| Host | `jack@jackyu66.com` |
| Key | `~/Project/deploy/zun_hk/id_ed25519_hk` |
| Dir | `/www/Project/nautilus_mm` |
| Experiment | `MM_EDGE_EXP_001` |
```bash
./scripts/deploy_server.sh
ssh -i ~/Project/deploy/zun_hk/id_ed25519_hk jack@jackyu66.com
nano /www/Project/nautilus_mm/.env # BINANCE_API_KEY / SECRET
systemctl --user start mm-edge-probe # 用户级 systemd（无需 sudo）
journalctl --user -u mm-edge-probe -f
./scripts/probe_status.sh
./scripts/pull_report.sh 2000
```
本地短测：`USE_PROXY=true ./scripts/run_probe.sh`
STATUS.md
+205
View File
@@ -0,0 +1,205 @@
# Project Status
## MM_EDGE_EXP_001 — CLOSED (FROZEN)
| Dimension | Status |
|-----------|--------|
| Research Phenomenon | **PASS** (conditional markout exists) |
| Economic Edge | **FAIL** |
| Prefill Predictability | **FAIL** (snapshot observability) |
| Strategy | **FROZEN** |
| Execution | **STOPPED** |
| Stage 3 | **LOCKED** |
EXP_001 Prefill strict conclusion:
> Under current **snapshot observability** (~1.66s), no CANDIDATE_V0_2_SIGNAL.
> This is **NOT** proof that pre-fill signal does not exist in the market.
Weak signals (research observations only — **not** trading filters):
- `depth_total_5`: ±3.8pp Path C separation, Economic gate FAIL
- `obi_change_5s`: +0.00027 USDT/fill economic Δ
- `pre_deteriorated_strict=False`: +0.00116 USDT/fill economic Δ
Reports: `logs/maker_edge/` — see sections below for detail.
---
## MM_EDGE_EXP_002 — Phase 1 STOPPED (artifact FROZEN)
| Dimension | Status |
|-----------|--------|
| Type | Data Collection / Observability |
| Strategy | **NONE** |
| Trading | **NO** |
| Execution | **STOPPED** 2026-09-10T08:45:24Z |
| Stage 3 | **LOCKED** |
| Gate 1 | **PASS** (smoke `EXP-002-RUN-001`) |
| Gate 2 | **PASS** |
| Gate 3 | **PASS** |
| Gate 4 | **BLOCKED** (no `fill_anchor`; not opened) |
| Long-run | **EXP-002-RUN-002** STOPPED + systemd **disabled** |
| Hypothesis Closure | **NO** — Gate 4 never ran |
Human decision 2026-09-10: stop further collection. Phase 1 is an observability artifact only.
This is **not** “market has no pre-fill signal” and **not** a v0.2 unlock.
`event-state-probe.service` (user): **inactive / disabled**. Do not restart `EXP-002-RUN-002`.
Ledger: `logs/event_state/EXP-002-RUN-002/` — 24 files, 2026-08-18 → 2026-09-10 (last day partial), **18.06 GiB**.
Event counts not fully censused; size-based estimate ~1.2e7 lines (volume gates clearly exceeded).
`fill_anchor` expected **0** (observability-only). Marker: `EXP-002-RUN-002.PHASE1_STOPPED.json`
Do **not** Path-C snoop this artifact. Frozen Gate 4 sample gates remain in `MM_EDGE_EXP_002.md`.
Spec: `MM_EDGE_EXP_002.md`
Smoke: `./scripts/smoke_test_event_state.sh`
Validate: `python scripts/validate_event_ledger.py --dir logs/event_state/EXP-002-RUN-001 --run-id EXP-002-RUN-001`
Logs: `logs/event_state/EXP-002-RUN-001/`
### EXP-002-RUN-001 Smoke (2026-08-18, ~12 min + restart)
Host: `jADUtR1041803` | Sessions: **2** (restart test) | Schema: `immutable_event_v1`
| Check | Result |
|-------|--------|
| Gate 1 Event Completeness | **PASS** (Phase 1 stream; no fill_anchor) |
| Gate 2 Temporal Integrity | **PASS** |
| Gate 3 Event Coverage | **PASS** |
| Restart contract | **PASS** (0 parse fail, 0 dup event_id, seq reset) |
| Gate 4 | **BLOCKED** |
Write rates:
- `aggressive_trade`: **3.21 / sec** (2300 events)
- `book_update`: **6.09 / sec** (4366 events)
- **total**: **9.30 / sec** (6666 market events)
Timestamp quality (100% exchange + local present):
- `local − exchange` lag: p50 **112.5ms**, p95 **237.7ms**, p99 **251.4ms**, max **443.4ms**
- `exchange > local + 50ms`: **0** violations
Event order (recorded, not sorted):
- `exchange_ts_ns` regressions: **1053** (max back **276ms**) — multi-source async; explicit in report
Raw schema sample (n=200 each): **0% missing** on core fields; **0** hollow book_update.
Manifest: `logs/event_state/EXP-002-RUN-001/EXP-002-RUN-001.manifest.json`
Report: `logs/event_state/EXP-002-RUN-001/Event_Ledger_Validation.json`
Long-run identity: **EXP-002-RUN-002** (separate from smoke) — **STOPPED** 2026-09-10. Status: `./scripts/event_state_status.sh`
---
## MM_EDGE_EXP_001 Detail
**Phase: Prefill Adverse-Selection Attribution v0.1** (probe **STOPPED**)
| Gate | Status |
|------|--------|
| Research Freeze | **ACTIVE** |
| Probe | **STOPPED** |
| Data Integrity | **PASS** |
| Maker-only | **PASS** (`TAKER=0`) |
| Account Reconciliation (RECON-01) | **PASS** |
| RECON-02 classification | **PASS** |
| Order-level closure (RECON-03) | **PASS** |
| Strict trade-level closure | **FAIL** (Testnet userTrades cutoff — **not** Alpha FAIL) |
| Economic Edge | **FAIL** (current execution economics) |
| Prefill Predictability (v0.1 strict) | **FAIL** (no CANDIDATE_V0_2_SIGNAL) |
| Stage 3 | **LOCKED** |
## Evidence taxonomy (4777 fills)
| Class | Count | Grade |
|-------|-------|-------|
| **MATCHED** | 3890 | Order + Trade (dual) |
| **VENUE_CONFIRMED_NO_TRADE_HISTORY** | 876 | Order FILLED, no trade row |
| **VENUE_PARTIAL_ORDER_CANCELED** | 11 | Partial fill + TTL cancel |
| DUPLICATE / MISMATCH / UNCONFIRMED | 0 | — |
`userTrades` cutoff: **2026-08-17T03:08 UTC** — see `TESTNET_LIMITATIONS.md`
Matched trade-level: qty residual **0**, |Δt| p50 **72ms**
## Separation
```
MakerAlpha (+0.008%) ≠ Account Δ (−61.66 = fee + realized)
Strict trade FAIL ≠ Maker Edge FAIL
887 orphans = VENUE-HISTORY-CUTOFF (now order-confirmed)
```
## Economic Attribution v0.1 (Hard Evidence Population only)
Population: **MATCHED=3890**
Core report: `logs/maker_edge/Economic_Attribution_v0_1.txt`
- Matched fills / paths / clusters: **3890 / 3886 / 3334**
- Fee total: **42.42 USDT**
- Realized component: **−8.52 USDT**
- Gross markout @30s: **−1.73 USDT**
- Net attributable @30s: **−52.67 USDT**
- Inventory carry:
- Max `|net BTC|`: **0.0059**
- TW `|net BTC|`: **0.0035**
- Turnover: **3.3316 BTC**
Counterfactual Attribution:
- Baseline matched net30: **−52.67 USDT**
- Exclude `Path C`: **−21.22 USDT**
- Exclude toxic: **−21.22 USDT**
- Exclude negative states: **−27.64 USDT**
Interpretation:
- v0.1 已完成 baseline 使命：**Maker markout phenomenon exists, economic edge not established**
- 当前主要拖累不是“假 alpha”，而是 **fee + realized / inventory economics**
## Metric Reconciliation (MATCHED=3890)
- Return-space MakerAlpha（fill-weighted）：-0.000693%
- Return-space MakerAlpha（notional-weighted）：-0.000816%
- Dollar-space gross markout @30s：-1.730780 USDT
结论：回报口径一致，但“加权方式”导致返回与美元金额的方向差异。
## Fee Sensitivity (counterfactual, fee only)
net_attr_30s @fee_factor:
1.00 → -52.674655 USDT
0.50 → -31.463582 USDT
0.25 → -20.858046 USDT
0.10 → -14.494724 USDT
0.00 → -10.252510 USDT
结论：即使假设 0 fee，net 仍 < 0，因此“真实拖累”不仅是 fee。
## Prefill Adverse-Selection Attribution v0.1 (strict contract)
Population: **MATCHED paths = 3886** (100% strict-prefill coverage)
Core report: `logs/maker_edge/Prefill_Adverse_Selection_Attribution_v0_1.txt`
Time contract: `feature_timestamp <= t_fill - 0.25s`
Feature source: sampled `mid_tick` / `inventory_tick` only (no fill-callback leakage)
Baseline (strict population):
- P(Path C): **33.20%**
- P(Toxic): **34.61%**
- P(Neg30s): **47.43%**
- P(Economic<0): **77.61%**
- Mean net_attr_30s: **−0.0136 USDT/fill**
- Feature age: mean **1681ms**, median **1660ms**
Conclusion Matrix (auto-grade):
- **CANDIDATE_V0_2_SIGNAL**: **0 / 19 features**
- **STATISTICAL_SIGNAL_ONLY**: spread_change_5s (Toxic/Neg30s), inventory/inventory_skew (Economic only)
- **NO_PREFILL_SIGNAL**: all others under strict gate
Unavailable under strict contract (deferred to EXP_002):
- `time_since_last_market_event`, intensity, large trades, fill-callback `market_event_before_fill`
## Forbidden (both experiments)
- Resume EXP_001 probe without new experiment ID
- Restart EXP_002 / `EXP-002-RUN-002` collection
- Enable trading under EXP_002
- Reclassify VENUE_CONFIRMED as MATCHED
- Unlock Stage 3 on observability or attribution alone
- Treat STATISTICAL_SIGNAL_ONLY as v0.2 candidate
- Design v0.2 or claim Hypothesis Closure without a later fill-authorized Gate 4
TESTNET_LIMITATIONS.md
+55
View File
@@ -0,0 +1,55 @@
# Testnet Limitations — MM_EDGE_EXP_001
Observed on **Binance USDT-M Futures Testnet** during Data Collection (Aug 2026).
## userTrades history cutoff
```
Endpoint: GET /fapi/v1/userTrades
Observed max trade time (UTC): 2026-08-17T03:08:23
Behavior: Pagination returns 3890 rows; no further trades via time/fromId
after cutoff, even while probe continues to produce fills until
2026-08-18.
```
**Impact:** Local jsonl fill count can exceed paginated `userTrades` count.
This is **not** evidence of duplicate local logging or fake fills.
## Order API remains available
```
Endpoint: GET /fapi/v1/order?orderId=
Behavior: Post-cutoff orders return status=FILLED, executedQty, avgPrice
while userTrades?orderId= returns 0 rows for the same orderId.
```
RECON-03 classifies these as:
```
VENUE_CONFIRMED_NO_TRADE_HISTORY
```
Evidence grade: **Order only** (not dual Order+Trade).
## Income ledger continues
`GET /fapi/v1/income` continues to record COMMISSION / REALIZED_PNL after
the userTrades cutoff. Account reconciliation (RECON-01) uses income, not
userTrades alone.
## Implications for future runs
1. **Real-time immutable ledger** — persist on every `OrderFilled`:
`venue_trade_id`, `venue_order_id`, `liquidity_side`, `commission`,
`exchange_ts`, `local_ts`. Do not rely on post-hoc userTrades backfill.
2. **Reports must use evidence taxonomy** — never compare raw fill count to
userTrades count without cutoff annotation.
3. **Strict trade-level closure** may remain FAIL on Testnet while
**order-level closure** can still PASS.
## Maker-only constraint
Post-only orders rejected with `-5022` when they would take. Verified:
`TAKER_FILLED_COUNT = 0` on all 3890 trades inside userTrades window.
configs/event_state_probe.yaml
+31
View File
@@ -0,0 +1,31 @@
# MM_EDGE_EXP_002 — Event-State Observability Probe
# Data collection only. NO trading.
experiment_id: MM_EDGE_EXP_002
probe_version: event_state_v0.1
experiment_type: Event-State Observability Probe
strategy: NONE
execution_trading: false
depends_on: MM_EDGE_EXP_001
quote: frozen
fee: frozen
exchange: frozen
trader_id: EVENT-STATE-002
symbol: BTCUSDT-PERP
account_type: USDT_FUTURES
environment: TESTNET
book_depth: 10
prefill_window_sec: 5.0
prefill_margin_sec: 0.25
large_trade_qty: 0.1
log_every_book_delta: true
log_dir: logs/event_state
# Gate 3 thresholds (frozen)
gate3_trade_coverage_min: 0.99
gate3_book_coverage_min: 0.99
gate3_timestamp_valid_min: 0.99
configs/maker_edge_probe.yaml
+24
View File
@@ -0,0 +1,24 @@
# Reference config (env vars in .env take precedence for secrets)
# Experiment identity — do not change mid-run to "make report look better"
experiment_id: MM_EDGE_EXP_001
probe_version: probe_v0.1
quote: frozen
fee: frozen
exchange: frozen
trader_id: MAKER-EDGE-001
symbol: BTCUSDT-PERP
account_type: USDT_FUTURES
environment: TESTNET # TESTNET | LIVE
order_qty: "0.001"
book_depth: 10
quote_offset_ticks: 1
max_quotes: 1
cooldown_secs: 60
obi_enter: 0.25
enable_trading: true
proxy: http://127.0.0.1:7897
log_dir: logs/maker_edge
deploy/event-state-probe.service
+32
View File
@@ -0,0 +1,32 @@
[Unit]
Description=MM_EDGE_EXP_002 Event-State Observability Probe (no trading)
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=jack
Group=jack
WorkingDirectory=/www/Project/nautilus_mm
Environment=PYTHONPATH=/www/Project/nautilus_mm/src
EnvironmentFile=-/www/Project/nautilus_mm/.env
Environment=ENABLE_TRADING=false
Environment=EXPERIMENT_ID=MM_EDGE_EXP_002
Environment=PROBE_VERSION=event_state_v0.1
Environment=LEDGER_RUN_ID=EXP-002-RUN-002
Environment=EVENT_STATE_LOG_DIR=/www/Project/nautilus_mm/logs/event_state/EXP-002-RUN-002
Environment=HTTP_PROXY=
Environment=HTTPS_PROXY=
Environment=http_proxy=
Environment=https_proxy=
ExecStart=/www/Project/nautilus_mm/scripts/run_event_state.sh
Restart=always
RestartSec=10
StandardOutput=journal
StandardError=journal
SyslogIdentifier=event-state-probe
NoNewPrivileges=true
PrivateTmp=true
[Install]
WantedBy=multi-user.target
deploy/event-state-probe.user.service
+30
View File
@@ -0,0 +1,30 @@
# 用户级 systemd — ~/.config/systemd/user/event-state-probe.service
# MM_EDGE_EXP_002 Phase 1 long-term Event-State collection. Trading = NO.
[Unit]
Description=MM_EDGE_EXP_002 Event-State Observability Probe (no trading)
After=network-online.target
[Service]
Type=simple
WorkingDirectory=/www/Project/nautilus_mm
Environment=PYTHONPATH=/www/Project/nautilus_mm/src
EnvironmentFile=-/www/Project/nautilus_mm/.env
# Layer 0 (systemd): override .env — EXP_002 never trades
Environment=ENABLE_TRADING=false
Environment=EXPERIMENT_ID=MM_EDGE_EXP_002
Environment=PROBE_VERSION=event_state_v0.1
Environment=LEDGER_RUN_ID=EXP-002-RUN-002
Environment=EVENT_STATE_LOG_DIR=/www/Project/nautilus_mm/logs/event_state/EXP-002-RUN-002
Environment=HTTP_PROXY=
Environment=HTTPS_PROXY=
Environment=http_proxy=
Environment=https_proxy=
ExecStart=/www/Project/nautilus_mm/scripts/run_event_state.sh
Restart=always
RestartSec=10
StandardOutput=journal
StandardError=journal
SyslogIdentifier=event-state-probe
[Install]
WantedBy=default.target
deploy/mm-edge-probe.service
+28
View File
@@ -0,0 +1,28 @@
[Unit]
Description=Maker Edge Probe MM_EDGE_EXP_001 (Research Freeze / Data Collection)
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=jack
Group=jack
WorkingDirectory=/www/Project/nautilus_mm
Environment=PYTHONPATH=/www/Project/nautilus_mm/src
EnvironmentFile=-/www/Project/nautilus_mm/.env
# 服务器直连交易所
Environment=HTTP_PROXY=
Environment=HTTPS_PROXY=
Environment=http_proxy=
Environment=https_proxy=
ExecStart=/www/Project/nautilus_mm/.venv/bin/python -m nautilus_mm.run_live
Restart=always
RestartSec=10
StandardOutput=journal
StandardError=journal
SyslogIdentifier=mm-edge-probe
NoNewPrivileges=true
PrivateTmp=true
[Install]
WantedBy=multi-user.target
deploy/mm-edge-probe.user.service
+21
View File
@@ -0,0 +1,21 @@
# 用户级 systemd（无需 sudo）— 安装到 ~/.config/systemd/user/
[Unit]
Description=Maker Edge Probe MM_EDGE_EXP_001
After=network-online.target
[Service]
Type=simple
WorkingDirectory=/www/Project/nautilus_mm
Environment=PYTHONPATH=/www/Project/nautilus_mm/src
EnvironmentFile=-/www/Project/nautilus_mm/.env
Environment=HTTP_PROXY=
Environment=HTTPS_PROXY=
ExecStart=/www/Project/nautilus_mm/.venv/bin/python -m nautilus_mm.run_live
Restart=always
RestartSec=10
StandardOutput=journal
StandardError=journal
SyslogIdentifier=mm-edge-probe
[Install]
WantedBy=default.target
deploy_server.sh
Executable
+95
View File
@@ -0,0 +1,95 @@
#!/usr/bin/env bash
# 部署 MM_EDGE_EXP_001 → jack@jackyu66.com:/www/Project/nautilus_mm
#
# 默认：
# SSH_HOST=jack@jackyu66.com
# SSH_KEY=~/Project/deploy/zun_hk/id_ed25519_hk
# REMOTE_DIR=/www/Project/nautilus_mm
#
# 覆盖：export SSH_HOST=... SSH_KEY=... REMOTE_DIR=...
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
SSH_HOST="${SSH_HOST:-jack@jackyu66.com}"
SSH_KEY="${SSH_KEY:-${HOME}/Project/deploy/zun_hk/id_ed25519_hk}"
REMOTE_DIR="${REMOTE_DIR:-/www/Project/nautilus_mm}"
if [[ ! -f "$SSH_KEY" ]]; then
echo "SSH key not found: $SSH_KEY"
exit 1
fi
chmod 400 "$SSH_KEY" 2>/dev/null || true
SSH_OPTS=(-i "$SSH_KEY" -o StrictHostKeyChecking=accept-new)
SSH=(ssh "${SSH_OPTS[@]}" "$SSH_HOST")
RSYNC_E="ssh ${SSH_OPTS[*]}"
echo "==> stop remote probe before sync (if running)"
"${SSH[@]}" "systemctl --user stop mm-edge-probe 2>/dev/null || true"
echo "==> sync $ROOT → $SSH_HOST:$REMOTE_DIR"
"${SSH[@]}" "mkdir -p '$REMOTE_DIR' '$REMOTE_DIR/logs/maker_edge'"
rsync -avz --delete \
-e "$RSYNC_E" \
--exclude '.venv' \
--exclude '__pycache__' \
--exclude '*.pyc' \
--exclude 'logs/maker_edge/*.jsonl' \
--exclude 'logs/maker_edge/*.txt' \
--exclude 'logs/maker_edge_smoke' \
--exclude '.env' \
"$ROOT/" "$SSH_HOST:$REMOTE_DIR/"
echo "==> remote setup (uv venv + user systemd)"
"${SSH[@]}" "REMOTE_DIR='$REMOTE_DIR' bash -s" <<'REMOTE'
set -euo pipefail
export PATH="$HOME/.local/bin:$PATH"
cd "$REMOTE_DIR"
if [[ ! -f .env ]]; then
cp .env.example .env
{
echo ""
echo "# Server Data Collection — MM_EDGE_EXP_001"
echo "EXPERIMENT_ID=MM_EDGE_EXP_001"
echo "PROBE_VERSION=probe_v0.1"
echo "EXCHANGE_NAME=binance_usdm"
echo "BINANCE_ENVIRONMENT=TESTNET"
echo "ENABLE_TRADING=false"
echo "QUOTE_TTL_SECS=30"
echo "MAX_ABS_INVENTORY=0.005"
echo "HTTP_PROXY="
echo "HTTPS_PROXY="
echo "MAKER_EDGE_LOG_DIR=${REMOTE_DIR}/logs/maker_edge"
} >> .env
echo "CREATED .env — fill BINANCE_API_KEY / BINANCE_API_SECRET"
else
echo ".env exists — left untouched"
fi
if [[ ! -x "$HOME/.local/bin/uv" ]]; then
curl -LsSf https://astral.sh/uv/install.sh | sh
fi
uv python install 3.12
rm -rf .venv
uv venv .venv --python 3.12
uv pip install -r requirements.txt --python .venv/bin/python
mkdir -p "$HOME/.config/systemd/user"
sed -e "s|/www/Project/nautilus_mm|${REMOTE_DIR}|g" \
deploy/mm-edge-probe.user.service > "$HOME/.config/systemd/user/mm-edge-probe.service"
systemctl --user daemon-reload
systemctl --user enable mm-edge-probe.service
loginctl enable-linger "$(whoami)" 2>/dev/null || true
echo "User systemd installed (not started — fill keys first)."
echo " nano $REMOTE_DIR/.env"
echo " systemctl --user start mm-edge-probe"
echo " journalctl --user -u mm-edge-probe -f"
REMOTE
echo ""
echo "==> done"
echo "1) ssh -i $SSH_KEY $SSH_HOST"
echo "2) nano $REMOTE_DIR/.env # TESTNET keys"
echo "3) systemctl --user start mm-edge-probe"
echo "4) ./scripts/probe_status.sh"
echo "5) ./scripts/pull_report.sh"
economic_attribution_v0_1.py
+533
View File
@@ -0,0 +1,533 @@
#!/usr/bin/env python3
"""
Economic Attribution v0.1
Hard Evidence Population only:
MATCHED = Local Fill ↔ Venue Trade dual evidence
Purpose:
Economic Attribution only.
No strategy modification.
No live execution.
No economic simulation.
"""
from __future__ import annotations
import argparse
import json
import math
import sys
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import numpy as np
import pandas as pd
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from reconcile_fills import load_local_fills, match, normalize_local, normalize_venue # noqa: E402
def _load_jsonl_df(log_dir: Path) -> pd.DataFrame:
rows: list[dict[str, Any]] = []
for f in sorted(log_dir.glob("*.jsonl")):
if f.name.startswith(("Account_", "Maker_", "RECON")):
continue
for line in f.open():
try:
e = json.loads(line)
except Exception:
continue
if isinstance(e, dict):
rows.append(e)
return pd.DataFrame(rows)
def _parse_fill_context(df: pd.DataFrame) -> pd.DataFrame:
if df.empty or "fill_context" not in df.columns:
return pd.DataFrame(columns=["fill_id"])
rows = []
for _, r in df.iterrows():
ctx = r.get("fill_context")
if not isinstance(ctx, dict):
continue
rows.append(
{
"fill_id": r.get("fill_id"),
"market_event_before_fill": ctx.get("market_event_before_fill"),
"trade_imbalance_5s": ctx.get("trade_imbalance_5s"),
"price_velocity_5s": ctx.get("price_velocity_5s"),
"fill_type": ctx.get("fill_type"),
}
)
return pd.DataFrame(rows)
def _fav_ret(side: pd.Series, fill: pd.Series, px: pd.Series) -> pd.Series:
raw = (pd.to_numeric(px, errors="coerce") - pd.to_numeric(fill, errors="coerce")) / pd.to_numeric(
fill, errors="coerce"
)
return pd.Series(np.where(side == "long", raw, -raw), index=side.index)
def _cluster_weight(frame: pd.DataFrame) -> pd.Series:
cnt = frame.groupby("event_cluster_id")["event_cluster_id"].transform("count")
return 1.0 / cnt.clip(lower=1)
def _pct(v: float | None) -> str:
if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))):
return "n/a"
return f"{v * 100:.4f}%"
def _num(v: float | None, digits: int = 4) -> str:
if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))):
return "n/a"
return f"{v:.{digits}f}"
def _mean(s: pd.Series) -> float | None:
s = pd.to_numeric(s, errors="coerce").dropna()
return None if s.empty else float(s.mean())
def _sum(s: pd.Series) -> float:
s = pd.to_numeric(s, errors="coerce").fillna(0.0)
return float(s.sum())
def _weighted_mean(v: pd.Series, w: pd.Series) -> float | None:
vv = pd.to_numeric(v, errors="coerce")
ww = pd.to_numeric(w, errors="coerce").fillna(0.0)
mask = vv.notna() & ww.notna()
vv = vv[mask]
ww = ww[mask]
if vv.empty or float(ww.sum()) == 0.0:
return None
return float((vv * ww).sum() / ww.sum())
def _prepare_paths(paths: pd.DataFrame) -> pd.DataFrame:
paths = paths.copy()
if "max_price" in paths.columns and "min_price" in paths.columns and "fill_price" in paths.columns:
rng = (pd.to_numeric(paths["max_price"], errors="coerce") - pd.to_numeric(paths["min_price"], errors="coerce")) / pd.to_numeric(
paths["fill_price"], errors="coerce"
)
med = float(rng.dropna().median()) if rng.notna().any() else 0.0
paths["vol_bucket"] = np.where(rng >= med, "high_vol", "low_vol")
if "price_velocity_5s" in paths.columns and pd.to_numeric(paths["price_velocity_5s"], errors="coerce").notna().any():
v = pd.to_numeric(paths["price_velocity_5s"], errors="coerce")
thr = float(v.abs().median()) * 0.5
paths["trend_bucket"] = np.where(v > thr, "trend_up", np.where(v < -thr, "trend_down", "range"))
if "spread" in paths.columns and "fill_price" in paths.columns and pd.to_numeric(paths["spread"], errors="coerce").notna().any():
sp = pd.to_numeric(paths["spread"], errors="coerce") / pd.to_numeric(paths["fill_price"], errors="coerce")
med = float(sp.dropna().median()) if sp.notna().any() else 0.0
paths["liq_bucket"] = np.where(sp <= med, "tight_spread", "wide_spread")
paths["toxicity_bucket"] = np.where(paths["path_type"].astype(str).str.startswith("C"), "toxic", "non_toxic")
return paths
def _inventory_metrics(matched: pd.DataFrame) -> dict[str, float | None]:
if matched.empty:
return {}
g = matched.sort_values("venue_time_ms").copy()
g["signed_qty"] = np.where(g["side"] == "long", g["qty"], -g["qty"])
g["net_btc"] = g["signed_qty"].cumsum()
g["abs_net_btc"] = g["net_btc"].abs()
times = pd.to_numeric(g["venue_time_ms"], errors="coerce").astype("float64") / 1000.0
dt = times.shift(-1) - times
dt = dt.fillna(0.0).clip(lower=0.0)
total_t = float(dt.sum())
tw_abs = float((g["abs_net_btc"] * dt).sum() / total_t) if total_t > 0 else None
tw_signed = float((g["net_btc"] * dt).sum() / total_t) if total_t > 0 else None
return {
"max_net_btc": float(g["net_btc"].max()),
"min_net_btc": float(g["net_btc"].min()),
"max_abs_net_btc": float(g["abs_net_btc"].max()),
"avg_abs_net_btc_per_fill": float(g["abs_net_btc"].mean()),
"time_weighted_abs_net_btc": tw_abs,
"time_weighted_signed_net_btc": tw_signed,
"long_qty": float(g.loc[g["signed_qty"] > 0, "signed_qty"].sum()),
"short_qty": float((-g.loc[g["signed_qty"] < 0, "signed_qty"]).sum()),
"turnover_btc": float(g["qty"].sum()),
}
def _bucket_table(paths: pd.DataFrame, bucket: str, title: str) -> list[dict[str, Any]]:
if bucket not in paths.columns or paths.empty:
return []
rows = []
for key, grp in paths.groupby(bucket):
notional = grp["notional_usdt"].sum()
clusters = grp["event_cluster_id"].nunique()
rows.append(
{
"dimension": title,
"bucket": str(key),
"fills": int(len(grp)),
"clusters": int(clusters),
"btc_qty": float(grp["qty"].sum()),
"notional_usdt": float(notional),
"fee_usdt": float(grp["commission_usdt"].sum()),
"fee_per_fill": float(grp["commission_usdt"].mean()) if len(grp) else None,
"fee_per_btc": float(grp["commission_usdt"].sum() / grp["qty"].sum()) if grp["qty"].sum() else None,
"markout_1s": _weighted_mean(grp["markout_1s"], grp["notional_usdt"]),
"markout_5s": _weighted_mean(grp["markout_5s"], grp["notional_usdt"]),
"markout_10s": _weighted_mean(grp["markout_10s"], grp["notional_usdt"]),
"markout_30s": _weighted_mean(grp["markout_30s"], grp["notional_usdt"]),
"markout_300s": _weighted_mean(grp["markout_300s"], grp["notional_usdt"]),
"gross_markout_30s_usdt": float(grp["gross_markout_30s_usdt"].sum()),
"realized_pnl_usdt": float(grp["realized_pnl_usdt"].sum()),
"net_attr_30s_usdt": float(grp["net_attr_30s_usdt"].sum()),
}
)
rows.sort(key=lambda x: (-x["fills"], x["bucket"]))
return rows
def _counterfactual(base: pd.DataFrame, exclude_col: str, exclude_values: set[str], label: str) -> dict[str, Any]:
kept = base[~base[exclude_col].astype(str).isin(exclude_values)].copy()
return {
"name": label,
"fills": int(len(kept)),
"clusters": int(kept["event_cluster_id"].nunique()) if not kept.empty else 0,
"btc_qty": float(kept["qty"].sum()) if not kept.empty else 0.0,
"fee_usdt": float(kept["commission_usdt"].sum()) if not kept.empty else 0.0,
"gross_markout_30s_usdt": float(kept["gross_markout_30s_usdt"].sum()) if not kept.empty else 0.0,
"realized_pnl_usdt": float(kept["realized_pnl_usdt"].sum()) if not kept.empty else 0.0,
"net_attr_30s_usdt": float(kept["net_attr_30s_usdt"].sum()) if not kept.empty else 0.0,
"markout_30s": _weighted_mean(kept["markout_30s"], kept["notional_usdt"]),
}
def main() -> int:
ap = argparse.ArgumentParser(description="Economic Attribution v0.1 (MATCHED only)")
ap.add_argument("--dir", default=str(ROOT / "logs" / "maker_edge"))
ap.add_argument("--out", default=str(ROOT / "logs" / "maker_edge" / "Economic_Attribution_v0_1.txt"))
ap.add_argument("--recon03", default=str(ROOT / "logs" / "maker_edge" / "RECONCILIATION_03.json"))
ap.add_argument("--account", default=str(ROOT / "logs" / "maker_edge" / "Account_Reconciliation.json"))
ap.add_argument("--venue-trades", default=str(ROOT / "logs" / "maker_edge" / "venue_trades.json"))
args = ap.parse_args()
log_dir = Path(args.dir)
df = _load_jsonl_df(log_dir)
fills = df[df["event"] == "fill"].copy()
paths = df[df["event"] == "fill_path"].copy()
inv = df[df["event"] == "inventory_tick"].copy()
venue_trades = json.loads(Path(args.venue_trades).read_text())
local_fills_raw = load_local_fills(log_dir)
locals_norm = [normalize_local(e, i) for i, e in enumerate(local_fills_raw)]
venues_norm = [normalize_venue(t, i) for i, t in enumerate(venue_trades)]
recon = match(locals_norm, venues_norm)
matched_fill_ids = {m["local"]["fill_id"] for m in recon["matched"]}
matched_trade_ids = {m["venue"]["venue_trade_id"] for m in recon["matched"]}
fills = fills[fills["fill_id"].isin(matched_fill_ids)].copy()
paths = paths[paths["fill_id"].isin(matched_fill_ids)].copy()
fc = _parse_fill_context(fills)
meta_cols = [
c
for c in [
"fill_id",
"side",
"fill_price",
"spread",
"spread_capture_pct",
"obi",
"trade_imbalance",
"bid_depth_5",
"ask_depth_5",
"book_age_ms",
"inventory",
"inventory_time",
"inventory_skew",
"pre_5s_deteriorated",
"mid",
"event_cluster_id",
"pair",
]
if c in fills.columns
]
meta = fills.drop_duplicates("fill_id")[meta_cols]
paths = paths.merge(meta, on="fill_id", how="left", suffixes=("", "_f"))
for col in ["event_cluster_id", "side", "fill_price", "spread_capture_pct", "mid"]:
alt = f"{col}_f"
if alt in paths.columns:
if col not in paths.columns:
paths[col] = paths[alt]
else:
paths[col] = paths[col].fillna(paths[alt])
if not fc.empty:
paths = paths.merge(fc, on="fill_id", how="left")
paths = _prepare_paths(paths)
venue = pd.DataFrame(venues_norm)
venue = venue[venue["venue_trade_id"].isin(matched_trade_ids)].copy()
venue = venue.rename(
columns={
"venue_trade_id": "trade_id_link",
"venue_order_id": "venue_order_id",
"qty": "qty",
"px": "venue_price",
"ts": "venue_ts",
}
)
raw_v = pd.DataFrame(venue_trades)
raw_v["trade_id_link"] = raw_v["id"].astype(str)
raw_v["venue_order_id"] = raw_v["orderId"].astype(str)
raw_v["commission_usdt"] = pd.to_numeric(raw_v["commission"], errors="coerce")
raw_v["realized_pnl_usdt"] = pd.to_numeric(raw_v["realizedPnl"], errors="coerce").fillna(0.0)
raw_v["venue_time_ms"] = pd.to_numeric(raw_v["time"], errors="coerce")
raw_v["qty"] = pd.to_numeric(raw_v["qty"], errors="coerce")
raw_v["venue_price"] = pd.to_numeric(raw_v["price"], errors="coerce")
raw_v["side"] = np.where(raw_v["buyer"].astype(bool), "long", "short")
raw_v = raw_v[raw_v["trade_id_link"].isin(matched_trade_ids)].copy()
matched_map = pd.DataFrame(
[
{
"fill_id": m["local"]["fill_id"],
"trade_id_link": m["venue"]["venue_trade_id"],
"venue_order_id": m["venue"]["venue_order_id"],
}
for m in recon["matched"]
]
)
paths = paths.merge(
matched_map.merge(
raw_v[
[
"trade_id_link",
"venue_order_id",
"commission_usdt",
"realized_pnl_usdt",
"venue_time_ms",
"qty",
"venue_price",
"side",
]
],
on=["trade_id_link", "venue_order_id"],
how="left",
),
on="fill_id",
how="left",
suffixes=("", "_venue"),
)
paths["fill_price"] = pd.to_numeric(paths["fill_price"], errors="coerce")
paths["qty"] = pd.to_numeric(paths["qty"], errors="coerce")
paths["notional_usdt"] = paths["fill_price"] * paths["qty"]
for sec, col in [(1, "after_1s_price"), (5, "after_5s_price"), (10, "after_10s_price"), (30, "after_30s_price"), (300, "after_5m_price")]:
paths[f"markout_{sec}s"] = _fav_ret(paths["side"], paths["fill_price"], paths[col])
paths["gross_markout_30s_usdt"] = paths["notional_usdt"] * paths["markout_30s"]
paths["net_attr_30s_usdt"] = (
paths["gross_markout_30s_usdt"]
- pd.to_numeric(paths["commission_usdt"], errors="coerce").fillna(0.0)
+ pd.to_numeric(paths["realized_pnl_usdt"], errors="coerce").fillna(0.0)
)
inventory_metrics = _inventory_metrics(
raw_v[
["venue_time_ms", "side", "qty", "commission_usdt", "realized_pnl_usdt", "venue_order_id", "trade_id_link"]
].copy()
)
n_matched_paths = len(paths)
n_matched_fills = len(fills)
n_matched_clusters = int(fills["event_cluster_id"].nunique()) if not fills.empty else 0
cluster_w = _cluster_weight(paths) if not paths.empty and "event_cluster_id" in paths.columns else pd.Series(dtype=float)
horizon_rows = []
for sec in (1, 5, 10, 30, 300):
col = f"markout_{sec}s"
valid = paths[col].notna()
sub = paths[valid]
w = sub["notional_usdt"]
horizon_rows.append(
{
"horizon": f"{sec}s",
"n": int(len(sub)),
"fill_w": _weighted_mean(sub[col], w),
"cluster_w": _weighted_mean(sub[col], _cluster_weight(sub) if not sub.empty else pd.Series(dtype=float)),
"gross_usdt": float((sub["notional_usdt"] * sub[col]).sum()) if not sub.empty else 0.0,
}
)
fee_total = float(paths["commission_usdt"].sum())
realized_total = float(paths["realized_pnl_usdt"].sum())
gross_30_total = float(paths["gross_markout_30s_usdt"].sum())
net_attr_30_total = float(paths["net_attr_30s_usdt"].sum())
total_qty = float(paths["qty"].sum())
total_notional = float(paths["notional_usdt"].sum())
bucket_rows: list[dict[str, Any]] = []
for col, title in [
("path_type", "PathType"),
("toxicity_bucket", "Toxicity"),
("vol_bucket", "Volatility"),
("liq_bucket", "Spread"),
("trend_bucket", "Trend"),
("market_event_before_fill", "FillContext"),
]:
bucket_rows.extend(_bucket_table(paths, col, title))
bucket_df = pd.DataFrame(bucket_rows)
negative_states: set[str] = set()
if not bucket_df.empty:
neg = bucket_df[(bucket_df["dimension"] != "PathType") & (bucket_df["markout_30s"] < 0)]
negative_states = set(neg["bucket"].astype(str))
counterfactuals = [
{
"name": "BASELINE",
"fills": int(len(paths)),
"clusters": int(paths["event_cluster_id"].nunique()) if not paths.empty else 0,
"btc_qty": total_qty,
"fee_usdt": fee_total,
"gross_markout_30s_usdt": gross_30_total,
"realized_pnl_usdt": realized_total,
"net_attr_30s_usdt": net_attr_30_total,
"markout_30s": _weighted_mean(paths["markout_30s"], paths["notional_usdt"]),
},
_counterfactual(paths, "path_type", {"C_toxic"}, "EXCLUDE_PATH_C"),
_counterfactual(paths, "toxicity_bucket", {"toxic"}, "EXCLUDE_TOXIC"),
_counterfactual(paths, "market_event_before_fill", negative_states, "EXCLUDE_NEGATIVE_STATE"),
]
account = json.loads(Path(args.account).read_text()) if Path(args.account).exists() else {}
recon03 = json.loads(Path(args.recon03).read_text()) if Path(args.recon03).exists() else {}
out_txt = Path(args.out)
out_json = out_txt.with_suffix(".json")
lines: list[str] = []
def p(s: str = "") -> None:
lines.append(s)
print(s)
p("=" * 72)
p("Economic Attribution v0.1")
p("=" * 72)
p("Experiment: MM_EDGE_EXP_001")
p("Population: MATCHED=3890")
p("Strategy: v0.1 FROZEN")
p("Execution: STOPPED")
p("Stage3: LOCKED")
p("Purpose: Economic Attribution only.")
p("No strategy modification. No live execution. No economic simulation.")
p()
p("Layer 1 — Hard Economic Evidence")
p("-" * 40)
p(f"Matched fills: {n_matched_fills}")
p(f"Matched paths: {n_matched_paths}")
p(f"Matched clusters: {n_matched_clusters}")
p(f"Fee total: {_num(fee_total, 6)} USDT")
p(f"Fee / fill: {_num(fee_total / max(n_matched_paths, 1), 6)} USDT")
p(f"Fee / BTC: {_num(fee_total / max(total_qty, 1e-12), 6)} USDT")
p(f"Fee / cluster: {_num(fee_total / max(n_matched_clusters, 1), 6)} USDT")
p(f"Realized component: {_num(realized_total, 6)} USDT")
p(f"Gross markout @30s: {_num(gross_30_total, 6)} USDT")
p(f"Net attributable @30s: {_num(net_attr_30_total, 6)} USDT")
p()
p("Markout by horizon (MATCHED only)")
p("-" * 40)
for row in horizon_rows:
p(
f"{row['horizon']:>5} n={row['n']:4d} fill-w={_pct(row['fill_w'])} "
f"cluster-w={_pct(row['cluster_w'])} gross={_num(row['gross_usdt'], 6)} USDT"
)
p()
p("Inventory carry / exposure")
p("-" * 40)
p(f"Max net BTC: {_num(inventory_metrics.get('max_net_btc'), 6)}")
p(f"Min net BTC: {_num(inventory_metrics.get('min_net_btc'), 6)}")
p(f"Max |net BTC|: {_num(inventory_metrics.get('max_abs_net_btc'), 6)}")
p(f"Average |net BTC|: {_num(inventory_metrics.get('avg_abs_net_btc_per_fill'), 6)}")
p(f"TW |net BTC|: {_num(inventory_metrics.get('time_weighted_abs_net_btc'), 6)}")
p(f"TW signed net BTC: {_num(inventory_metrics.get('time_weighted_signed_net_btc'), 6)}")
p(f"Long qty / Short qty: {_num(inventory_metrics.get('long_qty'), 6)} / {_num(inventory_metrics.get('short_qty'), 6)} BTC")
p(f"Inventory turnover: {_num(inventory_metrics.get('turnover_btc'), 6)} BTC")
p()
p("Slices (weighted by notional, MATCHED only)")
p("-" * 40)
for dim in ["PathType", "Toxicity", "Volatility", "Spread", "Trend", "FillContext"]:
sub = bucket_df[bucket_df["dimension"] == dim].copy()
if sub.empty:
continue
p(dim)
for _, r in sub.sort_values(["fills", "bucket"], ascending=[False, True]).iterrows():
p(
f" {r['bucket']}: n={int(r['fills'])} clusters={int(r['clusters'])} "
f"fee={_num(r['fee_usdt'], 4)} gross30={_num(r['gross_markout_30s_usdt'], 4)} "
f"realized={_num(r['realized_pnl_usdt'], 4)} net30={_num(r['net_attr_30s_usdt'], 4)} "
f"m30={_pct(r['markout_30s'])}"
)
p()
p("Layer 2 — Evidence Extension (excluded from core conclusion)")
p("-" * 40)
p(f"VENUE_CONFIRMED_NO_TRADE_HISTORY: {recon03.get('venue_confirmed_no_trade_history', 'n/a')}")
p(f"VENUE_PARTIAL_ORDER_CANCELED: {recon03.get('venue_partial_order_canceled', 'n/a')}")
p("These rows are order-confirmed, but not part of the Hard Evidence Population.")
p()
p("Layer 3 — Counterfactual Attribution (NOT backtest)")
p("-" * 40)
p("Observed vs Exclude-Bucket Attribution. These are contribution decompositions only.")
for row in counterfactuals:
p(
f"{row['name']}: fills={row['fills']} clusters={row['clusters']} "
f"fee={_num(row['fee_usdt'], 4)} gross30={_num(row['gross_markout_30s_usdt'], 4)} "
f"realized={_num(row['realized_pnl_usdt'], 4)} net30={_num(row['net_attr_30s_usdt'], 4)} "
f"m30={_pct(row['markout_30s'])}"
)
p()
p("Interpretation")
p("-" * 40)
p("Core conclusion is based on 3890 fully matched fills.")
p("Economic Attribution asks why MakerAlpha did not convert to money.")
p("It does NOT change quote logic, does NOT restart v0.1, and does NOT unlock Stage 3.")
p("=" * 72)
out_txt.write_text("\n".join(lines) + "\n", encoding="utf-8")
sidecar = {
"experiment_id": "MM_EDGE_EXP_001",
"population": {
"name": "MATCHED",
"fills": n_matched_fills,
"paths": n_matched_paths,
"clusters": n_matched_clusters,
},
"strategy": "v0.1 FROZEN",
"execution": "STOPPED",
"stage3": "LOCKED",
"fee_total_usdt": fee_total,
"fee_per_fill_usdt": fee_total / max(n_matched_paths, 1),
"fee_per_btc_usdt": fee_total / max(total_qty, 1e-12),
"fee_per_cluster_usdt": fee_total / max(n_matched_clusters, 1),
"realized_component_usdt": realized_total,
"gross_markout_30s_usdt": gross_30_total,
"net_attr_30s_usdt": net_attr_30_total,
"markout_by_horizon": horizon_rows,
"inventory_metrics": inventory_metrics,
"bucket_rows": bucket_rows,
"counterfactuals": counterfactuals,
"recon03_extension": {
"venue_confirmed_no_trade_history": recon03.get("venue_confirmed_no_trade_history"),
"venue_partial_order_canceled": recon03.get("venue_partial_order_canceled"),
},
"account_recon_ref": account,
}
out_json.write_text(json.dumps(sidecar, indent=2) + "\n", encoding="utf-8")
return 0
if __name__ == "__main__":
raise SystemExit(main())
requirements.txt
+6
View File
@@ -0,0 +1,6 @@
nautilus_trader==1.231.0
python-dotenv>=1.0.0
PyYAML>=6.0
ccxt>=4.0.0
pandas>=2.2,<3
numpy>=1.26,<2.1
scripts/Economic_Fee_Sensitivity_v0_1.txt
+20
View File
@@ -0,0 +1,20 @@
========================================================================
Economic Fee Sensitivity v0.1 (MATCHED=3890)
========================================================================
gross_markout_30s_usdt: -1.730780 USDT
fee_total_usdt: +42.422146 USDT
realized_component_usdt:-8.521730 USDT
Fee assumption → Net attributable @30s
------------------------------------------
fee_factor | fee_usdt_assumed | net_attr_30s_usdt
1.00 | +42.422146 | -52.674655
0.50 | +21.211073 | -31.463582
0.25 | +10.605536 | -20.858046
0.10 | +4.242215 | -14.494724
0.00 | +0.000000 | -10.252510
Interpretation:
- If net remains < 0 at fee_factor=0 → economics not salvageable by fee reduction alone.
- If fee reduction flips net > 0 → current venue/fee tier can be the dominant issue.
========================================================================
scripts/Economic_Metric_Reconciliation_v0_1.txt
+21
View File
@@ -0,0 +1,21 @@
========================================================================
Economic Metric Reconciliation v0.1 (MATCHED=3890)
========================================================================
Matched paths: 3886 (expected ~3886)
Definitions (same math as analyze_maker_edge):
- markout_30s_return = _fav_ret(side, fill_price, after_30s_price)
- gross_markout_usdt = sum(notional_usdt * markout_30s_return)
Return-space metrics (sign may differ due to weighting):
MakerAlpha fill-weighted mean return: -0.000693%
MakerAlpha notional-weighted mean return: -0.000816%
MakerAlpha cluster-weighted mean return: -0.000830%
Dollar-space metrics:
gross_markout_usdt (30s): -1.730780 USDT
total_notional_usdt: 212110.750 USDT
If fill-weighted return is + but gross_markout_usdt is negative,
it means notional weighting flips sign (alpha is conditionally realized).
========================================================================
scripts/analyze.sh
Executable
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
LOG_DIR="${MAKER_EDGE_LOG_DIR:-$ROOT/logs/maker_edge}"
# Prefer project venv python if present
PY="${ROOT}/.venv/bin/python"
if [[ ! -x "$PY" ]]; then
PY=python3
fi
export PYTHONPATH="${ROOT}/src${PYTHONPATH:+:$PYTHONPATH}"
exec "$PY" "$ROOT/scripts/analyze_maker_edge.py" --dir "$LOG_DIR" --report --min-fills "${1:-2000}"
scripts/analyze_maker_edge.py
+978
View File
@@ -0,0 +1,978 @@
#!/usr/bin/env python3
"""
Maker Edge Report v0.1 — Research Freeze / Data Collection Phase
固定格式（每次运行必须相同、可比较）：
Executive Summary
Section 1 — Data Integrity
Section 2 — Fill Alpha
Section 3 — Toxicity Profile
Section 4 — Observed Edge Attribution
Section 5 — Decision
研究对象：可验证的市场现象（不是策略）。
见 FREEZE.md — 只许数据字段/质量检查/报告解释；禁止新交易规则。
用法：
./scripts/analyze.sh 2000
python scripts/analyze_maker_edge.py --report --min-fills 2000
"""
from __future__ import annotations
import argparse
import json
import sys
import uuid
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import numpy as np
import pandas as pd
_ROOT = Path(__file__).resolve().parents[1]
_SRC = _ROOT / "src"
if str(_SRC) not in sys.path:
sys.path.insert(0, str(_SRC))
FEE = 0.00016
EXPECTED_SLIPPAGE = 0.00005
POSITIVE_EDGE_NET = 0.0002
CLUSTER_GAP_SEC = 30.0
TOXIC_FAIL_RATIO = 0.60
PASS_MIN_FILLS_DEFAULT = 2000
def _load_experiment_from_df(df: pd.DataFrame) -> dict[str, Any]:
"""优先用 jsonl 中的 experiment_start / 事件戳；否则回退环境默认。"""
try:
from nautilus_mm.experiment import load_experiment_meta
base = load_experiment_meta()
except Exception:
base = {
"experiment_id": "MM_EDGE_EXP_001",
"probe_version": "probe_v0.1",
"quote_assumption": "frozen",
"fee_model": "frozen",
"exchange_assumption": "frozen",
"exchange": "binance_usdm",
"environment": "TESTNET",
"symbol": "BTCUSDT-PERP",
}
if df.empty or "event" not in df.columns:
return base
starts = df[df["event"] == "experiment_start"]
if not starts.empty:
row = starts.iloc[-1]
for k in ("experiment_id", "probe_version", "exchange", "environment", "symbol"):
if k in row and pd.notna(row[k]):
base[k] = row[k]
return base
# 任意带 experiment_id 的事件
if "experiment_id" in df.columns and df["experiment_id"].notna().any():
base["experiment_id"] = df["experiment_id"].dropna().iloc[-1]
if "probe_version" in df.columns and df["probe_version"].notna().any():
base["probe_version"] = df["probe_version"].dropna().iloc[-1]
return base
def load_events(log_dir: Path) -> pd.DataFrame:
rows = []
files = sorted(log_dir.glob("*.jsonl"))
if not files:
raise FileNotFoundError(f"No jsonl in {log_dir}")
for f in files:
for line in f.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line:
continue
rows.append(json.loads(line))
return pd.DataFrame(rows)
def _fav_ret(side: pd.Series, fill: pd.Series, px: pd.Series) -> pd.Series:
raw = (px.astype(float) - fill.astype(float)) / fill.astype(float)
return pd.Series(np.where(side == "long", raw, -raw), index=side.index)
def _side_label(side: str) -> str:
return "Bid" if side == "long" else "Ask"
def _extract_fill_context(fills: pd.DataFrame) -> pd.DataFrame:
if fills.empty or "fill_context" not in fills.columns:
return pd.DataFrame()
rows = []
for _, r in fills.iterrows():
ctx = r.get("fill_context")
if not isinstance(ctx, dict):
continue
rows.append(
{
"fill_id": r.get("fill_id"),
"market_event_before_fill": ctx.get("market_event_before_fill"),
"trade_imbalance_5s": ctx.get("trade_imbalance_5s"),
"price_velocity_5s": ctx.get("price_velocity_5s"),
"fill_type": ctx.get("fill_type"),
}
)
return pd.DataFrame(rows)
def _median_safe(s: pd.Series) -> float | None:
s = pd.to_numeric(s, errors="coerce").dropna()
return float(s.median()) if len(s) else None
def _fmt_pct(x: float | None, digits: int = 4) -> str:
if x is None or (isinstance(x, float) and np.isnan(x)):
return "n/a"
return f"{x*100:+.{digits}f}%"
def _fmt_pp(x: float | None) -> str:
if x is None or (isinstance(x, float) and np.isnan(x)):
return "n/a"
return f"{x*100:+.1f}pp"
def _dist_stats(s: pd.Series) -> dict[str, float | None]:
s = pd.to_numeric(s, errors="coerce").dropna()
if s.empty:
return {"mean": None, "median": None, "p25": None, "p75": None, "n": 0}
return {
"mean": float(s.mean()),
"median": float(s.median()),
"p25": float(s.quantile(0.25)),
"p75": float(s.quantile(0.75)),
"n": int(len(s)),
}
def _print_dist(p, title: str, d: dict[str, float | None]) -> None:
if not d.get("n"):
p(f"{title}: n/a")
return
p(f"{title} (n={d['n']}):")
p(f" mean: {_fmt_pct(d['mean'])}")
p(f" median: {_fmt_pct(d['median'])}")
p(f" p25: {_fmt_pct(d['p25'])}")
p(f" p75: {_fmt_pct(d['p75'])}")
def _observation_window(n_fills: int, n_clusters: int) -> str:
if n_fills < 500:
return "anomaly-check only (<500 fills)"
if n_fills < 2000:
return "early look (500+) — do not over-interpret"
if n_fills < 10000:
return "preliminary judgment (2000+) — clusters still matter more than fills"
return "stability discussion eligible (10000+ fills)"
def assign_clusters_offline(fills: pd.DataFrame, gap_sec: float = CLUSTER_GAP_SEC) -> pd.DataFrame:
out = fills.copy()
if out.empty:
return out
if "event_cluster_id" in out.columns and out["event_cluster_id"].notna().any():
return out
if "ts_epoch" not in out.columns:
out["event_cluster_id"] = [f"na_{i}" for i in range(len(out))]
out["cluster_fill_index"] = 1
return out
out = out.sort_values("ts_epoch").reset_index(drop=True)
cids: list[str] = []
idxs: list[int] = []
cid = None
last_ts = -1e18
last_side = None
n = 0
for _, r in out.iterrows():
ts = float(r["ts_epoch"])
side = r.get("side")
if cid is None or side != last_side or (ts - last_ts) > gap_sec:
cid = uuid.uuid4().hex[:12]
n = 0
n += 1
cids.append(cid)
idxs.append(n)
last_ts = ts
last_side = side
out["event_cluster_id"] = cids
out["cluster_fill_index"] = idxs
return out
def classify_space(raw_capture: float, net_edge: float) -> str:
if raw_capture <= 0 or net_edge <= 0:
return "NO_EDGE"
if net_edge < POSITIVE_EDGE_NET:
return "EDGE_AFTER_COST"
return "POSITIVE_EDGE"
def build_mid_series(df: pd.DataFrame) -> pd.DataFrame:
parts = []
for ev in ("mid_tick", "inventory_tick"):
if "event" not in df.columns:
break
sub = df[df["event"] == ev]
if sub.empty or "mid" not in sub.columns or "ts_epoch" not in sub.columns:
continue
parts.append(sub[["ts_epoch", "mid"]].dropna())
if not parts:
return pd.DataFrame(columns=["ts_epoch", "mid"])
m = pd.concat(parts, ignore_index=True)
m["ts_epoch"] = pd.to_numeric(m["ts_epoch"], errors="coerce")
m["mid"] = pd.to_numeric(m["mid"], errors="coerce")
return m.dropna().sort_values("ts_epoch").drop_duplicates("ts_epoch").reset_index(drop=True)
def _cluster_weight(frame: pd.DataFrame) -> pd.Series:
if "event_cluster_id" not in frame.columns:
return pd.Series(1.0, index=frame.index)
cnt = frame.groupby("event_cluster_id")["event_cluster_id"].transform("count")
return 1.0 / cnt.clip(lower=1)
def _period_str(df: pd.DataFrame) -> str:
if df.empty or "ts_epoch" not in df.columns or df["ts_epoch"].isna().all():
return "n/a"
t0 = float(pd.to_numeric(df["ts_epoch"], errors="coerce").min())
t1 = float(pd.to_numeric(df["ts_epoch"], errors="coerce").max())
a = datetime.fromtimestamp(t0, tz=timezone.utc).strftime("%Y-%m-%d")
b = datetime.fromtimestamp(t1, tz=timezone.utc).strftime("%Y-%m-%d")
return f"{a} ~ {b}"
def _instrument(fills: pd.DataFrame, df: pd.DataFrame) -> str:
for src in (fills, df):
if not src.empty and "pair" in src.columns and src["pair"].notna().any():
return str(src["pair"].dropna().iloc[0])
return "BTCUSDT Perpetual (assumed)"
def _maker_alpha_frame(g: pd.DataFrame) -> tuple[pd.Series, pd.Series]:
"""Return (fill_ret, mkt_signed) for MakerAlpha = fill − market."""
mid0 = g["mid"].astype(float)
mid1 = g["after_30s_price"].astype(float)
mkt_ret = (mid1 - mid0) / mid0
mkt_signed = pd.Series(
np.where(g["side"] == "long", mkt_ret, -mkt_ret), index=g.index
)
fill_ret = _fav_ret(g["side"], g["fill_price"], g["after_30s_price"])
return fill_ret, mkt_signed
def report(df: pd.DataFrame, min_fills: int = PASS_MIN_FILLS_DEFAULT, out_path: Path | None = None) -> dict[str, Any]:
fills = df[df["event"] == "fill"].copy() if "event" in df.columns else pd.DataFrame()
paths = df[df["event"] == "fill_path"].copy() if "event" in df.columns else pd.DataFrame()
health = df[df["event"] == "health"].copy() if "event" in df.columns else pd.DataFrame()
exp = _load_experiment_from_df(df)
lines: list[str] = []
def p(s: str = "") -> None:
lines.append(s)
print(s)
if not fills.empty:
fills = assign_clusters_offline(fills)
if not paths.empty and not fills.empty and "fill_id" in fills.columns:
meta_cols = [
c
for c in [
"side",
"fill_price",
"fill_reason",
"spread",
"spread_capture_pct",
"obi",
"trade_imbalance",
"bid_depth_5",
"ask_depth_5",
"book_age_ms",
"inventory",
"pre_5s_deteriorated",
"mid",
"event_cluster_id",
"cluster_fill_index",
"ts_epoch",
"pair",
]
if c in fills.columns
]
meta = fills.drop_duplicates("fill_id")[["fill_id"] + meta_cols]
paths = paths.merge(meta, on="fill_id", how="left", suffixes=("", "_f"))
for col in ("side", "fill_price", "mid", "event_cluster_id", "spread", "spread_capture_pct"):
alt = f"{col}_f"
if alt in paths.columns:
if col not in paths.columns:
paths[col] = paths[alt]
else:
paths[col] = paths[col].fillna(paths[alt])
fc = _extract_fill_context(fills)
if not fc.empty:
paths = paths.merge(fc, on="fill_id", how="left")
n_fills = len(fills)
n_paths = len(paths)
n_clusters = int(fills["event_cluster_id"].nunique()) if n_fills and "event_cluster_id" in fills.columns else 0
cluster_fill_ratio = n_clusters / max(n_fills, 1)
# ---------- compute: integrity ----------
integrity_ok = True
integrity_notes: list[str] = []
healthy_ratio = gap_total = gap_win_max = None
lat_p50 = lat_p95 = lat_p99 = ba_med = None
if health.empty:
integrity_ok = False
integrity_notes.append("no health telemetry")
else:
healthy_ratio = float(health["healthy"].astype(bool).mean()) if "healthy" in health.columns else 0.0
gap_total = int(health["sequence_gap"].iloc[-1]) if "sequence_gap" in health.columns else 0
gap_win_max = (
int(pd.to_numeric(health.get("sequence_gap_window"), errors="coerce").fillna(0).max())
if "sequence_gap_window" in health.columns
else 0
)
lat_p50 = health["latency_ms_p50"].iloc[-1] if "latency_ms_p50" in health.columns else None
lat_p95 = health["latency_ms_p95"].iloc[-1] if "latency_ms_p95" in health.columns else None
lat_p99 = health["latency_ms_p99"].iloc[-1] if "latency_ms_p99" in health.columns else None
ba_series = (
fills["book_age_ms"]
if "book_age_ms" in fills.columns and fills["book_age_ms"].notna().any()
else health.get("book_age_ms")
)
ba_med = _median_safe(ba_series) if ba_series is not None else None
if healthy_ratio < 0.99:
integrity_ok = False
integrity_notes.append(f"healthy_ratio={healthy_ratio*100:.2f}% < 99%")
# Binance depth update ids are not contiguous — log only, do not INVALID.
if gap_win_max and gap_win_max > 0:
integrity_notes.append(
f"sequence_gap_window_max={gap_win_max} (observe-only; Binance ids skip)"
)
if ba_med is not None and ba_med > 500:
integrity_ok = False
integrity_notes.append(f"book_age_median={ba_med:.0f}ms > 500ms")
decision: dict[str, Any] = {
"integrity": integrity_ok,
"verdict": "INSUFFICIENT_DATA",
"reasons": [],
"space_class": None,
"benchmark_alpha": None,
"maker_alpha_mean": None,
"quality": {
"fills": n_fills,
"clusters": n_clusters,
"cluster_fill_ratio": cluster_fill_ratio,
"healthy_ratio": healthy_ratio,
},
}
if fills.empty:
p("=" * 72)
p("Maker Edge Report v0.1")
p("Phase: Research Freeze / Data Collection")
p("=" * 72)
p("\nExecutive Summary")
p(f" Experiment: {exp.get('experiment_id')}")
p(f" Version: {exp.get('probe_version')}")
p(" Quote: frozen")
p(" Fee: frozen")
p(" Exchange: frozen")
p(f" Period: {_period_str(df)}")
p(f" Instrument: {_instrument(fills, df)}")
p(" Samples:")
p(" fills: 0")
p(" clusters: 0")
p(" Decision: INSUFFICIENT_DATA")
p(" Reason: no fills yet — run probe")
decision["experiment"] = exp
_finish(lines, out_path, decision)
return decision
# ---------- compute: fill alpha table + distributions ----------
alpha_table: dict[str, dict[str, float | None]] = {
"Bid": {"fill_w": None, "cluster_w": None},
"Ask": {"fill_w": None, "cluster_w": None},
"Overall": {"fill_w": None, "cluster_w": None},
}
fill_alpha_dist: dict[str, float | None] = {}
cluster_alpha_dist: dict[str, float | None] = {}
fq_pass = None
bench_alpha = None
maker_alpha_mean = None
agree = None
pct_fills_positive_alpha = None
if not paths.empty and "after_30s_price" in paths.columns and "mid" in paths.columns and paths["mid"].notna().any():
for side_name, g in paths.groupby("side"):
label = _side_label(str(side_name))
fill_ret, mkt_signed = _maker_alpha_frame(g)
alpha = fill_ret - mkt_signed
w = _cluster_weight(g)
alpha_table[label]["fill_w"] = float(alpha.mean())
alpha_table[label]["cluster_w"] = float((alpha * w).sum() / w.sum()) if w.sum() else float(alpha.mean())
fill_ret, mkt_signed = _maker_alpha_frame(paths)
alpha = fill_ret - mkt_signed
w = _cluster_weight(paths)
alpha_table["Overall"]["fill_w"] = float(alpha.mean())
alpha_table["Overall"]["cluster_w"] = (
float((alpha * w).sum() / w.sum()) if w.sum() else float(alpha.mean())
)
maker_alpha_mean = alpha_table["Overall"]["cluster_w"]
fw, cw = alpha_table["Overall"]["fill_w"], alpha_table["Overall"]["cluster_w"]
agree = (fw > 0 and cw > 0) or (fw <= 0 and cw <= 0)
fill_alpha_dist = _dist_stats(alpha)
pct_fills_positive_alpha = float((alpha > 0).mean())
# per-cluster mean MakerAlpha（事件级分布）
if "event_cluster_id" in paths.columns:
tmp = paths.assign(_alpha=alpha)
cluster_means = tmp.groupby("event_cluster_id")["_alpha"].mean()
cluster_alpha_dist = _dist_stats(cluster_means)
mkt_fav = mkt_signed > 0
fill_fav = fill_ret > 0
bench_alpha = float(np.mean(fill_fav) - np.mean(mkt_fav))
fav30 = fill_ret
p30_clu = float((fav30 > 0).astype(float).mul(w).sum() / w.sum()) if w.sum() else float((fav30 > 0).mean())
fq_pass = p30_clu > 0.50
decision["fill_quality"] = fq_pass
decision["benchmark_alpha"] = bench_alpha
decision["maker_alpha_mean"] = maker_alpha_mean
# ---------- compute: toxicity + loss concentration ----------
toxicity: dict[str, dict[str, float | None]] = {}
toxic_bid_ratio = None
c_share = None
tox_dist: dict[str, Any] = {}
if not paths.empty:
for side_name, g in paths.groupby("side"):
label = _side_label(str(side_name))
row: dict[str, float | None] = {}
for hz, col in [
("1s", "after_1s_price"),
("5s", "after_5s_price"),
("10s", "after_10s_price"),
("30s", "after_30s_price"),
("300s", "after_5m_price"),
]:
if col in g.columns:
row[hz] = float(_fav_ret(g["side"], g["fill_price"], g[col]).mean())
else:
row[hz] = None
toxicity[label] = row
if "path_type" in paths.columns:
c_share = float((paths["path_type"].astype(str).str.startswith("C")).mean())
bid = paths[paths["side"] == "long"]
if len(bid):
toxic_bid_ratio = float((bid["path_type"].astype(str).str.startswith("C")).mean())
# 毒性分布：多少成交在 10s 不利；最差 20% 占总不利损失比例
if "after_10s_price" in paths.columns:
fav10 = _fav_ret(paths["side"], paths["fill_price"], paths["after_10s_price"])
adverse = fav10[fav10 < 0]
tox_dist["pct_adverse_10s"] = float((fav10 < 0).mean())
tox_dist["fav10"] = _dist_stats(fav10)
if len(adverse) >= 5:
worst_n = max(1, int(np.ceil(0.20 * len(fav10))))
worst = fav10.nsmallest(worst_n)
total_adv = float((-adverse).sum())
worst_adv = float((-worst.clip(upper=0)).sum())
tox_dist["worst20_share_of_adverse"] = (
worst_adv / total_adv if total_adv > 1e-12 else None
)
else:
tox_dist["worst20_share_of_adverse"] = None
# ---------- compute: cost / adverse ----------
space_class = None
adv_pass = None
raw_capture = net_edge = adv_mag = sc_mean = total_cost = None
if not paths.empty and "after_30s_price" in paths.columns:
fav30 = _fav_ret(paths["side"], paths["fill_price"], paths["after_30s_price"])
w = _cluster_weight(paths)
raw_capture = float((fav30 * w).sum() / w.sum()) if w.sum() else float(fav30.mean())
adv_mag = (
float((-fav30.clip(upper=0) * w).sum() / w.sum())
if w.sum()
else float((-fav30.clip(upper=0)).mean())
)
sc_mean = (
float(fills["spread_capture_pct"].mean())
if "spread_capture_pct" in fills.columns and fills["spread_capture_pct"].notna().any()
else 0.0
)
if "book_age_ms" in fills.columns and fills["book_age_ms"].notna().any():
latency_cost = float(fills["book_age_ms"].mean()) / 100.0 * 0.00002
else:
latency_cost = 0.00002
total_cost = 2 * FEE + EXPECTED_SLIPPAGE + latency_cost
net_edge = raw_capture - total_cost
space_class = classify_space(raw_capture, net_edge)
adv_ok = (adv_mag < abs(sc_mean)) if sc_mean != 0 else False
adv_pass = bool(adv_ok and space_class in ("POSITIVE_EDGE", "EDGE_AFTER_COST"))
decision["adverse"] = adv_pass
decision["space_class"] = space_class
# ---------- compute: attribution (facts only) ----------
attr_rows: list[tuple[str, str, int, float]] = []
stab_pass = None
state_coverage_ok = None
concentrated = False
positive_envs = 0
total_envs = 0
if not paths.empty and "after_30s_price" in paths.columns:
paths = paths.copy()
paths["_fav30"] = _fav_ret(paths["side"], paths["fill_price"], paths["after_30s_price"])
if "vol_proxy_5m" in paths.columns and paths["vol_proxy_5m"].notna().any():
med = paths["vol_proxy_5m"].median()
paths["vol_bucket"] = np.where(paths["vol_proxy_5m"] >= med, "high_vol", "low_vol")
elif "max_price" in paths.columns:
rng = (paths["max_price"] - paths["min_price"]) / paths["fill_price"]
paths["vol_bucket"] = np.where(rng >= rng.median(), "high_vol", "low_vol")
if "price_velocity_5s" in paths.columns and paths["price_velocity_5s"].notna().any():
v = paths["price_velocity_5s"].astype(float)
thr = v.abs().median() * 0.5
paths["trend_bucket"] = np.where(
v > thr, "trend_up", np.where(v < -thr, "trend_down", "range")
)
if "spread" in paths.columns and paths["spread"].notna().any():
sp_pct = paths["spread"] / paths["fill_price"]
paths["liq_bucket"] = np.where(sp_pct <= sp_pct.median(), "tight_spread", "wide_spread")
if "bid_depth_5" in paths.columns and "ask_depth_5" in paths.columns:
depth = paths["bid_depth_5"].fillna(0) + paths["ask_depth_5"].fillna(0)
if depth.gt(0).any():
paths["depth_bucket"] = np.where(depth >= depth[depth > 0].median(), "deep_book", "thin_book")
pos_counts: list[int] = []
for col, title in [
("vol_bucket", "Volatility"),
("trend_bucket", "Trend"),
("liq_bucket", "Liquidity(spread)"),
("depth_bucket", "Liquidity(depth)"),
("market_event_before_fill", "FillContext"),
("path_type", "PathType"),
]:
if col not in paths.columns or paths[col].isna().all():
continue
for idx, row in paths.groupby(col)["_fav30"].agg(["count", "mean"]).iterrows():
total_envs += 1
mean = float(row["mean"])
n = int(row["count"])
attr_rows.append((title, str(idx), n, mean))
if mean > 0:
positive_envs += 1
pos_counts.append(n)
state_coverage_ok = total_envs >= 4
if total_envs >= 2:
if pos_counts:
concentrated = (max(pos_counts) / max(sum(pos_counts), 1)) >= 0.70 and len(pos_counts) == 1
stab_pass = positive_envs >= 2 and not concentrated
else:
state_coverage_ok = False
decision["stability"] = stab_pass
decision["quality"]["state_buckets"] = len(attr_rows)
# ---------- decision ----------
independence_ok = (
n_clusters >= max(50, min_fills // 20) if n_fills >= min_fills else None
)
decision["independence"] = independence_ok
reasons: list[str] = []
min_paths = max(1, min_fills // 10)
sample_ok = n_fills >= min_fills and n_paths >= min_paths
gates = {
"integrity": integrity_ok,
"fill_quality": fq_pass,
"adverse": adv_pass,
"stability": stab_pass,
}
hard_fail = False
if not integrity_ok:
hard_fail = True
reasons.append("data integrity failed — stop interpretation")
if toxic_bid_ratio is not None and toxic_bid_ratio > TOXIC_FAIL_RATIO:
hard_fail = True
reasons.append(f"Bid toxic fill ratio {toxic_bid_ratio*100:.0f}% > {TOXIC_FAIL_RATIO*100:.0f}%")
if space_class == "NO_EDGE" and sample_ok:
hard_fail = True
reasons.append("edge disappears after cost / NO_EDGE")
if bench_alpha is not None and bench_alpha <= 0 and sample_ok:
reasons.append("benchmark-adjusted alpha negative")
if maker_alpha_mean is not None and maker_alpha_mean <= 0 and sample_ok:
reasons.append("MakerAlpha (fill−market) ≤ 0")
if adv_pass is False and sample_ok:
reasons.append("adverse selection ≥ spread capture")
if concentrated:
reasons.append("edge concentrated in single regime")
pass_extras = True
if bench_alpha is not None and bench_alpha <= 0:
pass_extras = False
if independence_ok is False:
pass_extras = False
reasons.append(f"insufficient independent clusters ({n_clusters})")
if space_class == "NO_EDGE":
pass_extras = False
all_gates = all(v is True for v in gates.values())
# Stage3 unlock checklist（严格）
stage3_unlock = {
"data_integrity": integrity_ok is True,
"cluster_weighted_alpha_gt_0": bool(maker_alpha_mean is not None and maker_alpha_mean > 0),
"benchmark_alpha_gt_0": bool(bench_alpha is not None and bench_alpha > 0),
"not_concentrated": not concentrated,
}
stage3_ready = all(stage3_unlock.values()) and sample_ok and all_gates and pass_extras
# 局部正 edge：归因桶分化或集中在单一正 regime
local_positive = positive_envs >= 1 and total_envs >= 2 and (
(positive_envs < total_envs) or concentrated
)
if not integrity_ok:
verdict = "INVALID"
reasons = ["Data Integrity FAIL — do not interpret Alpha; discard / keep collecting clean data"]
reasons.extend(integrity_notes)
elif not sample_ok or state_coverage_ok is False:
verdict = "COLLECTING"
reasons = []
if n_fills < min_fills:
reasons.append(f"fills {n_fills} < {min_fills}")
if n_paths < min_paths:
reasons.append(f"fill_paths {n_paths} < {min_paths}")
if n_clusters < max(50, min_fills // 20) and n_fills >= 500:
reasons.append(f"clusters {n_clusters} insufficient (independent liquidity events)")
if state_coverage_ok is False:
reasons.append("state coverage incomplete")
if not reasons:
reasons.append("Insufficient independent liquidity events")
elif hard_fail and not local_positive:
verdict = "FAIL"
if not reasons:
reasons.append("market hypothesis does not hold under current quote assumption")
elif stage3_ready:
verdict = "PASS"
reasons = [
"Maker alpha survives: cost",
"Maker alpha survives: benchmark",
"Maker alpha survives: cluster weighting",
"Maker alpha survives: multiple states",
]
elif local_positive and integrity_ok and sample_ok:
verdict = "PARTIAL_PASS"
reasons = [
"edge not universal — observed only in subset of states/events",
f"positive attribution buckets: {positive_envs}/{total_envs}",
]
if concentrated:
reasons.append("edge concentrated in one regime/event class")
if maker_alpha_mean is not None and maker_alpha_mean <= 0:
reasons.append("overall cluster-weighted MakerAlpha ≤ 0")
else:
verdict = "FAIL"
if not reasons:
reasons.append("gates failed under current quote assumption")
if adv_pass is False:
reasons.insert(0, "adverse selection")
if sc_mean is not None and abs(sc_mean) < 1e-8:
reasons.append("insufficient spread")
decision["verdict"] = verdict
decision["reasons"] = reasons
decision["stage3_unlock"] = stage3_unlock
decision["stage3_ready"] = stage3_ready
decision["experiment"] = exp
# ==================================================================
# PRINT — fixed format
# ==================================================================
p("=" * 72)
p("Maker Edge Report v0.1")
p("Phase: Research Freeze / Data Collection")
p("Object: verifiable market phenomenon (not a strategy)")
p("=" * 72)
# ----- Executive Summary -----
p("\nExecutive Summary")
p("-" * 40)
p(f"Experiment: {exp.get('experiment_id')}")
p(f"Version: {exp.get('probe_version')}")
p("Quote: frozen")
p("Fee: frozen")
p("Exchange: frozen")
p(f"Venue: {exp.get('exchange')} / {exp.get('environment')}")
p(f"Period: {_period_str(df if not df.empty else fills)}")
p(f"Instrument: {_instrument(fills, df)}")
p("Samples:")
p(f" fills: {n_fills}")
p(f" clusters: {n_clusters}")
p(f" paths: {n_paths}")
p(f" cluster/fill: {cluster_fill_ratio*100:.1f}%")
p(f"Observation window: {_observation_window(n_fills, n_clusters)}")
p(" (500=anomaly · 2000=preliminary · 10000=stability; clusters > fills)")
p(f"Decision: {verdict}")
p("Reason:")
for r in reasons:
p(f" - {r}")
p("Hypothesis under test: passive fills produce +MakerAlpha")
p(" under current BTC perp / venue / quote / execution — not strategy PnL.")
p("Read order: Integrity → distributions (not mean) → Cluster → Toxicity → Decision")
# ----- Section 1 -----
p("\n" + "=" * 72)
p("Section 1 — Data Integrity")
p("Question: Is the data trustworthy?")
p("=" * 72)
if health.empty:
p("Healthy: n/a (no health events)")
p("Sequence gap: n/a")
p("Latency: n/a")
p("Book freshness:n/a")
else:
p(f"Healthy: {healthy_ratio*100:.2f}%")
p(f"Sequence gap: total={gap_total} window_max={gap_win_max}")
p("Latency:")
p(f" p50: {lat_p50} ms")
p(f" p95: {lat_p95} ms")
p(f" p99: {lat_p99} ms")
p(f"Book freshness: median={ba_med:.1f} ms" if ba_med is not None else "Book freshness: n/a")
p(f"Integrity: [{'PASS' if integrity_ok else 'FAIL'}]")
for n in integrity_notes:
p(f" · {n}")
if not integrity_ok:
p("\n★ STOP — Data Integrity FAIL → Decision=INVALID.")
p(" Do not interpret Alpha. Bad book/latency/gap fills have no research value.")
# ----- Section 2 -----
p("\n" + "=" * 72)
p("Section 2 — Fill Alpha")
p("Question: Fill − Matched Market Move (not PnL)")
p("Priority: distribution (median/p25/p75) over mean")
p("=" * 72)
if not integrity_ok:
p("(skipped for decision — integrity INVALID; numbers below are not evidence)")
if alpha_table["Overall"]["fill_w"] is None:
p("(waiting for fill_path with mid + after_30s)")
else:
p(f"{'':12s} {'Fill weighted':>16s} {'Cluster weighted':>18s}")
for lab in ("Bid", "Ask", "Overall"):
fw = alpha_table[lab]["fill_w"]
cw = alpha_table[lab]["cluster_w"]
p(f"{lab+' Alpha':12s} {_fmt_pct(fw):>16s} {_fmt_pct(cw):>18s}")
p(f"Direction agree (fill-w vs cluster-w): {'YES' if agree else 'NO ★'}")
p(f"Benchmark P(+) Δ (fill − matched mid): {_fmt_pp(bench_alpha)}")
p(f"SPACE class: {space_class or 'PENDING'}")
if raw_capture is not None and net_edge is not None and total_cost is not None:
p(f"Raw capture@30s (cluster-w): {_fmt_pct(raw_capture)}")
p(f"Total cost (fee+slip+lat): {_fmt_pct(total_cost)}")
p(f"Net edge: {_fmt_pct(net_edge)}")
p("")
p("Fill Alpha distribution (do not trust mean alone):")
_print_dist(p, " per-fill MakerAlpha", fill_alpha_dist)
if pct_fills_positive_alpha is not None:
p(f" share of fills with +alpha: {pct_fills_positive_alpha*100:.1f}%")
if pct_fills_positive_alpha < 0.35 and (fill_alpha_dist.get("mean") or 0) > 0:
p(" ★ mean>0 but minority of fills — edge likely event-driven / fat tail")
p("")
p("Cluster Alpha distribution (independent liquidity events):")
_print_dist(p, " per-cluster mean MakerAlpha", cluster_alpha_dist)
if (
alpha_table["Overall"]["fill_w"] is not None
and alpha_table["Overall"]["cluster_w"] is not None
):
fw, cw = alpha_table["Overall"]["fill_w"], alpha_table["Overall"]["cluster_w"]
if fw > 0 >= cw:
p(" ★ Fill+ but Cluster≤0 — edge from few burst fills; unstable")
elif fw > 0 and cw > 0:
p(" Fill+ and Cluster+ — credibility higher")
# ----- Section 3 -----
p("\n" + "=" * 72)
p("Section 3 — Toxicity Profile")
p("Question: Are fills naturally on the wrong side? (record only — no quote changes)")
p("=" * 72)
if not toxicity:
p("(waiting for fill_path)")
else:
for label, row in toxicity.items():
p(f"\n{label}:")
p(" Immediate toxicity:")
for hz in ("1s", "5s", "10s"):
p(f" {hz}: {_fmt_pct(row.get(hz))}")
p(" Recovery:")
for hz in ("30s", "300s"):
p(f" {hz}: {_fmt_pct(row.get(hz))}")
# factual pattern note only
t10, t300 = row.get("10s"), row.get("300s")
if t10 is not None and t300 is not None:
if t10 < 0 < t300:
p(" Observed pattern: early toxicity + later recovery (fact; not a rule)")
elif t10 < 0 and t300 <= 0:
p(" Observed pattern: sustained adverse (fact; not a rule)")
elif t10 is not None and t10 > 0:
p(" Observed pattern: immediate favorable (fact; not a rule)")
if c_share is not None:
p(f"\nPath C (toxic) share: {c_share*100:.1f}%")
if toxic_bid_ratio is not None:
p(f"Bid toxic fill ratio: {toxic_bid_ratio*100:.1f}%")
if adv_mag is not None and sc_mean is not None:
p(f"mean_adverse vs |spread_capture|: {_fmt_pct(adv_mag)} vs {_fmt_pct(abs(sc_mean))}")
if tox_dist:
p("\nToxicity distribution:")
if tox_dist.get("pct_adverse_10s") is not None:
p(f" fills adverse@10s: {tox_dist['pct_adverse_10s']*100:.1f}%")
if tox_dist.get("fav10"):
_print_dist(p, " fav@10s", tox_dist["fav10"])
w20 = tox_dist.get("worst20_share_of_adverse")
if w20 is not None:
p(f" worst 20% of fills share of adverse loss: {w20*100:.1f}%")
if w20 >= 0.70:
p(" ★ losses concentrated — future value may be 'which quotes NOT to place'")
p(" (record only; no cancel/filter rules in freeze)")
# ----- Section 4 -----
p("\n" + "=" * 72)
p("Section 4 — Observed Edge Attribution")
p("Facts only. Not strategy recommendations. Not filter rules.")
p("=" * 72)
if not attr_rows:
p("(insufficient state slices)")
else:
cur_title = None
for title, idx, n, mean in attr_rows:
if title != cur_title:
p(f"\n{title}:")
cur_title = title
sign = "positive" if mean > 0 else ("negative" if mean < 0 else "flat")
p(f" {idx}: n={n} E[fav30]={_fmt_pct(mean)} ({sign})")
if concentrated:
p("\nObservation: positive mass concentrated in a single bucket (fact).")
# ----- Section 5 -----
p("\n" + "=" * 72)
p("Section 5 — Decision")
p("=" * 72)
p(f"Decision: {verdict}")
p("")
if verdict == "INVALID":
p("Reason:")
for r in reasons:
p(f" - {r}")
p("\nKeep collecting only after Data Integrity is clean.")
elif verdict == "COLLECTING":
p("Reason:")
for r in reasons:
p(f" - {r}")
p("\nDo not over-interpret before 2000 fills / adequate clusters.")
p("500 = anomaly check · 2000 = preliminary · 10000 = stability.")
elif verdict == "PASS":
p("Maker alpha survives:")
for r in reasons:
p(f" - {r.replace('Maker alpha survives: ', '')}")
p("\n→ Unlock Stage3 Economic Simulation → Symmetric MM")
elif verdict == "PARTIAL_PASS":
p("Partial: market hypothesis holds only in some states/events.")
for r in reasons:
p(f" - {r}")
p("\n→ Path: Event-driven LP (not all-day Symmetric MM)")
p(" Still locked: no new filters yet — attribution is observation only.")
else:
p("No maker edge under current quote assumption.")
p("Dominant reasons:")
for r in reasons:
p(f" - {r}")
p("\nConclusion = hypothesis false (not 'strategy failed'). Avoid futile tuning.")
p("\nStage3 Unlock Checklist (Economic Simulation):")
for k, v in stage3_unlock.items():
p(f" [{'OK' if v else '·'}] {k}")
p(f" Stage3 ready: {'YES' if stage3_ready else 'NO'}")
p("")
p("State machine:")
p(" FAIL → change hypothesis")
p(" PARTIAL_PASS → Event-driven LP")
p(" PASS → Economic Simulation → Symmetric MM")
p(" COLLECTING → keep collecting")
p("")
p("Action: run probe. Look at distributions first, Decision second.")
p("=" * 72)
_finish(lines, out_path, decision)
return decision
def _finish(lines: list[str], out_path: Path | None, decision: dict[str, Any]) -> None:
if out_path:
out_path.parent.mkdir(parents=True, exist_ok=True)
footer = {
"event": "maker_edge_decision",
"report": "Maker Edge Report v0.1",
"phase": "Research Freeze / Data Collection",
"verdict": decision.get("verdict"),
"experiment": decision.get("experiment"),
"space_class": decision.get("space_class"),
"benchmark_alpha": decision.get("benchmark_alpha"),
"maker_alpha_mean": decision.get("maker_alpha_mean"),
"stage3_ready": decision.get("stage3_ready"),
"stage3_unlock": decision.get("stage3_unlock"),
"quality": decision.get("quality"),
"gates": {
"integrity": decision.get("integrity"),
"independence": decision.get("independence"),
"fill_quality": decision.get("fill_quality"),
"adverse": decision.get("adverse"),
"stability": decision.get("stability"),
},
"reasons": decision.get("reasons"),
}
text = "\n".join(lines) + "\n\n---\n" + json.dumps(footer, ensure_ascii=False, indent=2) + "\n"
out_path.write_text(text, encoding="utf-8")
print(f"\nReport saved: {out_path}")
def main() -> None:
ap = argparse.ArgumentParser(description="Maker Edge Report v0.1 — Research Freeze")
ap.add_argument(
"--dir",
type=str,
default=str(Path(__file__).resolve().parents[1] / "logs" / "maker_edge"),
)
ap.add_argument("--min-fills", type=int, default=PASS_MIN_FILLS_DEFAULT)
ap.add_argument("--report", action="store_true")
args = ap.parse_args()
log_dir = Path(args.dir)
if not log_dir.exists():
print(f"日志目录不存在: {log_dir}")
return
try:
df = load_events(log_dir)
except FileNotFoundError as e:
print(e)
return
out = log_dir / "Maker_Edge_Report_v0.1.txt" if args.report else None
report(df, min_fills=args.min_fills, out_path=out)
if __name__ == "__main__":
main()
scripts/deploy_server.sh
Executable
+95
View File
@@ -0,0 +1,95 @@
#!/usr/bin/env bash
# 部署 MM_EDGE_EXP_001 → jack@jackyu66.com:/www/Project/nautilus_mm
#
# 默认：
# SSH_HOST=jack@jackyu66.com
# SSH_KEY=~/Project/deploy/zun_hk/id_ed25519_hk
# REMOTE_DIR=/www/Project/nautilus_mm
#
# 覆盖：export SSH_HOST=... SSH_KEY=... REMOTE_DIR=...
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
SSH_HOST="${SSH_HOST:-jack@jackyu66.com}"
SSH_KEY="${SSH_KEY:-${HOME}/Project/deploy/zun_hk/id_ed25519_hk}"
REMOTE_DIR="${REMOTE_DIR:-/www/Project/nautilus_mm}"
if [[ ! -f "$SSH_KEY" ]]; then
echo "SSH key not found: $SSH_KEY"
exit 1
fi
chmod 400 "$SSH_KEY" 2>/dev/null || true
SSH_OPTS=(-i "$SSH_KEY" -o StrictHostKeyChecking=accept-new)
SSH=(ssh "${SSH_OPTS[@]}" "$SSH_HOST")
RSYNC_E="ssh ${SSH_OPTS[*]}"
echo "==> stop remote probe before sync (if running)"
"${SSH[@]}" "systemctl --user stop mm-edge-probe 2>/dev/null || true"
echo "==> sync $ROOT → $SSH_HOST:$REMOTE_DIR"
"${SSH[@]}" "mkdir -p '$REMOTE_DIR' '$REMOTE_DIR/logs/maker_edge'"
rsync -avz --delete \
-e "$RSYNC_E" \
--exclude '.venv' \
--exclude '__pycache__' \
--exclude '*.pyc' \
--exclude 'logs/maker_edge/*.jsonl' \
--exclude 'logs/maker_edge/*.txt' \
--exclude 'logs/maker_edge_smoke' \
--exclude '.env' \
"$ROOT/" "$SSH_HOST:$REMOTE_DIR/"
echo "==> remote setup (uv venv + user systemd)"
"${SSH[@]}" "REMOTE_DIR='$REMOTE_DIR' bash -s" <<'REMOTE'
set -euo pipefail
export PATH="$HOME/.local/bin:$PATH"
cd "$REMOTE_DIR"
if [[ ! -f .env ]]; then
cp .env.example .env
{
echo ""
echo "# Server Data Collection — MM_EDGE_EXP_001"
echo "EXPERIMENT_ID=MM_EDGE_EXP_001"
echo "PROBE_VERSION=probe_v0.1"
echo "EXCHANGE_NAME=binance_usdm"
echo "BINANCE_ENVIRONMENT=TESTNET"
echo "ENABLE_TRADING=false"
echo "QUOTE_TTL_SECS=30"
echo "MAX_ABS_INVENTORY=0.005"
echo "HTTP_PROXY="
echo "HTTPS_PROXY="
echo "MAKER_EDGE_LOG_DIR=${REMOTE_DIR}/logs/maker_edge"
} >> .env
echo "CREATED .env — fill BINANCE_API_KEY / BINANCE_API_SECRET"
else
echo ".env exists — left untouched"
fi
if [[ ! -x "$HOME/.local/bin/uv" ]]; then
curl -LsSf https://astral.sh/uv/install.sh | sh
fi
uv python install 3.12
rm -rf .venv
uv venv .venv --python 3.12
uv pip install -r requirements.txt --python .venv/bin/python
mkdir -p "$HOME/.config/systemd/user"
sed -e "s|/www/Project/nautilus_mm|${REMOTE_DIR}|g" \
deploy/mm-edge-probe.user.service > "$HOME/.config/systemd/user/mm-edge-probe.service"
systemctl --user daemon-reload
systemctl --user enable mm-edge-probe.service
loginctl enable-linger "$(whoami)" 2>/dev/null || true
echo "User systemd installed (not started — fill keys first)."
echo " nano $REMOTE_DIR/.env"
echo " systemctl --user start mm-edge-probe"
echo " journalctl --user -u mm-edge-probe -f"
REMOTE
echo ""
echo "==> done"
echo "1) ssh -i $SSH_KEY $SSH_HOST"
echo "2) nano $REMOTE_DIR/.env # TESTNET keys"
echo "3) systemctl --user start mm-edge-probe"
echo "4) ./scripts/probe_status.sh"
echo "5) ./scripts/pull_report.sh"
scripts/economic_attribution_v0_1.py
+533
View File
@@ -0,0 +1,533 @@
#!/usr/bin/env python3
"""
Economic Attribution v0.1
Hard Evidence Population only:
MATCHED = Local Fill ↔ Venue Trade dual evidence
Purpose:
Economic Attribution only.
No strategy modification.
No live execution.
No economic simulation.
"""
from __future__ import annotations
import argparse
import json
import math
import sys
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import numpy as np
import pandas as pd
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from reconcile_fills import load_local_fills, match, normalize_local, normalize_venue # noqa: E402
def _load_jsonl_df(log_dir: Path) -> pd.DataFrame:
rows: list[dict[str, Any]] = []
for f in sorted(log_dir.glob("*.jsonl")):
if f.name.startswith(("Account_", "Maker_", "RECON")):
continue
for line in f.open():
try:
e = json.loads(line)
except Exception:
continue
if isinstance(e, dict):
rows.append(e)
return pd.DataFrame(rows)
def _parse_fill_context(df: pd.DataFrame) -> pd.DataFrame:
if df.empty or "fill_context" not in df.columns:
return pd.DataFrame(columns=["fill_id"])
rows = []
for _, r in df.iterrows():
ctx = r.get("fill_context")
if not isinstance(ctx, dict):
continue
rows.append(
{
"fill_id": r.get("fill_id"),
"market_event_before_fill": ctx.get("market_event_before_fill"),
"trade_imbalance_5s": ctx.get("trade_imbalance_5s"),
"price_velocity_5s": ctx.get("price_velocity_5s"),
"fill_type": ctx.get("fill_type"),
}
)
return pd.DataFrame(rows)
def _fav_ret(side: pd.Series, fill: pd.Series, px: pd.Series) -> pd.Series:
raw = (pd.to_numeric(px, errors="coerce") - pd.to_numeric(fill, errors="coerce")) / pd.to_numeric(
fill, errors="coerce"
)
return pd.Series(np.where(side == "long", raw, -raw), index=side.index)
def _cluster_weight(frame: pd.DataFrame) -> pd.Series:
cnt = frame.groupby("event_cluster_id")["event_cluster_id"].transform("count")
return 1.0 / cnt.clip(lower=1)
def _pct(v: float | None) -> str:
if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))):
return "n/a"
return f"{v * 100:.4f}%"
def _num(v: float | None, digits: int = 4) -> str:
if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))):
return "n/a"
return f"{v:.{digits}f}"
def _mean(s: pd.Series) -> float | None:
s = pd.to_numeric(s, errors="coerce").dropna()
return None if s.empty else float(s.mean())
def _sum(s: pd.Series) -> float:
s = pd.to_numeric(s, errors="coerce").fillna(0.0)
return float(s.sum())
def _weighted_mean(v: pd.Series, w: pd.Series) -> float | None:
vv = pd.to_numeric(v, errors="coerce")
ww = pd.to_numeric(w, errors="coerce").fillna(0.0)
mask = vv.notna() & ww.notna()
vv = vv[mask]
ww = ww[mask]
if vv.empty or float(ww.sum()) == 0.0:
return None
return float((vv * ww).sum() / ww.sum())
def _prepare_paths(paths: pd.DataFrame) -> pd.DataFrame:
paths = paths.copy()
if "max_price" in paths.columns and "min_price" in paths.columns and "fill_price" in paths.columns:
rng = (pd.to_numeric(paths["max_price"], errors="coerce") - pd.to_numeric(paths["min_price"], errors="coerce")) / pd.to_numeric(
paths["fill_price"], errors="coerce"
)
med = float(rng.dropna().median()) if rng.notna().any() else 0.0
paths["vol_bucket"] = np.where(rng >= med, "high_vol", "low_vol")
if "price_velocity_5s" in paths.columns and pd.to_numeric(paths["price_velocity_5s"], errors="coerce").notna().any():
v = pd.to_numeric(paths["price_velocity_5s"], errors="coerce")
thr = float(v.abs().median()) * 0.5
paths["trend_bucket"] = np.where(v > thr, "trend_up", np.where(v < -thr, "trend_down", "range"))
if "spread" in paths.columns and "fill_price" in paths.columns and pd.to_numeric(paths["spread"], errors="coerce").notna().any():
sp = pd.to_numeric(paths["spread"], errors="coerce") / pd.to_numeric(paths["fill_price"], errors="coerce")
med = float(sp.dropna().median()) if sp.notna().any() else 0.0
paths["liq_bucket"] = np.where(sp <= med, "tight_spread", "wide_spread")
paths["toxicity_bucket"] = np.where(paths["path_type"].astype(str).str.startswith("C"), "toxic", "non_toxic")
return paths
def _inventory_metrics(matched: pd.DataFrame) -> dict[str, float | None]:
if matched.empty:
return {}
g = matched.sort_values("venue_time_ms").copy()
g["signed_qty"] = np.where(g["side"] == "long", g["qty"], -g["qty"])
g["net_btc"] = g["signed_qty"].cumsum()
g["abs_net_btc"] = g["net_btc"].abs()
times = pd.to_numeric(g["venue_time_ms"], errors="coerce").astype("float64") / 1000.0
dt = times.shift(-1) - times
dt = dt.fillna(0.0).clip(lower=0.0)
total_t = float(dt.sum())
tw_abs = float((g["abs_net_btc"] * dt).sum() / total_t) if total_t > 0 else None
tw_signed = float((g["net_btc"] * dt).sum() / total_t) if total_t > 0 else None
return {
"max_net_btc": float(g["net_btc"].max()),
"min_net_btc": float(g["net_btc"].min()),
"max_abs_net_btc": float(g["abs_net_btc"].max()),
"avg_abs_net_btc_per_fill": float(g["abs_net_btc"].mean()),
"time_weighted_abs_net_btc": tw_abs,
"time_weighted_signed_net_btc": tw_signed,
"long_qty": float(g.loc[g["signed_qty"] > 0, "signed_qty"].sum()),
"short_qty": float((-g.loc[g["signed_qty"] < 0, "signed_qty"]).sum()),
"turnover_btc": float(g["qty"].sum()),
}
def _bucket_table(paths: pd.DataFrame, bucket: str, title: str) -> list[dict[str, Any]]:
if bucket not in paths.columns or paths.empty:
return []
rows = []
for key, grp in paths.groupby(bucket):
notional = grp["notional_usdt"].sum()
clusters = grp["event_cluster_id"].nunique()
rows.append(
{
"dimension": title,
"bucket": str(key),
"fills": int(len(grp)),
"clusters": int(clusters),
"btc_qty": float(grp["qty"].sum()),
"notional_usdt": float(notional),
"fee_usdt": float(grp["commission_usdt"].sum()),
"fee_per_fill": float(grp["commission_usdt"].mean()) if len(grp) else None,
"fee_per_btc": float(grp["commission_usdt"].sum() / grp["qty"].sum()) if grp["qty"].sum() else None,
"markout_1s": _weighted_mean(grp["markout_1s"], grp["notional_usdt"]),
"markout_5s": _weighted_mean(grp["markout_5s"], grp["notional_usdt"]),
"markout_10s": _weighted_mean(grp["markout_10s"], grp["notional_usdt"]),
"markout_30s": _weighted_mean(grp["markout_30s"], grp["notional_usdt"]),
"markout_300s": _weighted_mean(grp["markout_300s"], grp["notional_usdt"]),
"gross_markout_30s_usdt": float(grp["gross_markout_30s_usdt"].sum()),
"realized_pnl_usdt": float(grp["realized_pnl_usdt"].sum()),
"net_attr_30s_usdt": float(grp["net_attr_30s_usdt"].sum()),
}
)
rows.sort(key=lambda x: (-x["fills"], x["bucket"]))
return rows
def _counterfactual(base: pd.DataFrame, exclude_col: str, exclude_values: set[str], label: str) -> dict[str, Any]:
kept = base[~base[exclude_col].astype(str).isin(exclude_values)].copy()
return {
"name": label,
"fills": int(len(kept)),
"clusters": int(kept["event_cluster_id"].nunique()) if not kept.empty else 0,
"btc_qty": float(kept["qty"].sum()) if not kept.empty else 0.0,
"fee_usdt": float(kept["commission_usdt"].sum()) if not kept.empty else 0.0,
"gross_markout_30s_usdt": float(kept["gross_markout_30s_usdt"].sum()) if not kept.empty else 0.0,
"realized_pnl_usdt": float(kept["realized_pnl_usdt"].sum()) if not kept.empty else 0.0,
"net_attr_30s_usdt": float(kept["net_attr_30s_usdt"].sum()) if not kept.empty else 0.0,
"markout_30s": _weighted_mean(kept["markout_30s"], kept["notional_usdt"]),
}
def main() -> int:
ap = argparse.ArgumentParser(description="Economic Attribution v0.1 (MATCHED only)")
ap.add_argument("--dir", default=str(ROOT / "logs" / "maker_edge"))
ap.add_argument("--out", default=str(ROOT / "logs" / "maker_edge" / "Economic_Attribution_v0_1.txt"))
ap.add_argument("--recon03", default=str(ROOT / "logs" / "maker_edge" / "RECONCILIATION_03.json"))
ap.add_argument("--account", default=str(ROOT / "logs" / "maker_edge" / "Account_Reconciliation.json"))
ap.add_argument("--venue-trades", default=str(ROOT / "logs" / "maker_edge" / "venue_trades.json"))
args = ap.parse_args()
log_dir = Path(args.dir)
df = _load_jsonl_df(log_dir)
fills = df[df["event"] == "fill"].copy()
paths = df[df["event"] == "fill_path"].copy()
inv = df[df["event"] == "inventory_tick"].copy()
venue_trades = json.loads(Path(args.venue_trades).read_text())
local_fills_raw = load_local_fills(log_dir)
locals_norm = [normalize_local(e, i) for i, e in enumerate(local_fills_raw)]
venues_norm = [normalize_venue(t, i) for i, t in enumerate(venue_trades)]
recon = match(locals_norm, venues_norm)
matched_fill_ids = {m["local"]["fill_id"] for m in recon["matched"]}
matched_trade_ids = {m["venue"]["venue_trade_id"] for m in recon["matched"]}
fills = fills[fills["fill_id"].isin(matched_fill_ids)].copy()
paths = paths[paths["fill_id"].isin(matched_fill_ids)].copy()
fc = _parse_fill_context(fills)
meta_cols = [
c
for c in [
"fill_id",
"side",
"fill_price",
"spread",
"spread_capture_pct",
"obi",
"trade_imbalance",
"bid_depth_5",
"ask_depth_5",
"book_age_ms",
"inventory",
"inventory_time",
"inventory_skew",
"pre_5s_deteriorated",
"mid",
"event_cluster_id",
"pair",
]
if c in fills.columns
]
meta = fills.drop_duplicates("fill_id")[meta_cols]
paths = paths.merge(meta, on="fill_id", how="left", suffixes=("", "_f"))
for col in ["event_cluster_id", "side", "fill_price", "spread_capture_pct", "mid"]:
alt = f"{col}_f"
if alt in paths.columns:
if col not in paths.columns:
paths[col] = paths[alt]
else:
paths[col] = paths[col].fillna(paths[alt])
if not fc.empty:
paths = paths.merge(fc, on="fill_id", how="left")
paths = _prepare_paths(paths)
venue = pd.DataFrame(venues_norm)
venue = venue[venue["venue_trade_id"].isin(matched_trade_ids)].copy()
venue = venue.rename(
columns={
"venue_trade_id": "trade_id_link",
"venue_order_id": "venue_order_id",
"qty": "qty",
"px": "venue_price",
"ts": "venue_ts",
}
)
raw_v = pd.DataFrame(venue_trades)
raw_v["trade_id_link"] = raw_v["id"].astype(str)
raw_v["venue_order_id"] = raw_v["orderId"].astype(str)
raw_v["commission_usdt"] = pd.to_numeric(raw_v["commission"], errors="coerce")
raw_v["realized_pnl_usdt"] = pd.to_numeric(raw_v["realizedPnl"], errors="coerce").fillna(0.0)
raw_v["venue_time_ms"] = pd.to_numeric(raw_v["time"], errors="coerce")
raw_v["qty"] = pd.to_numeric(raw_v["qty"], errors="coerce")
raw_v["venue_price"] = pd.to_numeric(raw_v["price"], errors="coerce")
raw_v["side"] = np.where(raw_v["buyer"].astype(bool), "long", "short")
raw_v = raw_v[raw_v["trade_id_link"].isin(matched_trade_ids)].copy()
matched_map = pd.DataFrame(
[
{
"fill_id": m["local"]["fill_id"],
"trade_id_link": m["venue"]["venue_trade_id"],
"venue_order_id": m["venue"]["venue_order_id"],
}
for m in recon["matched"]
]
)
paths = paths.merge(
matched_map.merge(
raw_v[
[
"trade_id_link",
"venue_order_id",
"commission_usdt",
"realized_pnl_usdt",
"venue_time_ms",
"qty",
"venue_price",
"side",
]
],
on=["trade_id_link", "venue_order_id"],
how="left",
),
on="fill_id",
how="left",
suffixes=("", "_venue"),
)
paths["fill_price"] = pd.to_numeric(paths["fill_price"], errors="coerce")
paths["qty"] = pd.to_numeric(paths["qty"], errors="coerce")
paths["notional_usdt"] = paths["fill_price"] * paths["qty"]
for sec, col in [(1, "after_1s_price"), (5, "after_5s_price"), (10, "after_10s_price"), (30, "after_30s_price"), (300, "after_5m_price")]:
paths[f"markout_{sec}s"] = _fav_ret(paths["side"], paths["fill_price"], paths[col])
paths["gross_markout_30s_usdt"] = paths["notional_usdt"] * paths["markout_30s"]
paths["net_attr_30s_usdt"] = (
paths["gross_markout_30s_usdt"]
- pd.to_numeric(paths["commission_usdt"], errors="coerce").fillna(0.0)
+ pd.to_numeric(paths["realized_pnl_usdt"], errors="coerce").fillna(0.0)
)
inventory_metrics = _inventory_metrics(
raw_v[
["venue_time_ms", "side", "qty", "commission_usdt", "realized_pnl_usdt", "venue_order_id", "trade_id_link"]
].copy()
)
n_matched_paths = len(paths)
n_matched_fills = len(fills)
n_matched_clusters = int(fills["event_cluster_id"].nunique()) if not fills.empty else 0
cluster_w = _cluster_weight(paths) if not paths.empty and "event_cluster_id" in paths.columns else pd.Series(dtype=float)
horizon_rows = []
for sec in (1, 5, 10, 30, 300):
col = f"markout_{sec}s"
valid = paths[col].notna()
sub = paths[valid]
w = sub["notional_usdt"]
horizon_rows.append(
{
"horizon": f"{sec}s",
"n": int(len(sub)),
"fill_w": _weighted_mean(sub[col], w),
"cluster_w": _weighted_mean(sub[col], _cluster_weight(sub) if not sub.empty else pd.Series(dtype=float)),
"gross_usdt": float((sub["notional_usdt"] * sub[col]).sum()) if not sub.empty else 0.0,
}
)
fee_total = float(paths["commission_usdt"].sum())
realized_total = float(paths["realized_pnl_usdt"].sum())
gross_30_total = float(paths["gross_markout_30s_usdt"].sum())
net_attr_30_total = float(paths["net_attr_30s_usdt"].sum())
total_qty = float(paths["qty"].sum())
total_notional = float(paths["notional_usdt"].sum())
bucket_rows: list[dict[str, Any]] = []
for col, title in [
("path_type", "PathType"),
("toxicity_bucket", "Toxicity"),
("vol_bucket", "Volatility"),
("liq_bucket", "Spread"),
("trend_bucket", "Trend"),
("market_event_before_fill", "FillContext"),
]:
bucket_rows.extend(_bucket_table(paths, col, title))
bucket_df = pd.DataFrame(bucket_rows)
negative_states: set[str] = set()
if not bucket_df.empty:
neg = bucket_df[(bucket_df["dimension"] != "PathType") & (bucket_df["markout_30s"] < 0)]
negative_states = set(neg["bucket"].astype(str))
counterfactuals = [
{
"name": "BASELINE",
"fills": int(len(paths)),
"clusters": int(paths["event_cluster_id"].nunique()) if not paths.empty else 0,
"btc_qty": total_qty,
"fee_usdt": fee_total,
"gross_markout_30s_usdt": gross_30_total,
"realized_pnl_usdt": realized_total,
"net_attr_30s_usdt": net_attr_30_total,
"markout_30s": _weighted_mean(paths["markout_30s"], paths["notional_usdt"]),
},
_counterfactual(paths, "path_type", {"C_toxic"}, "EXCLUDE_PATH_C"),
_counterfactual(paths, "toxicity_bucket", {"toxic"}, "EXCLUDE_TOXIC"),
_counterfactual(paths, "market_event_before_fill", negative_states, "EXCLUDE_NEGATIVE_STATE"),
]
account = json.loads(Path(args.account).read_text()) if Path(args.account).exists() else {}
recon03 = json.loads(Path(args.recon03).read_text()) if Path(args.recon03).exists() else {}
out_txt = Path(args.out)
out_json = out_txt.with_suffix(".json")
lines: list[str] = []
def p(s: str = "") -> None:
lines.append(s)
print(s)
p("=" * 72)
p("Economic Attribution v0.1")
p("=" * 72)
p("Experiment: MM_EDGE_EXP_001")
p("Population: MATCHED=3890")
p("Strategy: v0.1 FROZEN")
p("Execution: STOPPED")
p("Stage3: LOCKED")
p("Purpose: Economic Attribution only.")
p("No strategy modification. No live execution. No economic simulation.")
p()
p("Layer 1 — Hard Economic Evidence")
p("-" * 40)
p(f"Matched fills: {n_matched_fills}")
p(f"Matched paths: {n_matched_paths}")
p(f"Matched clusters: {n_matched_clusters}")
p(f"Fee total: {_num(fee_total, 6)} USDT")
p(f"Fee / fill: {_num(fee_total / max(n_matched_paths, 1), 6)} USDT")
p(f"Fee / BTC: {_num(fee_total / max(total_qty, 1e-12), 6)} USDT")
p(f"Fee / cluster: {_num(fee_total / max(n_matched_clusters, 1), 6)} USDT")
p(f"Realized component: {_num(realized_total, 6)} USDT")
p(f"Gross markout @30s: {_num(gross_30_total, 6)} USDT")
p(f"Net attributable @30s: {_num(net_attr_30_total, 6)} USDT")
p()
p("Markout by horizon (MATCHED only)")
p("-" * 40)
for row in horizon_rows:
p(
f"{row['horizon']:>5} n={row['n']:4d} fill-w={_pct(row['fill_w'])} "
f"cluster-w={_pct(row['cluster_w'])} gross={_num(row['gross_usdt'], 6)} USDT"
)
p()
p("Inventory carry / exposure")
p("-" * 40)
p(f"Max net BTC: {_num(inventory_metrics.get('max_net_btc'), 6)}")
p(f"Min net BTC: {_num(inventory_metrics.get('min_net_btc'), 6)}")
p(f"Max |net BTC|: {_num(inventory_metrics.get('max_abs_net_btc'), 6)}")
p(f"Average |net BTC|: {_num(inventory_metrics.get('avg_abs_net_btc_per_fill'), 6)}")
p(f"TW |net BTC|: {_num(inventory_metrics.get('time_weighted_abs_net_btc'), 6)}")
p(f"TW signed net BTC: {_num(inventory_metrics.get('time_weighted_signed_net_btc'), 6)}")
p(f"Long qty / Short qty: {_num(inventory_metrics.get('long_qty'), 6)} / {_num(inventory_metrics.get('short_qty'), 6)} BTC")
p(f"Inventory turnover: {_num(inventory_metrics.get('turnover_btc'), 6)} BTC")
p()
p("Slices (weighted by notional, MATCHED only)")
p("-" * 40)
for dim in ["PathType", "Toxicity", "Volatility", "Spread", "Trend", "FillContext"]:
sub = bucket_df[bucket_df["dimension"] == dim].copy()
if sub.empty:
continue
p(dim)
for _, r in sub.sort_values(["fills", "bucket"], ascending=[False, True]).iterrows():
p(
f" {r['bucket']}: n={int(r['fills'])} clusters={int(r['clusters'])} "
f"fee={_num(r['fee_usdt'], 4)} gross30={_num(r['gross_markout_30s_usdt'], 4)} "
f"realized={_num(r['realized_pnl_usdt'], 4)} net30={_num(r['net_attr_30s_usdt'], 4)} "
f"m30={_pct(r['markout_30s'])}"
)
p()
p("Layer 2 — Evidence Extension (excluded from core conclusion)")
p("-" * 40)
p(f"VENUE_CONFIRMED_NO_TRADE_HISTORY: {recon03.get('venue_confirmed_no_trade_history', 'n/a')}")
p(f"VENUE_PARTIAL_ORDER_CANCELED: {recon03.get('venue_partial_order_canceled', 'n/a')}")
p("These rows are order-confirmed, but not part of the Hard Evidence Population.")
p()
p("Layer 3 — Counterfactual Attribution (NOT backtest)")
p("-" * 40)
p("Observed vs Exclude-Bucket Attribution. These are contribution decompositions only.")
for row in counterfactuals:
p(
f"{row['name']}: fills={row['fills']} clusters={row['clusters']} "
f"fee={_num(row['fee_usdt'], 4)} gross30={_num(row['gross_markout_30s_usdt'], 4)} "
f"realized={_num(row['realized_pnl_usdt'], 4)} net30={_num(row['net_attr_30s_usdt'], 4)} "
f"m30={_pct(row['markout_30s'])}"
)
p()
p("Interpretation")
p("-" * 40)
p("Core conclusion is based on 3890 fully matched fills.")
p("Economic Attribution asks why MakerAlpha did not convert to money.")
p("It does NOT change quote logic, does NOT restart v0.1, and does NOT unlock Stage 3.")
p("=" * 72)
out_txt.write_text("\n".join(lines) + "\n", encoding="utf-8")
sidecar = {
"experiment_id": "MM_EDGE_EXP_001",
"population": {
"name": "MATCHED",
"fills": n_matched_fills,
"paths": n_matched_paths,
"clusters": n_matched_clusters,
},
"strategy": "v0.1 FROZEN",
"execution": "STOPPED",
"stage3": "LOCKED",
"fee_total_usdt": fee_total,
"fee_per_fill_usdt": fee_total / max(n_matched_paths, 1),
"fee_per_btc_usdt": fee_total / max(total_qty, 1e-12),
"fee_per_cluster_usdt": fee_total / max(n_matched_clusters, 1),
"realized_component_usdt": realized_total,
"gross_markout_30s_usdt": gross_30_total,
"net_attr_30s_usdt": net_attr_30_total,
"markout_by_horizon": horizon_rows,
"inventory_metrics": inventory_metrics,
"bucket_rows": bucket_rows,
"counterfactuals": counterfactuals,
"recon03_extension": {
"venue_confirmed_no_trade_history": recon03.get("venue_confirmed_no_trade_history"),
"venue_partial_order_canceled": recon03.get("venue_partial_order_canceled"),
},
"account_recon_ref": account,
}
out_json.write_text(json.dumps(sidecar, indent=2) + "\n", encoding="utf-8")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/economic_fee_sensitivity_v0_1.py
+81
View File
@@ -0,0 +1,81 @@
#!/usr/bin/env python3
"""
Economic Fee Sensitivity v0.1 (MATCHED=3890)
Computes:
net_attr_30s(fee_factor) = gross_markout_30s_usdt - fee_factor * fee_total_usdt + realized_component_usdt
Assumption:
realized_component_usdt and gross_markout_30s_usdt are fixed (price/path unchanged).
Only fee scaling is applied as a counterfactual sensitivity.
This is NOT a strategy backtest and does NOT modify any execution logic.
"""
from __future__ import annotations
import argparse
import json
import math
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def main() -> int:
ap = argparse.ArgumentParser(description="Economic Fee Sensitivity v0.1")
ap.add_argument(
"--attribution",
default=str(ROOT / "logs" / "maker_edge" / "Economic_Attribution_v0_1.json"),
)
ap.add_argument(
"--out",
default=str(ROOT / "logs" / "maker_edge" / "Economic_Fee_Sensitivity_v0_1.txt"),
)
args = ap.parse_args()
data = json.loads(Path(args.attribution).read_text())
fee_total = float(data["fee_total_usdt"])
realized_total = float(data["realized_component_usdt"])
gross_markout = float(data["gross_markout_30s_usdt"])
factors = [1.0, 0.5, 0.25, 0.1, 0.0]
lines: list[str] = []
def p(s: str = "") -> None:
lines.append(s)
print(s)
p("=" * 72)
p("Economic Fee Sensitivity v0.1 (MATCHED=3890)")
p("=" * 72)
p(f"gross_markout_30s_usdt: {gross_markout:+.6f} USDT")
p(f"fee_total_usdt: {fee_total:+.6f} USDT")
p(f"realized_component_usdt:{realized_total:+.6f} USDT")
p()
p("Fee assumption → Net attributable @30s")
p("-" * 42)
header = ["fee_factor", "fee_usdt_assumed", "net_attr_30s_usdt"]
p(" | ".join(header))
for f in factors:
fee_assumed = f * fee_total
net = gross_markout - fee_assumed + realized_total
row = [f"{f:.2f}", f"{fee_assumed:+.6f}", f"{net:+.6f}"]
p(" | ".join(row))
p()
p("Interpretation:")
p("- If net remains < 0 at fee_factor=0 → economics not salvageable by fee reduction alone.")
p("- If fee reduction flips net > 0 → current venue/fee tier can be the dominant issue.")
p("=" * 72)
Path(args.out).write_text("\n".join(lines) + "\n", encoding="utf-8")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/economic_metric_reconciliation_v0_1.py
+204
View File
@@ -0,0 +1,204 @@
#!/usr/bin/env python3
"""
Metric Reconciliation v0.1 (MATCHED only)
Confirms consistency between:
- "MakerAlpha" reported in v0.1 research (return space)
- "Gross markout @30s" in Economic Attribution (dollar space)
- realized component used in Economic Attribution
Key point:
Same definition may flip sign depending on weighting:
fill-weighted mean return vs notional-weighted dollar markout
This script is read-only: it does NOT change any strategy/execution.
"""
from __future__ import annotations
import argparse
import json
import math
import sys
from collections import defaultdict
from pathlib import Path
import numpy as np
import pandas as pd
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from reconcile_fills import load_local_fills, match, normalize_local, normalize_venue # noqa: E402
def _load_jsonl_df(log_dir: Path) -> pd.DataFrame:
rows: list[dict] = []
for f in sorted(log_dir.glob("*.jsonl")):
if f.name.startswith(("Account_", "Maker_", "RECON")):
continue
for line in f.open():
try:
e = json.loads(line)
except Exception:
continue
if isinstance(e, dict):
rows.append(e)
return pd.DataFrame(rows)
def _fav_ret(side: pd.Series, fill: pd.Series, px: pd.Series) -> pd.Series:
# return space, signed by side
fill = pd.to_numeric(fill, errors="coerce")
px = pd.to_numeric(px, errors="coerce")
raw = (px - fill) / fill
return pd.Series(np.where(side == "long", raw, -raw), index=side.index)
def _weighted_mean(x: pd.Series, w: pd.Series) -> float | None:
xx = pd.to_numeric(x, errors="coerce")
ww = pd.to_numeric(w, errors="coerce")
mask = xx.notna() & ww.notna()
xx = xx[mask]
ww = ww[mask]
if xx.empty:
return None
sw = float(ww.sum())
if sw == 0:
return None
return float((xx * ww).sum() / sw)
def _cluster_weight(paths: pd.DataFrame) -> pd.Series:
if "event_cluster_id" not in paths.columns:
return pd.Series(1.0, index=paths.index)
cnt = paths.groupby("event_cluster_id")["event_cluster_id"].transform("count")
return 1.0 / cnt.clip(lower=1)
def main() -> int:
ap = argparse.ArgumentParser(description="Economic Metric Reconciliation v0.1")
ap.add_argument("--dir", default=str(ROOT / "logs" / "maker_edge"))
ap.add_argument("--out", default=str(ROOT / "logs" / "maker_edge" / "Economic_Metric_Reconciliation_v0_1.txt"))
ap.add_argument("--venue-trades", default=str(ROOT / "logs" / "maker_edge" / "venue_trades.json"))
ap.add_argument("--matched-take", type=int, default=3890)
args = ap.parse_args()
log_dir = Path(args.dir)
venue_trades_path = Path(args.venue_trades)
df = _load_jsonl_df(log_dir)
fills = df[df["event"] == "fill"].copy() if "event" in df.columns else pd.DataFrame()
paths = df[df["event"] == "fill_path"].copy() if "event" in df.columns else pd.DataFrame()
# Hard matched population via RECON-02/03 evidence: use existing matcher logic.
venue_trades = json.loads(venue_trades_path.read_text())
local_fills_raw = load_local_fills(log_dir)
locals_norm = [normalize_local(e, i) for i, e in enumerate(local_fills_raw)]
venues_norm = [normalize_venue(t, i) for i, t in enumerate(venue_trades)]
recon = match(locals_norm, venues_norm)
matched_fill_ids = {m["local"]["fill_id"] for m in recon["matched"]}
matched_trade_ids = {m["venue"]["venue_trade_id"] for m in recon["matched"]}
fills = fills[fills["fill_id"].isin(matched_fill_ids)].copy()
paths = paths[paths["fill_id"].isin(matched_fill_ids)].copy()
# Build after_30s already present in fill_path fields.
# MakerAlpha in analyze_maker_edge uses after_30s_price and _fav_ret definition.
# We'll recompute:
# return space:
# maker_alpha_fill_weighted = mean(markout_30s)
# maker_alpha_notional_weighted_return = (gross_markout_usdt / total_notional)
# gross_markout_usdt = sum(notional * markout_30s)
#
if paths.empty:
raise SystemExit("No matched paths loaded")
# Merge meta from fills (side, fill_price, event_cluster_id, notional proxy)
meta_cols = [
c
for c in [
"fill_id",
"side",
"fill_price",
"amount",
"event_cluster_id",
"spread_capture_pct",
"pair",
]
if c in fills.columns
]
meta = fills.drop_duplicates("fill_id")[meta_cols]
paths = paths.merge(meta, on="fill_id", how="left", suffixes=("", "_m"))
# If fill_path already had these columns, merge created *_m alternates.
for col in ["side", "fill_price", "amount", "event_cluster_id"]:
alt = f"{col}_m"
if alt in paths.columns:
if col not in paths.columns:
paths[col] = paths[alt]
else:
paths[col] = paths[col].fillna(paths[alt])
# Ensure required fields
paths["side"] = paths["side"].astype(str)
paths["fill_price"] = pd.to_numeric(paths["fill_price"], errors="coerce")
paths["qty"] = pd.to_numeric(paths["amount"], errors="coerce")
paths["notional_usdt"] = paths["fill_price"] * paths["qty"]
paths["after_30s_price"] = pd.to_numeric(paths["after_30s_price"], errors="coerce")
paths["markout_30s_return"] = _fav_ret(paths["side"], paths["fill_price"], paths["after_30s_price"])
gross_markout_usdt = float((paths["notional_usdt"] * paths["markout_30s_return"]).sum())
total_notional = float(paths["notional_usdt"].sum())
maker_alpha_fill_weighted = float(paths["markout_30s_return"].mean())
maker_alpha_notional_weighted_return = float(gross_markout_usdt / total_notional) if total_notional else None
cw = _cluster_weight(paths)
maker_alpha_cluster_weighted_return = _weighted_mean(paths["markout_30s_return"], cw)
# realized component from userTrades is already in Economic Attribution.
# Here we only validate return space; realized component sign conventions are asserted elsewhere.
out = Path(args.out)
lines: list[str] = []
def p(s: str = "") -> None:
lines.append(s)
print(s)
p("=" * 72)
p("Economic Metric Reconciliation v0.1 (MATCHED=3890)")
p("=" * 72)
p(f"Matched paths: {len(paths)} (expected ~3886)")
p()
p("Definitions (same math as analyze_maker_edge):")
p("- markout_30s_return = _fav_ret(side, fill_price, after_30s_price)")
p("- gross_markout_usdt = sum(notional_usdt * markout_30s_return)")
p()
p("Return-space metrics (sign may differ due to weighting):")
p(f"MakerAlpha fill-weighted mean return: {_pct(maker_alpha_fill_weighted)}")
p(f"MakerAlpha notional-weighted mean return: {_pct(maker_alpha_notional_weighted_return)}")
p(f"MakerAlpha cluster-weighted mean return: {_pct(maker_alpha_cluster_weighted_return)}")
p()
p("Dollar-space metrics:")
p(f"gross_markout_usdt (30s): {gross_markout_usdt:+.6f} USDT")
p(f"total_notional_usdt: {total_notional:.3f} USDT")
p()
p("If fill-weighted return is + but gross_markout_usdt is negative,")
p("it means notional weighting flips sign (alpha is conditionally realized).")
p("=" * 72)
out.write_text("\n".join(lines) + "\n", encoding="utf-8")
return 0
def _pct(v: float | None) -> str:
if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))):
return "n/a"
return f"{v*100:.6f}%"
if __name__ == "__main__":
raise SystemExit(main())
scripts/event_state_status.sh
Executable
+64
View File
@@ -0,0 +1,64 @@
#!/usr/bin/env bash
# Remote EXP_002 long-run status (read-only). Does not analyze Path C.
set -euo pipefail
SSH_HOST="${SSH_HOST:-jack@jackyu66.com}"
SSH_KEY="${SSH_KEY:-${HOME}/Project/deploy/zun_hk/id_ed25519_hk}"
REMOTE_DIR="${REMOTE_DIR:-/www/Project/nautilus_mm}"
SSH_OPTS=(-o StrictHostKeyChecking=accept-new)
if [[ -n "$SSH_KEY" ]]; then
chmod 400 "$SSH_KEY" 2>/dev/null || true
SSH_OPTS+=(-i "$SSH_KEY")
fi
ssh "${SSH_OPTS[@]}" "$SSH_HOST" "REMOTE_DIR='$REMOTE_DIR' bash -s" <<'EOF'
set -euo pipefail
echo "=== systemd --user event-state-probe ==="
systemctl --user is-active event-state-probe || true
systemctl --user show event-state-probe -p Environment --no-pager 2>/dev/null | tr ' ' '\n' | grep -E 'ENABLE_TRADING|EXPERIMENT_ID|LEDGER_RUN_ID' || true
echo ""
echo "=== mm-edge-probe (EXP_001) ==="
systemctl --user is-active mm-edge-probe || true
echo ""
LOG="$REMOTE_DIR/logs/event_state/EXP-002-RUN-002"
echo "=== ledger $LOG ==="
if [[ ! -d "$LOG" ]]; then
echo "no log dir yet"
exit 0
fi
python3 - <<PY
import json
from pathlib import Path
log = Path("$LOG")
starts = trades = books = fills = parse_fail = 0
run_id = None
for f in sorted(log.glob("*.jsonl")):
for line in f.open():
s = line.strip()
if not s:
continue
try:
ev = json.loads(s)
except Exception:
parse_fail += 1
continue
run_id = ev.get("run_id") or run_id
e = ev.get("event")
if e == "experiment_start":
starts += 1
elif e == "fill_anchor":
fills += 1
elif e == "market_event":
t = ev.get("event_type")
if t == "aggressive_trade":
trades += 1
elif t == "book_update":
books += 1
print(f"run_id={run_id} starts={starts} trades={trades} books={books} fill_anchors={fills} parse_fail={parse_fail}")
print("Gate 4 remains BLOCKED until fill_anchors exist. Do not Path-C snoop.")
PY
echo ""
echo "=== journal (last 15) ==="
journalctl --user -u event-state-probe -n 15 --no-pager || true
EOF
scripts/prefill_adverse_selection_attribution_v0_1.py
+595
View File
@@ -0,0 +1,595 @@
#!/usr/bin/env python3
"""
Prefill Adverse-Selection Attribution v0.1
Experiment: MM_EDGE_EXP_001
Population: frozen historical fills
Strategy: v0.1 FROZEN
Execution: STOPPED
Purpose:
Pre-fill adverse-selection predictability audit
NOT:
strategy
backtest
optimization
model training
Hard contract:
feature_timestamp <= t_fill - margin_sec
This script intentionally prefers strict no-leakage over feature richness.
"""
from __future__ import annotations
import argparse
import json
import math
import sys
from pathlib import Path
from typing import Any
import numpy as np
import pandas as pd
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from reconcile_fills import load_local_fills, match, normalize_local, normalize_venue # noqa: E402
def _load_jsonl_df(log_dir: Path) -> pd.DataFrame:
rows: list[dict[str, Any]] = []
for f in sorted(log_dir.glob("*.jsonl")):
if f.name.startswith(("Account_", "Maker_", "RECON")):
continue
for line in f.open():
try:
e = json.loads(line)
except Exception:
continue
if isinstance(e, dict):
rows.append(e)
return pd.DataFrame(rows)
def _pct(v: float | None) -> str:
if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))):
return "n/a"
return f"{v*100:.2f}%"
def _num(v: float | None, digits: int = 4) -> str:
if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))):
return "n/a"
return f"{v:.{digits}f}"
def _fav_ret(side: pd.Series, fill: pd.Series, px: pd.Series) -> pd.Series:
fill = pd.to_numeric(fill, errors="coerce")
px = pd.to_numeric(px, errors="coerce")
raw = (px - fill) / fill
return pd.Series(np.where(side == "long", raw, -raw), index=side.index)
def _weighted_mean(x: pd.Series, w: pd.Series) -> float | None:
xx = pd.to_numeric(x, errors="coerce")
ww = pd.to_numeric(w, errors="coerce")
mask = xx.notna() & ww.notna()
xx = xx[mask]
ww = ww[mask]
if xx.empty:
return None
sw = float(ww.sum())
if sw == 0.0:
return None
return float((xx * ww).sum() / sw)
def _sample_grade(n: int) -> str:
if n < 30:
return "LOW_N"
if n < 100:
return "WEAK_EVIDENCE"
return "USABLE"
def _grade_probability(delta_pp: float, n_best: int) -> str:
if n_best < 30:
return "LOW_N"
if delta_pp < 5.0:
return "NO_PREFILL_SIGNAL"
if n_best < 100 or delta_pp < 10.0:
return "STATISTICAL_SIGNAL_ONLY"
return "CANDIDATE_V0_2_SIGNAL"
def _grade_economic(delta_usdt_per_fill: float, n_best: int) -> str:
if n_best < 30:
return "LOW_N"
if abs(delta_usdt_per_fill) < 0.003:
return "NO_PREFILL_SIGNAL"
if n_best < 100 or abs(delta_usdt_per_fill) < 0.008:
return "STATISTICAL_SIGNAL_ONLY"
return "CANDIDATE_V0_2_SIGNAL"
def _state_table_num(df: pd.DataFrame, feature: str, labels: list[str]) -> tuple[list[dict[str, Any]], dict[str, str]]:
s = pd.to_numeric(df[feature], errors="coerce")
valid = df[s.notna()].copy()
valid[feature] = s[s.notna()]
if valid.empty:
return [], {k: "NO_DATA" for k in labels + ["Economic"]}
q30 = float(valid[feature].quantile(0.30))
q70 = float(valid[feature].quantile(0.70))
# if no spread, collapse
if math.isclose(q30, q70):
valid["_state"] = "all"
else:
valid["_state"] = np.where(
valid[feature] <= q30,
"low",
np.where(valid[feature] >= q70, "high", "mid"),
)
base = {
lab: float(valid[lab].mean()) for lab in labels
}
base["economic_mean"] = float(valid["net_attr_30s_usdt"].mean())
rows = []
grades: dict[str, str] = {}
for state, g in valid.groupby("_state"):
row = {
"feature": feature,
"state": str(state),
"n": int(len(g)),
"sample_grade": _sample_grade(int(len(g))),
"median": float(g[feature].median()),
"p25": float(g[feature].quantile(0.25)),
"p75": float(g[feature].quantile(0.75)),
"net_attr_mean": float(g["net_attr_30s_usdt"].mean()),
}
for lab in labels:
row[f"p_{lab}"] = float(g[lab].mean())
row[f"delta_{lab}_pp"] = (row[f"p_{lab}"] - base[lab]) * 100.0
row["delta_economic_per_fill"] = row["net_attr_mean"] - base["economic_mean"]
rows.append(row)
# grade by strongest state-vs-baseline shift
for lab in labels:
best = max(rows, key=lambda r: abs(r[f"delta_{lab}_pp"]))
grades[lab] = _grade_probability(abs(best[f"delta_{lab}_pp"]), int(best["n"]))
best_e = max(rows, key=lambda r: abs(r["delta_economic_per_fill"]))
grades["Economic"] = _grade_economic(abs(best_e["delta_economic_per_fill"]), int(best_e["n"]))
return rows, grades
def _state_table_cat(df: pd.DataFrame, feature: str, labels: list[str]) -> tuple[list[dict[str, Any]], dict[str, str]]:
valid = df[df[feature].notna()].copy()
if valid.empty:
return [], {k: "NO_DATA" for k in labels + ["Economic"]}
base = {
lab: float(valid[lab].mean()) for lab in labels
}
base["economic_mean"] = float(valid["net_attr_30s_usdt"].mean())
rows = []
grades: dict[str, str] = {}
for state, g in valid.groupby(feature):
n = int(len(g))
row = {
"feature": feature,
"state": str(state),
"n": n,
"sample_grade": _sample_grade(n),
"median": None,
"p25": None,
"p75": None,
"net_attr_mean": float(g["net_attr_30s_usdt"].mean()),
}
for lab in labels:
row[f"p_{lab}"] = float(g[lab].mean())
row[f"delta_{lab}_pp"] = (row[f"p_{lab}"] - base[lab]) * 100.0
row["delta_economic_per_fill"] = row["net_attr_mean"] - base["economic_mean"]
rows.append(row)
for lab in labels:
best = max(rows, key=lambda r: abs(r[f"delta_{lab}_pp"]))
grades[lab] = _grade_probability(abs(best[f"delta_{lab}_pp"]), int(best["n"]))
best_e = max(rows, key=lambda r: abs(r["delta_economic_per_fill"]))
grades["Economic"] = _grade_economic(abs(best_e["delta_economic_per_fill"]), int(best_e["n"]))
return rows, grades
def main() -> int:
ap = argparse.ArgumentParser(description="Prefill Adverse-Selection Attribution v0.1")
ap.add_argument("--dir", default=str(ROOT / "logs" / "maker_edge"))
ap.add_argument("--venue-trades", default=str(ROOT / "logs" / "maker_edge" / "venue_trades.json"))
ap.add_argument("--out", default=str(ROOT / "logs" / "maker_edge" / "Prefill_Adverse_Selection_Attribution_v0_1.txt"))
ap.add_argument("--margin-sec", type=float, default=0.25)
args = ap.parse_args()
log_dir = Path(args.dir)
df = _load_jsonl_df(log_dir)
fills = df[df["event"] == "fill"].copy()
paths = df[df["event"] == "fill_path"].copy()
state_ticks = df[df["event"].isin(["mid_tick", "inventory_tick"])].copy()
venue_trades = json.loads(Path(args.venue_trades).read_text())
local_fills_raw = load_local_fills(log_dir)
locals_norm = [normalize_local(e, i) for i, e in enumerate(local_fills_raw)]
venues_norm = [normalize_venue(t, i) for i, t in enumerate(venue_trades)]
recon = match(locals_norm, venues_norm)
matched_fill_ids = {m["local"]["fill_id"] for m in recon["matched"]}
matched_trade_ids = {m["venue"]["venue_trade_id"] for m in recon["matched"]}
fills = fills[fills["fill_id"].isin(matched_fill_ids)].copy()
paths = paths[paths["fill_id"].isin(matched_fill_ids)].copy()
# merge labels/path info
fill_meta_cols = [
c
for c in [
"fill_id",
"side",
"fill_price",
"amount",
"quote_fill_time",
"ts_epoch",
"event_cluster_id",
"pair",
]
if c in fills.columns
]
meta = fills.drop_duplicates("fill_id")[fill_meta_cols]
paths = paths.merge(meta, on="fill_id", how="left", suffixes=("", "_f"))
for col in ["side", "fill_price", "amount", "event_cluster_id", "quote_fill_time", "ts_epoch"]:
alt = f"{col}_f"
if alt in paths.columns:
if col not in paths.columns:
paths[col] = paths[alt]
else:
paths[col] = paths[col].fillna(paths[alt])
paths["fill_ts"] = pd.to_datetime(paths["quote_fill_time"], utc=True, errors="coerce")
# Prefer fill-event epoch seconds. astype(int64)/1e9 breaks when pandas stores UTC as us.
fill_epoch = pd.to_numeric(paths["ts_epoch"], errors="coerce")
iso_epoch = paths["fill_ts"].map(lambda ts: ts.timestamp() if pd.notna(ts) else np.nan)
paths["fill_ts_epoch"] = fill_epoch.fillna(iso_epoch)
paths["qty"] = pd.to_numeric(paths["amount"], errors="coerce")
paths["fill_price"] = pd.to_numeric(paths["fill_price"], errors="coerce")
paths["after_10s_price"] = pd.to_numeric(paths["after_10s_price"], errors="coerce")
paths["after_30s_price"] = pd.to_numeric(paths["after_30s_price"], errors="coerce")
paths["notional_usdt"] = paths["qty"] * paths["fill_price"]
paths["markout_10s"] = _fav_ret(paths["side"], paths["fill_price"], paths["after_10s_price"])
paths["markout_30s"] = _fav_ret(paths["side"], paths["fill_price"], paths["after_30s_price"])
paths["path_c"] = paths["path_type"].astype(str).eq("C_toxic")
paths["toxic"] = (paths["markout_10s"] < 0) & (paths["markout_30s"] < 0)
paths["negative_30s"] = paths["markout_30s"] < 0
# attach trade economics
raw_v = pd.DataFrame(venue_trades)
raw_v["trade_id_link"] = raw_v["id"].astype(str)
raw_v["commission_usdt"] = pd.to_numeric(raw_v["commission"], errors="coerce")
raw_v["realized_pnl_usdt"] = pd.to_numeric(raw_v["realizedPnl"], errors="coerce").fillna(0.0)
matched_map = pd.DataFrame(
[
{
"fill_id": m["local"]["fill_id"],
"trade_id_link": m["venue"]["venue_trade_id"],
}
for m in recon["matched"]
]
)
paths = paths.merge(
matched_map.merge(raw_v[["trade_id_link", "commission_usdt", "realized_pnl_usdt"]], on="trade_id_link", how="left"),
on="fill_id",
how="left",
)
paths["gross_markout_30s_usdt"] = paths["notional_usdt"] * paths["markout_30s"]
paths["net_attr_30s_usdt"] = (
paths["gross_markout_30s_usdt"]
- pd.to_numeric(paths["commission_usdt"], errors="coerce").fillna(0.0)
+ pd.to_numeric(paths["realized_pnl_usdt"], errors="coerce").fillna(0.0)
)
paths["economic_negative"] = paths["net_attr_30s_usdt"] < 0
# strict prefill state from sampled historical ticks only
state_ticks = state_ticks.copy()
state_ticks["ts_epoch"] = pd.to_numeric(state_ticks["ts_epoch"], errors="coerce")
state_ticks = state_ticks.dropna(subset=["ts_epoch"]).sort_values("ts_epoch").drop_duplicates("ts_epoch")
keep_cols = [
c
for c in [
"ts_epoch",
"mid",
"spread",
"bid_depth_1",
"ask_depth_1",
"bid_depth_5",
"ask_depth_5",
"obi",
"delta",
"trade_imbalance",
"delta_efficiency",
"inventory",
"inventory_time",
"inventory_skew",
]
if c in state_ticks.columns
]
states = state_ticks[keep_cols].copy()
num_cols = [c for c in keep_cols if c != "ts_epoch"]
for col in num_cols:
states[col] = pd.to_numeric(states[col], errors="coerce")
# 5s lag features using sampled states
lag_df = states[["ts_epoch"] + [c for c in ["mid", "spread", "obi", "bid_depth_5", "ask_depth_5", "trade_imbalance", "delta"] if c in states.columns]].copy()
lag_df["lag_ts"] = lag_df["ts_epoch"] + 5.0
lag_cols = {c: f"{c}_past5s" for c in lag_df.columns if c not in {"ts_epoch", "lag_ts"}}
lag_df = lag_df.rename(columns=lag_cols)
paths = paths[paths["fill_ts_epoch"].notna()].copy()
fill_states = paths[["fill_id", "fill_ts_epoch", "side"]].copy().sort_values("fill_ts_epoch")
fill_states["feature_cutoff_ts"] = fill_states["fill_ts_epoch"] - float(args.margin_sec)
# latest sampled tick strictly before fill-margin
snap = pd.merge_asof(
fill_states.sort_values("feature_cutoff_ts"),
states.sort_values("ts_epoch"),
left_on="feature_cutoff_ts",
right_on="ts_epoch",
direction="backward",
)
snap = snap[snap["ts_epoch"].notna()].copy()
snap = pd.merge_asof(
snap.sort_values("ts_epoch"),
lag_df.sort_values("lag_ts"),
left_on="ts_epoch",
right_on="lag_ts",
direction="backward",
)
if "ts_epoch_x" in snap.columns:
snap = snap.rename(columns={"ts_epoch_x": "ts_epoch"})
# derived strict-prefill features
snap["spread_pct"] = snap["spread"] / snap["mid"]
snap["depth_total_5"] = snap["bid_depth_5"] + snap["ask_depth_5"]
snap["depth_imbalance_5"] = (snap["bid_depth_5"] - snap["ask_depth_5"]) / snap["depth_total_5"]
snap["price_velocity_5s"] = (snap["mid"] - snap["mid_past5s"]) / snap["mid_past5s"]
snap["spread_change_5s"] = snap["spread_pct"] - (snap["spread_past5s"] / snap["mid_past5s"])
snap["obi_change_5s"] = snap["obi"] - snap["obi_past5s"]
snap["depth_total_5_past"] = snap["bid_depth_5_past5s"] + snap["ask_depth_5_past5s"]
snap["depth_change_5s"] = snap["depth_total_5"] - snap["depth_total_5_past"]
snap["trade_imbalance_change_5s"] = snap["trade_imbalance"] - snap["trade_imbalance_past5s"]
snap["delta_change_5s"] = snap["delta"] - snap["delta_past5s"]
snap["pre_deteriorated_strict"] = np.where(
snap["side"].eq("long"),
(snap["price_velocity_5s"] < 0) | (snap["depth_change_5s"] < 0),
(snap["price_velocity_5s"] > 0) | (snap["depth_change_5s"] < 0),
)
snap["feature_age_ms"] = (snap["fill_ts_epoch"] - snap["ts_epoch"]) * 1000.0
snap = snap.rename(columns={"ts_epoch": "feature_ts_epoch"})
snap_feature_cols = [
"fill_id",
"feature_ts_epoch",
"feature_cutoff_ts",
"mid",
"spread",
"bid_depth_1",
"ask_depth_1",
"bid_depth_5",
"ask_depth_5",
"obi",
"delta",
"trade_imbalance",
"delta_efficiency",
"inventory",
"inventory_time",
"inventory_skew",
"mid_past5s",
"spread_past5s",
"obi_past5s",
"bid_depth_5_past5s",
"ask_depth_5_past5s",
"trade_imbalance_past5s",
"delta_past5s",
"spread_pct",
"depth_total_5",
"depth_imbalance_5",
"price_velocity_5s",
"spread_change_5s",
"depth_total_5_past",
"depth_change_5s",
"obi_change_5s",
"trade_imbalance_change_5s",
"delta_change_5s",
"pre_deteriorated_strict",
"feature_age_ms",
]
snap_feature_cols = [c for c in snap_feature_cols if c in snap.columns]
rename_map = {
c: f"strict_{c}"
for c in snap_feature_cols
if c not in {"fill_id", "feature_ts_epoch", "feature_cutoff_ts", "feature_age_ms", "pre_deteriorated_strict"}
}
rename_map["feature_ts_epoch"] = "strict_feature_ts_epoch"
rename_map["feature_cutoff_ts"] = "strict_feature_cutoff_ts"
rename_map["feature_age_ms"] = "strict_feature_age_ms"
rename_map["pre_deteriorated_strict"] = "strict_pre_deteriorated"
snap_merge = snap[snap_feature_cols].rename(columns=rename_map)
pref = paths.merge(snap_merge, on="fill_id", how="left")
pref = pref[pref["strict_feature_ts_epoch"].notna()].copy()
labels = ["path_c", "toxic", "negative_30s"]
numeric_features = [
"strict_obi",
"strict_delta",
"strict_trade_imbalance",
"strict_spread_pct",
"strict_bid_depth_5",
"strict_ask_depth_5",
"strict_depth_total_5",
"strict_depth_imbalance_5",
"strict_price_velocity_5s",
"strict_spread_change_5s",
"strict_depth_change_5s",
"strict_obi_change_5s",
"strict_trade_imbalance_change_5s",
"strict_delta_change_5s",
"strict_inventory",
"strict_inventory_skew",
"strict_inventory_time",
"strict_feature_age_ms",
]
cat_features = ["strict_pre_deteriorated"]
result_rows: list[dict[str, Any]] = []
matrix_rows: list[dict[str, Any]] = []
for feat in numeric_features:
if feat not in pref.columns:
continue
rows, grades = _state_table_num(pref, feat, labels)
result_rows.extend(rows)
matrix_rows.append(
{
"feature": feat,
"Path C": grades["path_c"],
"Toxic": grades["toxic"],
"Neg30s": grades["negative_30s"],
"Economic": grades["Economic"],
}
)
for feat in cat_features:
if feat not in pref.columns:
continue
rows, grades = _state_table_cat(pref, feat, labels)
result_rows.extend(rows)
matrix_rows.append(
{
"feature": feat,
"Path C": grades["path_c"],
"Toxic": grades["toxic"],
"Neg30s": grades["negative_30s"],
"Economic": grades["Economic"],
}
)
baseline = {
"path_c": float(pref["path_c"].mean()),
"toxic": float(pref["toxic"].mean()),
"negative_30s": float(pref["negative_30s"].mean()),
"economic_negative": float(pref["economic_negative"].mean()),
"net_attr_30s_usdt_mean": float(pref["net_attr_30s_usdt"].mean()),
"markout_30s_mean": float(pref["markout_30s"].mean()),
}
coverage = {
"matched_paths": int(len(paths)),
"strict_prefill_rows": int(len(pref)),
"strict_prefill_coverage_pct": float(len(pref) / max(len(paths), 1) * 100.0),
"mean_feature_age_ms": float(pref["strict_feature_age_ms"].mean()),
"median_feature_age_ms": float(pref["strict_feature_age_ms"].median()),
}
out_txt = Path(args.out)
out_json = out_txt.with_suffix(".json")
lines: list[str] = []
def p(s: str = "") -> None:
lines.append(s)
print(s)
p("=" * 72)
p("Prefill Adverse-Selection Attribution v0.1")
p("=" * 72)
p("Experiment: MM_EDGE_EXP_001")
p("Population: frozen historical fills (Hard core = MATCHED only)")
p("Strategy: v0.1 FROZEN")
p("Execution: STOPPED")
p("Purpose: Pre-fill adverse-selection predictability audit")
p("NOT: strategy / backtest / optimization / model training")
p()
p("Time Contract")
p("-" * 40)
p(f"feature_timestamp <= t_fill - {args.margin_sec:.2f}s")
p("Only sampled historical mid_tick / inventory_tick states are used.")
p("Fill-callback contemporaneous fields are intentionally excluded to avoid leakage.")
p()
p("Unavailable under strict contract in v0.1")
p("-" * 40)
p("- event intensity / large trades / time_since_last_market_event")
p("- fill-callback market_event_before_fill")
p("- any future path / realized / cancel-after-fill info as features")
p()
p("Baseline labels (MATCHED only)")
p("-" * 40)
p(f"P(Path C): {_pct(baseline['path_c'])}")
p(f"P(Toxic): {_pct(baseline['toxic'])}")
p(f"P(Neg30s): {_pct(baseline['negative_30s'])}")
p(f"P(Economic<0): {_pct(baseline['economic_negative'])}")
p(f"Mean net_attr_30s: {_num(baseline['net_attr_30s_usdt_mean'], 6)} USDT/fill")
p(f"Mean markout_30s: {_pct(baseline['markout_30s_mean'])}")
p(f"Matched path rows: {coverage['matched_paths']}")
p(f"Strict prefill rows: {coverage['strict_prefill_rows']}")
p(f"Strict coverage: {coverage['strict_prefill_coverage_pct']:.1f}%")
p(f"Feature age ms: mean={coverage['mean_feature_age_ms']:.1f} median={coverage['median_feature_age_ms']:.1f}")
p()
p("Sample-size policy")
p("-" * 40)
p("n < 30 exploratory only (LOW_N)")
p("n < 100 weak evidence (WEAK_EVIDENCE)")
p("n >= 100 usable attribution (USABLE)")
p()
p("Conclusion Matrix")
p("-" * 40)
p("feature | Path C | Toxic | Neg30s | Economic")
for row in matrix_rows:
p(f"{row['feature']} | {row['Path C']} | {row['Toxic']} | {row['Neg30s']} | {row['Economic']}")
p()
p("State tables")
p("-" * 40)
for feat in [r["feature"] for r in matrix_rows]:
sub = [r for r in result_rows if r["feature"] == feat]
if not sub:
continue
p(feat)
for r in sub:
med = _num(r["median"], 6) if r["median"] is not None else "n/a"
p(
f" {r['state']}: n={r['n']} [{r['sample_grade']}] median={med} "
f"P(C)={_pct(r['p_path_c'])} Δ={r['delta_path_c_pp']:+.1f}pp "
f"P(Toxic)={_pct(r['p_toxic'])} Δ={r['delta_toxic_pp']:+.1f}pp "
f"P(Neg30)={_pct(r['p_negative_30s'])} Δ={r['delta_negative_30s_pp']:+.1f}pp "
f"E[net30]={_num(r['net_attr_mean'], 5)} Δ={_num(r['delta_economic_per_fill'], 5)}"
)
p()
p("Interpretation")
p("-" * 40)
p("Only pre-fill observable states count as candidate signals.")
p("A feature may separate Path C statistically but still fail Economic relevance.")
p("Only rows graded CANDIDATE_V0_2_SIGNAL with usable n should enter v0.2 hypothesis design.")
p("=" * 72)
out_txt.write_text("\n".join(lines) + "\n", encoding="utf-8")
out_json.write_text(
json.dumps(
{
"experiment_id": "MM_EDGE_EXP_001",
"purpose": "prefill adverse-selection predictability audit",
"population": {
"matched_rows": int(len(pref)),
"margin_sec": float(args.margin_sec),
},
"baseline": baseline,
"coverage": coverage,
"matrix": matrix_rows,
"states": result_rows,
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/probe_status.sh
Executable
+53
View File
@@ -0,0 +1,53 @@
#!/usr/bin/env bash
# 远程查看探针状态 + fills/clusters 粗计数
# 用法：export SSH_HOST=user@ip [SSH_KEY=...] ./scripts/probe_status.sh
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
SSH_HOST="${SSH_HOST:-jack@jackyu66.com}"
SSH_KEY="${SSH_KEY:-${HOME}/Project/deploy/zun_hk/id_ed25519_hk}"
REMOTE_DIR="${REMOTE_DIR:-/www/Project/nautilus_mm}"
SSH_OPTS=(-o StrictHostKeyChecking=accept-new)
if [[ -n "$SSH_KEY" ]]; then
chmod 400 "$SSH_KEY" 2>/dev/null || true
SSH_OPTS+=(-i "$SSH_KEY")
fi
ssh "${SSH_OPTS[@]}" "$SSH_HOST" "REMOTE_DIR='$REMOTE_DIR' bash -s" <<'EOF'
set -euo pipefail
echo "=== systemd --user ==="
systemctl --user is-active mm-edge-probe || true
systemctl --user status mm-edge-probe --no-pager -l | head -20 || true
echo ""
echo "=== experiment (.env) ==="
grep -E '^(EXPERIMENT_ID|PROBE_VERSION|BINANCE_ENVIRONMENT|ENABLE_TRADING)=' "$REMOTE_DIR/.env" 2>/dev/null || true
grep -E '^BINANCE_API_KEY=.+' "$REMOTE_DIR/.env" >/dev/null && echo "API key: SET" || echo "API key: EMPTY"
echo ""
echo "=== fills / clusters (jsonl) ==="
cd "$REMOTE_DIR/logs/maker_edge" 2>/dev/null || { echo "no log dir"; exit 0; }
python3 - <<'PY'
import json
from pathlib import Path
fills=0
cids=set()
health=0
for f in sorted(Path('.').glob('*.jsonl')):
for line in f.read_text().splitlines():
try:
ev=json.loads(line)
except Exception:
continue
if ev.get('event')=='fill':
fills+=1
if ev.get('event_cluster_id'):
cids.add(ev['event_cluster_id'])
elif ev.get('event')=='health':
health+=1
print(f"fills={fills} clusters={len(cids)} health_ticks={health}")
print(f"cluster/fill={len(cids)/fills*100:.1f}%" if fills else "cluster/fill=n/a")
PY
echo ""
echo "=== recent journal (--user) ==="
journalctl --user -u mm-edge-probe -n 30 --no-pager || true
EOF
scripts/pull_report.sh
Executable
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env bash
# 从服务器拉取 jsonl + 本地生成 Maker Edge Report
# 用法：export SSH_HOST=user@ip [SSH_KEY=...] ./scripts/pull_report.sh [min_fills]
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
SSH_HOST="${SSH_HOST:-jack@jackyu66.com}"
SSH_KEY="${SSH_KEY:-${HOME}/Project/deploy/zun_hk/id_ed25519_hk}"
REMOTE_DIR="${REMOTE_DIR:-/www/Project/nautilus_mm}"
LOCAL_LOG="${LOCAL_LOG:-$ROOT/logs/maker_edge}"
SSH_OPTS=(-o StrictHostKeyChecking=accept-new)
if [[ -n "$SSH_KEY" ]]; then
chmod 400 "$SSH_KEY" 2>/dev/null || true
SSH_OPTS+=(-i "$SSH_KEY")
fi
mkdir -p "$LOCAL_LOG"
echo "==> pull logs from $SSH_HOST"
rsync -avz -e "ssh ${SSH_OPTS[*]}" \
"$SSH_HOST:$REMOTE_DIR/logs/maker_edge/" "$LOCAL_LOG/"
echo "==> analyze"
export PYTHONPATH="${ROOT}/src${PYTHONPATH:+:$PYTHONPATH}"
exec "$ROOT/scripts/analyze.sh" "${1:-2000}"
scripts/reconcile_account.py
+402
View File
@@ -0,0 +1,402 @@
#!/usr/bin/env python3
"""
Account Ledger Reconciliation — MM_EDGE_EXP_001
Separates:
MakerAlpha (research markout) ≠ Account Equity (wallet economics)
Pulls paginated Binance Futures:
- /fapi/v1/userTrades (maker flag, commission per fill)
- /fapi/v1/income (REALIZED_PNL, COMMISSION, FUNDING_FEE, …)
- /fapi/v2/account (wallet + unrealized + position)
Hard gate:
TAKER_FILLED_COUNT == 0 else Maker-only = INVALID
Equity identity (target error ≈ 0):
StartWallet + Σincome_types + (EndUnrealized − StartUnrealized*)
+ Transfers/Adjustments ≈ EndMarginBalance
* StartUnrealized often unknown → report EndUnrealized separately.
"""
from __future__ import annotations
import argparse
import hashlib
import hmac
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path
from dotenv import load_dotenv
_ROOT = Path(__file__).resolve().parents[1]
load_dotenv(_ROOT / ".env")
def _env(name: str, default: str = "") -> str:
return os.getenv(name, default).strip()
def _base_url() -> str:
env = _env("BINANCE_ENVIRONMENT", "TESTNET").upper()
if env == "TESTNET":
return "https://testnet.binancefuture.com"
if env == "LIVE":
return "https://fapi.binance.com"
raise SystemExit(f"BINANCE_ENVIRONMENT must be TESTNET|LIVE, got {env!r}")
def _signed_get(path: str, params: dict | None = None) -> object:
key = _env("BINANCE_API_KEY")
sec = _env("BINANCE_API_SECRET")
if not key or not sec:
raise SystemExit("BINANCE_API_KEY / BINANCE_API_SECRET required")
params = dict(params or {})
params["timestamp"] = int(time.time() * 1000)
params["recvWindow"] = 60_000
qs = urllib.parse.urlencode(params)
sig = hmac.new(sec.encode(), qs.encode(), hashlib.sha256).hexdigest()
url = f"{_base_url()}{path}?{qs}&signature={sig}"
req = urllib.request.Request(url, headers={"X-MBX-APIKEY": key})
try:
with urllib.request.urlopen(req, timeout=60) as r:
return json.loads(r.read().decode())
except urllib.error.HTTPError as e:
body = e.read().decode(errors="replace")
raise RuntimeError(f"HTTP {e.code} {path} params={params} body={body}") from e
def _fetch_user_trades(symbol: str, start_ms: int, end_ms: int) -> list[dict]:
"""Paginate userTrades by time windows (dedupe by trade id).
Note: Testnet userTrades can stop returning rows after ~7d of dense history
even while orders/income continue — RECON-02 must flag that gap separately.
"""
out: dict[int, dict] = {}
cursor = start_ms
safety = 0
while cursor < end_ms and safety < 2000:
safety += 1
chunk_end = min(cursor + 7 * 86400_000 - 1, end_ms)
batch = _signed_get(
"/fapi/v1/userTrades",
{
"symbol": symbol,
"startTime": cursor,
"endTime": chunk_end,
"limit": 1000,
},
)
assert isinstance(batch, list)
if not batch:
cursor = chunk_end + 1
continue
for t in batch:
out[int(t["id"])] = t
last_t = int(batch[-1]["time"])
if len(batch) < 1000:
cursor = max(last_t + 1, chunk_end + 1)
else:
nxt = last_t + 1
if nxt <= cursor:
nxt = cursor + 1
cursor = nxt
time.sleep(0.08)
return sorted(out.values(), key=lambda x: (int(x["time"]), int(x["id"])))
def _fetch_income(start_ms: int, end_ms: int) -> list[dict]:
"""Paginate income by time only."""
out: list[dict] = []
seen: set[tuple] = set()
cursor = start_ms
safety = 0
while cursor < end_ms and safety < 2000:
safety += 1
chunk_end = min(cursor + 7 * 86400_000 - 1, end_ms)
batch = _signed_get(
"/fapi/v1/income",
{"startTime": cursor, "endTime": chunk_end, "limit": 1000},
)
assert isinstance(batch, list)
if not batch:
cursor = chunk_end + 1
continue
for row in batch:
key = (
row.get("tranId"),
row.get("time"),
row.get("incomeType"),
row.get("income"),
row.get("asset"),
row.get("symbol"),
)
if key in seen:
continue
seen.add(key)
out.append(row)
last_t = int(batch[-1]["time"])
if len(batch) < 1000:
cursor = max(last_t + 1, chunk_end + 1)
else:
cursor = last_t + 1
time.sleep(0.08)
return out
def _ms_iso(ms: int) -> str:
return datetime.fromtimestamp(ms / 1000, tz=timezone.utc).isoformat()
def load_jsonl_fill_count(log_dir: Path) -> int:
n = 0
if not log_dir.exists():
return 0
for f in sorted(log_dir.glob("*.jsonl")):
for line in f.open():
try:
e = json.loads(line)
except Exception:
continue
if isinstance(e, dict) and e.get("event") == "fill":
n += 1
return n
def main() -> int:
ap = argparse.ArgumentParser(description="Maker Edge account reconciliation")
ap.add_argument(
"--start-wallet",
type=float,
default=float(_env("RECON_START_WALLET", "5000")),
help="Observed starting USDT wallet (default 5000 testnet grant)",
)
ap.add_argument(
"--symbol",
default=_env("RECON_SYMBOL", "BTCUSDT"),
help="Futures symbol for userTrades (default BTCUSDT)",
)
ap.add_argument(
"--since-days",
type=float,
default=float(_env("RECON_SINCE_DAYS", "14")),
)
ap.add_argument(
"--out",
default=str(_ROOT / "logs" / "maker_edge" / "Account_Reconciliation.txt"),
)
args = ap.parse_args()
end_ms = int(time.time() * 1000)
start_ms = end_ms - int(args.since_days * 86400 * 1000)
print(f"[recon] env={_env('BINANCE_ENVIRONMENT','TESTNET')} base={_base_url()}")
print(f"[recon] window {_ms_iso(start_ms)} → {_ms_iso(end_ms)}")
print("[recon] pulling userTrades (paginated)…")
trades = _fetch_user_trades(args.symbol, start_ms, end_ms)
print(f"[recon] userTrades={len(trades)}")
print("[recon] pulling income (paginated)…")
income = _fetch_income(start_ms, end_ms)
print(f"[recon] income rows={len(income)}")
acct = _signed_get("/fapi/v2/account")
assert isinstance(acct, dict)
# --- Maker-only hard check ---
maker_n = sum(1 for t in trades if t.get("maker") is True)
taker_n = sum(1 for t in trades if t.get("maker") is False)
unknown_n = len(trades) - maker_n - taker_n
maker_only_ok = taker_n == 0 and unknown_n == 0 and len(trades) > 0
maker_only_status = "PASS" if maker_only_ok else ("INVALID" if taker_n > 0 else "NEED VERIFY")
fee_by_asset: dict[str, float] = defaultdict(float)
notional = 0.0
buy_qty = sell_qty = 0.0
for t in trades:
fee_by_asset[t.get("commissionAsset") or "?"] += float(t.get("commission") or 0)
q = float(t.get("qty") or 0)
px = float(t.get("price") or 0)
notional += abs(q * px)
if t.get("buyer"):
buy_qty += q
else:
sell_qty += q
net_qty = buy_qty - sell_qty
income_by: dict[str, float] = defaultdict(float)
for row in income:
income_by[str(row.get("incomeType"))] += float(row.get("income") or 0)
wallet = float(acct.get("totalWalletBalance") or 0)
upnl = float(acct.get("totalUnrealizedProfit") or 0)
margin = float(acct.get("totalMarginBalance") or 0)
avail = float(acct.get("availableBalance") or 0)
positions = []
for p in acct.get("positions") or []:
amt = float(p.get("positionAmt") or 0)
if abs(amt) > 1e-12:
positions.append(
{
"symbol": p.get("symbol"),
"amt": amt,
"entry": float(p.get("entryPrice") or 0),
"unrealized": float(p.get("unrealizedProfit") or 0),
}
)
start_wallet = float(args.start_wallet)
income_sum = sum(income_by.values())
# Identity without known start upnl:
# EndWallet ≈ StartWallet + Σ income (transfers included in income types if any)
implied_end_wallet = start_wallet + income_sum
wallet_gap = wallet - implied_end_wallet
equity_now = margin # wallet + upnl
equity_vs_start = equity_now - start_wallet
jsonl_fills = load_jsonl_fill_count(_ROOT / "logs" / "maker_edge")
lines: list[str] = []
def p(s: str = "") -> None:
lines.append(s)
print(s)
p("=" * 72)
p("Account Reconciliation — MM_EDGE_EXP_001 / probe_v0.1")
p("Research markout (MakerAlpha) ≠ Account equity")
p("=" * 72)
p()
p("Status Snapshot")
p("-" * 40)
p("Maker Phenomenon PARTIAL_PASS")
p("Data Integrity PASS (from Maker Edge Report)")
p(f"Maker-only constraint {maker_only_status}")
p("Account Reconciliation NOT COMPLETE" if abs(wallet_gap) > 0.5 else "Account Reconciliation CLOSE")
p("Economic Edge UNKNOWN")
p("Stage 3 LOCKED")
p("Probe STOPPED (no further volume until ledger closes)")
p()
p("Section A — Maker-only hard check (exchange userTrades)")
p("-" * 40)
p(f"Symbol: {args.symbol}")
p(f"Exchange trades: {len(trades)}")
p(f"Jsonl fills (local): {jsonl_fills}")
p(f"MAKER fills: {maker_n}")
p(f"TAKER fills: {taker_n}")
p(f"Unknown liquidity: {unknown_n}")
p(f"TAKER_FILLED_COUNT: {taker_n}")
if taker_n > 0:
p("→ INVALID: sample contaminated by taker fills")
elif maker_only_ok:
p("→ PASS: all exchange trades marked maker=true")
else:
p("→ NEED VERIFY")
p(f"Buy qty / Sell qty: {buy_qty:.6f} / {sell_qty:.6f}")
p(f"Net inventory (qty): {net_qty:.6f}")
p(f"Gross notional: {notional:.4f} USDT")
for asset, fee in sorted(fee_by_asset.items()):
p(f"Commission ({asset}): {fee}")
p()
p("Section B — Income ledger (paginated, full window)")
p("-" * 40)
for k, v in sorted(income_by.items(), key=lambda kv: -abs(kv[1])):
p(f" {k:24s} {v:+.8f}")
p(f" {'Σ income':24s} {income_sum:+.8f}")
p()
p("Section C — Account snapshot (now)")
p("-" * 40)
p(f"totalWalletBalance: {wallet:.8f}")
p(f"totalUnrealizedProfit: {upnl:.8f}")
p(f"totalMarginBalance: {margin:.8f} ← equity")
p(f"availableBalance: {avail:.8f}")
if positions:
p("Open positions:")
for pos in positions:
p(
f" {pos['symbol']} amt={pos['amt']} entry={pos['entry']} "
f"upnl={pos['unrealized']}"
)
else:
p("Open positions: (none)")
p()
p("Section D — Equity bridge (attempt)")
p("-" * 40)
p(f"Start wallet (assumed): {start_wallet:.8f}")
p(f"+ Σ income: {income_sum:+.8f}")
p(f"= Implied end wallet: {implied_end_wallet:.8f}")
p(f"Actual end wallet: {wallet:.8f}")
p(f"Wallet residual gap: {wallet_gap:+.8f}")
p(f"End unrealized: {upnl:+.8f}")
p(f"End equity: {equity_now:.8f}")
p(f"Equity − start wallet: {equity_vs_start:+.8f}")
p()
p("Interpretation:")
p(" - Do NOT equate EquityΔ with MakerAlpha failure/success.")
p(" - Residual gap means incomplete history, wrong start, or missing")
p(" transfer/adjustment types — Account Reconciliation stays open.")
p(" - Inventory drift (net qty / open position) can dominate economics")
p(" even when per-fill markout is slightly positive.")
p()
p("Section E — Next required chain")
p("-" * 40)
p("QuoteIntent → Submitted → Accepted → Filled")
p(" → fill_px/qty → liquidity=MAKER → fee")
p(" → position Δ → realized → funding → equity")
p("Daily: StartEquity + TradingPnL + Fees + Funding + uPnL + Transfers = EndEquity")
p("Target residual ≈ 0 before any Stage3 unlock / further volume.")
p("=" * 72)
out = Path(args.out)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text("\n".join(lines) + "\n", encoding="utf-8")
# machine-readable sidecar
sidecar = out.with_suffix(".json")
sidecar.write_text(
json.dumps(
{
"experiment_id": "MM_EDGE_EXP_001",
"maker_only_status": maker_only_status,
"taker_filled_count": taker_n,
"maker_filled_count": maker_n,
"exchange_trades": len(trades),
"jsonl_fills": jsonl_fills,
"income_by_type": dict(income_by),
"income_sum": income_sum,
"start_wallet_assumed": start_wallet,
"end_wallet": wallet,
"end_unrealized": upnl,
"end_equity": equity_now,
"wallet_residual_gap": wallet_gap,
"net_qty": net_qty,
"fee_by_asset": dict(fee_by_asset),
"positions": positions,
"probe": "STOPPED",
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
print(f"[recon] saved {out}")
print(f"[recon] saved {sidecar}")
return 0 if maker_only_ok or taker_n == 0 else 2
if __name__ == "__main__":
sys.exit(main())
scripts/reconcile_fills.py
+594
View File
@@ -0,0 +1,594 @@
#!/usr/bin/env python3
"""
RECONCILIATION-02 — Local Fill ↔ Venue Trade 1:1 / quantity-level closure
Does NOT resume the probe. Does NOT change quote logic.
Gate: 100% of local fills and venue trades classified into:
MATCHED | DUPLICATE | ORPHAN_LOCAL | ORPHAN_VENUE | MISMATCH | MALFORMED
Primary link: venue_trade_id when present.
Fallback (historical jsonl has trade_id=None):
venue_order_id + side + qty + price + timestamp window
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path
_ROOT = Path(__file__).resolve().parents[1]
_SRC = _ROOT / "src"
if str(_SRC) not in sys.path:
sys.path.insert(0, str(_SRC))
# reuse pagination from recon-01
sys.path.insert(0, str(_ROOT / "scripts"))
from reconcile_account import _env, _fetch_user_trades, _signed_get # noqa: E402
PX_TICK = 0.1 # BTCUSDT tick
QTY_EPS = 1e-8
TIME_MATCH_SEC = 30.0
TIME_DUP_SEC = 2.0
def _parse_iso(s: str | None) -> float | None:
if not s:
return None
try:
return datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp()
except Exception:
return None
def _ms_ts(ms: int | None) -> float | None:
if ms is None:
return None
return int(ms) / 1000.0
def load_local_fills(log_dir: Path) -> list[dict]:
fills: list[dict] = []
for f in sorted(log_dir.glob("*.jsonl")):
if f.name.startswith("Account_") or f.name.startswith("Maker_") or f.name.startswith("RECON"):
continue
for line in f.open():
try:
e = json.loads(line)
except Exception:
continue
if not isinstance(e, dict) or e.get("event") != "fill":
continue
fills.append(e)
return fills
def normalize_local(e: dict, idx: int) -> dict:
px = float(e.get("fill_price") or 0)
qty = float(e.get("amount") or 0)
side = e.get("side") # long / short
venue_oid = e.get("venue_order_id")
if venue_oid is not None:
venue_oid = str(venue_oid)
trade_id = e.get("venue_trade_id") or e.get("trade_id")
if trade_id in (None, "None", ""):
trade_id = None
else:
trade_id = str(trade_id)
ts = _parse_iso(e.get("quote_fill_time"))
malformed = []
if px <= 0:
malformed.append("bad_price")
if qty <= 0:
malformed.append("bad_qty")
if side not in ("long", "short"):
malformed.append("bad_side")
if not venue_oid:
malformed.append("missing_venue_order_id")
return {
"idx": idx,
"fill_id": e.get("fill_id"),
"client_order_id": e.get("client_order_id"),
"venue_order_id": venue_oid,
"venue_trade_id": trade_id,
"side": side,
"px": px,
"qty": qty,
"ts": ts,
"ts_iso": e.get("quote_fill_time"),
"commission": e.get("commission"),
"malformed": malformed,
"raw_keys": sorted(e.keys()),
}
def normalize_venue(t: dict, idx: int) -> dict:
buyer = bool(t.get("buyer"))
side = "long" if buyer else "short"
return {
"idx": idx,
"venue_trade_id": str(t.get("id")),
"venue_order_id": str(t.get("orderId")),
"side": side,
"px": float(t.get("price") or 0),
"qty": float(t.get("qty") or 0),
"ts": _ms_ts(t.get("time")),
"ts_iso": datetime.fromtimestamp(int(t["time"]) / 1000, tz=timezone.utc).isoformat()
if t.get("time")
else None,
"commission": float(t.get("commission") or 0),
"commission_asset": t.get("commissionAsset"),
"maker": t.get("maker"),
"symbol": t.get("symbol"),
}
def _compatible(loc: dict, ven: dict) -> tuple[bool, str]:
if loc["side"] != ven["side"]:
return False, "side"
if abs(loc["qty"] - ven["qty"]) > QTY_EPS:
return False, "qty"
if abs(loc["px"] - ven["px"]) > PX_TICK + 1e-9:
return False, "price"
if loc["ts"] is not None and ven["ts"] is not None:
if abs(loc["ts"] - ven["ts"]) > TIME_MATCH_SEC:
return False, "time"
return True, "ok"
def match(locals_: list[dict], venues: list[dict]) -> dict:
"""Greedy unique matching. Each venue trade consumed at most once."""
used_v: set[int] = set()
used_l: set[int] = set()
matched: list[dict] = []
mismatch: list[dict] = []
duplicate: list[dict] = []
loc_by_tid: dict[str, list[dict]] = defaultdict(list)
ven_by_tid: dict[str, dict] = {}
for v in venues:
ven_by_tid[v["venue_trade_id"]] = v
for loc in locals_:
if loc["venue_trade_id"]:
loc_by_tid[loc["venue_trade_id"]].append(loc)
# Pass 1: explicit venue_trade_id
for tid, locs in loc_by_tid.items():
v = ven_by_tid.get(tid)
if v is None:
continue
primary, *rest = locs
ok, why = _compatible(primary, v)
rec = {"local": primary, "venue": v, "link": "venue_trade_id", "compat": why}
if ok:
matched.append(rec)
else:
rec["mismatch_reason"] = why
mismatch.append(rec)
used_v.add(v["idx"])
used_l.add(primary["idx"])
for d in rest:
duplicate.append(
{"local": d, "venue": v, "link": "venue_trade_id_dup", "reason": "same venue_trade_id"}
)
used_l.add(d["idx"])
# Pass 2: same venue_order_id, greedy best (qty, px, time)
loc_by_oid: dict[str, list[dict]] = defaultdict(list)
ven_by_oid: dict[str, list[dict]] = defaultdict(list)
for loc in locals_:
if loc["idx"] in used_l or loc["malformed"]:
continue
if loc["venue_order_id"]:
loc_by_oid[loc["venue_order_id"]].append(loc)
for v in venues:
if v["idx"] in used_v:
continue
ven_by_oid[v["venue_order_id"]].append(v)
def score(loc: dict, v: dict) -> float:
ok, _ = _compatible(loc, v)
if not ok:
return 1e18
dt = 0.0
if loc["ts"] is not None and v["ts"] is not None:
dt = abs(loc["ts"] - v["ts"])
return dt + abs(loc["px"] - v["px"]) * 1e-6
for oid, locs in loc_by_oid.items():
cands = [v for v in ven_by_oid.get(oid, []) if v["idx"] not in used_v]
remaining = [x for x in locs if x["idx"] not in used_l]
for loc in sorted(remaining, key=lambda x: x["ts"] or 0):
best = None
best_s = 1e18
for v in cands:
if v["idx"] in used_v:
continue
s = score(loc, v)
if s < best_s:
best_s = s
best = v
if best is None or best_s >= 1e17:
continue
matched.append({"local": loc, "venue": best, "link": "order_id+px+qty+time", "compat": "ok"})
used_l.add(loc["idx"])
used_v.add(best["idx"])
# Pass 3: remaining locals that share (oid, px, qty) with an already-matched
# local → DUPLICATE (restart / double-log of same execution)
matched_sig: dict[tuple, dict] = {}
for m in matched:
loc = m["local"]
v = m["venue"]
matched_sig[(loc["venue_order_id"], round(loc["px"], 2), round(loc["qty"], 8), loc["side"])] = v
for loc in locals_:
if loc["idx"] in used_l or loc["malformed"]:
continue
key = (loc["venue_order_id"], round(loc["px"], 2), round(loc["qty"], 8), loc["side"])
v = matched_sig.get(key)
if v is None:
continue
dt_ok = True
if loc["ts"] is not None and v["ts"] is not None:
dt_ok = abs(loc["ts"] - v["ts"]) <= TIME_MATCH_SEC
if not dt_ok:
continue
duplicate.append(
{
"local": loc,
"venue": v,
"link": "dup_of_matched",
"reason": "same order/px/qty/side as a matched fill",
}
)
used_l.add(loc["idx"])
# Pass 4: global leftover by px+qty+side+time (order id mismatch)
leftover_v = [v for v in venues if v["idx"] not in used_v]
leftover_l = [x for x in locals_ if x["idx"] not in used_l and not x["malformed"]]
for loc in leftover_l:
best = None
best_s = 1e18
for v in leftover_v:
if v["idx"] in used_v:
continue
s = score(loc, v)
if s < best_s:
best_s = s
best = v
if best is None or best_s >= 1e17:
continue
matched.append({"local": loc, "venue": best, "link": "global_px_qty_time", "compat": "ok"})
used_l.add(loc["idx"])
used_v.add(best["idx"])
malformed = [x for x in locals_ if x["malformed"]]
for x in malformed:
used_l.add(x["idx"])
orphan_local = [x for x in locals_ if x["idx"] not in used_l]
orphan_venue = [v for v in venues if v["idx"] not in used_v]
return {
"matched": matched,
"duplicate": duplicate,
"mismatch": mismatch,
"malformed": malformed,
"orphan_local": orphan_local,
"orphan_venue": orphan_venue,
}
def _qty(xs, key="qty") -> float:
return sum(float(x[key]) for x in xs)
def audit_orphan_orders(orphans: list[dict], symbol: str, max_checks: int = 40) -> dict:
"""Cross-check orphan locals against /fapi/v1/order and /userTrades?orderId=."""
stats = {
"checked": 0,
"order_filled_no_trades": 0,
"order_missing": 0,
"order_other": 0,
"trades_found": 0,
}
samples: list[dict] = []
for loc in orphans[:max_checks]:
oid = loc["venue_order_id"]
if not oid:
continue
stats["checked"] += 1
try:
order = _signed_get("/fapi/v1/order", {"symbol": symbol, "orderId": oid})
except Exception as exc:
stats["order_missing"] += 1
samples.append({"oid": oid, "fill_id": loc["fill_id"], "order": "ERR", "detail": str(exc)})
continue
st = order.get("status")
try:
tr = _signed_get("/fapi/v1/userTrades", {"symbol": symbol, "orderId": oid})
except Exception:
tr = []
ntr = len(tr) if isinstance(tr, list) else 0
if st == "FILLED" and ntr == 0:
stats["order_filled_no_trades"] += 1
elif ntr > 0:
stats["trades_found"] += 1
else:
stats["order_other"] += 1
if len(samples) < 8:
samples.append(
{
"oid": oid,
"fill_id": loc["fill_id"],
"status": st,
"execQty": order.get("executedQty"),
"avgPrice": order.get("avgPrice"),
"userTrades_n": ntr,
}
)
stats["samples"] = samples
return stats
def write_report(
out: Path,
result: dict,
n_local: int,
n_venue: int,
*,
venue_t_max: str | None = None,
orphan_audit: dict | None = None,
) -> None:
m = result["matched"]
d = result["duplicate"]
mm = result["mismatch"]
mal = result["malformed"]
ol = result["orphan_local"]
ov = result["orphan_venue"]
loc_explained = len(m) + len(d) + len(mm) + len(mal) + len(ol)
ven_explained = len(m) + len(mm) + len(ov) # dups share venue; orphans leftover
# every local in exactly one bucket
# every venue in matched, mismatch, or orphan_venue (dups don't extra-count venue)
m_qty_l = sum(x["local"]["qty"] for x in m)
m_qty_v = sum(x["venue"]["qty"] for x in m)
m_fee_v = sum(x["venue"]["commission"] for x in m)
dt = [
abs(x["local"]["ts"] - x["venue"]["ts"])
for x in m
if x["local"]["ts"] is not None and x["venue"]["ts"] is not None
]
dt.sort()
def pctile(a, q):
if not a:
return None
i = min(len(a) - 1, max(0, int(round(q * (len(a) - 1)))))
return a[i]
unexplained_local = n_local - (len(m) + len(d) + len(mm) + len(mal))
# orphan_local IS unexplained in the sense of no venue link, but classified
classified_local = len(m) + len(d) + len(mm) + len(mal) + len(ol)
classified_venue = len({x["venue"]["idx"] for x in m + mm} | {x["idx"] for x in ov})
gate = (
classified_local == n_local
and classified_venue == n_venue
and len(ol) == 0
and len(ov) == 0
and len(mm) == 0
and len(mal) == 0
)
# 100% explainable ≠ zero orphans. User asked 100% explainable.
# We treat orphans as classified. Gate PASS if all rows classified (always if logic sound)
# Strict gate: no orphans/mismatch/malformed
explainable = classified_local == n_local and classified_venue == n_venue
lines = []
def p(s: str = "") -> None:
lines.append(s)
p("=" * 72)
p("RECONCILIATION-02 — Local Fill ↔ Venue Trade")
p("MM_EDGE_EXP_001 / probe_v0.1 / TESTNET BTCUSDT")
p("Probe remains STOPPED")
p("=" * 72)
p()
p("Counts")
p("-" * 40)
p(f"Local JSONL fills: {n_local}")
p(f"Venue userTrades: {n_venue}")
p(f" MATCHED: {len(m)}")
p(f" DUPLICATE (local): {len(d)}")
p(f" MISMATCH: {len(mm)}")
p(f" MALFORMED (local): {len(mal)}")
p(f" ORPHAN_LOCAL: {len(ol)}")
p(f" ORPHAN_VENUE: {len(ov)}")
p(f"Local classified: {classified_local}/{n_local}")
p(f"Venue classified: {classified_venue}/{n_venue}")
venue_t_max_ts = None
if venue_t_max:
p(f"Venue history max (UTC): {venue_t_max}")
try:
venue_t_max_ts = datetime.fromisoformat(venue_t_max).timestamp()
except Exception:
venue_t_max_ts = None
if ol and venue_t_max_ts:
orphan_after = sum(1 for x in ol if x["ts"] is not None and x["ts"] > venue_t_max_ts)
orphan_before = len(ol) - orphan_after
p(f"Orphan after venue cutoff: {orphan_after} (userTrades history gap on testnet)")
p(f"Orphan before cutoff: {orphan_before}")
if orphan_audit:
p()
p("Orphan order audit (sample)")
p("-" * 40)
p(f" checked: {orphan_audit.get('checked')}")
p(f" order FILLED, 0 trades: {orphan_audit.get('order_filled_no_trades')}")
p(f" userTrades found: {orphan_audit.get('trades_found')}")
for s in orphan_audit.get("samples") or []:
p(f" oid={s.get('oid')} status={s.get('status')} exec={s.get('execQty')} trades={s.get('userTrades_n')}")
p()
p("Quantity (BTC)")
p("-" * 40)
p(f"Matched local qty: {m_qty_l:.6f}")
p(f"Matched venue qty: {m_qty_v:.6f}")
p(f"Qty residual: {m_qty_l - m_qty_v:+.8f}")
p(f"Orphan local qty: {sum(x['qty'] for x in ol):.6f}")
p(f"Orphan venue qty: {sum(x['qty'] for x in ov):.6f}")
p(f"Duplicate local qty: {sum(x['local']['qty'] for x in d):.6f}")
p()
p("Fee / time (matched only)")
p("-" * 40)
p(f"Venue commission sum: {m_fee_v:.8f} USDT")
if dt:
p(f"|Δt| n={len(dt)} p50={pctile(dt,0.5):.3f}s p95={pctile(dt,0.95):.3f}s max={dt[-1]:.3f}s")
p()
p("Link methods (matched)")
p("-" * 40)
by = defaultdict(int)
for x in m:
by[x["link"]] += 1
for k, v in sorted(by.items(), key=lambda kv: -kv[1]):
p(f" {k:28s} {v}")
p()
p("Gate")
p("-" * 40)
p(f"100% classified: {'PASS' if explainable else 'FAIL'}")
p(f"Strict (no orphan/mismatch/malformed): {'PASS' if gate else 'FAIL'}")
p("Do not resume probe until strict gate PASS or leftovers 100% attributed.")
p()
def dump_sample(title: str, rows: list, kind: str, n: int = 8) -> None:
if not rows:
return
p(f"Samples — {title} (showing {min(n, len(rows))}/{len(rows)})")
p("-" * 40)
for row in rows[:n]:
if kind == "match":
loc, v = row["local"], row["venue"]
p(
f" fill={loc['fill_id']} oid={loc['venue_order_id']} "
f"tid={v['venue_trade_id']} px={loc['px']}/{v['px']} "
f"qty={loc['qty']}/{v['qty']} link={row['link']}"
)
elif kind == "dup":
loc, v = row["local"], row["venue"]
p(
f" fill={loc['fill_id']} oid={loc['venue_order_id']} "
f"tid={v['venue_trade_id']} reason={row.get('reason')}"
)
elif kind == "local":
p(
f" fill={row['fill_id']} oid={row['venue_order_id']} "
f"px={row['px']} qty={row['qty']} side={row['side']} ts={row['ts_iso']}"
)
elif kind == "venue":
p(
f" tid={row['venue_trade_id']} oid={row['venue_order_id']} "
f"px={row['px']} qty={row['qty']} side={row['side']} ts={row['ts_iso']}"
)
p()
dump_sample("ORPHAN_LOCAL", ol, "local")
dump_sample("ORPHAN_VENUE", ov, "venue")
dump_sample("DUPLICATE", d, "dup")
dump_sample("MISMATCH", mm, "match")
p("=" * 72)
out.write_text("\n".join(lines) + "\n", encoding="utf-8")
print("\n".join(lines))
sidecar = {
"experiment_id": "MM_EDGE_EXP_001",
"recon": "RECONCILIATION-02",
"n_local": n_local,
"n_venue": n_venue,
"matched": len(m),
"duplicate": len(d),
"mismatch": len(mm),
"malformed": len(mal),
"orphan_local": len(ol),
"orphan_venue": len(ov),
"classified_local": classified_local,
"classified_venue": classified_venue,
"qty_matched_local": m_qty_l,
"qty_matched_venue": m_qty_v,
"qty_orphan_local": sum(x["qty"] for x in ol),
"qty_orphan_venue": sum(x["qty"] for x in ov),
"qty_duplicate_local": sum(x["local"]["qty"] for x in d),
"fee_matched_venue": m_fee_v,
"strict_gate": gate,
"classified_gate": explainable,
"dt_p50_sec": pctile(dt, 0.5),
"dt_p95_sec": pctile(dt, 0.95),
"orphan_local_oids": [x["venue_order_id"] for x in ol[:50]],
"orphan_venue_tids": [x["venue_trade_id"] for x in ov[:50]],
"venue_history_max": venue_t_max,
"orphan_audit": orphan_audit,
"probe": "STOPPED",
}
out.with_suffix(".json").write_text(json.dumps(sidecar, indent=2) + "\n")
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--dir", default=str(_ROOT / "logs" / "maker_edge"))
ap.add_argument("--symbol", default=_env("RECON_SYMBOL", "BTCUSDT"))
ap.add_argument("--since-days", type=float, default=20.0)
ap.add_argument("--trades-cache", default="")
ap.add_argument("--fetch", action="store_true", help="Fetch userTrades from exchange")
ap.add_argument("--out", default="")
args = ap.parse_args()
log_dir = Path(args.dir)
cache = Path(args.trades_cache) if args.trades_cache else log_dir / "venue_trades.json"
if args.fetch or not cache.exists():
import time
end_ms = int(time.time() * 1000)
start_ms = end_ms - int(args.since_days * 86400 * 1000)
print(f"[recon-02] fetching userTrades {args.symbol} …")
trades = _fetch_user_trades(args.symbol, start_ms, end_ms)
cache.write_text(json.dumps(trades))
print(f"[recon-02] cached {len(trades)} trades → {cache}")
else:
trades = json.loads(cache.read_text())
print(f"[recon-02] loaded {len(trades)} trades from {cache}")
raw_fills = load_local_fills(log_dir)
locals_ = [normalize_local(e, i) for i, e in enumerate(raw_fills)]
venues = [normalize_venue(t, i) for i, t in enumerate(trades)]
print(f"[recon-02] local fills={len(locals_)} venue={len(venues)}")
result = match(locals_, venues)
venue_t_max = None
if venues:
venue_t_max = datetime.fromtimestamp(
max(int(t["time"]) for t in trades) / 1000, tz=timezone.utc
).isoformat()
orphan_audit = audit_orphan_orders(result["orphan_local"], args.symbol)
out = Path(args.out) if args.out else log_dir / "RECONCILIATION_02.txt"
write_report(
out,
result,
len(locals_),
len(venues),
venue_t_max=venue_t_max,
orphan_audit=orphan_audit,
)
print(f"[recon-02] saved {out}")
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/reconcile_order_evidence.py
+260
View File
@@ -0,0 +1,260 @@
#!/usr/bin/env python3
"""
RECONCILIATION-03 — Order-level evidence for ORPHAN_LOCAL (post userTrades cutoff)
Does NOT resume probe. Does NOT reclassify as MATCHED.
For each ORPHAN_LOCAL from RECON-02, query /fapi/v1/order and validate:
status == FILLED
executedQty ~= sum(local qty per order)
avgPrice ~= local weighted avg
side consistent
Reclassify passing rows as:
VENUE_CONFIRMED_NO_TRADE_HISTORY
(Order evidence only — no userTrades row on Testnet after cutoff)
See TESTNET_LIMITATIONS.md
"""
from __future__ import annotations
import argparse
import json
import sys
import time
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path
_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(_ROOT / "scripts"))
sys.path.insert(0, str(_ROOT / "src"))
from reconcile_fills import ( # noqa: E402
load_local_fills,
match,
normalize_local,
normalize_venue,
)
from reconcile_account import _env, _fetch_user_trades, _signed_get # noqa: E402
PX_TICK = 0.1
QTY_EPS = 1e-8
def _order_side_to_local(side: str) -> str:
return "long" if side.upper() == "BUY" else "short"
def fetch_order(symbol: str, order_id: str, cache: dict) -> dict | None:
if order_id in cache:
return cache[order_id]
try:
o = _signed_get("/fapi/v1/order", {"symbol": symbol, "orderId": order_id})
except Exception as exc:
cache[order_id] = {"_error": str(exc)}
return cache[order_id]
cache[order_id] = o if isinstance(o, dict) else {"_error": "bad_response"}
time.sleep(0.05)
return cache[order_id]
def validate_order_group(fills: list[dict], order: dict) -> tuple[str, list[str]]:
"""Return (classification, reasons)."""
reasons: list[str] = []
if order.get("_error"):
return "ORPHAN_LOCAL_UNCONFIRMED", [f"order_api_error:{order['_error']}"]
st = order.get("status")
exec_qty = float(order.get("executedQty") or 0)
avg_px = float(order.get("avgPrice") or 0)
local_qty = sum(f["qty"] for f in fills)
if exec_qty <= 0:
return "ORPHAN_LOCAL_UNCONFIRMED", [f"status={st} executedQty=0"]
# Partial fill then TTL cancel: status=CANCELED but executedQty>0
if st not in ("FILLED", "CANCELED"):
return "ORPHAN_LOCAL_UNCONFIRMED", [f"status={st}"]
if abs(local_qty - exec_qty) > QTY_EPS:
reasons.append(f"qty local={local_qty} order={exec_qty}")
wavg = sum(f["px"] * f["qty"] for f in fills) / local_qty if local_qty else 0
if avg_px > 0 and abs(wavg - avg_px) > PX_TICK + 1e-6:
reasons.append(f"px local_wavg={wavg:.2f} order_avg={avg_px:.2f}")
order_side = _order_side_to_local(str(order.get("side", "")))
for f in fills:
if f["side"] != order_side:
reasons.append(f"side local={f['side']} order={order_side}")
break
if reasons:
return "ORDER_MISMATCH", reasons
if st == "CANCELED":
return "VENUE_PARTIAL_ORDER_CANCELED", []
return "VENUE_CONFIRMED_NO_TRADE_HISTORY", []
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--dir", default=str(_ROOT / "logs" / "maker_edge"))
ap.add_argument("--symbol", default=_env("RECON_SYMBOL", "BTCUSDT"))
ap.add_argument("--trades-cache", default="")
ap.add_argument("--out", default="")
args = ap.parse_args()
log_dir = Path(args.dir)
cache_path = Path(args.trades_cache) if args.trades_cache else log_dir / "venue_trades.json"
trades = json.loads(cache_path.read_text()) if cache_path.exists() else []
raw = load_local_fills(log_dir)
locals_ = [normalize_local(e, i) for i, e in enumerate(raw)]
venues = [normalize_venue(t, i) for i, t in enumerate(trades)]
r02 = match(locals_, venues)
orphans = r02["orphan_local"]
by_oid: dict[str, list[dict]] = defaultdict(list)
for f in orphans:
if f.get("venue_order_id"):
by_oid[f["venue_order_id"]].append(f)
order_cache: dict[str, dict] = {}
fill_class: dict[str, tuple[str, list[str], dict | None]] = {}
counts = defaultdict(int)
order_rows: list[dict] = []
for oid, fills in sorted(by_oid.items()):
order = fetch_order(args.symbol, oid, order_cache)
cls, reasons = validate_order_group(fills, order or {})
counts[cls] += len(fills)
order_rows.append(
{
"venue_order_id": oid,
"classification": cls,
"n_local_fills": len(fills),
"local_qty": sum(f["qty"] for f in fills),
"order_executedQty": order.get("executedQty") if order else None,
"order_avgPrice": order.get("avgPrice") if order else None,
"order_status": order.get("status") if order else None,
"order_updateTime": order.get("updateTime") if order else None,
"reasons": reasons,
}
)
for f in fills:
fill_class[f["fill_id"]] = (cls, reasons, order)
# Summary from RECON-02 matched
n_matched = len(r02["matched"])
n_dup = len(r02["duplicate"])
n_mismatch = len(r02["mismatch"])
n_mal = len(r02["malformed"])
n_confirmed = counts["VENUE_CONFIRMED_NO_TRADE_HISTORY"]
n_partial_canceled = counts["VENUE_PARTIAL_ORDER_CANCELED"]
n_order_mismatch = counts["ORDER_MISMATCH"]
n_unconfirmed = counts["ORPHAN_LOCAL_UNCONFIRMED"]
n_local = len(locals_)
venue_t_max = None
if trades:
venue_t_max = datetime.fromtimestamp(
max(int(t["time"]) for t in trades) / 1000, tz=timezone.utc
).isoformat()
order_evidence_ok = (
n_unconfirmed == 0
and n_order_mismatch == 0
and (n_confirmed + n_partial_canceled) == len(orphans)
)
classified = (
n_matched + n_dup + n_mismatch + n_mal
+ n_confirmed + n_partial_canceled + n_order_mismatch + n_unconfirmed
)
lines: list[str] = []
def p(s: str = "") -> None:
lines.append(s)
print(s)
p("=" * 72)
p("RECONCILIATION-03 — Order-level evidence (ORPHAN_LOCAL backfill)")
p("MM_EDGE_EXP_001 / probe_v0.1 / TESTNET BTCUSDT")
p("Probe remains STOPPED")
p("=" * 72)
p()
p("Prior RECON-02 (trade-level)")
p("-" * 40)
p(f"MATCHED (Order+Trade): {n_matched}")
p(f"DUPLICATE: {n_dup}")
p(f"MISMATCH: {n_mismatch}")
p(f"MALFORMED: {n_mal}")
p(f"ORPHAN_LOCAL (pre-03): {len(orphans)}")
if venue_t_max:
p(f"userTrades history max (UTC): {venue_t_max}")
p()
p("RECON-03 order-level reclassification")
p("-" * 40)
p(f"VENUE_CONFIRMED_NO_TRADE_HISTORY: {n_confirmed}")
p(f"VENUE_PARTIAL_ORDER_CANCELED: {n_partial_canceled}")
p(f"ORDER_MISMATCH: {n_order_mismatch}")
p(f"ORPHAN_LOCAL_UNCONFIRMED: {n_unconfirmed}")
p(f"Unique orders checked: {len(by_oid)}")
p()
p("Evidence grades (permanent taxonomy)")
p("-" * 40)
p("MATCHED = Order + Trade row (dual evidence)")
p("VENUE_CONFIRMED_NO_TRADE_HISTORY = Order FILLED, no userTrades row")
p("VENUE_PARTIAL_ORDER_CANCELED = Partial fill, order later CANCELED (TTL)")
p("ORDER_MISMATCH = Order exists but qty/px/side disagree")
p("ORPHAN_LOCAL_UNCONFIRMED = No reliable order evidence")
p()
p("Gates")
p("-" * 40)
p(f"RECON-02 classification (all buckets): {'PASS' if classified == n_local else 'FAIL'}")
p(f"Order-level closure (887 backfill): {'PASS' if order_evidence_ok else 'FAIL'}")
p(f"Strict trade-level closure: FAIL (by design until live trade_id ledger)")
p()
p("Testnet limitation")
p("-" * 40)
p("userTrades history is NOT guaranteed complete after observed cutoff.")
p("Order-level FILLED status remains queryable via /fapi/v1/order.")
p("Do NOT treat VENUE_CONFIRMED fills as fake or duplicate.")
p()
fails = [r for r in order_rows if r["classification"] in ("ORDER_MISMATCH", "ORPHAN_LOCAL_UNCONFIRMED")]
if fails:
p(f"Non-confirmed orders (showing {min(8, len(fails))}/{len(fails)})")
p("-" * 40)
for r in fails[:8]:
p(
f" oid={r['venue_order_id']} cls={r['classification']} "
f"local_qty={r['local_qty']} exec={r['order_executedQty']} reasons={r['reasons']}"
)
p()
p("=" * 72)
out = Path(args.out) if args.out else log_dir / "RECONCILIATION_03.txt"
out.write_text("\n".join(lines) + "\n", encoding="utf-8")
sidecar = {
"experiment_id": "MM_EDGE_EXP_001",
"recon": "RECONCILIATION-03",
"n_local": n_local,
"matched_trade_level": n_matched,
"orphan_local_pre03": len(orphans),
"venue_confirmed_no_trade_history": n_confirmed,
"venue_partial_order_canceled": n_partial_canceled,
"order_mismatch": n_order_mismatch,
"orphan_local_unconfirmed": n_unconfirmed,
"unique_orders_checked": len(by_oid),
"userTrades_cutoff_utc": venue_t_max,
"recon02_classification_pass": classified == n_local,
"order_level_closure_pass": order_evidence_ok,
"strict_trade_level_pass": False,
"probe": "STOPPED",
"order_rows": order_rows,
}
out.with_suffix(".json").write_text(json.dumps(sidecar, indent=2) + "\n")
print(f"[recon-03] saved {out}")
return 0 if order_evidence_ok else 1
if __name__ == "__main__":
sys.exit(main())
scripts/record_l2_ccxt.py
+83
View File
@@ -0,0 +1,83 @@
#!/usr/bin/env python3
"""
CCXT 轻量 L2 录音机（不依赖 Nautilus）
用途：在 Nautilus 节点未就绪时，先用代理拉 Binance USDT-M 盘口 + trades，
写入与 Maker Edge 相同的 jsonl schema（book history + 模拟 quote 心跳）。
用法：
cd nautilus_mm
source .venv/bin/activate
export PYTHONPATH=src
export HTTPS_PROXY=http://127.0.0.1:7897
python scripts/record_l2_ccxt.py
"""
from __future__ import annotations
import os
import sys
import time
from pathlib import Path
_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(_ROOT / "src"))
import ccxt # type: ignore
from nautilus_mm.recorder import MakerEdgeLogger
def main() -> None:
proxy = os.getenv("HTTPS_PROXY") or os.getenv("HTTP_PROXY") or "http://127.0.0.1:7897"
symbol = os.getenv("CCXT_SYMBOL", "BTC/USDT:USDT")
poll = float(os.getenv("POLL_SECS", "2"))
log_dir = os.getenv("MAKER_EDGE_LOG_DIR", str(_ROOT / "logs" / "maker_edge"))
ex = ccxt.binanceusdm(
{
"enableRateLimit": True,
"proxies": {"http": proxy, "https": proxy},
"options": {"defaultType": "future"},
}
)
lg = MakerEdgeLogger(log_dir=log_dir, levels=10)
last_mid = None
print(f"[ccxt-recorder] {symbol} proxy={proxy} log={log_dir}")
print("Ctrl+C to stop. This mode records book only (no live orders).")
while True:
try:
ob = ex.fetch_order_book(symbol, limit=10)
trades = ex.fetch_trades(symbol, limit=100)
snap = MakerEdgeLogger.snapshot_from_orderbook(
ob, levels=10, recent_trades=trades, last_mid=last_mid
)
if snap.mid:
last_mid = snap.mid
now = time.time()
lg.record_book(snap, now=now)
# 心跳 quote（不挂单，仅记录可报价位置）
if snap.best_bid:
lg.write(
{
"event": "book_tick",
"pair": symbol,
"inventory": 0,
**snap.to_book_fields(),
}
)
lg.update_paths(symbol, snap.mid or 0, now=now)
print(
f"\r mid={snap.mid:.1f} spread={snap.spread:.2f} obi={snap.obi:+.3f} "
f"timb={snap.trade_imbalance:+.3f} pending_fills={lg.pending_count}",
end="",
flush=True,
)
except Exception as e:
print(f"\nerror: {e}")
time.sleep(poll)
if __name__ == "__main__":
main()
scripts/run_event_state.sh
Executable
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
cd "$ROOT"
# Preserve systemd/caller identity before .env (which still belongs to EXP_001)
PRESERVE_RUN_ID="${LEDGER_RUN_ID:-}"
PRESERVE_LOG_DIR="${EVENT_STATE_LOG_DIR:-}"
if [[ -f .env ]]; then
set -a
# shellcheck disable=SC1091
source .env
set +a
fi
# Layer 1 (script): force EXP_002 contract after .env
export EXPERIMENT_ID=MM_EDGE_EXP_002
export PROBE_VERSION=event_state_v0.1
export ENABLE_TRADING=false
export LEDGER_RUN_ID="${PRESERVE_RUN_ID:-${LEDGER_RUN_ID:-EXP-002-RUN-002}}"
export EVENT_STATE_LOG_DIR="${PRESERVE_LOG_DIR:-$ROOT/logs/event_state/$LEDGER_RUN_ID}"
export PYTHONPATH="${PYTHONPATH:-$ROOT/src}"
mkdir -p "$EVENT_STATE_LOG_DIR"
echo "[run_event_state] EXP_002 observability | trading=NO | run=$LEDGER_RUN_ID | log=$EVENT_STATE_LOG_DIR"
exec "$ROOT/.venv/bin/python" -m nautilus_mm.run_event_state
scripts/run_probe.sh
Executable
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
cd "$ROOT"
if [[ ! -d .venv ]]; then
python3 -m venv .venv
.venv/bin/pip install -U pip
.venv/bin/pip install -r requirements.txt
fi
# shellcheck disable=SC1091
source .venv/bin/activate
export PYTHONPATH="${ROOT}/src:${PYTHONPATH:-}"
if [[ -f .env ]]; then
set -a
# shellcheck disable=SC1091
source .env
set +a
fi
# 本地可开代理；服务器 systemd 直连，勿强制 7897
if [[ "${USE_PROXY:-}" == "1" || "${USE_PROXY:-}" == "true" ]]; then
export HTTP_PROXY="${HTTP_PROXY:-http://127.0.0.1:7897}"
export HTTPS_PROXY="${HTTPS_PROXY:-http://127.0.0.1:7897}"
echo "[run_probe] proxy=$HTTPS_PROXY"
elif [[ -n "${HTTPS_PROXY:-}${HTTP_PROXY:-}" ]]; then
echo "[run_probe] proxy=${HTTPS_PROXY:-$HTTP_PROXY}"
else
echo "[run_probe] direct (no proxy)"
fi
exec python -m nautilus_mm.run_live
scripts/smoke_test_event_state.sh
Executable
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/env bash
# MM_EDGE_EXP_002 smoke test — 10–15 min, restart in the middle, NO trading.
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
cd "$ROOT"
RUN_ID="${LEDGER_RUN_ID:-EXP-002-RUN-001}"
SESSION_SECS="${SESSION_SECS:-360}" # 6 min × 2 = 12 min collect
LOG_DIR="${EVENT_STATE_LOG_DIR:-$ROOT/logs/event_state/$RUN_ID}"
PYTHON="${ROOT}/.venv/bin/python"
if [[ -f .env ]]; then
set -a
# shellcheck disable=SC1091
source .env
set +a
fi
export EXPERIMENT_ID=MM_EDGE_EXP_002
export PROBE_VERSION=event_state_v0.1
export ENABLE_TRADING=false
export LEDGER_RUN_ID="$RUN_ID"
export EVENT_STATE_LOG_DIR="$LOG_DIR"
export PYTHONPATH="$ROOT/src"
mkdir -p "$LOG_DIR"
run_session() {
local label="$1"
export LEDGER_SESSION_ID="$(python3 -c 'import uuid; print(uuid.uuid4().hex[:12])')"
echo "[smoke] session ${label} start session_id=${LEDGER_SESSION_ID} secs=${SESSION_SECS}"
"$PYTHON" -m nautilus_mm.run_event_state &
local pid=$!
echo "[smoke] pid=${pid}"
sleep "$SESSION_SECS"
echo "[smoke] session ${label} stopping pid=${pid}"
kill -INT "$pid" 2>/dev/null || true
# allow experiment_stop flush
local i=0
while kill -0 "$pid" 2>/dev/null && [[ $i -lt 30 ]]; do
sleep 1
i=$((i + 1))
done
if kill -0 "$pid" 2>/dev/null; then
echo "[smoke] SIGINT timeout — SIGTERM"
kill -TERM "$pid" 2>/dev/null || true
sleep 3
fi
if kill -0 "$pid" 2>/dev/null; then
echo "[smoke] SIGTERM timeout — SIGKILL"
kill -KILL "$pid" 2>/dev/null || true
fi
wait "$pid" 2>/dev/null || true
echo "[smoke] session ${label} stopped"
}
echo "[smoke] RUN_ID=${RUN_ID} log=${LOG_DIR} trading=NO"
run_session A
echo "[smoke] restart gap 5s"
sleep 5
run_session B
echo "[smoke] validating ledger"
"$PYTHON" "$ROOT/scripts/validate_event_ledger.py" \
--dir "$LOG_DIR" \
--run-id "$RUN_ID" \
--out "$LOG_DIR/Event_Ledger_Validation.json"
echo "[smoke] done"
scripts/validate_event_ledger.py
+453
View File
@@ -0,0 +1,453 @@
#!/usr/bin/env python3
"""
Validate MM_EDGE_EXP_002 Immutable Event Ledger.
Phase 1 smoke: Gates 1–3 plus ledger engineering contract.
Gate 4 (predictability) is blocked until fill anchors exist.
"""
from __future__ import annotations
import argparse
import json
import math
import random
from collections import Counter
from pathlib import Path
from typing import Any
TRADE_REQUIRED = [
"event_type",
"exchange_ts_ns",
"local_ts_epoch",
"local_ts_ns",
"trade_side",
"trade_qty",
"trade_price",
"best_bid", # optional on trade; counted separately
]
TRADE_CORE = [
"event_type",
"exchange_ts_ns",
"local_ts_epoch",
"local_ts_ns",
"trade_side",
"trade_qty",
"trade_price",
"price",
"quantity",
"best_bid",
"best_ask",
"mid",
"spread",
]
BOOK_CORE = [
"event_type",
"exchange_ts_ns",
"local_ts_epoch",
"local_ts_ns",
"best_bid",
"best_ask",
"mid",
"spread",
"bid_depth_1",
"ask_depth_1",
"bid_depth_5",
"ask_depth_5",
]
BOOK_DELTA_KEYS = [
"bid_depth_delta_1",
"ask_depth_delta_1",
"bid_move",
"ask_move",
"spread_change",
]
def _pctile(xs: list[float], q: float) -> float | None:
if not xs:
return None
ys = sorted(xs)
if len(ys) == 1:
return ys[0]
i = (len(ys) - 1) * q
lo = math.floor(i)
hi = math.ceil(i)
if lo == hi:
return ys[lo]
return ys[lo] * (hi - i) + ys[hi] * (i - lo)
def _num(v: float | None, digits: int = 3) -> str:
if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))):
return "n/a"
return f"{v:.{digits}f}"
def load_jsonl(log_dir: Path) -> tuple[list[dict[str, Any]], int, int]:
rows: list[dict[str, Any]] = []
parse_fail = 0
empty = 0
for f in sorted(log_dir.glob("*.jsonl")):
for line in f.open():
s = line.strip()
if not s:
empty += 1
continue
try:
e = json.loads(s)
except Exception:
parse_fail += 1
continue
if isinstance(e, dict):
rows.append(e)
else:
parse_fail += 1
return rows, parse_fail, empty
def _present(ev: dict[str, Any], key: str) -> bool:
v = ev.get(key)
return v is not None and v != ""
def _hollow_book(ev: dict[str, Any]) -> bool:
depths = [
ev.get("bid_depth_1"),
ev.get("ask_depth_1"),
ev.get("bid_depth_5"),
ev.get("ask_depth_5"),
ev.get("mid"),
]
nums = []
for d in depths:
try:
nums.append(float(d))
except (TypeError, ValueError):
nums.append(0.0)
return all(abs(x) < 1e-12 for x in nums)
def main() -> int:
ap = argparse.ArgumentParser(description="Validate EXP_002 event ledger / smoke contract")
ap.add_argument("--dir", default="logs/event_state")
ap.add_argument("--out", default="")
ap.add_argument("--run-id", default="")
ap.add_argument("--sample", type=int, default=200)
ap.add_argument("--latency-tolerance-ms", type=float, default=50.0)
ap.add_argument("--seed", type=int, default=42)
args = ap.parse_args()
log_dir = Path(args.dir)
rows, parse_fail, empty_lines = load_jsonl(log_dir)
if args.run_id:
rows = [r for r in rows if r.get("run_id") == args.run_id]
market = [r for r in rows if r.get("event") == "market_event"]
trades = [r for r in market if r.get("event_type") == "aggressive_trade"]
books = [r for r in market if r.get("event_type") == "book_update"]
starts = [r for r in rows if r.get("event") == "experiment_start"]
stops = [r for r in rows if r.get("event") == "experiment_stop"]
anchors = [r for r in rows if r.get("event") == "fill_anchor"]
run_ids = sorted({r.get("run_id") for r in rows if r.get("run_id")})
sessions = [r.get("session_id") for r in starts]
# Duration from first/last local_ts
local_epochs = [float(r["local_ts_epoch"]) for r in market if r.get("local_ts_epoch") is not None]
duration_s = (max(local_epochs) - min(local_epochs)) if len(local_epochs) >= 2 else 0.0
if duration_s <= 0:
duration_s = 1.0
rates = {
"aggressive_trade_per_sec": len(trades) / duration_s,
"book_update_per_sec": len(books) / duration_s,
"total_market_events_per_sec": len(market) / duration_s,
"duration_sec": duration_s,
}
# Timestamp quality
ex_ok = sum(1 for r in market if r.get("exchange_ts_ns") is not None)
loc_ok = sum(1 for r in market if r.get("local_ts_epoch") is not None and r.get("local_ts_ns") is not None)
latencies_ms: list[float] = []
skew_violations = 0
for r in market:
ex = r.get("exchange_ts_ns")
loc = r.get("local_ts_ns")
if ex is None or loc is None:
continue
lag_ms = (float(loc) - float(ex)) / 1e6
latencies_ms.append(lag_ms)
if float(ex) > float(loc) + args.latency_tolerance_ms * 1e6:
skew_violations += 1
ts_quality = {
"exchange_ts_ns_pct": (ex_ok / len(market)) if market else 0.0,
"local_ts_pct": (loc_ok / len(market)) if market else 0.0,
"latency_n": len(latencies_ms),
"latency_ms_p50": _pctile(latencies_ms, 0.50),
"latency_ms_p95": _pctile(latencies_ms, 0.95),
"latency_ms_p99": _pctile(latencies_ms, 0.99),
"latency_ms_max": max(latencies_ms) if latencies_ms else None,
"latency_ms_min": min(latencies_ms) if latencies_ms else None,
"exchange_after_local_violations": skew_violations,
"tolerance_ms": args.latency_tolerance_ms,
}
# Event order: exchange_ts regression (do not silently sort)
regressions = 0
max_back_ns = 0
prev_ex = None
for r in market:
ex = r.get("exchange_ts_ns")
if ex is None:
continue
ex = int(ex)
if prev_ex is not None and ex < prev_ex:
regressions += 1
max_back_ns = max(max_back_ns, prev_ex - ex)
prev_ex = ex
# Schema completeness (sample)
rng = random.Random(args.seed)
n_trade_s = min(args.sample, len(trades))
n_book_s = min(args.sample, len(books))
trade_sample = rng.sample(trades, n_trade_s) if n_trade_s else []
book_sample = rng.sample(books, n_book_s) if n_book_s else []
def missing_rate(sample: list[dict], keys: list[str]) -> dict[str, float]:
if not sample:
return {k: 1.0 for k in keys}
out = {}
for k in keys:
miss = sum(1 for e in sample if not _present(e, k))
out[k] = miss / len(sample)
return out
trade_missing = missing_rate(trade_sample, TRADE_CORE)
book_missing = missing_rate(book_sample, BOOK_CORE)
book_delta_key_miss = 0.0
if book_sample:
book_delta_key_miss = sum(
1 for e in book_sample if any(k not in e for k in BOOK_DELTA_KEYS)
) / len(book_sample)
hollow = sum(1 for e in book_sample if _hollow_book(e))
# Restart / integrity
event_ids = [r.get("event_id") for r in market if r.get("event_id")]
dup_ids = [k for k, v in Counter(event_ids).items() if v > 1]
seq_ok = True
seq_notes = []
by_session: dict[str, list[int]] = {}
for r in rows:
sid = r.get("session_id")
seq = r.get("event_seq")
if sid is None or seq is None:
continue
by_session.setdefault(str(sid), []).append(int(seq))
for sid, seqs in by_session.items():
if seqs != list(range(1, len(seqs) + 1)) and seqs != sorted(seqs):
# allow gaps only if we filtered; within session expect 1..n
expected = list(range(min(seqs), max(seqs) + 1))
if seqs != expected:
seq_ok = False
seq_notes.append(f"{sid}: not contiguous {seqs[:5]}...{seqs[-3:]}")
if seqs and seqs[0] != 1:
seq_notes.append(f"{sid}: seq starts at {seqs[0]} (expected 1 after restart)")
seq_reset_expected = len(sessions) >= 2 and all(
(by_session.get(str(s), [None])[0] == 1) for s in sessions if s
)
# Gates
gate1_pass: bool | None
if anchors:
reconstruct_fail = 0
for anc in anchors:
fill_ts = float(anc["fill_ts_epoch"])
start = float(anc.get("window_start_epoch", fill_ts - 5.0))
cutoff = float(anc.get("feature_cutoff_epoch", fill_ts - 0.25))
window = []
for r in market:
ex = r.get("exchange_ts_ns")
ts = float(ex) / 1e9 if ex is not None else r.get("local_ts_epoch")
if ts is None:
continue
if start <= float(ts) < cutoff:
window.append(r)
if not window:
reconstruct_fail += 1
gate1_pass = reconstruct_fail == 0
gate1_status = "PASS" if gate1_pass else "FAIL"
else:
# Phase 1: stream completeness stands in for fill reconstruction
stream_ok = parse_fail == 0 and len(market) > 0 and loc_ok == len(market)
gate1_pass = stream_ok
gate1_status = (
"PASS (Phase 1 stream completeness; no fill_anchor — expected)"
if stream_ok
else "FAIL (stream incomplete)"
)
gate2_ok = (
ts_quality["exchange_ts_ns_pct"] >= 0.99
and ts_quality["local_ts_pct"] >= 0.99
and skew_violations == 0
)
gate2_status = "PASS" if gate2_ok else "FAIL"
schema_ok = (
all(v == 0.0 for v in trade_missing.values())
and all(v == 0.0 for v in book_missing.values())
and book_delta_key_miss == 0.0
and hollow == 0
and len(trades) > 0
and len(books) > 0
)
gate3_ok = schema_ok and ts_quality["exchange_ts_ns_pct"] >= 0.99
gate3_status = "PASS" if gate3_ok else "FAIL"
restart_ok = (
parse_fail == 0
and len(dup_ids) == 0
and len(starts) >= 1
and (len(starts) == 1 or (len(stops) >= len(starts) - 1 and seq_reset_expected))
)
integrity = {
"parse_fail_lines": parse_fail,
"empty_lines": empty_lines,
"duplicate_event_ids": len(dup_ids),
"experiment_start_count": len(starts),
"experiment_stop_count": len(stops),
"sessions": sessions,
"seq_contiguous_ok": seq_ok,
"seq_reset_expected": seq_reset_expected,
"seq_notes": seq_notes[:8],
"restart_contract": "PASS" if restart_ok else "FAIL",
}
run_id = args.run_id or (run_ids[0] if len(run_ids) == 1 else ",".join(run_ids) or "UNSET")
start0 = starts[0] if starts else {}
manifest = {
"run_id": run_id,
"start_ts": start0.get("local_ts"),
"end_ts": stops[-1].get("local_ts") if stops else (rows[-1].get("local_ts") if rows else None),
"host": start0.get("host"),
"commit": start0.get("commit"),
"config_hash": start0.get("config_hash"),
"schema_version": start0.get("schema_version"),
"event_count": len(rows),
"trade_event_count": len(trades),
"book_event_count": len(books),
"session_count": len(sessions),
}
report = {
"experiment_id": start0.get("experiment_id", "MM_EDGE_EXP_002"),
"run_id": run_id,
"purpose": "ledger smoke / Gates 1-3",
"gate4_predictability": "BLOCKED",
"gates": {
"gate1_event_completeness": gate1_status,
"gate2_temporal_integrity": gate2_status,
"gate3_event_coverage": gate3_status,
},
"manifest": manifest,
"rates": rates,
"timestamp_quality": ts_quality,
"order": {
"exchange_ts_regressions": regressions,
"max_regression_ns": max_back_ns,
"max_regression_ms": max_back_ns / 1e6 if regressions else 0.0,
"note": "regressions recorded, not silently sorted",
},
"schema": {
"trade_sample_n": n_trade_s,
"book_sample_n": n_book_s,
"trade_missing_rate": trade_missing,
"book_missing_rate": book_missing,
"hollow_book_in_sample": hollow,
},
"integrity": integrity,
"counts": {
"total_rows": len(rows),
"market_events": len(market),
"aggressive_trades": len(trades),
"book_updates": len(books),
"fill_anchors": len(anchors),
},
}
lines = [
"=" * 68,
"MM_EDGE_EXP_002 Ledger Smoke / Gates 1–3",
"=" * 68,
f"run_id: {run_id}",
f"sessions: {len(sessions)} {sessions}",
f"host/commit:{start0.get('host')} / {str(start0.get('commit') or '')[:12]}",
f"config_hash:{start0.get('config_hash')}",
f"schema: {start0.get('schema_version')}",
"",
"Gate 1 Event Completeness: " + gate1_status,
"Gate 2 Temporal Integrity: " + gate2_status,
"Gate 3 Event Coverage: " + gate3_status,
"Gate 4 Predictability: BLOCKED",
"",
"1. Event write rates",
"-" * 40,
f"duration_sec: {_num(duration_s, 1)}",
f"aggressive_trade / sec: {_num(rates['aggressive_trade_per_sec'], 3)}",
f"book_update / sec: {_num(rates['book_update_per_sec'], 3)}",
f"total market events / sec: {_num(rates['total_market_events_per_sec'], 3)}",
f"counts: trades={len(trades)} books={len(books)} total={len(market)}",
"",
"2. Timestamp quality",
"-" * 40,
f"exchange_ts_ns != null: {ts_quality['exchange_ts_ns_pct']*100:.2f}%",
f"local_ts_ns != null: {ts_quality['local_ts_pct']*100:.2f}%",
f"exchange > local+tol: {skew_violations} (tol={args.latency_tolerance_ms}ms)",
f"local-exchange lag ms: p50={_num(ts_quality['latency_ms_p50'])} "
f"p95={_num(ts_quality['latency_ms_p95'])} p99={_num(ts_quality['latency_ms_p99'])} "
f"max={_num(ts_quality['latency_ms_max'])}",
"",
"3. Event order (exchange_ts_ns regression, not sorted)",
"-" * 40,
f"regressions: {regressions} max_back_ms={_num(max_back_ns/1e6 if regressions else 0.0)}",
"",
"4. Raw event completeness (sample)",
"-" * 40,
f"trade sample={n_trade_s} missing={trade_missing}",
f"book sample={n_book_s} missing={book_missing}",
f"hollow book_update (all depth/mid empty): {hollow}",
"",
"5. Restart / immutable integrity",
"-" * 40,
f"parse_fail_lines={parse_fail} empty_lines={empty_lines}",
f"duplicate_event_ids={len(dup_ids)}",
f"start={len(starts)} stop={len(stops)} seq_ok={seq_ok} seq_reset_expected={seq_reset_expected}",
f"restart_contract={integrity['restart_contract']}",
"",
"Gate 4 remains BLOCKED until fill_anchor exists. Do not resume trading.",
"=" * 68,
]
text = "\n".join(lines) + "\n"
print(text)
out_json = Path(args.out) if args.out else log_dir / "Event_Ledger_Validation.json"
out_txt = out_json.with_suffix(".txt")
out_json.parent.mkdir(parents=True, exist_ok=True)
out_json.write_text(json.dumps(report, indent=2, default=str) + "\n", encoding="utf-8")
out_txt.write_text(text, encoding="utf-8")
(log_dir / f"{run_id.replace('/', '_')}.manifest.json").write_text(
json.dumps(manifest, indent=2, default=str) + "\n", encoding="utf-8"
)
ok = gate1_pass is not False and gate2_ok and gate3_ok and restart_ok and parse_fail == 0
return 0 if ok else 1
if __name__ == "__main__":
raise SystemExit(main())
src/nautilus_mm/__init__.py
+3
View File
@@ -0,0 +1,3 @@
"""Nautilus MM — Trading OS Execution Reality Layer (Maker Edge)."""
__version__ = "0.1.0"
src/nautilus_mm/book_utils.py
+124
View File
@@ -0,0 +1,124 @@
"""Order book → MicroSnapshot helpers (Nautilus OrderBook / dict)."""
from __future__ import annotations
from typing import Any, Optional
from nautilus_mm.recorder import MicroSnapshot
def snapshot_from_nautilus_book(
book,
levels: int = 10,
recent_buy_qty: float = 0.0,
recent_sell_qty: float = 0.0,
last_mid: float | None = None,
liq_low: float | None = None,
liq_high: float | None = None,
) -> MicroSnapshot:
"""Convert nautilus_trader OrderBook to MicroSnapshot."""
try:
bids = list(book.bids())[:levels] if callable(getattr(book, "bids", None)) else []
asks = list(book.asks())[:levels] if callable(getattr(book, "asks", None)) else []
except Exception:
# Some versions expose bid/ask sequences differently
bids = getattr(book, "bids", [])[:levels] or []
asks = getattr(book, "asks", [])[:levels] or []
def _px_qty(level) -> tuple[float, float]:
# Nautilus BookLevel: price is attribute, size() is method
if hasattr(level, "price") and hasattr(level, "size"):
size = level.size() if callable(level.size) else level.size
return float(level.price), float(size)
if isinstance(level, (list, tuple)) and len(level) >= 2:
return float(level[0]), float(level[1])
return 0.0, 0.0
if not bids or not asks:
# try best bid/ask API
try:
bb = float(book.best_bid_price()) if book.best_bid_price() is not None else 0.0
ba = float(book.best_ask_price()) if book.best_ask_price() is not None else 0.0
bs = float(book.best_bid_size() or 0)
az = float(book.best_ask_size() or 0)
if bb and ba:
mid = (bb + ba) / 2
return MicroSnapshot(
best_bid=bb,
best_ask=ba,
mid=mid,
spread=ba - bb,
bid_depth_1=bs,
ask_depth_1=az,
bid_depth_5=bs,
ask_depth_5=az,
bid_depth=bs,
ask_depth=az,
obi=((bs - az) / (bs + az)) if (bs + az) else 0.0,
)
except Exception:
return MicroSnapshot()
return MicroSnapshot()
bid_levels = [_px_qty(x) for x in bids]
ask_levels = [_px_qty(x) for x in asks]
best_bid, bid1 = bid_levels[0]
best_ask, ask1 = ask_levels[0]
mid = (best_bid + best_ask) / 2.0
spread = best_ask - best_bid
def depth(lvls, n):
return sum(q for _, q in lvls[:n])
bid_depth_5 = depth(bid_levels, 5)
ask_depth_5 = depth(ask_levels, 5)
bid_depth = depth(bid_levels, levels)
ask_depth = depth(ask_levels, levels)
tot = bid_depth + ask_depth
obi = ((bid_depth - ask_depth) / tot) if tot else 0.0
delta = recent_buy_qty - recent_sell_qty
timb_den = recent_buy_qty + recent_sell_qty
trade_imbalance = (delta / timb_den) if timb_den else 0.0
de = 0.0
if last_mid and mid and abs(delta) > 1e-12:
de = ((mid - last_mid) / last_mid) / delta
liq_dist = 0.0
if liq_low and liq_high and mid and (liq_high - liq_low) > 0:
liq_dist = ((mid - liq_low) / (liq_high - liq_low)) * 2 - 1
return MicroSnapshot(
best_bid=best_bid,
best_ask=best_ask,
mid=mid,
spread=spread,
bid_depth_1=bid1,
ask_depth_1=ask1,
bid_depth_5=bid_depth_5,
ask_depth_5=ask_depth_5,
bid_depth=bid_depth,
ask_depth=ask_depth,
obi=obi,
delta=delta,
trade_imbalance=trade_imbalance,
delta_efficiency=de,
liquidation_distance=liq_dist,
)
def snapshot_from_ccxt_ob(
ob: dict[str, Any],
levels: int = 10,
recent_trades: list | None = None,
last_mid: float | None = None,
) -> MicroSnapshot:
from nautilus_mm.recorder import MakerEdgeLogger
return MakerEdgeLogger.snapshot_from_orderbook(
ob,
levels=levels,
recent_trades=recent_trades,
last_mid=last_mid,
)
src/nautilus_mm/event_ledger.py
+403
View File
@@ -0,0 +1,403 @@
"""
Immutable Event Ledger — MM_EDGE_EXP_002
Raw Event > Derived Feature
Stores immutable market events for later reconstruction of pre-fill windows.
Features are computed offline; this module only persists observability data.
"""
from __future__ import annotations
import hashlib
import json
import os
import socket
import subprocess
import time
import uuid
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from nautilus_mm.experiment import load_experiment_meta, stamp_event
from nautilus_mm.recorder import MicroSnapshot
LEDGER_SCHEMA_VERSION = "immutable_event_v1"
def _git_commit(root: Path) -> str | None:
try:
r = subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=root,
capture_output=True,
text=True,
timeout=3,
check=False,
)
if r.returncode == 0:
return r.stdout.strip() or None
except Exception:
return None
return None
def load_run_identity(*, extra_config: dict[str, Any] | None = None) -> dict[str, Any]:
"""Freeze one Ledger Integrity ID per collection run (sessions share the same run_id)."""
root = Path(__file__).resolve().parents[2]
cfg = {
"prefill_window_sec": os.getenv("PREFILL_WINDOW_SEC", "5.0"),
"prefill_margin_sec": os.getenv("PREFILL_MARGIN_SEC", "0.25"),
"large_trade_qty": os.getenv("LARGE_TRADE_QTY", "0.1"),
"book_depth": os.getenv("BOOK_DEPTH", "10"),
"symbol": os.getenv("SYMBOL", "BTCUSDT-PERP"),
"environment": os.getenv("BINANCE_ENVIRONMENT", "TESTNET"),
"schema_version": LEDGER_SCHEMA_VERSION,
}
if extra_config:
cfg.update({k: str(v) for k, v in extra_config.items()})
payload = json.dumps(cfg, sort_keys=True, default=str)
return {
"run_id": os.getenv("LEDGER_RUN_ID", "EXP-002-RUN-UNSET"),
"session_id": os.getenv("LEDGER_SESSION_ID") or uuid.uuid4().hex[:12],
"schema_version": LEDGER_SCHEMA_VERSION,
"host": socket.gethostname(),
"commit": _git_commit(root),
"config_hash": hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16],
"config_snapshot": cfg,
}
def _utc_iso(ts: float | None = None) -> str:
t = datetime.fromtimestamp(ts or time.time(), tz=timezone.utc)
return t.isoformat()
@dataclass
class EventTimingState:
last_trade_ts: float | None = None
last_large_trade_ts: float | None = None
last_book_event_ts: float | None = None
last_tob_change_ts: float | None = None
last_spread_change_ts: float | None = None
last_mid_change_ts: float | None = None
last_depth_change_ts: float | None = None
class ImmutableEventLedger:
"""Append-only JSONL ledger for market events."""
def __init__(
self,
log_dir: str | Path | None = None,
*,
prefill_window_sec: float = 5.0,
prefill_margin_sec: float = 0.25,
large_trade_qty: float = 0.1,
book_levels: int = 10,
) -> None:
root = Path(__file__).resolve().parents[2]
self.log_dir = Path(log_dir) if log_dir else root / "logs" / "event_state"
self.log_dir.mkdir(parents=True, exist_ok=True)
self.prefill_window_sec = float(prefill_window_sec)
self.prefill_margin_sec = float(prefill_margin_sec)
self.large_trade_qty = float(large_trade_qty)
self.book_levels = int(book_levels)
self.experiment = load_experiment_meta()
self.run_identity = load_run_identity(
extra_config={
"prefill_window_sec": self.prefill_window_sec,
"prefill_margin_sec": self.prefill_margin_sec,
"large_trade_qty": self.large_trade_qty,
"book_levels": self.book_levels,
}
)
self._timing = EventTimingState()
self._prev_snap: MicroSnapshot | None = None
self._event_seq = 0
self._session_event_count = 0
def _file(self) -> Path:
return self.log_dir / f"{datetime.now(timezone.utc).strftime('%Y%m%d')}.jsonl"
def _next_seq(self) -> int:
self._event_seq += 1
return self._event_seq
def _timing_fields(self, now: float, *, is_trade: bool = False, is_large_trade: bool = False) -> dict[str, Any]:
def _since(last: float | None) -> float | None:
if last is None:
return None
return (now - last) * 1000.0
fields = {
"time_since_last_trade_ms": _since(self._timing.last_trade_ts),
"time_since_last_large_trade_ms": _since(self._timing.last_large_trade_ts),
"time_since_last_book_event_ms": _since(self._timing.last_book_event_ts),
"time_since_last_tob_change_ms": _since(self._timing.last_tob_change_ts),
"time_since_last_spread_change_ms": _since(self._timing.last_spread_change_ts),
"time_since_last_mid_change_ms": _since(self._timing.last_mid_change_ts),
"time_since_last_depth_change_ms": _since(self._timing.last_depth_change_ts),
}
self._timing.last_book_event_ts = now
if is_trade:
self._timing.last_trade_ts = now
if is_large_trade:
self._timing.last_large_trade_ts = now
return fields
def write(self, event: dict[str, Any]) -> None:
# Receive time is always local wall clock. Never copy exchange_ts into local_ts.
now = time.time()
now_ns = time.time_ns()
event["ledger"] = LEDGER_SCHEMA_VERSION
event["schema_version"] = LEDGER_SCHEMA_VERSION
event["run_id"] = self.run_identity["run_id"]
event["session_id"] = self.run_identity["session_id"]
event["local_ts_epoch"] = now
event["local_ts_ns"] = now_ns
event["local_ts"] = _utc_iso(now)
event["event_seq"] = self._next_seq()
self._session_event_count += 1
stamp_event(event, self.experiment)
with self._file().open("a", encoding="utf-8") as f:
f.write(json.dumps(event, ensure_ascii=False, default=str) + "\n")
f.flush()
def write_experiment_start(self, extra: dict | None = None) -> None:
ev = {
"event": "experiment_start",
"experiment_type": "Event-State Observability Probe",
"strategy": "NONE",
"execution_trading": False,
"prefill_window_sec": self.prefill_window_sec,
"prefill_margin_sec": self.prefill_margin_sec,
"large_trade_qty": self.large_trade_qty,
**self.experiment,
**{k: v for k, v in self.run_identity.items() if k != "config_snapshot"},
"config_snapshot": self.run_identity["config_snapshot"],
}
if extra:
ev.update(extra)
self.write(ev)
def write_experiment_stop(self, extra: dict | None = None) -> None:
ev = {
"event": "experiment_stop",
"session_event_count": self._session_event_count,
"run_id": self.run_identity["run_id"],
"session_id": self.run_identity["session_id"],
"execution_trading": False,
}
if extra:
ev.update(extra)
self.write(ev)
def log_aggressive_trade(
self,
*,
pair: str,
price: float,
qty: float,
trade_side: str,
exchange_ts_ns: int | None,
local_ts_epoch: float | None = None,
aggressor_side: str | None = None,
trade_id: str | None = None,
snap: MicroSnapshot | None = None,
) -> str:
now = local_ts_epoch or time.time()
notional = price * qty
is_large = qty >= self.large_trade_qty
event_id = uuid.uuid4().hex[:16]
ev = {
"event": "market_event",
"event_id": event_id,
"event_type": "aggressive_trade",
"pair": pair,
"price": price,
"quantity": qty,
"trade_qty": qty,
"trade_price": price,
"trade_notional": notional,
"trade_side": trade_side,
"aggressor_side": aggressor_side or trade_side,
"large_trade_flag": is_large,
"trade_id": trade_id,
"exchange_ts_ns": exchange_ts_ns,
}
if snap is not None:
ev.update(self._snap_book_fields(snap))
ev.update(self._depth_deltas(self._prev_snap, snap))
ev.update(self._timing_fields(now, is_trade=True, is_large_trade=is_large))
self.write(ev)
return event_id
def _snap_book_fields(self, snap: MicroSnapshot) -> dict[str, Any]:
return {
"best_bid": snap.best_bid,
"best_ask": snap.best_ask,
"mid": snap.mid,
"spread": snap.spread,
"bid_depth_1": snap.bid_depth_1,
"ask_depth_1": snap.ask_depth_1,
"bid_depth_5": snap.bid_depth_5,
"ask_depth_5": snap.ask_depth_5,
"bid_depth": snap.bid_depth,
"ask_depth": snap.ask_depth,
"obi": snap.obi,
}
def _depth_deltas(self, prev: MicroSnapshot | None, cur: MicroSnapshot) -> dict[str, Any]:
if prev is None:
return {
"bid_depth_delta_1": None,
"ask_depth_delta_1": None,
"bid_depth_delta_5": None,
"ask_depth_delta_5": None,
"bid_depth_delta": None,
"ask_depth_delta": None,
}
return {
"bid_depth_delta_1": cur.bid_depth_1 - prev.bid_depth_1,
"ask_depth_delta_1": cur.ask_depth_1 - prev.ask_depth_1,
"bid_depth_delta_5": cur.bid_depth_5 - prev.bid_depth_5,
"ask_depth_delta_5": cur.ask_depth_5 - prev.ask_depth_5,
"bid_depth_delta": cur.bid_depth - prev.bid_depth,
"ask_depth_delta": cur.ask_depth - prev.ask_depth,
}
def log_book_state(
self,
*,
pair: str,
snap: MicroSnapshot,
exchange_ts_ns: int | None,
local_ts_epoch: float | None = None,
sequence: int | None = None,
delta_count: int | None = None,
event_type: str = "book_update",
) -> str:
now = local_ts_epoch or time.time()
prev = self._prev_snap
event_id = uuid.uuid4().hex[:16]
bid_move = None
ask_move = None
mid_move = None
spread_change = None
if prev and prev.mid > 0:
bid_move = snap.best_bid - prev.best_bid
ask_move = snap.best_ask - prev.best_ask
mid_move = snap.mid - prev.mid
spread_change = snap.spread - prev.spread
depth_deltas = self._depth_deltas(prev, snap)
ev = {
"event": "market_event",
"event_id": event_id,
"event_type": event_type,
"pair": pair,
"exchange_ts_ns": exchange_ts_ns,
"sequence": sequence,
"delta_count": delta_count,
**self._snap_book_fields(snap),
**depth_deltas,
"bid_move": bid_move,
"ask_move": ask_move,
"mid_move": mid_move,
"spread_change": spread_change,
}
ev.update(self._timing_fields(now))
if prev is not None:
if bid_move not in (None, 0.0) or ask_move not in (None, 0.0):
self._timing.last_tob_change_ts = now
if spread_change not in (None, 0.0):
self._timing.last_spread_change_ts = now
if mid_move not in (None, 0.0):
self._timing.last_mid_change_ts = now
if any(
depth_deltas[k] not in (None, 0.0)
for k in (
"bid_depth_delta_1",
"ask_depth_delta_1",
"bid_depth_delta_5",
"ask_depth_delta_5",
)
):
self._timing.last_depth_change_ts = now
self._prev_snap = snap
self.write(ev)
return event_id
def log_fill_anchor(
self,
*,
fill_id: str,
fill_ts_epoch: float,
exchange_ts_ns: int | None,
side: str,
fill_price: float,
fill_qty: float,
order_id: str | None = None,
venue_order_id: str | None = None,
venue_trade_id: str | None = None,
pair: str | None = None,
snap: MicroSnapshot | None = None,
extra: dict | None = None,
) -> str:
"""
Anchor for offline [-prefill_window_sec, fill) reconstruction.
EXP_002 Phase 1 may not emit these (no trading). Schema is frozen for
future fill-anchored analysis (Gate 4).
"""
window_id = uuid.uuid4().hex[:16]
window_start = fill_ts_epoch - self.prefill_window_sec
feature_cutoff = fill_ts_epoch - self.prefill_margin_sec
ev = {
"event": "fill_anchor",
"window_id": window_id,
"fill_id": fill_id,
"fill_ts_epoch": fill_ts_epoch,
"fill_ts": _utc_iso(fill_ts_epoch),
"exchange_ts_ns": exchange_ts_ns,
"window_start_epoch": window_start,
"feature_cutoff_epoch": feature_cutoff,
"prefill_window_sec": self.prefill_window_sec,
"prefill_margin_sec": self.prefill_margin_sec,
"side": side,
"fill_price": fill_price,
"fill_qty": fill_qty,
"order_id": order_id,
"venue_order_id": venue_order_id,
"venue_trade_id": venue_trade_id,
"pair": pair,
}
if snap is not None:
ev.update(self._snap_book_fields(snap))
if extra:
ev.update(extra)
self.write(ev)
return window_id
def events_in_window(self, events: list[dict], fill_ts_epoch: float) -> list[dict]:
"""Offline helper: filter market_event rows in [-window, fill-margin)."""
start = fill_ts_epoch - self.prefill_window_sec
cutoff = fill_ts_epoch - self.prefill_margin_sec
out = []
for ev in events:
if ev.get("event") != "market_event":
continue
ts = ev.get("exchange_ts_ns")
if ts is not None:
ts_epoch = float(ts) / 1e9
else:
ts_epoch = float(ev.get("local_ts_epoch", 0.0))
if start <= ts_epoch < cutoff:
out.append(ev)
return out
src/nautilus_mm/experiment.py
+63
View File
@@ -0,0 +1,63 @@
"""
实验身份绑定 — Research Freeze / Data Collection
每条 jsonl 与每份 Maker Edge Report 必须绑定同一 Experiment ID，
避免 v2/v3 混淆「哪个实验验证出了什么」。
冻结字段（运行期不可为「结果好看」而改）：
quote / fee / exchange / probe version
"""
from __future__ import annotations
import os
from typing import Any
# 默认实验身份（可用环境变量覆盖 ID，其余保持 frozen 语义）
DEFAULT_EXPERIMENT_ID = "MM_EDGE_EXP_001"
DEFAULT_PROBE_VERSION = "probe_v0.1"
EXP_002_ID = "MM_EDGE_EXP_002"
EXP_002_PROBE_VERSION = "event_state_v0.1"
def load_experiment_meta() -> dict[str, Any]:
"""从环境变量加载实验元数据；冻结维度固定为 frozen。"""
exp_id = os.getenv("EXPERIMENT_ID", DEFAULT_EXPERIMENT_ID)
probe_version = os.getenv("PROBE_VERSION", DEFAULT_PROBE_VERSION)
if exp_id == EXP_002_ID:
phase = "Event-State Observability / Data Collection"
experiment_type = "Event-State Observability Probe"
else:
phase = "Research Freeze / Data Collection"
experiment_type = "Maker Edge Phenomenon Probe"
return {
"experiment_id": exp_id,
"probe_version": probe_version,
"experiment_type": experiment_type,
"quote_assumption": "frozen",
"fee_model": "frozen",
"exchange_assumption": "frozen",
"exchange": os.getenv("EXCHANGE_NAME", "binance_usdm"),
"environment": os.getenv("BINANCE_ENVIRONMENT", "TESTNET").upper(),
"symbol": os.getenv("SYMBOL", "BTCUSDT-PERP"),
"phase": phase,
"depends_on": "MM_EDGE_EXP_001" if exp_id == EXP_002_ID else None,
}
def stamp_event(event: dict[str, Any], meta: dict[str, Any] | None = None) -> dict[str, Any]:
"""给单条事件打上实验身份（不覆盖已有显式字段）。"""
m = meta or load_experiment_meta()
event.setdefault("experiment_id", m["experiment_id"])
event.setdefault("probe_version", m["probe_version"])
event.setdefault("experiment", {
"quote": m["quote_assumption"],
"fee": m["fee_model"],
"exchange": m["exchange_assumption"],
"venue": m["exchange"],
"environment": m["environment"],
"symbol": m["symbol"],
})
return event
src/nautilus_mm/health.py
+147
View File
@@ -0,0 +1,147 @@
"""
Phase 0 — 连接 / 数据健康度
关键指标：
sequence_gap — order book 失真信号（>0 需警惕）
latency_ms — p50 / p95 / p99 / max（做市看尾部）
book_age_ms — quote/fill 使用盘口时的新鲜度
book/trade update rate
"""
from __future__ import annotations
import time
from collections import deque
from dataclasses import dataclass, field
from typing import Optional
def empty_market_state_snapshot() -> dict:
"""预留给 Market Pulse；探针阶段全部为 null，不做预测/下单决策。"""
return {
"regime": None,
"trend_state": None,
"liquidity_state": None,
"volatility_state": None,
}
def _percentile(sorted_vals: list[float], q: float) -> Optional[float]:
if not sorted_vals:
return None
if len(sorted_vals) == 1:
return sorted_vals[0]
idx = min(len(sorted_vals) - 1, max(0, int(round(q * (len(sorted_vals) - 1)))))
return sorted_vals[idx]
@dataclass
class ConnectionHealth:
window_sec: float = 60.0
report_every_sec: float = 30.0
_book_ts: deque = field(default_factory=lambda: deque(maxlen=50_000))
_trade_ts: deque = field(default_factory=lambda: deque(maxlen=50_000))
_latencies_ms: deque = field(default_factory=lambda: deque(maxlen=5_000))
_seq_gaps: int = 0
_seq_gaps_window: deque = field(default_factory=lambda: deque(maxlen=10_000))
_last_seq: Optional[int] = None
_last_report: float = 0.0
_book_count: int = 0
_trade_count: int = 0
_last_book_wall: float = 0.0 # 本地收到最新 book 的时间
def on_book(self, seq: int | None = None, event_ts_ns: int | None = None) -> None:
now = time.time()
self._book_ts.append(now)
self._last_book_wall = now
self._book_count += 1
if event_ts_ns is not None and event_ts_ns > 0:
lat = (now * 1e9 - event_ts_ns) / 1e6
if -1000 < lat < 60_000:
self._latencies_ms.append(lat)
if seq is not None:
if self._last_seq is not None and seq > self._last_seq + 1:
gap = seq - self._last_seq - 1
# Binance L2 update ids often jump across snapshot/reconnect;
# only count modest gaps as packet loss. Huge jumps → reset.
if gap <= 1000:
self._seq_gaps += gap
self._seq_gaps_window.append((now, gap))
self._last_seq = seq
self._trim(now)
def on_trade(self, event_ts_ns: int | None = None) -> None:
now = time.time()
self._trade_ts.append(now)
self._trade_count += 1
if event_ts_ns is not None and event_ts_ns > 0:
lat = (now * 1e9 - event_ts_ns) / 1e6
if -1000 < lat < 60_000:
self._latencies_ms.append(lat)
self._trim(now)
def book_age_ms(self, now: float | None = None) -> Optional[float]:
"""当前时刻距离最近一次 book 更新的年龄（ms）。"""
if self._last_book_wall <= 0:
return None
now = now or time.time()
return max(0.0, (now - self._last_book_wall) * 1000.0)
def _trim(self, now: float) -> None:
cut = now - self.window_sec
while self._book_ts and self._book_ts[0] < cut:
self._book_ts.popleft()
while self._trade_ts and self._trade_ts[0] < cut:
self._trade_ts.popleft()
while self._seq_gaps_window and self._seq_gaps_window[0][0] < cut:
self._seq_gaps_window.popleft()
def snapshot(self) -> dict:
now = time.time()
self._trim(now)
w = max(self.window_sec, 1e-6)
lat = sorted(self._latencies_ms)
gaps_in_window = sum(g for _, g in self._seq_gaps_window)
book_age = self.book_age_ms(now)
# Binance depth update ids are not contiguous; gap is observe-only.
# Healthy = sufficient book rate + fresh book.
healthy = (
len(self._book_ts) / w >= 0.5
and (book_age is None or book_age < 500.0)
)
return {
"event": "health",
"window_sec": self.window_sec,
"book_update_rate": len(self._book_ts) / w,
"trade_update_rate": len(self._trade_ts) / w,
"latency_ms_mean": (sum(lat) / len(lat)) if lat else None,
"latency_ms_p50": _percentile(lat, 0.50),
"latency_ms_p95": _percentile(lat, 0.95),
"latency_ms_p99": _percentile(lat, 0.99),
"latency_ms_max": lat[-1] if lat else None,
"sequence_gap": self._seq_gaps, # 累计
"sequence_gap_window": gaps_in_window, # 近窗
"book_age_ms": book_age,
"book_total": self._book_count,
"trade_total": self._trade_count,
"healthy": healthy,
}
def maybe_report(self) -> Optional[dict]:
now = time.time()
if now - self._last_report < self.report_every_sec:
return None
self._last_report = now
return self.snapshot()
def allow_quoting(self, max_book_age_ms: float = 500.0) -> bool:
"""Gate new quotes on freshness + update rate (not Binance seq jumps)."""
age = self.book_age_ms()
if age is None or age >= max_book_age_ms:
return False
now = time.time()
self._trim(now)
w = max(self.window_sec, 1e-6)
if len(self._book_ts) / w < 0.5:
return False
return True
src/nautilus_mm/recorder.py
+861
View File
@@ -0,0 +1,861 @@
"""
Maker Edge 事件记录器 — Execution Reality Layer（Nautilus / CCXT 共用）
事件：
- quote_created / quote_canceled / quote_filled
- fill / fill_path / fill_exit
默认输出：nautilus_mm/logs/maker_edge/YYYYMMDD.jsonl
（与 Freqtrade MakerEdgeProbe schema 对齐，可用同一 analyze 脚本）
"""
from __future__ import annotations
import json
import logging
import time
import uuid
from collections import deque
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Optional
from nautilus_mm.experiment import load_experiment_meta, stamp_event
logger = logging.getLogger(__name__)
def empty_market_state_snapshot() -> dict:
"""Market Pulse 预留位；探针阶段保持 null，不做交易决策。"""
return {
"regime": None,
"trend_state": None,
"liquidity_state": None,
"volatility_state": None,
}
def _utc_now() -> datetime:
return datetime.now(timezone.utc)
def _iso(ts: datetime | float | None = None) -> str:
if ts is None:
t = _utc_now()
elif isinstance(ts, (int, float)):
t = datetime.fromtimestamp(ts, tz=timezone.utc)
else:
t = ts if ts.tzinfo else ts.replace(tzinfo=timezone.utc)
return t.isoformat()
@dataclass
class MicroSnapshot:
best_bid: float = 0.0
best_ask: float = 0.0
mid: float = 0.0
spread: float = 0.0
bid_depth_1: float = 0.0
ask_depth_1: float = 0.0
bid_depth_5: float = 0.0
ask_depth_5: float = 0.0
bid_depth: float = 0.0 # top-N
ask_depth: float = 0.0
obi: float = 0.0
delta: float = 0.0
trade_imbalance: float = 0.0 # (buy-sell)/(buy+sell) on recent trades
delta_efficiency: float = 0.0
liquidation_distance: float = 0.0
def to_book_fields(self) -> dict[str, float]:
return {
"bid_price": self.best_bid,
"ask_price": self.best_ask,
"mid": self.mid,
"spread": self.spread,
"bid_depth_1": self.bid_depth_1,
"ask_depth_1": self.ask_depth_1,
"bid_depth_5": self.bid_depth_5,
"ask_depth_5": self.ask_depth_5,
"bid_depth": self.bid_depth,
"ask_depth": self.ask_depth,
"obi": self.obi,
"delta": self.delta,
"trade_imbalance": self.trade_imbalance,
"delta_efficiency": self.delta_efficiency,
"liquidation_distance": self.liquidation_distance,
# 兼容旧字段
"buy1_depth": self.bid_depth_1,
"sell1_depth": self.ask_depth_1,
}
@dataclass
class ActiveQuote:
quote_id: str
pair: str
side: str # bid / ask
quote_price: float
created_ts: float
reason: str = ""
trade_id: Optional[int] = None
status: str = "open" # open / filled / canceled
@dataclass
class PendingFillPath:
fill_id: str
pair: str
side: str
fill_price: float
fill_ts: float
quote_id: Optional[str] = None
exit_reason: Optional[str] = None
# horizon prices: +1s +5s +10s +30s +60s +300s
after_1s_price: Optional[float] = None
after_5s_price: Optional[float] = None
after_10s_price: Optional[float] = None
after_30s_price: Optional[float] = None
after_1m_price: Optional[float] = None
after_5m_price: Optional[float] = None
# running extrema
min_price: float = 0.0
max_price: float = 0.0
# time-MAE / MFE at horizons
mae_1s: Optional[float] = None
mae_5s: Optional[float] = None
mae_10s: Optional[float] = None
mae_30s: Optional[float] = None
mae_1m: Optional[float] = None
mae_5m: Optional[float] = None
mfe_1s: Optional[float] = None
mfe_5s: Optional[float] = None
mfe_10s: Optional[float] = None
mfe_30s: Optional[float] = None
mfe_1m: Optional[float] = None
mfe_5m: Optional[float] = None
done: bool = False
def __post_init__(self):
self.min_price = self.fill_price
self.max_price = self.fill_price
def signed_excursions(self) -> tuple[float, float]:
"""Return (mae, mfe) at current min/max. mae<=0 adverse, mfe>=0 favorable."""
if self.side == "long":
mae = (self.min_price - self.fill_price) / self.fill_price
mfe = (self.max_price - self.fill_price) / self.fill_price
else:
mae = (self.fill_price - self.max_price) / self.fill_price
mfe = (self.fill_price - self.min_price) / self.fill_price
return mae, mfe
def fav_ret_at(self, px: Optional[float]) -> Optional[float]:
if px is None or self.fill_price <= 0:
return None
raw = (px - self.fill_price) / self.fill_price
return raw if self.side == "long" else -raw
def classify_path_type(p: "PendingFillPath") -> str:
"""
成交后路径形态（决定未来 Quote Logic）：
A_immediate_edge — 立即有利（1s/5s 已正，30s 仍正）
B_drawdown_then_recover — 先亏后赚（早期 MAE，末期有利）
C_toxic — 成交即错误（持续不利）
D_mixed — 其它
"""
r1 = p.fav_ret_at(p.after_1s_price)
r5 = p.fav_ret_at(p.after_5s_price)
r30 = p.fav_ret_at(p.after_30s_price)
r300 = p.fav_ret_at(p.after_5m_price)
mae30 = p.mae_30s or 0.0
if r30 is not None and r30 > 0 and (r1 or 0) >= 0 and (r5 or 0) >= 0:
return "A_immediate_edge"
if mae30 < -1e-6 and r300 is not None and r300 > 0:
return "B_drawdown_then_recover"
if r30 is not None and r30 < 0 and (r300 is None or r300 <= 0):
return "C_toxic"
return "D_mixed"
class MakerEdgeLogger:
def __init__(
self,
log_dir: str | Path | None = None,
levels: int = 10,
book_history_sec: float = 30.0,
cluster_gap_sec: float = 30.0,
mid_tick_every_sec: float = 1.0,
):
# .../nautilus_mm/src/nautilus_mm/recorder.py → parents[2] = nautilus_mm
root = Path(__file__).resolve().parents[2]
self.log_dir = Path(log_dir) if log_dir else root / "logs" / "maker_edge"
self.log_dir.mkdir(parents=True, exist_ok=True)
self.levels = levels
self.book_history_sec = book_history_sec
self.cluster_gap_sec = cluster_gap_sec
self.mid_tick_every_sec = mid_tick_every_sec
self._pending: dict[str, PendingFillPath] = {}
self._quotes: dict[str, ActiveQuote] = {} # quote_id -> ActiveQuote
self._quotes_by_trade: dict[int, str] = {} # trade_id -> quote_id
self._book_hist: deque[tuple[float, MicroSnapshot]] = deque(maxlen=2000)
# inventory tracking for future quote engine
self._inv: float = 0.0
self._inv_nonzero_since: Optional[float] = None
self._inv_target: float = 0.0 # flat target; skew = inv - target
# liquidity-event clustering (样本独立性)
self._cluster_id: Optional[str] = None
self._cluster_side: Optional[str] = None
self._cluster_last_ts: float = 0.0
self._cluster_start_mid: Optional[float] = None
self._cluster_n: int = 0
self._last_mid_tick_ts: float = 0.0
self.experiment = load_experiment_meta()
def _file(self) -> Path:
return self.log_dir / f"{_utc_now().strftime('%Y%m%d')}.jsonl"
def update_inventory(self, inventory: float, now: float | None = None) -> dict:
"""更新库存并返回 inventory / inventory_time / inventory_skew。"""
now = now or time.time()
self._inv = float(inventory)
if abs(self._inv) < 1e-12:
self._inv_nonzero_since = None
inv_time = 0.0
else:
if self._inv_nonzero_since is None:
self._inv_nonzero_since = now
inv_time = now - self._inv_nonzero_since
skew = self._inv - self._inv_target
return {
"inventory": self._inv,
"inventory_time": inv_time,
"inventory_skew": skew,
}
def _attach_common(
self,
ev: dict[str, Any],
inventory: float | None = None,
state: dict | None = None,
now: float | None = None,
) -> dict[str, Any]:
now = now or time.time()
if inventory is not None:
ev.update(self.update_inventory(inventory, now=now))
# 始终带 market_state_snapshot（可被 state 覆盖内部字段）
mss = empty_market_state_snapshot()
if state:
for k in mss:
if k in state and state[k] is not None:
mss[k] = state[k]
# 兼容旧扁平字段
for k, v in state.items():
if k not in mss and k != "market_state_snapshot":
ev.setdefault(k, v)
ev["market_state_snapshot"] = mss
return ev
def write(self, event: dict[str, Any]) -> None:
event.setdefault("ts", _iso())
event.setdefault("ts_epoch", time.time())
if "market_state_snapshot" not in event:
event["market_state_snapshot"] = empty_market_state_snapshot()
stamp_event(event, self.experiment)
with self._file().open("a", encoding="utf-8") as f:
f.write(json.dumps(event, ensure_ascii=False, default=str) + "\n")
def write_experiment_start(self, extra: dict | None = None) -> None:
"""探针启动时写入一次，绑定本轮 Data Collection。"""
ev = {
"event": "experiment_start",
**self.experiment,
}
if extra:
ev.update(extra)
self.write(ev)
# ------------------------------------------------------------------ #
# Snapshot
# ------------------------------------------------------------------ #
@staticmethod
def snapshot_from_orderbook(
ob: dict,
levels: int = 10,
recent_trades: list | None = None,
last_mid: float | None = None,
liq_proxy_low: float | None = None,
liq_proxy_high: float | None = None,
) -> MicroSnapshot:
bids = (ob.get("bids") or [])[:levels]
asks = (ob.get("asks") or [])[:levels]
if not bids or not asks:
return MicroSnapshot()
best_bid = float(bids[0][0])
best_ask = float(asks[0][0])
mid = (best_bid + best_ask) / 2.0
spread = best_ask - best_bid
def depth(levels_side, n):
return sum(float(x[1]) for x in levels_side[:n])
bid_depth_1 = depth(bids, 1)
ask_depth_1 = depth(asks, 1)
bid_depth_5 = depth(bids, 5)
ask_depth_5 = depth(asks, 5)
bid_depth = depth(bids, levels)
ask_depth = depth(asks, levels)
tot = bid_depth + ask_depth
obi = ((bid_depth - ask_depth) / tot) if tot > 0 else 0.0
buy_v = sell_v = 0.0
if recent_trades:
for t in recent_trades:
amt = float(t.get("amount") or t.get("qty") or 0.0)
side = (t.get("side") or "").lower()
if side in ("buy", "b"):
buy_v += amt
elif side in ("sell", "s"):
sell_v += amt
delta = buy_v - sell_v
timb_den = buy_v + sell_v
trade_imbalance = ((buy_v - sell_v) / timb_den) if timb_den > 0 else 0.0
de = 0.0
if last_mid and mid and abs(delta) > 1e-12:
de = ((mid - last_mid) / last_mid) / delta
liq_dist = 0.0
if liq_proxy_low and liq_proxy_high and mid:
rng = liq_proxy_high - liq_proxy_low
if rng > 0:
liq_dist = ((mid - liq_proxy_low) / rng) * 2 - 1
return MicroSnapshot(
best_bid=best_bid,
best_ask=best_ask,
mid=mid,
spread=spread,
bid_depth_1=bid_depth_1,
ask_depth_1=ask_depth_1,
bid_depth_5=bid_depth_5,
ask_depth_5=ask_depth_5,
bid_depth=bid_depth,
ask_depth=ask_depth,
obi=obi,
delta=delta,
trade_imbalance=trade_imbalance,
delta_efficiency=de,
liquidation_distance=liq_dist,
)
def record_book(
self,
snap: MicroSnapshot,
now: float | None = None,
*,
emit_mid_tick: bool = True,
pair: str | None = None,
) -> None:
now = now or time.time()
self._book_hist.append((now, snap))
# trim old
cutoff = now - self.book_history_sec
while self._book_hist and self._book_hist[0][0] < cutoff:
self._book_hist.popleft()
# mid 时间序列：供 Fill vs Random Benchmark（研究保护栏）
if (
emit_mid_tick
and snap.mid > 0
and (now - self._last_mid_tick_ts) >= self.mid_tick_every_sec
):
self._last_mid_tick_ts = now
self.write(
{
"event": "mid_tick",
"pair": pair,
"mid": snap.mid,
"best_bid": snap.best_bid,
"best_ask": snap.best_ask,
"spread": snap.spread,
"ts_epoch": now,
}
)
def assign_event_cluster(
self,
side: str,
mid: float,
now: float | None = None,
) -> dict[str, Any]:
"""
将连续同侧成交归入同一流动性事件（event_cluster_id）。
规则（研究保护栏，非策略）：
- 同 side
- 与上一笔间隔 < cluster_gap_sec
→ 同一 cluster；否则新开 cluster。
统计时应用 cluster 加权，避免「暴跌连续 50 笔 Bid」当成 50 个独立样本。
"""
now = now or time.time()
new_cluster = (
self._cluster_id is None
or self._cluster_side != side
or (now - self._cluster_last_ts) > self.cluster_gap_sec
)
if new_cluster:
self._cluster_id = uuid.uuid4().hex[:12]
self._cluster_side = side
self._cluster_start_mid = mid if mid > 0 else None
self._cluster_n = 0
self._cluster_n += 1
self._cluster_last_ts = now
mid_move = None
if self._cluster_start_mid and mid > 0:
mid_move = (mid - self._cluster_start_mid) / self._cluster_start_mid
return {
"event_cluster_id": self._cluster_id,
"cluster_fill_index": self._cluster_n,
"cluster_mid_move_from_start": mid_move,
}
def book_at(self, target_ts: float) -> Optional[MicroSnapshot]:
"""取最接近 target_ts 的历史盘口（用于成交前5s）。"""
if not self._book_hist:
return None
best = min(self._book_hist, key=lambda x: abs(x[0] - target_ts))
return best[1]
def build_fill_context(self, side: str, now: float | None = None) -> dict:
"""
成交主动性上下文：区分「砸盘后吸收」vs「下跌接刀」。
不接 Market Pulse，仅用本地 book history + trade imbalance。
"""
now = now or time.time()
cur = self.book_at(now)
past = self.book_at(now - 5.0)
fill_type = "bid" if side == "long" else "ask"
ctx: dict[str, Any] = {
"fill_type": fill_type,
"market_event_before_fill": "unknown",
"trade_imbalance_5s": None,
"price_velocity_5s": None,
}
if not cur or not past or past.mid <= 0:
return {"fill_context": ctx}
vel = (cur.mid - past.mid) / past.mid
# 用当前与 5s 前 imbalance 的平均作代理
timb = (cur.trade_imbalance + past.trade_imbalance) / 2.0
ctx["trade_imbalance_5s"] = timb
ctx["price_velocity_5s"] = vel
if fill_type == "bid":
# 卖压后吸收：价格下行/企稳 + 卖向 imbalance，但盘口未继续恶化太狠
if timb < -0.2 and vel < 0:
if abs(vel) < 0.0003:
ctx["market_event_before_fill"] = "sell_pressure_absorbing"
else:
ctx["market_event_before_fill"] = "sell_pressure_falling"
elif vel < -0.0005:
ctx["market_event_before_fill"] = "momentum_down_catching_knife"
elif timb > 0.15:
ctx["market_event_before_fill"] = "buy_support"
else:
ctx["market_event_before_fill"] = "neutral"
else:
if timb > 0.2 and vel > 0:
if abs(vel) < 0.0003:
ctx["market_event_before_fill"] = "buy_pressure_absorbing"
else:
ctx["market_event_before_fill"] = "buy_pressure_rising"
elif vel > 0.0005:
ctx["market_event_before_fill"] = "momentum_up_chasing"
elif timb < -0.15:
ctx["market_event_before_fill"] = "sell_resistance"
else:
ctx["market_event_before_fill"] = "neutral"
return {"fill_context": ctx}
def book_deterioration(self, side: str, now: float | None = None, lookback: float = 5.0) -> dict:
"""
成交前 lookback 秒盘口是否恶化。
long: bid_depth 下降 / ask_depth 上升 / mid 下跌 → 恶化
"""
now = now or time.time()
cur = self.book_at(now)
past = self.book_at(now - lookback)
if not cur or not past or past.mid <= 0:
return {"book_ok": False}
mid_chg = (cur.mid - past.mid) / past.mid
bid5_chg = (cur.bid_depth_5 - past.bid_depth_5) / past.bid_depth_5 if past.bid_depth_5 else 0.0
ask5_chg = (cur.ask_depth_5 - past.ask_depth_5) / past.ask_depth_5 if past.ask_depth_5 else 0.0
obi_chg = cur.obi - past.obi
if side == "long":
deteriorated = (mid_chg < -0.00005) or (bid5_chg < -0.15) or (obi_chg < -0.1)
else:
deteriorated = (mid_chg > 0.00005) or (ask5_chg < -0.15) or (obi_chg > 0.1)
return {
"book_ok": True,
"pre_5s_mid_chg": mid_chg,
"pre_5s_bid_depth_5_chg": bid5_chg,
"pre_5s_ask_depth_5_chg": ask5_chg,
"pre_5s_obi_chg": obi_chg,
"pre_5s_deteriorated": bool(deteriorated),
"pre_5s_bid_depth_1": past.bid_depth_1,
"pre_5s_ask_depth_1": past.ask_depth_1,
"pre_5s_bid_depth_5": past.bid_depth_5,
"pre_5s_ask_depth_5": past.ask_depth_5,
"pre_5s_obi": past.obi,
"pre_5s_spread": past.spread,
"pre_5s_trade_imbalance": past.trade_imbalance,
}
# ------------------------------------------------------------------ #
# Quote lifecycle
# ------------------------------------------------------------------ #
def create_quote(
self,
pair: str,
side: str,
quote_price: float,
inventory: float,
snap: MicroSnapshot,
reason: str = "",
trade_id: Optional[int] = None,
state: dict | None = None,
extra: dict | None = None,
) -> str:
qid = uuid.uuid4().hex[:16]
now = time.time()
q = ActiveQuote(
quote_id=qid,
pair=pair,
side=side,
quote_price=quote_price,
created_ts=now,
reason=reason,
trade_id=trade_id,
status="open",
)
self._quotes[qid] = q
if trade_id is not None:
self._quotes_by_trade[trade_id] = qid
ev = {
"event": "quote_created",
"quote_id": qid,
"pair": pair,
"side": side,
"quote_price": quote_price,
"quote_created_time": _iso(now),
"quote_created_epoch": now,
"reason": reason,
"trade_id": trade_id,
"status": "open",
"filled": False,
}
ev.update(snap.to_book_fields())
self._attach_common(ev, inventory=inventory, state=state, now=now)
if extra:
ev.update(extra)
self.write(ev)
return qid
def cancel_quote(
self,
quote_id: str | None = None,
trade_id: Optional[int] = None,
reason: str = "timeout",
snap: MicroSnapshot | None = None,
) -> None:
q = None
if quote_id and quote_id in self._quotes:
q = self._quotes[quote_id]
elif trade_id is not None and trade_id in self._quotes_by_trade:
q = self._quotes.get(self._quotes_by_trade[trade_id])
if q is None or q.status != "open":
return
now = time.time()
q.status = "canceled"
ev = {
"event": "quote_canceled",
"quote_id": q.quote_id,
"pair": q.pair,
"side": q.side,
"quote_price": q.quote_price,
"quote_created_time": _iso(q.created_ts),
"quote_cancel_time": _iso(now),
"quote_cancel_epoch": now,
"time_alive_sec": now - q.created_ts,
"cancel_reason": reason,
"filled": False,
"status": "canceled",
"trade_id": q.trade_id,
}
if snap:
ev.update(snap.to_book_fields())
self.write(ev)
def bind_trade(self, quote_id: str, trade_id: int) -> None:
if quote_id in self._quotes:
self._quotes[quote_id].trade_id = trade_id
self._quotes_by_trade[trade_id] = quote_id
# ------------------------------------------------------------------ #
# Fill + path
# ------------------------------------------------------------------ #
def log_fill(
self,
pair: str,
side: str,
fill_price: float,
amount: float,
inventory: float,
snap: MicroSnapshot | None,
order_type: str = "limit",
quote_id: str | None = None,
trade_id: Optional[int] = None,
fill_reason: str = "maker_hit",
state: dict | None = None,
extra: dict | None = None,
quote_terminal: bool = True,
) -> str:
"""Record a fill. Always writes even if snap is None (book unavailable).
quote_terminal=False keeps quote open for partial fills so later slices
retain quote_id linkage until the order closes.
"""
now = time.time()
fill_id = uuid.uuid4().hex[:16]
snap = snap or MicroSnapshot()
# resolve quote lifecycle
q: Optional[ActiveQuote] = None
if quote_id and quote_id in self._quotes:
q = self._quotes[quote_id]
elif trade_id is not None and trade_id in self._quotes_by_trade:
q = self._quotes.get(self._quotes_by_trade[trade_id])
time_to_fill = None
quote_created_time = None
quote_price = fill_price
if q is not None:
if quote_terminal:
q.status = "filled"
time_to_fill = now - q.created_ts
quote_created_time = _iso(q.created_ts)
quote_price = q.quote_price
quote_id = q.quote_id
det = self.book_deterioration(side, now=now, lookback=5.0)
fctx = self.build_fill_context(side, now=now)
mid_for_cluster = snap.mid if snap.mid > 0 else fill_price
cluster = self.assign_event_cluster(side, mid_for_cluster, now=now)
ev = {
"event": "fill",
"fill_id": fill_id,
"quote_id": quote_id,
"pair": pair,
"side": side,
"fill_price": fill_price,
"quote_price": quote_price,
"amount": amount,
"order_type": order_type,
"fill_reason": fill_reason,
"quote_created_time": quote_created_time,
"quote_fill_time": _iso(now),
"time_to_fill": time_to_fill,
"trade_id": trade_id,
"filled": True,
"quote_terminal": quote_terminal,
"book_available": bool(snap.mid > 0),
}
ev.update(snap.to_book_fields())
ev.update(det)
ev.update(fctx)
ev.update(cluster)
# Effective spread capture proxy: 相对 mid 的被动成交优势
if snap.mid > 0:
if side == "long":
ev["spread_capture_pct"] = (snap.mid - fill_price) / snap.mid
else:
ev["spread_capture_pct"] = (fill_price - snap.mid) / snap.mid
self._attach_common(ev, inventory=inventory, state=state, now=now)
if extra:
ev.update(extra)
self.write(ev)
# also emit quote_filled lifecycle event (only when order fully done)
if q is not None and quote_terminal:
self.write(
{
"event": "quote_filled",
"quote_id": q.quote_id,
"fill_id": fill_id,
"pair": pair,
"side": q.side,
"quote_price": q.quote_price,
"quote_created_time": _iso(q.created_ts),
"quote_fill_time": _iso(now),
"time_to_fill": time_to_fill,
"fill_reason": fill_reason,
"filled": True,
"status": "filled",
"trade_id": trade_id,
**snap.to_book_fields(),
**det,
}
)
self._pending[fill_id] = PendingFillPath(
fill_id=fill_id,
pair=pair,
side=side,
fill_price=fill_price,
fill_ts=now,
quote_id=quote_id,
)
return fill_id
def attach_exit_reason(self, fill_id: str, exit_reason: str) -> None:
if fill_id in self._pending:
self._pending[fill_id].exit_reason = exit_reason
# also write lightweight annotation
self.write(
{
"event": "fill_exit",
"fill_id": fill_id,
"exit_reason": exit_reason,
}
)
def update_paths(self, pair: str, last_price: float, now: float | None = None) -> None:
now = now or time.time()
finished = []
for fid, p in self._pending.items():
if p.pair != pair or p.done:
continue
p.min_price = min(p.min_price, last_price)
p.max_price = max(p.max_price, last_price)
mae, mfe = p.signed_excursions()
age = now - p.fill_ts
def mark(horizon_attr_price, horizon_mae, horizon_mfe, sec, price_val):
if getattr(p, horizon_attr_price) is None and age >= sec:
setattr(p, horizon_attr_price, price_val)
setattr(p, horizon_mae, mae)
setattr(p, horizon_mfe, mfe)
mark("after_1s_price", "mae_1s", "mfe_1s", 1, last_price)
mark("after_5s_price", "mae_5s", "mfe_5s", 5, last_price)
mark("after_10s_price", "mae_10s", "mfe_10s", 10, last_price)
mark("after_30s_price", "mae_30s", "mfe_30s", 30, last_price)
mark("after_1m_price", "mae_1m", "mfe_1m", 60, last_price)
if p.after_5m_price is None and age >= 300:
p.after_5m_price = last_price
p.mae_5m = mae
p.mfe_5m = mfe
p.done = True
# Price MAE absolute
if p.side == "long":
price_mae = p.min_price - p.fill_price
price_mfe = p.max_price - p.fill_price
else:
price_mae = p.fill_price - p.max_price
price_mfe = p.fill_price - p.min_price
fav_30 = p.fav_ret_at(p.after_30s_price) or 0.0
fav_1 = p.fav_ret_at(p.after_1s_price)
fav_5 = p.fav_ret_at(p.after_5s_price)
fav_10 = p.fav_ret_at(p.after_10s_price)
fav_60 = p.fav_ret_at(p.after_1m_price)
fav_300 = p.fav_ret_at(p.after_5m_price) or 0.0
vol_proxy = abs(p.max_price - p.min_price) / p.fill_price if p.fill_price else 0.0
toxicity_score = max(0.0, -fav_30) / max(vol_proxy, 1e-8)
mfe_gt_mae_30 = (p.mfe_30s or 0.0) > abs(p.mae_30s or 0.0)
path_type = classify_path_type(p)
# 路径点（供形态分析 / 复现）
price_path = {
"t0": p.fill_price,
"t1s": p.after_1s_price,
"t5s": p.after_5s_price,
"t10s": p.after_10s_price,
"t30s": p.after_30s_price,
"t60s": p.after_1m_price,
"t300s": p.after_5m_price,
}
ret_path = {
"t1s": fav_1,
"t5s": fav_5,
"t10s": fav_10,
"t30s": fav_30,
"t60s": fav_60,
"t300s": fav_300,
}
self.write(
{
"event": "fill_path",
"fill_id": p.fill_id,
"quote_id": p.quote_id,
"pair": p.pair,
"side": p.side,
"fill_price": p.fill_price,
"exit_reason": p.exit_reason,
"after_1s_price": p.after_1s_price,
"after_5s_price": p.after_5s_price,
"after_10s_price": p.after_10s_price,
"after_30s_price": p.after_30s_price,
"after_1m_price": p.after_1m_price,
"after_5m_price": p.after_5m_price,
"price_path": price_path,
"ret_path": ret_path,
"path_type": path_type,
"min_price": p.min_price,
"max_price": p.max_price,
"mae_1s": p.mae_1s,
"mae_5s": p.mae_5s,
"mae_10s": p.mae_10s,
"mae_30s": p.mae_30s,
"mae_1m": p.mae_1m,
"mae_5m": p.mae_5m,
"mfe_1s": p.mfe_1s,
"mfe_5s": p.mfe_5s,
"mfe_10s": p.mfe_10s,
"mfe_30s": p.mfe_30s,
"mfe_1m": p.mfe_1m,
"mfe_5m": p.mfe_5m,
"price_mae": price_mae,
"price_mfe": price_mfe,
"price_mae_pct": mae,
"price_mfe_pct": mfe,
"fav_ret_30s": fav_30,
"vol_proxy_5m": vol_proxy,
"toxicity_score": toxicity_score,
"mfe_gt_mae_30s": mfe_gt_mae_30,
}
)
finished.append(fid)
for fid in finished:
self._pending.pop(fid, None)
@property
def pending_count(self) -> int:
return len(self._pending)
# 兼容旧 API
def log_quote(self, *args, **kwargs):
"""Deprecated wrapper → create_quote for live quotes; heartbeat uses book only."""
return self.create_quote(*args, **kwargs)
src/nautilus_mm/run_event_state.py
+169
View File
@@ -0,0 +1,169 @@
#!/usr/bin/env python3
"""
Launch Event-State Observability Probe (MM_EDGE_EXP_002)
Data collection only — NO trading, NO strategy, NO Stage 3 unlock.
Environment:
EXPERIMENT_ID=MM_EDGE_EXP_002
PROBE_VERSION=event_state_v0.1
ENABLE_TRADING=false (hard-enforced; any true value is ignored)
Usage:
cd nautilus_mm
source .venv/bin/activate
export PYTHONPATH=src
python -m nautilus_mm.run_event_state
"""
from __future__ import annotations
import os
import sys
from pathlib import Path
from dotenv import load_dotenv
_ROOT = Path(__file__).resolve().parents[2]
_SRC = _ROOT / "src"
if str(_SRC) not in sys.path:
sys.path.insert(0, str(_SRC))
load_dotenv(_ROOT / ".env")
from nautilus_trader.adapters.binance import BINANCE
from nautilus_trader.adapters.binance import BinanceAccountType
from nautilus_trader.adapters.binance import BinanceDataClientConfig
from nautilus_trader.adapters.binance import BinanceExecClientConfig
from nautilus_trader.adapters.binance import BinanceInstrumentProviderConfig
from nautilus_trader.adapters.binance import BinanceLiveDataClientFactory
from nautilus_trader.adapters.binance import BinanceLiveExecClientFactory
from nautilus_trader.adapters.binance.common.enums import BinanceEnvironment
from nautilus_trader.config import LiveDataEngineConfig
from nautilus_trader.config import LiveExecEngineConfig
from nautilus_trader.config import LoggingConfig
from nautilus_trader.config import TradingNodeConfig
from nautilus_trader.live.node import TradingNode
from nautilus_trader.model.identifiers import ClientId
from nautilus_trader.model.identifiers import InstrumentId
from nautilus_trader.model.identifiers import TraderId
from nautilus_mm.experiment import load_experiment_meta
from nautilus_mm.strategies.event_state_probe import EventStateProbe
from nautilus_mm.strategies.event_state_probe import EventStateProbeConfig
def _env_bool(name: str, default: bool = False) -> bool:
v = os.getenv(name)
if v is None:
return default
return v.strip().lower() in ("1", "true", "yes", "y")
def _resolve_environment() -> BinanceEnvironment:
raw = os.getenv("BINANCE_ENVIRONMENT", "TESTNET")
env_name = raw.strip().upper()
if env_name not in ("TESTNET", "LIVE"):
print(f"ERROR: BINANCE_ENVIRONMENT must be TESTNET or LIVE, got {raw!r}")
sys.exit(1)
if env_name == "LIVE" and not _env_bool("I_UNDERSTAND_LIVE", False):
print("ERROR: LIVE blocked for EXP_002 unless I_UNDERSTAND_LIVE=yes")
sys.exit(1)
return BinanceEnvironment.LIVE if env_name == "LIVE" else BinanceEnvironment.TESTNET
def main() -> None:
# Layer 2: runner hard-forces trading off even if .env / systemd is wrong
if _env_bool("ENABLE_TRADING", False):
print(
"WARNING: ENABLE_TRADING=true ignored — MM_EDGE_EXP_002 is observability-only"
)
os.environ["ENABLE_TRADING"] = "false"
os.environ["EXPERIMENT_ID"] = "MM_EDGE_EXP_002"
os.environ.setdefault("PROBE_VERSION", "event_state_v0.1")
exp_id = os.getenv("EXPERIMENT_ID", "MM_EDGE_EXP_002")
if exp_id != "MM_EDGE_EXP_002":
print(
f"WARNING: EXPERIMENT_ID={exp_id!r} — expected MM_EDGE_EXP_002 for this runner"
)
api_key = os.getenv("BINANCE_API_KEY", "")
api_secret = os.getenv("BINANCE_API_SECRET", "")
environment = _resolve_environment()
symbol = os.getenv("SYMBOL", "BTCUSDT-PERP")
instrument_id = InstrumentId.from_str(f"{symbol}.{BINANCE}")
log_dir = os.getenv("EVENT_STATE_LOG_DIR", str(_ROOT / "logs" / "event_state"))
if not api_key or not api_secret:
print("ERROR: set BINANCE_API_KEY / BINANCE_API_SECRET in nautilus_mm/.env")
sys.exit(1)
config_node = TradingNodeConfig(
trader_id=TraderId("EVENT-STATE-002"),
logging=LoggingConfig(log_level="INFO", log_colors=True, use_pyo3=True),
data_engine=LiveDataEngineConfig(external_clients=[ClientId(BINANCE)]),
exec_engine=LiveExecEngineConfig(
reconciliation=False,
open_check_interval_secs=0.0,
graceful_shutdown_on_exception=True,
),
data_clients={
BINANCE: BinanceDataClientConfig(
api_key=api_key,
api_secret=api_secret,
account_type=BinanceAccountType.USDT_FUTURES,
environment=environment,
instrument_provider=BinanceInstrumentProviderConfig(
load_ids=frozenset([instrument_id]),
),
),
},
exec_clients={
BINANCE: BinanceExecClientConfig(
api_key=api_key,
api_secret=api_secret,
account_type=BinanceAccountType.USDT_FUTURES,
environment=environment,
instrument_provider=BinanceInstrumentProviderConfig(
load_ids=frozenset([instrument_id]),
),
max_retries=3,
),
},
timeout_connection=30.0,
timeout_reconciliation=10.0,
timeout_portfolio=10.0,
timeout_disconnection=10.0,
timeout_post_stop=5.0,
)
node = TradingNode(config=config_node)
strat_config = EventStateProbeConfig(
instrument_id=instrument_id,
book_depth=int(os.getenv("BOOK_DEPTH", "10")),
log_dir=log_dir,
prefill_window_sec=float(os.getenv("PREFILL_WINDOW_SEC", "5.0")),
prefill_margin_sec=float(os.getenv("PREFILL_MARGIN_SEC", "0.25")),
large_trade_qty=float(os.getenv("LARGE_TRADE_QTY", "0.1")),
log_every_book_delta=_env_bool("LOG_EVERY_BOOK_DELTA", True),
)
node.trader.add_strategy(EventStateProbe(config=strat_config))
node.add_data_client_factory(BINANCE, BinanceLiveDataClientFactory)
node.add_exec_client_factory(BINANCE, BinanceLiveExecClientFactory)
node.build()
exp = load_experiment_meta()
print(
f"[event_state] Experiment={exp['experiment_id']} {exp['probe_version']} | "
f"type=Event-State Observability | trading=NO | {symbol} env={environment} | "
f"run={os.getenv('LEDGER_RUN_ID', 'EXP-002-RUN-UNSET')} | log={log_dir}"
)
try:
node.run()
finally:
node.dispose()
if __name__ == "__main__":
main()
src/nautilus_mm/run_live.py
+184
View File
@@ -0,0 +1,184 @@
#!/usr/bin/env python3
"""
启动 Nautilus TradingNode + MakerEdgeProbe（Binance USDT-M）
环境变量（或 .env）：
BINANCE_API_KEY
BINANCE_API_SECRET
BINANCE_ENVIRONMENT=TESTNET|LIVE (仅允许这两个值；默认 TESTNET)
I_UNDERSTAND_LIVE=yes (LIVE 必填)
ENABLE_TRADING=false (默认关闭；显式 true 才挂单)
HTTP_PROXY / HTTPS_PROXY (可选)
用法：
cd nautilus_mm
source .venv/bin/activate
export PYTHONPATH=src
python -m nautilus_mm.run_live
"""
from __future__ import annotations
import os
import sys
from decimal import Decimal
from pathlib import Path
from dotenv import load_dotenv
# ensure src on path when run as script
_ROOT = Path(__file__).resolve().parents[2]
_SRC = _ROOT / "src"
if str(_SRC) not in sys.path:
sys.path.insert(0, str(_SRC))
load_dotenv(_ROOT / ".env")
from nautilus_trader.adapters.binance import BINANCE
from nautilus_trader.adapters.binance import BinanceAccountType
from nautilus_trader.adapters.binance import BinanceDataClientConfig
from nautilus_trader.adapters.binance import BinanceExecClientConfig
from nautilus_trader.adapters.binance import BinanceInstrumentProviderConfig
from nautilus_trader.adapters.binance import BinanceLiveDataClientFactory
from nautilus_trader.adapters.binance import BinanceLiveExecClientFactory
from nautilus_trader.adapters.binance.common.enums import BinanceEnvironment
from nautilus_trader.config import LiveDataEngineConfig
from nautilus_trader.config import LiveExecEngineConfig
from nautilus_trader.config import LoggingConfig
from nautilus_trader.config import TradingNodeConfig
from nautilus_trader.live.node import TradingNode
from nautilus_trader.model.identifiers import ClientId
from nautilus_trader.model.identifiers import InstrumentId
from nautilus_trader.model.identifiers import TraderId
from nautilus_mm.experiment import load_experiment_meta
from nautilus_mm.strategies.maker_edge_probe import MakerEdgeProbe
from nautilus_mm.strategies.maker_edge_probe import MakerEdgeProbeConfig
def _env_bool(name: str, default: bool = False) -> bool:
v = os.getenv(name)
if v is None:
return default
return v.strip().lower() in ("1", "true", "yes", "y")
def _resolve_environment() -> BinanceEnvironment:
raw = os.getenv("BINANCE_ENVIRONMENT", "TESTNET")
env_name = raw.strip().upper()
if env_name not in ("TESTNET", "LIVE"):
print(
f"ERROR: BINANCE_ENVIRONMENT must be exactly TESTNET or LIVE, got {raw!r}"
)
sys.exit(1)
if env_name == "LIVE":
if not _env_bool("I_UNDERSTAND_LIVE", False):
print(
"ERROR: LIVE trading blocked. Set I_UNDERSTAND_LIVE=yes "
"only after you accept real-money risk."
)
sys.exit(1)
return BinanceEnvironment.LIVE
return BinanceEnvironment.TESTNET
def main() -> None:
api_key = os.getenv("BINANCE_API_KEY", "")
api_secret = os.getenv("BINANCE_API_SECRET", "")
environment = _resolve_environment()
symbol = os.getenv("SYMBOL", "BTCUSDT-PERP")
instrument_id = InstrumentId.from_str(f"{symbol}.{BINANCE}")
order_qty = Decimal(os.getenv("ORDER_QTY", "0.001"))
enable_trading = _env_bool("ENABLE_TRADING", False)
max_abs_inventory = Decimal(os.getenv("MAX_ABS_INVENTORY", "0.005"))
quote_ttl_secs = float(os.getenv("QUOTE_TTL_SECS", "30"))
log_dir = os.getenv("MAKER_EDGE_LOG_DIR", str(_ROOT / "logs" / "maker_edge"))
# 代理：Nautilus/httpx 会读 HTTP(S)_PROXY；这里仅提示
proxy = os.getenv("HTTPS_PROXY") or os.getenv("HTTP_PROXY") or ""
if proxy:
print(f"[nautilus_mm] using proxy: {proxy}")
if not api_key or not api_secret:
print("ERROR: set BINANCE_API_KEY / BINANCE_API_SECRET in nautilus_mm/.env")
print("For TESTNET keys: https://testnet.binancefuture.com/")
sys.exit(1)
config_node = TradingNodeConfig(
trader_id=TraderId("MAKER-EDGE-001"),
logging=LoggingConfig(log_level="INFO", log_colors=True, use_pyo3=True),
data_engine=LiveDataEngineConfig(external_clients=[ClientId(BINANCE)]),
exec_engine=LiveExecEngineConfig(
reconciliation=True,
open_check_interval_secs=5.0,
graceful_shutdown_on_exception=True,
),
data_clients={
BINANCE: BinanceDataClientConfig(
api_key=api_key,
api_secret=api_secret,
account_type=BinanceAccountType.USDT_FUTURES,
environment=environment,
instrument_provider=BinanceInstrumentProviderConfig(
load_ids=frozenset([instrument_id]),
),
),
},
exec_clients={
BINANCE: BinanceExecClientConfig(
api_key=api_key,
api_secret=api_secret,
account_type=BinanceAccountType.USDT_FUTURES,
environment=environment,
instrument_provider=BinanceInstrumentProviderConfig(
load_ids=frozenset([instrument_id]),
),
max_retries=3,
),
},
timeout_connection=30.0,
timeout_reconciliation=10.0,
timeout_portfolio=10.0,
timeout_disconnection=10.0,
timeout_post_stop=5.0,
)
node = TradingNode(config=config_node)
strat_config = MakerEdgeProbeConfig(
instrument_id=instrument_id,
order_qty=order_qty,
book_depth=10,
quote_offset_ticks=1,
max_quotes=1,
quote_ttl_secs=quote_ttl_secs,
cooldown_secs=float(os.getenv("COOLDOWN_SECS", "60")),
book_sample_secs=2.0,
log_dir=log_dir,
obi_enter=float(os.getenv("OBI_ENTER", "0.25")),
enable_trading=enable_trading,
max_abs_inventory=max_abs_inventory,
)
strategy = MakerEdgeProbe(config=strat_config)
node.trader.add_strategy(strategy)
node.add_data_client_factory(BINANCE, BinanceLiveDataClientFactory)
node.add_exec_client_factory(BINANCE, BinanceLiveExecClientFactory)
node.build()
exp = load_experiment_meta()
print(
f"[nautilus_mm] Experiment={exp['experiment_id']} {exp['probe_version']} "
f"quote/fee/exchange=frozen | {symbol} env={environment} "
f"trading={enable_trading} ttl={quote_ttl_secs}s max_inv={max_abs_inventory} "
f"log={log_dir}"
)
try:
node.run()
finally:
node.dispose()
if __name__ == "__main__":
main()
src/nautilus_mm/strategies/__init__.py
+9
View File
@@ -0,0 +1,9 @@
from nautilus_mm.strategies.event_state_probe import EventStateProbe, EventStateProbeConfig
from nautilus_mm.strategies.maker_edge_probe import MakerEdgeProbe, MakerEdgeProbeConfig
__all__ = [
"MakerEdgeProbe",
"MakerEdgeProbeConfig",
"EventStateProbe",
"EventStateProbeConfig",
]
src/nautilus_mm/strategies/event_state_probe.py
+187
View File
@@ -0,0 +1,187 @@
"""
Event-State Observability Probe — MM_EDGE_EXP_002
Type: Data Collection / Observability Experiment
Strategy: NONE (no quotes, no orders, no trading)
Purpose: Capture immutable pre-fill Event State
EXP_001 remains FROZEN. This probe never submits orders.
"""
from __future__ import annotations
import time
from collections import deque
from nautilus_trader.common.enums import LogColor
from nautilus_trader.config import PositiveInt, StrategyConfig
from nautilus_trader.model.data import OrderBookDeltas, TradeTick
from nautilus_trader.model.enums import AggressorSide, BookType
from nautilus_trader.model.identifiers import InstrumentId
from nautilus_trader.model.instruments import Instrument
from nautilus_trader.trading.strategy import Strategy
from nautilus_mm.book_utils import snapshot_from_nautilus_book
from nautilus_mm.event_ledger import ImmutableEventLedger
from nautilus_mm.health import ConnectionHealth
class EventStateProbeConfig(StrategyConfig, frozen=True):
instrument_id: InstrumentId
book_depth: PositiveInt = 10
log_dir: str = ""
prefill_window_sec: float = 5.0
prefill_margin_sec: float = 0.25
large_trade_qty: float = 0.1
# Log every book delta batch (raw). Do not downsample.
log_every_book_delta: bool = True
class EventStateProbe(Strategy):
"""Read-only market observability — immutable event ledger only."""
def __init__(self, config: EventStateProbeConfig) -> None:
super().__init__(config)
self.instrument: Instrument | None = None
self._ledger = ImmutableEventLedger(
log_dir=config.log_dir or None,
prefill_window_sec=float(config.prefill_window_sec),
prefill_margin_sec=float(config.prefill_margin_sec),
large_trade_qty=float(config.large_trade_qty),
book_levels=int(config.book_depth),
)
self._health = ConnectionHealth(window_sec=60.0, report_every_sec=30.0)
self._last_mid: float | None = None
self._recent_buys: deque[tuple[float, float]] = deque(maxlen=500)
self._recent_sells: deque[tuple[float, float]] = deque(maxlen=500)
def on_start(self) -> None:
self.instrument = self.cache.instrument(self.config.instrument_id)
if self.instrument is None:
self.log.error(f"Instrument not found: {self.config.instrument_id}")
self.stop()
return
self.subscribe_order_book_deltas(
instrument_id=self.config.instrument_id,
book_type=BookType.L2_MBP,
depth=int(self.config.book_depth),
)
self.subscribe_trade_ticks(self.config.instrument_id)
exp = self._ledger.experiment
ident = self._ledger.run_identity
self._ledger.write_experiment_start(
extra={
"instrument_id": str(self.config.instrument_id),
"log_dir": str(self._ledger.log_dir),
"log_every_book_delta": bool(self.config.log_every_book_delta),
"depends_on": "MM_EDGE_EXP_001 / v0.1 FROZEN",
}
)
self.log.info(
f"EXP_002 Event-State Observability | {exp['experiment_id']} | "
f"{exp['probe_version']} | run={ident['run_id']} session={ident['session_id']} | "
f"trading=NO | log={self._ledger.log_dir}",
LogColor.GREEN,
)
def submit_order(self, *args, **kwargs): # noqa: ANN002
raise RuntimeError(
"MM_EDGE_EXP_002 forbids submit_order — observability probe, trading=NO"
)
def submit_order_list(self, *args, **kwargs): # noqa: ANN002
raise RuntimeError(
"MM_EDGE_EXP_002 forbids submit_order_list — observability probe, trading=NO"
)
def on_stop(self) -> None:
try:
self._ledger.write_experiment_stop()
except Exception as exc:
self.log.warning(f"experiment_stop write failed: {exc}")
self.log.info("EventStateProbe stopped (no orders were submitted)")
def _trade_qty_window(self, window_sec: float = 20.0) -> tuple[float, float]:
now = time.time()
buy = sum(q for t, q in self._recent_buys if now - t <= window_sec)
sell = sum(q for t, q in self._recent_sells if now - t <= window_sec)
return buy, sell
def _current_snap(self):
book = self.cache.order_book(self.config.instrument_id)
if book is None:
return None
buy, sell = self._trade_qty_window()
snap = snapshot_from_nautilus_book(
book,
levels=int(self.config.book_depth),
recent_buy_qty=buy,
recent_sell_qty=sell,
last_mid=self._last_mid,
)
if snap.mid:
self._last_mid = snap.mid
return snap
def on_order_book_deltas(self, deltas: OrderBookDeltas) -> None:
seq = getattr(deltas, "sequence", None)
ts_event = getattr(deltas, "ts_event", None)
self._health.on_book(seq=int(seq) if seq is not None else None, event_ts_ns=ts_event)
report = self._health.maybe_report()
if report:
self._ledger.write({**report, "event": "phase0_health"})
if not self.config.log_every_book_delta:
return
snap = self._current_snap()
if snap is None or snap.mid <= 0:
return
delta_count = len(getattr(deltas, "deltas", []) or [])
exchange_ts = int(ts_event) if ts_event is not None else None
self._ledger.log_book_state(
pair=str(self.config.instrument_id),
snap=snap,
exchange_ts_ns=exchange_ts,
local_ts_epoch=time.time(),
sequence=int(seq) if seq is not None else None,
delta_count=delta_count,
event_type="book_update",
)
def on_trade_tick(self, tick: TradeTick) -> None:
ts_event = getattr(tick, "ts_event", None)
self._health.on_trade(event_ts_ns=ts_event)
qty = float(tick.size)
price = float(tick.price)
now = time.time()
trade_side = "unknown"
aggressor = str(getattr(tick, "aggressor_side", ""))
try:
if tick.aggressor_side == AggressorSide.BUYER:
trade_side = "buy"
self._recent_buys.append((now, qty))
elif tick.aggressor_side == AggressorSide.SELLER:
trade_side = "sell"
self._recent_sells.append((now, qty))
except Exception:
trade_side = "unknown"
exchange_ts = int(ts_event) if ts_event is not None else None
trade_id = str(getattr(tick, "trade_id", "") or getattr(tick, "id", "") or "")
self._ledger.log_aggressive_trade(
pair=str(self.config.instrument_id),
price=price,
qty=qty,
trade_side=trade_side,
exchange_ts_ns=exchange_ts,
local_ts_epoch=now,
aggressor_side=aggressor,
trade_id=trade_id or None,
snap=self._current_snap(),
)
src/nautilus_mm/strategies/maker_edge_probe.py
+430
View File
@@ -0,0 +1,430 @@
"""
MakerEdgeProbe — Nautilus 事件驱动探针（v0）
实验冻结见 nautilus_mm/FREEZE.md — 三不动：
1. 不动 Quote Logic（无动态 spread / inv skew / AI / Pulse）
2. 不动成本模型
3. 不动 PASS/COLLECTING/FAIL 定义
只记录：quote / fill / outcome + Phase0 健康度。
market_state_snapshot 必须保持 null，禁止注入交易决策。
Stage3+ 未解锁前禁止进化为本文件的「聪明报价」。
安全闸（非报价进化）：TTL 撤单、健康度 gate、库存上限、fill 必记。
"""
from __future__ import annotations
import time
from collections import deque
from decimal import Decimal
from typing import Optional
from nautilus_trader.common.enums import LogColor
from nautilus_trader.config import PositiveInt, StrategyConfig
from nautilus_trader.model.data import OrderBookDeltas, TradeTick
from nautilus_trader.model.enums import BookType, OrderSide, TimeInForce
from nautilus_trader.model.events import OrderCanceled, OrderDenied, OrderFilled, OrderRejected
from nautilus_trader.model.identifiers import InstrumentId
from nautilus_trader.model.instruments import Instrument
from nautilus_trader.model.objects import Price, Quantity
from nautilus_trader.trading.strategy import Strategy
from nautilus_mm.book_utils import snapshot_from_nautilus_book
from nautilus_mm.health import ConnectionHealth, empty_market_state_snapshot
from nautilus_mm.recorder import MakerEdgeLogger
class MakerEdgeProbeConfig(StrategyConfig, frozen=True):
instrument_id: InstrumentId
order_qty: Decimal = Decimal("0.001")
book_depth: PositiveInt = 10
quote_offset_ticks: PositiveInt = 1
max_quotes: PositiveInt = 1
quote_ttl_secs: float = 30.0
cooldown_secs: float = 60.0
book_sample_secs: float = 2.0
log_dir: str = ""
# 探针：仅在 OBI 极端时挂一侧（吸收叙事），避免噪音
obi_enter: float = 0.25
enable_trading: bool = False # False = 只录盘口不挂单
# 风险熔断：|inventory| 达上限后只允许减仓方向挂单
max_abs_inventory: Decimal = Decimal("0.005")
class MakerEdgeProbe(Strategy):
def __init__(self, config: MakerEdgeProbeConfig) -> None:
super().__init__(config)
self.instrument: Instrument | None = None
self._logger = MakerEdgeLogger(
log_dir=config.log_dir or None,
levels=int(config.book_depth),
)
self._last_mid: float | None = None
self._last_book_sample = 0.0
self._last_quote_ts = 0.0
self._recent_buys = deque(maxlen=200)
self._recent_sells = deque(maxlen=200)
self._quote_id_by_client: dict[str, str] = {}
self._quote_submit_ts: dict[str, float] = {}
self._fill_id_by_client: dict[str, str] = {}
self._liq_high = 0.0
self._liq_low = 0.0
self._health = ConnectionHealth(window_sec=60.0, report_every_sec=30.0)
self._quoting_halted = False
def on_start(self) -> None:
self.instrument = self.cache.instrument(self.config.instrument_id)
if self.instrument is None:
self.log.error(f"Instrument not found: {self.config.instrument_id}")
self.stop()
return
# 启动清场：避免上次硬杀残留挂单污染实验
try:
self.cancel_all_orders(self.config.instrument_id)
self.log.info("Startup cancel_all_orders issued", LogColor.BLUE)
except Exception as exc:
self.log.warning(f"Startup cancel_all failed: {exc}")
self.subscribe_order_book_deltas(
instrument_id=self.config.instrument_id,
book_type=BookType.L2_MBP,
depth=int(self.config.book_depth),
)
self.subscribe_trade_ticks(self.config.instrument_id)
exp = self._logger.experiment
self._logger.write_experiment_start(
extra={
"instrument_id": str(self.config.instrument_id),
"enable_trading": bool(self.config.enable_trading),
"quote_ttl_secs": float(self.config.quote_ttl_secs),
"max_abs_inventory": str(self.config.max_abs_inventory),
"log_dir": str(self._logger.log_dir),
}
)
self.log.info(
f"Experiment {exp['experiment_id']} | {exp['probe_version']} | "
f"quote/fee/exchange=frozen | log={self._logger.log_dir} | "
f"trading={self.config.enable_trading} ttl={self.config.quote_ttl_secs}s "
f"max_inv={self.config.max_abs_inventory}",
LogColor.GREEN,
)
self.log.info(
"Research Freeze: Data Collection only — no Pulse / no quote evolution",
LogColor.BLUE,
)
def on_stop(self) -> None:
try:
self.cancel_all_orders(self.config.instrument_id)
except Exception as exc:
self.log.warning(f"Stop cancel_all failed: {exc}")
self.log.info("MakerEdgeProbe stopped")
# ------------------------------------------------------------------ #
def _trade_qty_window(self, window_sec: float = 20.0) -> tuple[float, float]:
now = time.time()
buy = sum(q for t, q in self._recent_buys if now - t <= window_sec)
sell = sum(q for t, q in self._recent_sells if now - t <= window_sec)
return buy, sell
def _current_snap(self):
book = self.cache.order_book(self.config.instrument_id)
if book is None:
return None
buy, sell = self._trade_qty_window()
snap = snapshot_from_nautilus_book(
book,
levels=int(self.config.book_depth),
recent_buy_qty=buy,
recent_sell_qty=sell,
last_mid=self._last_mid,
liq_low=self._liq_low or None,
liq_high=self._liq_high or None,
)
if snap.mid:
self._last_mid = snap.mid
self._liq_high = max(self._liq_high or snap.mid, snap.mid)
self._liq_low = min(self._liq_low or snap.mid, snap.mid) if self._liq_low else snap.mid
return snap
def _inventory(self) -> float:
try:
positions = self.cache.positions_open(instrument_id=self.config.instrument_id)
except TypeError:
positions = [
p
for p in self.cache.positions_open()
if p.instrument_id == self.config.instrument_id
]
if not positions:
return 0.0
inv = 0.0
for pos in positions:
qty = float(pos.quantity)
inv += -qty if pos.is_short else qty
return inv
def _open_orders(self):
try:
return list(self.cache.orders_open(instrument_id=self.config.instrument_id))
except TypeError:
return [
o
for o in self.cache.orders_open()
if o.instrument_id == self.config.instrument_id
]
def _expire_stale_quotes(self, now: float) -> None:
"""Cancel GTC quotes older than quote_ttl_secs."""
ttl = float(self.config.quote_ttl_secs)
if ttl <= 0:
return
for order in self._open_orders():
cid = order.client_order_id.value
submitted = self._quote_submit_ts.get(cid)
if submitted is None:
# 非本进程跟踪的单（启动残留等）— 一并撤掉
self.log.warning(f"TTL cancel untracked open order {cid}")
self.cancel_order(order)
continue
if now - submitted >= ttl:
self.log.info(f"TTL cancel {cid} age={now - submitted:.1f}s", LogColor.YELLOW)
self.cancel_order(order)
def _inventory_allows(self, side: OrderSide, inv: float) -> bool:
max_abs = float(self.config.max_abs_inventory)
if max_abs <= 0:
return True
if abs(inv) < max_abs:
return True
# 超限：只允许减仓方向
if inv >= max_abs and side == OrderSide.SELL:
return True
if inv <= -max_abs and side == OrderSide.BUY:
return True
return False
def _release_quote_client(self, cid: str, reason: str, snap=None) -> None:
qid = self._quote_id_by_client.pop(cid, None)
self._quote_submit_ts.pop(cid, None)
if qid is not None:
self._logger.cancel_quote(quote_id=qid, reason=reason, snap=snap)
def on_order_book_deltas(self, deltas: OrderBookDeltas) -> None:
now = time.time()
# Phase 0 health
seq = getattr(deltas, "sequence", None)
ts_event = getattr(deltas, "ts_event", None)
self._health.on_book(seq=int(seq) if seq is not None else None, event_ts_ns=ts_event)
report = self._health.maybe_report()
if report:
self._logger.write(report)
gap_w = report.get("sequence_gap_window", 0)
color = LogColor.RED if gap_w or not report.get("healthy") else LogColor.CYAN
self.log.info(
f"Phase0 book/s={report['book_update_rate']:.1f} "
f"trade/s={report['trade_update_rate']:.1f} "
f"lat_p50/p99/max={report['latency_ms_p50']}/"
f"{report['latency_ms_p99']}/{report['latency_ms_max']} "
f"gap_win={gap_w} book_age_ms={report.get('book_age_ms')}",
color,
)
# TTL 撤单：与报价逻辑无关的生命周期闭环
self._expire_stale_quotes(now)
snap = self._current_snap()
if snap is None or snap.mid <= 0:
return
if now - self._last_book_sample >= float(self.config.book_sample_secs):
self._last_book_sample = now
self._logger.record_book(
snap,
now=now,
emit_mid_tick=True,
pair=str(self.config.instrument_id),
)
inv_fields = self._logger.update_inventory(self._inventory(), now=now)
self._logger.write(
{
"event": "inventory_tick",
"pair": str(self.config.instrument_id),
**inv_fields,
"market_state_snapshot": empty_market_state_snapshot(),
**snap.to_book_fields(),
}
)
# 推进 fill path
self._logger.update_paths(str(self.config.instrument_id), snap.mid, now=now)
if not self.config.enable_trading:
return
if not self._health.allow_quoting():
if not self._quoting_halted:
self._quoting_halted = True
self.log.warning("Quoting halted: health gate (stale book / low update rate)")
return
if self._quoting_halted:
self._quoting_halted = False
self.log.info("Quoting resumed: health OK", LogColor.GREEN)
if now - self._last_quote_ts < float(self.config.cooldown_secs):
return
if len(self._open_orders()) >= int(self.config.max_quotes):
return
if self.instrument is None:
return
# 简单吸收探针：OBI 极端 → 挂被动单
tick = float(self.instrument.price_increment)
offset = int(self.config.quote_offset_ticks) * tick
qty = self.instrument.make_qty(self.config.order_qty)
inv = self._inventory()
if snap.obi >= float(self.config.obi_enter):
side = OrderSide.SELL
if not self._inventory_allows(side, inv):
return
price = self.instrument.make_price(snap.best_ask + offset)
self._submit_quote(side, price, qty, snap, reason="obi_bid_thick")
elif snap.obi <= -float(self.config.obi_enter):
side = OrderSide.BUY
if not self._inventory_allows(side, inv):
return
price = self.instrument.make_price(snap.best_bid - offset)
self._submit_quote(side, price, qty, snap, reason="obi_ask_thick")
def on_trade_tick(self, tick: TradeTick) -> None:
now = time.time()
self._health.on_trade(event_ts_ns=getattr(tick, "ts_event", None))
qty = float(tick.size)
# Aggressor side
try:
from nautilus_trader.model.enums import AggressorSide
if tick.aggressor_side == AggressorSide.BUYER:
self._recent_buys.append((now, qty))
elif tick.aggressor_side == AggressorSide.SELLER:
self._recent_sells.append((now, qty))
except Exception:
self._recent_buys.append((now, qty * 0.5))
self._recent_sells.append((now, qty * 0.5))
snap_mid = self._last_mid or float(tick.price)
self._logger.update_paths(str(self.config.instrument_id), snap_mid, now=now)
def _submit_quote(self, side: OrderSide, price: Price, qty: Quantity, snap, reason: str) -> None:
assert self.instrument is not None
order = self.order_factory.limit(
instrument_id=self.config.instrument_id,
order_side=side,
quantity=qty,
price=price,
time_in_force=TimeInForce.GTC,
post_only=True,
)
qside = "bid" if side == OrderSide.BUY else "ask"
cid = order.client_order_id.value
# 先 submit，成功后再记 quote（避免幽灵 quote_created）
try:
self.submit_order(order)
except Exception as exc:
self.log.error(f"submit_order failed: {exc}")
return
qid = self._logger.create_quote(
pair=str(self.config.instrument_id),
side=qside,
quote_price=float(price),
inventory=self._inventory(),
snap=snap,
reason=reason,
state=empty_market_state_snapshot(), # 故意不接 Market Pulse
extra={"book_age_ms": self._health.book_age_ms()},
)
self._quote_id_by_client[cid] = qid
self._quote_submit_ts[cid] = time.time()
self._last_quote_ts = time.time()
self.log.info(f"QUOTE {qside} {price} qty={qty} reason={reason}", LogColor.BLUE)
def on_order_canceled(self, event: OrderCanceled) -> None:
cid = event.client_order_id.value
snap = self._current_snap()
self._release_quote_client(cid, reason="canceled", snap=snap)
def on_order_rejected(self, event: OrderRejected) -> None:
cid = event.client_order_id.value
reason = getattr(event, "reason", None) or "rejected"
self.log.warning(f"OrderRejected {cid}: {reason}")
snap = self._current_snap()
self._release_quote_client(cid, reason=f"rejected:{reason}", snap=snap)
def on_order_denied(self, event: OrderDenied) -> None:
cid = event.client_order_id.value
reason = getattr(event, "reason", None) or "denied"
self.log.warning(f"OrderDenied {cid}: {reason}")
snap = self._current_snap()
self._release_quote_client(cid, reason=f"denied:{reason}", snap=snap)
def on_order_filled(self, event: OrderFilled) -> None:
cid = event.client_order_id.value
qid = self._quote_id_by_client.get(cid)
snap = self._current_snap()
# snap 缺失仍必须记 fill（book 字段可空）
side = "long" if event.order_side == OrderSide.BUY else "short"
det = self._logger.book_deterioration(side)
fill_reason = "toxic_passive" if det.get("pre_5s_deteriorated") else "maker_hit"
order = self.cache.order(event.client_order_id)
terminal = True
if order is not None:
terminal = bool(order.is_closed) or float(order.leaves_qty) <= 0
fill_id = self._logger.log_fill(
pair=str(self.config.instrument_id),
side=side,
fill_price=float(event.last_px),
amount=float(event.last_qty),
inventory=self._inventory(),
snap=snap,
order_type="limit",
quote_id=qid,
fill_reason=fill_reason,
state=empty_market_state_snapshot(),
quote_terminal=terminal,
extra={
"client_order_id": cid,
"venue_order_id": str(event.venue_order_id),
"trade_id": str(event.trade_id),
"venue_trade_id": str(event.trade_id),
"exchange_ts_ns": int(event.ts_event) if getattr(event, "ts_event", None) else None,
"local_ts": time.time(),
"book_age_ms": self._health.book_age_ms(),
"leaves_qty": float(order.leaves_qty) if order is not None else None,
# Maker-only hard evidence (do not trust post_only param alone)
"liquidity_side": str(event.liquidity_side),
"is_maker": event.liquidity_side.name == "MAKER"
if hasattr(event.liquidity_side, "name")
else str(event.liquidity_side) == "MAKER",
"commission": float(event.commission) if event.commission is not None else None,
"commission_currency": (
str(event.commission.currency) if event.commission is not None else None
),
"post_only": True,
"execution_type": "TRADE",
},
)
self._fill_id_by_client[cid] = fill_id
if terminal:
self._quote_id_by_client.pop(cid, None)
self._quote_submit_ts.pop(cid, None)
self.log.info(
f"FILL {side} px={event.last_px} qty={event.last_qty} "
f"reason={fill_reason} terminal={terminal} book={'ok' if snap else 'none'}",
LogColor.YELLOW,
)