From e2fbe1c2b305b0108ca2587cb36baf3157eda95c Mon Sep 17 00:00:00 2001 From: jackyu66git Date: Thu, 10 Sep 2026 16:53:22 +0800 Subject: [PATCH] Snapshot nautilus_mm after EXP_002 Phase 1 stop. Keep frozen research conclusions and code; raw ledgers and secrets stay out of git. Co-authored-by: Cursor --- .env.example | 41 + .gitignore | 11 + Economic_Attribution_v0_1.json | 531 ++++++++++ Economic_Attribution_v0_1.txt | 95 ++ FREEZE.md | 342 ++++++ MM_EDGE_EXP_002.md | 214 ++++ README.md | 158 +++ STATUS.md | 205 ++++ TESTNET_LIMITATIONS.md | 55 + configs/event_state_probe.yaml | 31 + configs/maker_edge_probe.yaml | 24 + deploy/event-state-probe.service | 32 + deploy/event-state-probe.user.service | 30 + deploy/mm-edge-probe.service | 28 + deploy/mm-edge-probe.user.service | 21 + deploy_server.sh | 95 ++ economic_attribution_v0_1.py | 533 ++++++++++ requirements.txt | 6 + scripts/Economic_Fee_Sensitivity_v0_1.txt | 20 + .../Economic_Metric_Reconciliation_v0_1.txt | 21 + scripts/analyze.sh | 11 + scripts/analyze_maker_edge.py | 978 ++++++++++++++++++ scripts/deploy_server.sh | 95 ++ scripts/economic_attribution_v0_1.py | 533 ++++++++++ scripts/economic_fee_sensitivity_v0_1.py | 81 ++ .../economic_metric_reconciliation_v0_1.py | 204 ++++ scripts/event_state_status.sh | 64 ++ ...fill_adverse_selection_attribution_v0_1.py | 595 +++++++++++ scripts/probe_status.sh | 53 + scripts/pull_report.sh | 25 + scripts/reconcile_account.py | 402 +++++++ scripts/reconcile_fills.py | 594 +++++++++++ scripts/reconcile_order_evidence.py | 260 +++++ scripts/record_l2_ccxt.py | 83 ++ scripts/run_event_state.sh | 27 + scripts/run_probe.sh | 34 + scripts/smoke_test_event_state.sh | 68 ++ scripts/validate_event_ledger.py | 453 ++++++++ src/nautilus_mm/__init__.py | 3 + src/nautilus_mm/book_utils.py | 124 +++ src/nautilus_mm/event_ledger.py | 403 ++++++++ src/nautilus_mm/experiment.py | 63 ++ src/nautilus_mm/health.py | 147 +++ src/nautilus_mm/recorder.py | 861 +++++++++++++++ src/nautilus_mm/run_event_state.py | 169 +++ src/nautilus_mm/run_live.py | 184 ++++ src/nautilus_mm/strategies/__init__.py | 9 + .../strategies/event_state_probe.py | 187 ++++ .../strategies/maker_edge_probe.py | 430 ++++++++ 49 files changed, 9633 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 Economic_Attribution_v0_1.json create mode 100644 Economic_Attribution_v0_1.txt create mode 100644 FREEZE.md create mode 100644 MM_EDGE_EXP_002.md create mode 100644 README.md create mode 100644 STATUS.md create mode 100644 TESTNET_LIMITATIONS.md create mode 100644 configs/event_state_probe.yaml create mode 100644 configs/maker_edge_probe.yaml create mode 100644 deploy/event-state-probe.service create mode 100644 deploy/event-state-probe.user.service create mode 100644 deploy/mm-edge-probe.service create mode 100644 deploy/mm-edge-probe.user.service create mode 100755 deploy_server.sh create mode 100644 economic_attribution_v0_1.py create mode 100644 requirements.txt create mode 100644 scripts/Economic_Fee_Sensitivity_v0_1.txt create mode 100644 scripts/Economic_Metric_Reconciliation_v0_1.txt create mode 100755 scripts/analyze.sh create mode 100644 scripts/analyze_maker_edge.py create mode 100755 scripts/deploy_server.sh create mode 100644 scripts/economic_attribution_v0_1.py create mode 100644 scripts/economic_fee_sensitivity_v0_1.py create mode 100644 scripts/economic_metric_reconciliation_v0_1.py create mode 100755 scripts/event_state_status.sh create mode 100644 scripts/prefill_adverse_selection_attribution_v0_1.py create mode 100755 scripts/probe_status.sh create mode 100755 scripts/pull_report.sh create mode 100644 scripts/reconcile_account.py create mode 100644 scripts/reconcile_fills.py create mode 100644 scripts/reconcile_order_evidence.py create mode 100644 scripts/record_l2_ccxt.py create mode 100755 scripts/run_event_state.sh create mode 100755 scripts/run_probe.sh create mode 100755 scripts/smoke_test_event_state.sh create mode 100644 scripts/validate_event_ledger.py create mode 100644 src/nautilus_mm/__init__.py create mode 100644 src/nautilus_mm/book_utils.py create mode 100644 src/nautilus_mm/event_ledger.py create mode 100644 src/nautilus_mm/experiment.py create mode 100644 src/nautilus_mm/health.py create mode 100644 src/nautilus_mm/recorder.py create mode 100644 src/nautilus_mm/run_event_state.py create mode 100644 src/nautilus_mm/run_live.py create mode 100644 src/nautilus_mm/strategies/__init__.py create mode 100644 src/nautilus_mm/strategies/event_state_probe.py create mode 100644 src/nautilus_mm/strategies/maker_edge_probe.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..12553cc --- /dev/null +++ b/.env.example @@ -0,0 +1,41 @@ +# Binance Futures API (prefer TESTNET first) +BINANCE_API_KEY= +BINANCE_API_SECRET= +# Must be exactly TESTNET or LIVE (case-insensitive). Anything else exits. +BINANCE_ENVIRONMENT=TESTNET +# Required only for LIVE: +# I_UNDERSTAND_LIVE=yes + +# Proxy: 本地可开；服务器请留空（直连） +# 本地: USE_PROXY=true ./scripts/run_probe.sh +HTTP_PROXY= +HTTPS_PROXY= + +# Instrument +SYMBOL=BTCUSDT-PERP +ORDER_QTY=0.001 +# Default false — set true explicitly to place post-only quotes +ENABLE_TRADING=false +COOLDOWN_SECS=60 +OBI_ENTER=0.25 +QUOTE_TTL_SECS=30 +# Absolute inventory circuit breaker (BTC). Beyond: reduce-only side. +MAX_ABS_INVENTORY=0.005 + +# Logs (shared schema with Freqtrade MakerEdgeProbe) +MAKER_EDGE_LOG_DIR= + +# Experiment identity (bind every fill + every report) +EXPERIMENT_ID=MM_EDGE_EXP_001 +PROBE_VERSION=probe_v0.1 +EXCHANGE_NAME=binance_usdm + +# --- MM_EDGE_EXP_002 (Event-State Observability — use run_event_state.sh) --- +# EXPERIMENT_ID=MM_EDGE_EXP_002 +# PROBE_VERSION=event_state_v0.1 +# ENABLE_TRADING=false # hard-enforced; never true for EXP_002 +# EVENT_STATE_LOG_DIR=logs/event_state +# PREFILL_WINDOW_SEC=5.0 +# PREFILL_MARGIN_SEC=0.25 +# LARGE_TRADE_QTY=0.1 +# LOG_EVERY_BOOK_DELTA=true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..48f9871 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +.venv/ +.env +.env.* +!.env.example +__pycache__/ +*.pyc +logs/ +.DS_Store +*.egg-info/ +dist/ +build/ diff --git a/Economic_Attribution_v0_1.json b/Economic_Attribution_v0_1.json new file mode 100644 index 0000000..a4cbbe0 --- /dev/null +++ b/Economic_Attribution_v0_1.json @@ -0,0 +1,531 @@ +{ + "experiment_id": "MM_EDGE_EXP_001", + "population": { + "name": "MATCHED", + "fills": 3890, + "paths": 3886, + "clusters": 3334 + }, + "strategy": "v0.1 FROZEN", + "execution": "STOPPED", + "stage3": "LOCKED", + "fee_total_usdt": 42.42214561, + "fee_per_fill_usdt": 0.010916661248069994, + "fee_per_btc_usdt": 12.74857122550787, + "fee_per_cluster_usdt": 0.012724098863227356, + "realized_component_usdt": -8.52172967, + "gross_markout_30s_usdt": -1.7307799999996094, + "net_attr_30s_usdt": -52.67465527999961, + "markout_by_horizon": [ + { + "horizon": "1s", + "n": 3886, + "fill_w": 1.9522065688461466e-05, + "cluster_w": 1.9672552540924558e-05, + "gross_usdt": 4.140839999999786 + }, + { + "horizon": "5s", + "n": 3886, + "fill_w": 1.6056729777556372e-05, + "cluster_w": 1.563711857682482e-05, + "gross_usdt": 3.4058050000001323 + }, + { + "horizon": "10s", + "n": 3886, + "fill_w": 1.1123222170478668e-05, + "cluster_w": 1.1253447279750312e-05, + "gross_usdt": 2.359355000000128 + }, + { + "horizon": "30s", + "n": 3886, + "fill_w": -8.159793870873894e-06, + "cluster_w": -8.29897884011306e-06, + "gross_usdt": -1.7307799999996094 + }, + { + "horizon": "300s", + "n": 3886, + "fill_w": -3.581687863688742e-05, + "cluster_w": -3.542059919144397e-05, + "gross_usdt": -7.597144999999726 + } + ], + "inventory_metrics": { + "max_net_btc": 0.0059, + "min_net_btc": -0.0058000000000000005, + "max_abs_net_btc": 0.0059, + "avg_abs_net_btc_per_fill": 0.0029282519280205655, + "time_weighted_abs_net_btc": 0.0035138619697350987, + "time_weighted_signed_net_btc": 0.0009845445677824191, + "long_qty": 1.6646, + "short_qty": 1.6670000000000003, + "turnover_btc": 3.331600000000001 + }, + "bucket_rows": [ + { + "dimension": "PathType", + "bucket": "A_immediate_edge", + "fills": 1481, + "clusters": 1279, + "btc_qty": 1.2579000000000002, + "notional_usdt": 80131.73112, + "fee_usdt": 16.02634448, + "fee_per_fill": 0.010821299446320053, + "fee_per_btc": 12.74055527466412, + "markout_1s": 0.00010135972212851294, + "markout_5s": 0.00012093929414163527, + "markout_10s": 0.00011803045894311667, + "markout_30s": 0.00015268377244562442, + "markout_300s": 0.00013524292871909752, + "gross_markout_30s_usdt": 12.23481500000004, + "realized_pnl_usdt": -1.7373983300000004, + "net_attr_30s_usdt": -5.52892780999996 + }, + { + "dimension": "PathType", + "bucket": "C_toxic", + "fills": 1290, + "clusters": 1114, + "btc_qty": 1.1028000000000002, + "notional_usdt": 70343.84668, + "fee_usdt": 14.068767789999999, + "fee_per_fill": 0.010906021542635659, + "fee_per_btc": 12.757315732680446, + "markout_1s": -2.26781740733745e-05, + "markout_5s": -5.864727612589716e-05, + "markout_10s": -9.311823718992184e-05, + "markout_30s": -0.0002122337020878937, + "markout_300s": -0.0004798649291039827, + "gross_markout_30s_usdt": -14.92933499999959, + "realized_pnl_usdt": -2.45849123, + "net_attr_30s_usdt": -31.45659401999959 + }, + { + "dimension": "PathType", + "bucket": "B_drawdown_then_recover", + "fills": 872, + "clusters": 767, + "btc_qty": 0.7574000000000001, + "notional_usdt": 48282.0527, + "fee_usdt": 9.656409660000001, + "fee_per_fill": 0.011073864288990827, + "fee_per_btc": 12.749418616318987, + "markout_1s": -4.145950488969804e-05, + "markout_5s": -3.18002221144232e-05, + "markout_10s": -1.0215286476420607e-05, + "markout_30s": -8.034869652509012e-06, + "markout_300s": 0.00040945162631827824, + "gross_markout_30s_usdt": -0.3879400000000708, + "realized_pnl_usdt": -2.83361474, + "net_attr_30s_usdt": -12.87796440000007 + }, + { + "dimension": "PathType", + "bucket": "D_mixed", + "fills": 243, + "clusters": 215, + "btc_qty": 0.20950000000000005, + "notional_usdt": 13353.11977, + "fee_usdt": 2.6706236800000003, + "fee_per_fill": 0.010990220905349795, + "fee_per_btc": 12.747607064439139, + "markout_1s": -2.877754462019724e-05, + "markout_5s": -4.676173139724288e-05, + "markout_10s": -4.12787430573779e-06, + "markout_30s": 0.00010122578268464162, + "markout_300s": -0.00033310680025451153, + "gross_markout_30s_usdt": 1.3516800000000115, + "realized_pnl_usdt": -1.49222537, + "net_attr_30s_usdt": -2.8111690499999886 + }, + { + "dimension": "Toxicity", + "bucket": "non_toxic", + "fills": 2596, + "clusters": 2236, + "btc_qty": 2.2248, + "notional_usdt": 141766.90359, + "fee_usdt": 28.35337782, + "fee_per_fill": 0.010921948312788905, + "fee_per_btc": 12.744236704422867, + "markout_1s": 4.046155946657924e-05, + "markout_5s": 5.3124388057320254e-05, + "markout_10s": 6.284717923844347e-05, + "markout_30s": 9.310039695986549e-05, + "markout_300s": 0.00018451697355013032, + "gross_markout_30s_usdt": 13.198554999999981, + "realized_pnl_usdt": -6.063238439999999, + "net_attr_30s_usdt": -21.21806126000002 + }, + { + "dimension": "Toxicity", + "bucket": "toxic", + "fills": 1290, + "clusters": 1114, + "btc_qty": 1.1028000000000002, + "notional_usdt": 70343.84668, + "fee_usdt": 14.068767789999999, + "fee_per_fill": 0.010906021542635659, + "fee_per_btc": 12.757315732680446, + "markout_1s": -2.26781740733745e-05, + "markout_5s": -5.864727612589716e-05, + "markout_10s": -9.311823718992184e-05, + "markout_30s": -0.0002122337020878937, + "markout_300s": -0.0004798649291039827, + "gross_markout_30s_usdt": -14.92933499999959, + "realized_pnl_usdt": -2.45849123, + "net_attr_30s_usdt": -31.45659401999959 + }, + { + "dimension": "Volatility", + "bucket": "high_vol", + "fills": 1944, + "clusters": 1704, + "btc_qty": 1.7018000000000004, + "notional_usdt": 108527.27282000001, + "fee_usdt": 21.705452540000003, + "fee_per_fill": 0.011165356244855968, + "fee_per_btc": 12.754408590903747, + "markout_1s": 2.410343439052789e-05, + "markout_5s": 2.200234962100767e-05, + "markout_10s": 1.3732354654038277e-05, + "markout_30s": -1.8503321311044688e-05, + "markout_300s": -7.311572283918717e-05, + "gross_markout_30s_usdt": -2.0081149999998673, + "realized_pnl_usdt": -2.5832574499999996, + "net_attr_30s_usdt": -26.296824989999866 + }, + { + "dimension": "Volatility", + "bucket": "low_vol", + "fills": 1942, + "clusters": 1630, + "btc_qty": 1.6258000000000004, + "notional_usdt": 103583.47745, + "fee_usdt": 20.71669307, + "fee_per_fill": 0.010667710128733266, + "fee_per_btc": 12.74246098536105, + "markout_1s": 1.4722039050446979e-05, + "markout_5s": 9.827339504906817e-06, + "markout_10s": 8.389561939738874e-06, + "markout_30s": 2.677405768059181e-06, + "markout_300s": 3.2621515353484367e-06, + "gross_markout_30s_usdt": 0.2773350000002581, + "realized_pnl_usdt": -5.9384722199999995, + "net_attr_30s_usdt": -26.377830289999743 + }, + { + "dimension": "Trend", + "bucket": "range", + "fills": 1521, + "clusters": 1383, + "btc_qty": 1.3480000000000003, + "notional_usdt": 85940.04824, + "fee_usdt": 17.18800828, + "fee_per_fill": 0.011300465667324127, + "fee_per_btc": 12.750747982195842, + "markout_1s": -1.477018021278579e-06, + "markout_5s": -8.501973351929517e-06, + "markout_10s": -1.3186808981480409e-05, + "markout_30s": -3.577395013107376e-05, + "markout_300s": -8.103131360308704e-05, + "gross_markout_30s_usdt": -3.074414999999833, + "realized_pnl_usdt": -5.27847826, + "net_attr_30s_usdt": -25.540901539999833 + }, + { + "dimension": "Trend", + "bucket": "trend_up", + "fills": 1255, + "clusters": 1097, + "btc_qty": 1.0467, + "notional_usdt": 66723.59572000001, + "fee_usdt": 13.34471747, + "fee_per_fill": 0.010633241011952193, + "fee_per_btc": 12.749324037451037, + "markout_1s": 3.411746587448147e-05, + "markout_5s": 3.5157502749763345e-05, + "markout_10s": 2.718513563945042e-05, + "markout_30s": -2.7502264831481206e-06, + "markout_300s": -1.9947965717947442e-06, + "gross_markout_30s_usdt": -0.18350500000001263, + "realized_pnl_usdt": 0.07479931999999989, + "net_attr_30s_usdt": -13.453423150000011 + }, + { + "dimension": "Trend", + "bucket": "trend_down", + "fills": 1110, + "clusters": 972, + "btc_qty": 0.9329000000000001, + "notional_usdt": 59447.10631, + "fee_usdt": 11.88941986, + "fee_per_fill": 0.010711189063063063, + "fee_per_btc": 12.744581262729124, + "markout_1s": 3.349759346764102e-05, + "markout_5s": 3.012139885602704e-05, + "markout_10s": 2.823922145589404e-05, + "markout_30s": 2.568905527607399e-05, + "markout_300s": -8.414370876041091e-06, + "gross_markout_30s_usdt": 1.5271400000002369, + "realized_pnl_usdt": -3.3180507300000004, + "net_attr_30s_usdt": -13.680330589999762 + }, + { + "dimension": "FillContext", + "bucket": "neutral", + "fills": 1388, + "clusters": 1213, + "btc_qty": 1.1838000000000002, + "notional_usdt": 75485.50388999999, + "fee_usdt": 15.09709914, + "fee_per_fill": 0.01087687257925072, + "fee_per_btc": 12.7530825646224, + "markout_1s": 1.647018216652519e-05, + "markout_5s": -3.5886360432144643e-06, + "markout_10s": -2.1338136688432127e-05, + "markout_30s": -6.403666599409166e-05, + "markout_300s": -0.00017397764237140476, + "gross_markout_30s_usdt": -4.833839999999636, + "realized_pnl_usdt": 0.9367158200000003, + "net_attr_30s_usdt": -18.994223319999634 + }, + { + "dimension": "FillContext", + "bucket": "sell_resistance", + "fills": 979, + "clusters": 861, + "btc_qty": 0.8544, + "notional_usdt": 54478.163700000005, + "fee_usdt": 10.89563175, + "fee_per_fill": 0.011129348059244126, + "fee_per_btc": 12.752377984550561, + "markout_1s": 2.2746912080660278e-05, + "markout_5s": 3.396737104044108e-05, + "markout_10s": 4.7584661889033e-05, + "markout_30s": 6.382970283559833e-05, + "markout_300s": 0.00015825056159152353, + "gross_markout_30s_usdt": 3.4773250000000804, + "realized_pnl_usdt": -6.22020069, + "net_attr_30s_usdt": -13.63850743999992 + }, + { + "dimension": "FillContext", + "bucket": "buy_support", + "fills": 957, + "clusters": 842, + "btc_qty": 0.8356000000000002, + "notional_usdt": 53220.48314, + "fee_usdt": 10.64409564, + "fee_per_fill": 0.01112235699059561, + "fee_per_btc": 12.73826668262326, + "markout_1s": 8.405128507064573e-06, + "markout_5s": 1.8714880836011024e-05, + "markout_10s": 2.0467871310649115e-05, + "markout_30s": 1.984809114230057e-05, + "markout_300s": 5.777324478456582e-05, + "gross_markout_30s_usdt": 1.0563249999999906, + "realized_pnl_usdt": -4.60049982, + "net_attr_30s_usdt": -14.188270460000009 + }, + { + "dimension": "FillContext", + "bucket": "buy_pressure_absorbing", + "fills": 288, + "clusters": 242, + "btc_qty": 0.23299999999999998, + "notional_usdt": 14849.31926, + "fee_usdt": 2.96986344, + "fee_per_fill": 0.010312025833333334, + "fee_per_btc": 12.74619502145923, + "markout_1s": 3.085831693539742e-05, + "markout_5s": 9.949951065962983e-07, + "markout_10s": -5.396206963900885e-06, + "markout_30s": -8.174078412265757e-05, + "markout_300s": -0.00018505090717539603, + "gross_markout_30s_usdt": -1.2137950000000812, + "realized_pnl_usdt": 0.5555894899999999, + "net_attr_30s_usdt": -3.628068950000081 + }, + { + "dimension": "FillContext", + "bucket": "sell_pressure_absorbing", + "fills": 246, + "clusters": 207, + "btc_qty": 0.19330000000000003, + "notional_usdt": 12320.81768, + "fee_usdt": 2.46416312, + "fee_per_fill": 0.01001692325203252, + "fee_per_btc": 12.74786921883083, + "markout_1s": 3.3994902844800725e-05, + "markout_5s": 1.3797785537899414e-05, + "markout_10s": -3.266666307814853e-05, + "markout_30s": -5.2299288629677764e-05, + "markout_300s": -0.0003042407652930987, + "gross_markout_30s_usdt": -0.6443699999999568, + "realized_pnl_usdt": 0.9138141899999999, + "net_attr_30s_usdt": -2.194718929999957 + }, + { + "dimension": "FillContext", + "bucket": "sell_pressure_falling", + "fills": 10, + "clusters": 10, + "btc_qty": 0.010000000000000002, + "notional_usdt": 638.0611, + "fee_usdt": 0.12761222, + "fee_per_fill": 0.012761222, + "fee_per_btc": 12.761221999999997, + "markout_1s": 0.0002923701194132765, + "markout_5s": 0.0002596146356515865, + "markout_10s": 0.0005288678466685525, + "markout_30s": 8.886296312371227e-05, + "markout_300s": -2.1392935566876843e-05, + "gross_markout_30s_usdt": 0.056699999999975284, + "realized_pnl_usdt": 0.02931205, + "net_attr_30s_usdt": -0.041600170000024736 + }, + { + "dimension": "FillContext", + "bucket": "buy_pressure_rising", + "fills": 8, + "clusters": 8, + "btc_qty": 0.0075, + "notional_usdt": 479.4465, + "fee_usdt": 0.0958893, + "fee_per_fill": 0.0119861625, + "fee_per_btc": 12.78524, + "markout_1s": 0.00011800065283618207, + "markout_5s": 4.155833862592348e-05, + "markout_10s": -0.00018521357440294395, + "markout_30s": -0.0002589965720888399, + "markout_300s": -0.0008807030607168815, + "gross_markout_30s_usdt": -0.12417499999999199, + "realized_pnl_usdt": -0.00064907, + "net_attr_30s_usdt": -0.22071336999999197 + }, + { + "dimension": "FillContext", + "bucket": "momentum_down_catching_knife", + "fills": 5, + "clusters": 5, + "btc_qty": 0.005, + "notional_usdt": 317.4407, + "fee_usdt": 0.06348814, + "fee_per_fill": 0.012697627999999999, + "fee_per_btc": 12.697628, + "markout_1s": -8.804794092254599e-05, + "markout_5s": 0.00041661324461542815, + "markout_10s": 0.0007213945785779604, + "markout_30s": 0.000589716441527518, + "markout_300s": 0.0009308825238855407, + "gross_markout_30s_usdt": 0.18720000000000436, + "realized_pnl_usdt": -0.13601287, + "net_attr_30s_usdt": -0.012301009999995626 + }, + { + "dimension": "FillContext", + "bucket": "momentum_up_chasing", + "fills": 5, + "clusters": 5, + "btc_qty": 0.005, + "notional_usdt": 321.51430000000005, + "fee_usdt": 0.06430286, + "fee_per_fill": 0.012860572, + "fee_per_btc": 12.860572000000001, + "markout_1s": 0.00036950144985773765, + "markout_5s": 0.001018928240516915, + "markout_10s": 0.0009125566110123747, + "markout_30s": 0.0009575001796187784, + "markout_300s": 0.0014820491654648018, + "gross_markout_30s_usdt": 0.30785000000000584, + "realized_pnl_usdt": 0.00020122999999998975, + "net_attr_30s_usdt": 0.24374837000000582 + } + ], + "counterfactuals": [ + { + "name": "BASELINE", + "fills": 3886, + "clusters": 3332, + "btc_qty": 3.327600000000001, + "fee_usdt": 42.42214561, + "gross_markout_30s_usdt": -1.7307799999996094, + "realized_pnl_usdt": -8.52172967, + "net_attr_30s_usdt": -52.67465527999961, + "markout_30s": -8.159793870873894e-06 + }, + { + "name": "EXCLUDE_PATH_C", + "fills": 2596, + "clusters": 2236, + "btc_qty": 2.2248, + "fee_usdt": 28.35337782, + "gross_markout_30s_usdt": 13.198554999999981, + "realized_pnl_usdt": -6.063238439999999, + "net_attr_30s_usdt": -21.21806126000002, + "markout_30s": 9.310039695986549e-05 + }, + { + "name": "EXCLUDE_TOXIC", + "fills": 2596, + "clusters": 2236, + "btc_qty": 2.2248, + "fee_usdt": 28.35337782, + "gross_markout_30s_usdt": 13.198554999999981, + "realized_pnl_usdt": -6.063238439999999, + "net_attr_30s_usdt": -21.21806126000002, + "markout_30s": 9.310039695986549e-05 + }, + { + "name": "EXCLUDE_NEGATIVE_STATE", + "fills": 1956, + "clusters": 1723, + "btc_qty": 1.7100000000000002, + "fee_usdt": 21.79513061, + "gross_markout_30s_usdt": 5.085400000000057, + "realized_pnl_usdt": -10.9272001, + "net_attr_30s_usdt": -27.636930709999945, + "markout_30s": 4.666546513967971e-05 + } + ], + "recon03_extension": { + "venue_confirmed_no_trade_history": 876, + "venue_partial_order_canceled": 11 + }, + "account_recon_ref": { + "experiment_id": "MM_EDGE_EXP_001", + "maker_only_status": "PASS", + "taker_filled_count": 0, + "maker_filled_count": 3890, + "exchange_trades": 3890, + "jsonl_fills": 4777, + "income_by_type": { + "COMMISSION": -52.04535895999991, + "REALIZED_PNL": -9.671389249999995, + "FUNDING_FEE": 0.05826837000000001 + }, + "income_sum": -61.658479839999906, + "start_wallet_assumed": 5000.0, + "end_wallet": 4938.32625704, + "end_unrealized": 0.40078863, + "end_equity": 4938.72704567, + "wallet_residual_gap": -0.015263119999872288, + "net_qty": -0.0023999999999990695, + "fee_by_asset": { + "USDT": 42.47422358999987 + }, + "positions": [ + { + "symbol": "BTCUSDT", + "amt": 0.0032, + "entry": 64233.4, + "unrealized": 0.40078863 + } + ], + "probe": "STOPPED" + } +} diff --git a/Economic_Attribution_v0_1.txt b/Economic_Attribution_v0_1.txt new file mode 100644 index 0000000..1184fc2 --- /dev/null +++ b/Economic_Attribution_v0_1.txt @@ -0,0 +1,95 @@ +======================================================================== +Economic Attribution v0.1 +======================================================================== +Experiment: MM_EDGE_EXP_001 +Population: MATCHED=3890 +Strategy: v0.1 FROZEN +Execution: STOPPED +Stage3: LOCKED +Purpose: Economic Attribution only. +No strategy modification. No live execution. No economic simulation. + +Layer 1 — Hard Economic Evidence +---------------------------------------- +Matched fills: 3890 +Matched paths: 3886 +Matched clusters: 3334 +Fee total: 42.422146 USDT +Fee / fill: 0.010917 USDT +Fee / BTC: 12.748571 USDT +Fee / cluster: 0.012724 USDT +Realized component: -8.521730 USDT +Gross markout @30s: -1.730780 USDT +Net attributable @30s: -52.674655 USDT + +Markout by horizon (MATCHED only) +---------------------------------------- + 1s n=3886 fill-w=0.0020% cluster-w=0.0020% gross=4.140840 USDT + 5s n=3886 fill-w=0.0016% cluster-w=0.0016% gross=3.405805 USDT + 10s n=3886 fill-w=0.0011% cluster-w=0.0011% gross=2.359355 USDT + 30s n=3886 fill-w=-0.0008% cluster-w=-0.0008% gross=-1.730780 USDT + 300s n=3886 fill-w=-0.0036% cluster-w=-0.0035% gross=-7.597145 USDT + +Inventory carry / exposure +---------------------------------------- +Max net BTC: 0.005900 +Min net BTC: -0.005800 +Max |net BTC|: 0.005900 +Average |net BTC|: 0.002928 +TW |net BTC|: 0.003514 +TW signed net BTC: 0.000985 +Long qty / Short qty: 1.664600 / 1.667000 BTC +Inventory turnover: 3.331600 BTC + +Slices (weighted by notional, MATCHED only) +---------------------------------------- +PathType + A_immediate_edge: n=1481 clusters=1279 fee=16.0263 gross30=12.2348 realized=-1.7374 net30=-5.5289 m30=0.0153% + C_toxic: n=1290 clusters=1114 fee=14.0688 gross30=-14.9293 realized=-2.4585 net30=-31.4566 m30=-0.0212% + B_drawdown_then_recover: n=872 clusters=767 fee=9.6564 gross30=-0.3879 realized=-2.8336 net30=-12.8780 m30=-0.0008% + D_mixed: n=243 clusters=215 fee=2.6706 gross30=1.3517 realized=-1.4922 net30=-2.8112 m30=0.0101% + +Toxicity + non_toxic: n=2596 clusters=2236 fee=28.3534 gross30=13.1986 realized=-6.0632 net30=-21.2181 m30=0.0093% + toxic: n=1290 clusters=1114 fee=14.0688 gross30=-14.9293 realized=-2.4585 net30=-31.4566 m30=-0.0212% + +Volatility + high_vol: n=1944 clusters=1704 fee=21.7055 gross30=-2.0081 realized=-2.5833 net30=-26.2968 m30=-0.0019% + low_vol: n=1942 clusters=1630 fee=20.7167 gross30=0.2773 realized=-5.9385 net30=-26.3778 m30=0.0003% + +Trend + range: n=1521 clusters=1383 fee=17.1880 gross30=-3.0744 realized=-5.2785 net30=-25.5409 m30=-0.0036% + trend_up: n=1255 clusters=1097 fee=13.3447 gross30=-0.1835 realized=0.0748 net30=-13.4534 m30=-0.0003% + trend_down: n=1110 clusters=972 fee=11.8894 gross30=1.5271 realized=-3.3181 net30=-13.6803 m30=0.0026% + +FillContext + neutral: n=1388 clusters=1213 fee=15.0971 gross30=-4.8338 realized=0.9367 net30=-18.9942 m30=-0.0064% + sell_resistance: n=979 clusters=861 fee=10.8956 gross30=3.4773 realized=-6.2202 net30=-13.6385 m30=0.0064% + buy_support: n=957 clusters=842 fee=10.6441 gross30=1.0563 realized=-4.6005 net30=-14.1883 m30=0.0020% + buy_pressure_absorbing: n=288 clusters=242 fee=2.9699 gross30=-1.2138 realized=0.5556 net30=-3.6281 m30=-0.0082% + sell_pressure_absorbing: n=246 clusters=207 fee=2.4642 gross30=-0.6444 realized=0.9138 net30=-2.1947 m30=-0.0052% + sell_pressure_falling: n=10 clusters=10 fee=0.1276 gross30=0.0567 realized=0.0293 net30=-0.0416 m30=0.0089% + buy_pressure_rising: n=8 clusters=8 fee=0.0959 gross30=-0.1242 realized=-0.0006 net30=-0.2207 m30=-0.0259% + momentum_down_catching_knife: n=5 clusters=5 fee=0.0635 gross30=0.1872 realized=-0.1360 net30=-0.0123 m30=0.0590% + momentum_up_chasing: n=5 clusters=5 fee=0.0643 gross30=0.3079 realized=0.0002 net30=0.2437 m30=0.0958% + +Layer 2 — Evidence Extension (excluded from core conclusion) +---------------------------------------- +VENUE_CONFIRMED_NO_TRADE_HISTORY: 876 +VENUE_PARTIAL_ORDER_CANCELED: 11 +These rows are order-confirmed, but not part of the Hard Evidence Population. + +Layer 3 — Counterfactual Attribution (NOT backtest) +---------------------------------------- +Observed vs Exclude-Bucket Attribution. These are contribution decompositions only. +BASELINE: fills=3886 clusters=3332 fee=42.4221 gross30=-1.7308 realized=-8.5217 net30=-52.6747 m30=-0.0008% +EXCLUDE_PATH_C: fills=2596 clusters=2236 fee=28.3534 gross30=13.1986 realized=-6.0632 net30=-21.2181 m30=0.0093% +EXCLUDE_TOXIC: fills=2596 clusters=2236 fee=28.3534 gross30=13.1986 realized=-6.0632 net30=-21.2181 m30=0.0093% +EXCLUDE_NEGATIVE_STATE: fills=1956 clusters=1723 fee=21.7951 gross30=5.0854 realized=-10.9272 net30=-27.6369 m30=0.0047% + +Interpretation +---------------------------------------- +Core conclusion is based on 3890 fully matched fills. +Economic Attribution asks why MakerAlpha did not convert to money. +It does NOT change quote logic, does NOT restart v0.1, and does NOT unlock Stage 3. +======================================================================== diff --git a/FREEZE.md b/FREEZE.md new file mode 100644 index 0000000..af27f79 --- /dev/null +++ b/FREEZE.md @@ -0,0 +1,342 @@ +# Research Freeze / Data Collection Phase + +**研究框架已冻结。** 研究对象：可验证的市场现象（不是策略）。 + +验证的假设是： + +> 在当前 BTC 永续、当前交易所、当前报价假设、当前执行条件下，被动成交是否产生正向 Maker Alpha。 + +不是：「我的策略有没有赚钱」。 + +| 报告结论 | 含义 | 下一步 | +|----------|------|--------| +| **FAIL** | 这个市场假设不成立 | 换假设（Carry / Basis / Funding…） | +| **PARTIAL_PASS** | 优势仅局部存在 | Event-driven LP | +| **PASS** | 普遍可捕获 | Economic Simulation → Symmetric MM | +| **COLLECTING** | 样本不足 | 继续采集 | + +三个答案都推进系统。只有盈利才算成功 —— 错误。 + +--- + +## 状态机（锁定） + +``` +Research Freeze + | + v +Data Collection + | + v +Maker Edge Report v0.1 + | + +---- FAIL --------------→ 换假设 + | + +---- PARTIAL_PASS ------→ Event-driven LP + | + +---- PASS --------------→ Economic Simulation + | ↓ + | Symmetric MM / Quote Engine + | + +---- COLLECTING --------→ 继续采集 +``` + +--- + +## 观察窗口（不要过早解释） + +| 规模 | 用途 | +|------|------| +| **~500 fills** | 发现异常 / 管道是否工作 | +| **~2000 fills** | 初步判断（Decision 门槛默认） | +| **~10000 fills** | 才讨论稳定性 | + +**cluster 数量比 fill 数量更重要。** +5000 fills / 80 clusters ≠ 3000 fills / 900 clusters。 + +第一份报告最值得看的不是 Decision，而是三个**分布**：Fill Alpha / Cluster Alpha / Toxicity。 + +--- + +## Unlock Stage3（Economic Simulation）必须同时满足 + +``` +Data Integrity PASS +AND cluster-weighted MakerAlpha > 0 +AND benchmark-adjusted alpha > 0 +AND edge not concentrated in one event/regime +``` + +否则：继续 Data Collection，或判定 FAIL / PARTIAL_PASS。 +**禁止**用改 Quote Logic / 改成本假设 来「凑」解锁。 + +--- + +## 冻结期间只允许 / 禁止 + +### ✅ 允许 + +- 数据字段（不改变报价行为） +- 数据质量检查 +- 报告解释能力（分布、归因、可比格式） + +### ❌ 禁止（直到 Gate 解锁） + +- 新交易规则 / 动态 spread / inventory skew +- 新过滤条件（Pulse / AI） +- 新收益优化参数（为结果改 fee/slip/latency） + +### 三不动 + +1. Quote Logic +2. 成本模型：`Net = Raw − Fee − Slip − Latency` +3. 失败定义：`PASS` / `PARTIAL_PASS` / `COLLECTING` / `FAIL` + +--- + +## Report v0.1 固定格式 + +``` +Executive Summary +Section 1 — Data Integrity +Section 2 — Fill Alpha (+ distribution: mean/median/p25/p75) +Section 3 — Toxicity Profile (+ loss concentration) +Section 4 — Observed Edge Attribution (事实，非策略建议) +Section 5 — Decision + Stage3 unlock checklist +``` + +第一份报告不期待 PASS。价值在于：市场在哪些情况下愿意付给流动性提供者溢价。 + +--- + +## Experiment ID（强制绑定） + +每轮 Data Collection 绑定固定身份，写入每条 jsonl + 每份报告： + +``` +Experiment: MM_EDGE_EXP_001 +Version: probe_v0.1 +Quote: frozen +Fee: frozen +Exchange: frozen +``` + +环境变量：`EXPERIMENT_ID` / `PROBE_VERSION`（见 `.env.example`）。 +换实验假设时换新 ID（如 `MM_EDGE_EXP_002`），禁止在同一 ID 下改报价逻辑后重解释旧数据。 + +报告阅读顺序：**Integrity → 分布（非均值）→ Cluster → Toxicity → Decision**。 +Integrity FAIL → `Decision=INVALID`，不解释 Alpha。 + +--- + +## Post-Report Phase (2026-08-18) + +Maker Edge Report → **PARTIAL_PASS**. +Account wallet moved ≈ −62 USDT vs assumed 5000 start — **not** Maker Edge FAIL evidence; +also **not** ignorable. Research markout ≠ account equity. + +**Probe volume STOPPED** until: + +1. Maker-only hard check: `TAKER_FILLED_COUNT == 0` (exchange `userTrades.maker`) +2. Order→Fill→Fee→Position→Funding→Equity ledger residual ≈ 0 + +See `STATUS.md` and `scripts/reconcile_account.py`. + +Stage3 remains **LOCKED**. Economic Edge = **UNKNOWN**. + +--- + +## Economic Attribution Phase (2026-08-18) + +`MM_EDGE_EXP_001 / probe_v0.1` is now a **FROZEN BASELINE**. + +Execution state: + +- Probe: **STOPPED** +- Strategy modifications: **forbidden** +- Purpose: **Economic Attribution only** + +First hard-evidence population: + +``` +MATCHED = 3890 +``` + +This means: + +- Local Fill +- Venue Trade +- venue_trade_id +- quantity closure +- price verification +- fee verification +- maker-only verification + +Economic Attribution v0.1 must: + +1. Use **MATCHED only** for core conclusions +2. Keep `VENUE_CONFIRMED_NO_TRADE_HISTORY` / `VENUE_PARTIAL_ORDER_CANCELED` + as extension evidence, not core population +3. Treat counterfactuals as **attribution**, not backtest / simulation + +Current baseline conclusion: + +> Maker markout exists, but economic edge is **not established** under v0.1. + +Known explanation path: + +``` +MakerAlpha + ↓ +matched fills + ↓ +fee + realized / inventory economics + ↓ +account outcome +``` + +Next allowed work: + +- Fee attribution +- Inventory carry / exposure attribution +- Counterfactual attribution (`Path C`, toxic, negative states) +- Real-time immutable fill ledger design for future runs + +Still forbidden: + +- Resume v0.1 live execution +- Change quote offset / TTL / cooldown as a shortcut +- Unlock Stage3 from attribution alone + +--- + +## Prefill Adverse-Selection Attribution Phase (2026-08-18) + +Purpose: **Pre-fill adverse-selection predictability audit** — NOT strategy / backtest / optimization / model training. + +Hard contract: + +``` +feature_timestamp <= t_fill - 0.25s +``` + +Features: sampled `mid_tick` / `inventory_tick` only. +Forbidden as features: fill price, post-fill states, Path A/B/C/D, markout, future book/trade/inventory, realized PnL, cancel-after-fill. + +Population: **MATCHED paths = 3886** (100% strict coverage) + +Executability gate (three tiers): + +| Grade | Meaning | +|-------|---------| +| `NO_PREFILL_SIGNAL` | P(C) / economic almost unchanged | +| `STATISTICAL_SIGNAL_ONLY` | probability shift, economic improvement insufficient | +| `CANDIDATE_V0_2_SIGNAL` | probability + economic separation — only this enters v0.2 hypothesis | + +Sample-size policy: + +- n < 30 → `LOW_N` (exploratory only) +- n < 100 → `WEAK_EVIDENCE` +- n >= 100 → `USABLE` + +v0.1 strict conclusion: + +> **0 / 19 features** reach `CANDIDATE_V0_2_SIGNAL`. +> Fill 前可观测信号不足以支撑 v0.2 设计。Stage 3 remains **LOCKED**. + +Deferred (requires richer pre-fill event log): + +- `time_since_last_market_event` +- intensity / large trades +- fill-callback `market_event_before_fill` + +Still forbidden: + +- Treat `STATISTICAL_SIGNAL_ONLY` as v0.2 candidate +- Resume probe or design v0.2 strategy without new experiment ID + prefill signal pass + +--- + +## MM_EDGE_EXP_002 — Event-State Observability (2026-08-18) + +**Type:** Data Collection / Observability Experiment +**NOT:** strategy experiment, backtest, optimization, model training + +``` +MM_EDGE_EXP_001 → phenomenon PASS, economic FAIL, prefill FAIL (snapshot) +MM_EDGE_EXP_002 → close observability gap (immutable event ledger) + → Gate 4 only after Gates 1–3 + fill anchors + → (only then) v0.2 hypothesis allowed +``` + +Identity: + +| Field | Value | +|-------|-------| +| Experiment | MM_EDGE_EXP_002 | +| Probe | event_state_v0.1 | +| Strategy | NONE | +| Trading | NO | +| Stage 3 | LOCKED | +| Depends on | MM_EDGE_EXP_001 FROZEN | + +Core change: **Immutable Event Ledger** — raw events first, features offline later. + +Pre-fill window schema (frozen): + +``` +[-5s, fill - 250ms) → all market_event rows +fill_anchor → immutable fill metadata (Gate 4; Phase 1 may have none) +``` + +Success gates: + +1. **Event Completeness** — PASS (smoke) +2. **Temporal Integrity** — PASS (smoke) +3. **Event Coverage** — PASS (smoke) +4. **Predictability** — BLOCKED until frozen sample gates in `MM_EDGE_EXP_002.md` + +Long-run: `EXP-002-RUN-002` / `event-state-probe.service` / trading=NO. + +Frozen before long-run (do not change after seeing more data): + +- Phase 1 dataset: ≥ 7 days AND ≥ 5,000,000 market events +- Gate 4: ≥ 2,000 MATCHED fill_anchors AND ≥ 500 clusters (requires later fill-authorized phase) + +No daily Path C analysis during collection. + +Still forbidden: + +- Resume EXP_001 or modify v0.1 quote logic +- Enable trading under EXP_002 +- Unlock Stage 3 from observability data alone +- Peek at Gate 4 before the frozen sample threshold + +--- + +## MM_EDGE_EXP_002 Phase 1 STOPPED (2026-09-10) + +Human decision: **stop collection**. Do not open a fill-authorized phase. Do not design v0.2. + +| Field | Value | +|-------|-------| +| Run | `EXP-002-RUN-002` | +| Stopped | **2026-09-10T08:45:24Z** | +| systemd | user `event-state-probe` **inactive / disabled** | +| Ledger | `logs/event_state/EXP-002-RUN-002/` (24 files, 18.06 GiB, 2026-08-18 → 2026-09-10 partial) | +| Trading | **NO** | +| Stage 3 | **LOCKED** | +| Gate 4 | **BLOCKED** (no `fill_anchor`) | +| Hypothesis Closure | **NO** | + +Phase 1 volume thresholds are met as an **observability freeze** only. +That does **not** unlock Gate 4 or Stage 3. + +Marker: `logs/event_state/EXP-002-RUN-002/EXP-002-RUN-002.PHASE1_STOPPED.json` + +Still forbidden: + +- Restart `EXP-002-RUN-002` +- Enable trading under EXP_002 +- Path-C / Gate 4 peek on this artifact +- Treat this stop as “no pre-fill signal exists” diff --git a/MM_EDGE_EXP_002.md b/MM_EDGE_EXP_002.md new file mode 100644 index 0000000..a5f752d --- /dev/null +++ b/MM_EDGE_EXP_002.md @@ -0,0 +1,214 @@ +# MM_EDGE_EXP_002 — Event-State Observability Probe + +## Experiment Identity + +``` +Experiment: MM_EDGE_EXP_002 +Type: Data Collection / Observability Experiment +Strategy: NONE +Execution: STOPPED (no trading) +Purpose: Capture immutable pre-fill Event State +Dependency: MM_EDGE_EXP_001 / v0.1 FROZEN +Stage 3: LOCKED +``` + +**NOT:** strategy experiment, backtest, optimization, model training, v0.2 design. + +--- + +## Why EXP_002 Exists + +EXP_001 Prefill Audit conclusion: + +> Under **current snapshot observability** (~1.66s sampled mid/inventory state), no sufficient pre-fill signal was found. + +This must **not** be interpreted as: + +> The market has no pre-fill adverse-selection information. + +EXP_001 strategy is **event-driven**, but observability was **snapshot-driven**. Information between snapshots (e.g. aggressive sweep 100ms before fill) is lost. + +EXP_002 closes the **observability gap**, not the **strategy gap**. + +--- + +## Core Design Principle + +``` +Raw Event > Derived Feature +``` + +Store immutable events. Features are computed offline later. + +--- + +## Immutable Event Ledger + +Each market event records (minimum): + +| Field | Description | +|-------|-------------| +| `exchange_ts_ns` | Exchange event time | +| `local_ts_epoch` / `local_ts` | Local receive time | +| `event_type` | `aggressive_trade`, `book_update`, … | +| `best_bid` / `best_ask` / `mid` / `spread` | Top-of-book | +| `bid_depth_*` / `ask_depth_*` | Depth levels | +| `*_delta` | Depth / spread / mid changes | +| `time_since_last_*` | Event timing state | + +Fill anchor schema (frozen, for Gate 4 when fills exist): + +``` +[-5s, fill - 250ms) → all market_event rows +fill_anchor → immutable fill metadata +``` + +--- + +## Event Categories (Priority) + +1. **Aggressive Trade** — side, qty, notional, large_trade_flag, intensity proxy +2. **Book Depletion** — depth deltas, level removal velocity +3. **Spread / TOB Event** — spread change, bid/ask/mid move +4. **Event Timing** — time_since_last_trade / large_trade / depth_change / spread_change + +--- + +## Success Gates (frozen before collection) + +### Gate 1 — Event Completeness + +For each `fill_anchor`: + +``` +100% reconstructable pre-fill event sequence in [-5s, fill - margin) +``` + +Phase 1 (observability-only): **N/A** until fill anchors exist. + +### Gate 2 — Temporal Integrity + +``` +all event_ts < fill_ts +feature_cutoff = fill_ts - 250ms +``` + +### Gate 3 — Event Coverage + +| Metric | Threshold | +|--------|-----------| +| trade events present | ≥ 99% of sessions with trades | +| book events present | ≥ 99% of sessions with book updates | +| timestamp valid | ≥ 99% rows with exchange_ts_ns or local_ts | + +### Gate 4 — Predictability (BLOCKED until sample freeze + fill anchors) + +Do **not** inspect Path C daily during collection (researcher degrees of freedom). + +Frozen sample thresholds (**set 2026-08-18, before long-run start**): + +**Phase 1 Event-State dataset freeze** (observability-only, no fills): + +| Metric | Minimum | +|--------|---------| +| Calendar span | **≥ 7 days** | +| `market_event` count | **≥ 5,000,000** | +| `aggressive_trade` | **≥ 1,500,000** | +| `book_update` | **≥ 3,000,000** | +| timestamp valid | **≥ 99%** | +| parse_fail_lines | **0** | + +Reaching this freeze **does not** unlock Gate 4. It only freezes the Event-State stream as a research artifact. + +**Gate 4 (requires a later fill-authorized phase, not this systemd job):** + +| Metric | Minimum | +|--------|---------| +| MATCHED `fill_anchor` | **≥ 2,000** | +| `event_cluster_id` | **≥ 500** | +| reconstructable `[-5s, fill−250ms)` | **100%** of MATCHED anchors | +| venue_trade_id + exchange_ts_ns on fill | **100%** | + +Then, **once**: + +``` +Event State → P(Path C) → Economic separation +→ CANDIDATE_V0_2_SIGNAL only if both probability and economic gates pass +``` + +Future fill collection (if ever authorized) **must** write: + +``` +Fill → venue_trade_id → exchange_ts_ns → Event Ledger → [-5s, fill_ts) +``` + +Do not resume snapshot-only fills. + +Only **CANDIDATE_V0_2_SIGNAL** after Gate 4 may enter v0.2 hypothesis design. + +--- + +## Research Chain + +``` +EXP_001 Maker Edge Phenomenon + ↓ + conditional markout exists + ↓ + Economic FAIL + ↓ + Prefill audit (snapshot) → NO SIGNAL + ↓ +EXP_002 Event-State Observability + ↓ + Gate 1–3 PASS? + ↓ + Gate 4 predictability + ↓ + (only then) v0.2 hypothesis +``` + +--- + +## Running + +```bash +cd nautilus_mm +cp .env.example .env # set EXP_002 block +export PYTHONPATH=src +./scripts/run_event_state.sh +``` + +Long-run (systemd, trading=NO): + +``` +LEDGER_RUN_ID=EXP-002-RUN-002 +logs/event_state/EXP-002-RUN-002/ +``` + +```bash +systemctl --user start event-state-probe +./scripts/event_state_status.sh # counts only — not Path C analysis +``` + +**Phase 1 STOPPED 2026-09-10T08:45:24Z** — user unit `event-state-probe` is **disabled**. +Do not start it again on `EXP-002-RUN-002`. Ledger is a frozen observability artifact. + +--- + +## Forbidden + +- Resume EXP_001 probe or modify v0.1 quote logic +- Restart EXP_002 / `EXP-002-RUN-002` collection +- Enable trading under EXP_002 +- Unlock Stage 3 from observability data alone +- Treat weak EXP_001 statistical signals as v0.2 filters +- Claim Hypothesis Closure or design v0.2 without a later fill-authorized Gate 4 + +--- + +## Hypothesis Closure (if Gate 4 also fails) + +> Conditional Maker phenomenon exists, but is not sufficiently predictable pre-fill to be monetizable under this venue/execution model. + +That would be a **strong Research FAIL / Hypothesis Closure** — not "try one more parameter." diff --git a/README.md b/README.md new file mode 100644 index 0000000..a0cfb1c --- /dev/null +++ b/README.md @@ -0,0 +1,158 @@ +# nautilus_mm — Research Freeze / Data Collection Phase + +Freqtrade 保留缠论 / 中低频；**Maker / L2 / Fill 事件**迁到 NautilusTrader。 + +**研究对象：可验证的市场现象（不是策略）。** + +``` +Trading OS + ├─ Market Intelligence (Market Pulse) ← Stage5 才接（Quote Adjustment） + ├─ Execution Reality Layer (本仓库) ← Fill Alpha Dataset + ├─ Freqtrade ← Chan / 中低频 + └─ NautilusTrader ← 事件驱动执行 +``` + +## 冻结研究路径（禁止跳级） + +``` +Stage 0 Data Integrity + ↓ +Stage 1 Fill Alpha ← 当前 + ↓ +Stage 2 Maker Edge Report ← 当前（决策门） + ↓ +Stage 3 Economic Simulation ← LOCKED until Edge PASS + ↓ +Stage 4 Quote Engine + ↓ +Stage 5 Market Regime Adaptation (Market Pulse → Quote Adjustment) +``` + +| Stage | 目标 | 状态 | +|-------|------|------| +| **0** | WS/L2 可信：seq gap / latency / book_age | 探针已记 | +| **1** | Fill Alpha Dataset：真实成交 + 路径 | **进行中** | +| **2** | Maker Edge 决策门：PASS / FAIL / COLLECTING | **进行中** | +| **3** | 经济仿真：quote→fill→inventory→exit（partial/cancel/funding/fee） | **未解锁** | +| **4** | Quote Engine | 未开始 | +| **5** | Regime Adaptation | 未开始 | + +原则：**先证明成交有优势，再谈账户收益，再设计报价。** +禁止现在加：Quote Engine / AI / Market Pulse 交易信号 / 参数优化。 + +### 核心问题（交给 ~2000 真实 fills） + +> 个人开发者在 BTC 永续上，通过被动流动性提供，是否能获得统计优势？ +> 若有：优势来自哪里？ + +可能结果： + +| 情况 | 含义 | 下一步 | +|------|------|--------| +| **A** 全市场 PASS | 稳定被动流动性优势 | Symmetric MM | +| **B** 仅特定状态 PASS | 高波动 / 吸收 / 震荡等 | Event-driven LP（更可能） | +| **C** 全部 FAIL | 普通 Maker edge 不存在 | Cash Carry / Basis / Funding / 跨所 | + +**最值得等待的不是 PASS，而是 Edge 来自哪里。** 第一份 FAIL 也是高价值结果。 + +### 防自我欺骗（已内建） + +1. **Cluster-weighted** — 暴跌 50 笔 Bid ≠ 50 独立样本；同时看 fill-w 与 cluster-w，方向一致才可信 +2. **Matched Mid / Maker Alpha** — `MakerAlpha = Fill Outcome − Market Move`（剥离方向收益） +3. **Adverse Selection** — 成交是否天然站在错误一侧；spread capture 挡不住毒流 + +Stage 3（Economic Simulation）只在 Edge PASS 后做：partial fill、cancel latency、inventory limit、position aging、funding、fee tier → 真实账户收益分布。 + +## Maker Edge Report v0.1 + +```bash +./scripts/analyze.sh # 默认 min-fills=2000 +./scripts/analyze.sh 500 # 早期预览（仍为 COLLECTING） +``` + +报告结构：Data Integrity → Sample Independence → Fill Quality + Benchmark → Adverse Selection（`POSITIVE_EDGE` / `EDGE_AFTER_COST` / `NO_EDGE`）→ Path Attribution → State Stability → Decision。 + +## 快速开始 + +```bash +cd nautilus_mm +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +cp .env.example .env # TESTNET key；代理默认 7897 + +./scripts/run_probe.sh # 攒真实 fills +./scripts/analyze.sh 2000 # 决策门报告 +``` + +探针只记 quote / fill / outcome。`market_state_snapshot` 预留为 null。 + +## 目录 + +``` +nautilus_mm/ +├── configs/ +├── logs/maker_edge/ # Fill Alpha Dataset (jsonl) +├── scripts/ +│ ├── run_probe.sh +│ ├── analyze.sh +│ ├── analyze_maker_edge.py +│ └── record_l2_ccxt.py +└── src/nautilus_mm/ + ├── recorder.py + ├── health.py + ├── book_utils.py + ├── run_live.py + └── strategies/maker_edge_probe.py +``` + +## 与 Freqtrade + +| | Freqtrade | nautilus_mm | +|--|-----------|-------------| +| 用途 | 缠论 / 中低频 | Maker / L2 / Edge 验证 | +| 驱动 | K 线 | order book / fill 事件 | + +## 安全 + +- 默认 `BINANCE_ENVIRONMENT=TESTNET` +- `ENABLE_TRADING=false` 可只订数据 +- **独立 `.venv`**，勿与 freqtrade 混装 + +## 实验冻结（见 [FREEZE.md](FREEZE.md)） + +**三不动：** Quote Logic / 成本模型 / PASS·COLLECTING·FAIL + +| ✅ 冻结期允许 | ❌ Gate 解锁前禁止 | +|-------------|-------------------| +| 数据字段 | 新交易规则 | +| 数据质量检查 | 新过滤条件 | +| 报告解释能力 | 新收益优化参数 | + +Decision：`PASS` / `PARTIAL_PASS` / `COLLECTING` / `FAIL` +观察窗：500 异常 · 2000 初步 · 10000 稳定性；**clusters > fills**。 +第一份报告先看分布（Fill / Cluster / Toxicity），再看 Decision。 + +## 服务器 Data Collection（zun_hk） + +默认： + +| | | +|--|--| +| Host | `jack@jackyu66.com` | +| Key | `~/Project/deploy/zun_hk/id_ed25519_hk` | +| Dir | `/www/Project/nautilus_mm` | +| Experiment | `MM_EDGE_EXP_001` | + +```bash +./scripts/deploy_server.sh + +ssh -i ~/Project/deploy/zun_hk/id_ed25519_hk jack@jackyu66.com +nano /www/Project/nautilus_mm/.env # BINANCE_API_KEY / SECRET +systemctl --user start mm-edge-probe # 用户级 systemd（无需 sudo） +journalctl --user -u mm-edge-probe -f + +./scripts/probe_status.sh +./scripts/pull_report.sh 2000 +``` + +本地短测：`USE_PROXY=true ./scripts/run_probe.sh` diff --git a/STATUS.md b/STATUS.md new file mode 100644 index 0000000..074d5f7 --- /dev/null +++ b/STATUS.md @@ -0,0 +1,205 @@ +# Project Status + +## MM_EDGE_EXP_001 — CLOSED (FROZEN) + +| Dimension | Status | +|-----------|--------| +| Research Phenomenon | **PASS** (conditional markout exists) | +| Economic Edge | **FAIL** | +| Prefill Predictability | **FAIL** (snapshot observability) | +| Strategy | **FROZEN** | +| Execution | **STOPPED** | +| Stage 3 | **LOCKED** | + +EXP_001 Prefill strict conclusion: + +> Under current **snapshot observability** (~1.66s), no CANDIDATE_V0_2_SIGNAL. +> This is **NOT** proof that pre-fill signal does not exist in the market. + +Weak signals (research observations only — **not** trading filters): + +- `depth_total_5`: ±3.8pp Path C separation, Economic gate FAIL +- `obi_change_5s`: +0.00027 USDT/fill economic Δ +- `pre_deteriorated_strict=False`: +0.00116 USDT/fill economic Δ + +Reports: `logs/maker_edge/` — see sections below for detail. + +--- + +## MM_EDGE_EXP_002 — Phase 1 STOPPED (artifact FROZEN) + +| Dimension | Status | +|-----------|--------| +| Type | Data Collection / Observability | +| Strategy | **NONE** | +| Trading | **NO** | +| Execution | **STOPPED** 2026-09-10T08:45:24Z | +| Stage 3 | **LOCKED** | +| Gate 1 | **PASS** (smoke `EXP-002-RUN-001`) | +| Gate 2 | **PASS** | +| Gate 3 | **PASS** | +| Gate 4 | **BLOCKED** (no `fill_anchor`; not opened) | +| Long-run | **EXP-002-RUN-002** STOPPED + systemd **disabled** | +| Hypothesis Closure | **NO** — Gate 4 never ran | + +Human decision 2026-09-10: stop further collection. Phase 1 is an observability artifact only. +This is **not** “market has no pre-fill signal” and **not** a v0.2 unlock. + +`event-state-probe.service` (user): **inactive / disabled**. Do not restart `EXP-002-RUN-002`. + +Ledger: `logs/event_state/EXP-002-RUN-002/` — 24 files, 2026-08-18 → 2026-09-10 (last day partial), **18.06 GiB**. +Event counts not fully censused; size-based estimate ~1.2e7 lines (volume gates clearly exceeded). +`fill_anchor` expected **0** (observability-only). Marker: `EXP-002-RUN-002.PHASE1_STOPPED.json` + +Do **not** Path-C snoop this artifact. Frozen Gate 4 sample gates remain in `MM_EDGE_EXP_002.md`. + +Spec: `MM_EDGE_EXP_002.md` +Smoke: `./scripts/smoke_test_event_state.sh` +Validate: `python scripts/validate_event_ledger.py --dir logs/event_state/EXP-002-RUN-001 --run-id EXP-002-RUN-001` +Logs: `logs/event_state/EXP-002-RUN-001/` + +### EXP-002-RUN-001 Smoke (2026-08-18, ~12 min + restart) + +Host: `jADUtR1041803` | Sessions: **2** (restart test) | Schema: `immutable_event_v1` + +| Check | Result | +|-------|--------| +| Gate 1 Event Completeness | **PASS** (Phase 1 stream; no fill_anchor) | +| Gate 2 Temporal Integrity | **PASS** | +| Gate 3 Event Coverage | **PASS** | +| Restart contract | **PASS** (0 parse fail, 0 dup event_id, seq reset) | +| Gate 4 | **BLOCKED** | + +Write rates: +- `aggressive_trade`: **3.21 / sec** (2300 events) +- `book_update`: **6.09 / sec** (4366 events) +- **total**: **9.30 / sec** (6666 market events) + +Timestamp quality (100% exchange + local present): +- `local − exchange` lag: p50 **112.5ms**, p95 **237.7ms**, p99 **251.4ms**, max **443.4ms** +- `exchange > local + 50ms`: **0** violations + +Event order (recorded, not sorted): +- `exchange_ts_ns` regressions: **1053** (max back **276ms**) — multi-source async; explicit in report + +Raw schema sample (n=200 each): **0% missing** on core fields; **0** hollow book_update. + +Manifest: `logs/event_state/EXP-002-RUN-001/EXP-002-RUN-001.manifest.json` +Report: `logs/event_state/EXP-002-RUN-001/Event_Ledger_Validation.json` + +Long-run identity: **EXP-002-RUN-002** (separate from smoke) — **STOPPED** 2026-09-10. Status: `./scripts/event_state_status.sh` + +--- + +## MM_EDGE_EXP_001 Detail + +**Phase: Prefill Adverse-Selection Attribution v0.1** (probe **STOPPED**) + +| Gate | Status | +|------|--------| +| Research Freeze | **ACTIVE** | +| Probe | **STOPPED** | +| Data Integrity | **PASS** | +| Maker-only | **PASS** (`TAKER=0`) | +| Account Reconciliation (RECON-01) | **PASS** | +| RECON-02 classification | **PASS** | +| Order-level closure (RECON-03) | **PASS** | +| Strict trade-level closure | **FAIL** (Testnet userTrades cutoff — **not** Alpha FAIL) | +| Economic Edge | **FAIL** (current execution economics) | +| Prefill Predictability (v0.1 strict) | **FAIL** (no CANDIDATE_V0_2_SIGNAL) | +| Stage 3 | **LOCKED** | + +## Evidence taxonomy (4777 fills) + +| Class | Count | Grade | +|-------|-------|-------| +| **MATCHED** | 3890 | Order + Trade (dual) | +| **VENUE_CONFIRMED_NO_TRADE_HISTORY** | 876 | Order FILLED, no trade row | +| **VENUE_PARTIAL_ORDER_CANCELED** | 11 | Partial fill + TTL cancel | +| DUPLICATE / MISMATCH / UNCONFIRMED | 0 | — | + +`userTrades` cutoff: **2026-08-17T03:08 UTC** — see `TESTNET_LIMITATIONS.md` + +Matched trade-level: qty residual **0**, |Δt| p50 **72ms** + +## Separation + +``` +MakerAlpha (+0.008%) ≠ Account Δ (−61.66 = fee + realized) +Strict trade FAIL ≠ Maker Edge FAIL +887 orphans = VENUE-HISTORY-CUTOFF (now order-confirmed) +``` + +## Economic Attribution v0.1 (Hard Evidence Population only) + +Population: **MATCHED=3890** +Core report: `logs/maker_edge/Economic_Attribution_v0_1.txt` + +- Matched fills / paths / clusters: **3890 / 3886 / 3334** +- Fee total: **42.42 USDT** +- Realized component: **−8.52 USDT** +- Gross markout @30s: **−1.73 USDT** +- Net attributable @30s: **−52.67 USDT** +- Inventory carry: + - Max `|net BTC|`: **0.0059** + - TW `|net BTC|`: **0.0035** + - Turnover: **3.3316 BTC** + +Counterfactual Attribution: +- Baseline matched net30: **−52.67 USDT** +- Exclude `Path C`: **−21.22 USDT** +- Exclude toxic: **−21.22 USDT** +- Exclude negative states: **−27.64 USDT** + +Interpretation: +- v0.1 已完成 baseline 使命：**Maker markout phenomenon exists, economic edge not established** +- 当前主要拖累不是“假 alpha”，而是 **fee + realized / inventory economics** + +## Metric Reconciliation (MATCHED=3890) +- Return-space MakerAlpha（fill-weighted）：-0.000693% +- Return-space MakerAlpha（notional-weighted）：-0.000816% +- Dollar-space gross markout @30s：-1.730780 USDT +结论：回报口径一致，但“加权方式”导致返回与美元金额的方向差异。 + +## Fee Sensitivity (counterfactual, fee only) +net_attr_30s @fee_factor: + 1.00 → -52.674655 USDT + 0.50 → -31.463582 USDT + 0.25 → -20.858046 USDT + 0.10 → -14.494724 USDT + 0.00 → -10.252510 USDT +结论：即使假设 0 fee，net 仍 < 0，因此“真实拖累”不仅是 fee。 + +## Prefill Adverse-Selection Attribution v0.1 (strict contract) + +Population: **MATCHED paths = 3886** (100% strict-prefill coverage) +Core report: `logs/maker_edge/Prefill_Adverse_Selection_Attribution_v0_1.txt` + +Time contract: `feature_timestamp <= t_fill - 0.25s` +Feature source: sampled `mid_tick` / `inventory_tick` only (no fill-callback leakage) + +Baseline (strict population): +- P(Path C): **33.20%** +- P(Toxic): **34.61%** +- P(Neg30s): **47.43%** +- P(Economic<0): **77.61%** +- Mean net_attr_30s: **−0.0136 USDT/fill** +- Feature age: mean **1681ms**, median **1660ms** + +Conclusion Matrix (auto-grade): +- **CANDIDATE_V0_2_SIGNAL**: **0 / 19 features** +- **STATISTICAL_SIGNAL_ONLY**: spread_change_5s (Toxic/Neg30s), inventory/inventory_skew (Economic only) +- **NO_PREFILL_SIGNAL**: all others under strict gate + +Unavailable under strict contract (deferred to EXP_002): +- `time_since_last_market_event`, intensity, large trades, fill-callback `market_event_before_fill` + +## Forbidden (both experiments) + +- Resume EXP_001 probe without new experiment ID +- Restart EXP_002 / `EXP-002-RUN-002` collection +- Enable trading under EXP_002 +- Reclassify VENUE_CONFIRMED as MATCHED +- Unlock Stage 3 on observability or attribution alone +- Treat STATISTICAL_SIGNAL_ONLY as v0.2 candidate +- Design v0.2 or claim Hypothesis Closure without a later fill-authorized Gate 4 diff --git a/TESTNET_LIMITATIONS.md b/TESTNET_LIMITATIONS.md new file mode 100644 index 0000000..1b00f03 --- /dev/null +++ b/TESTNET_LIMITATIONS.md @@ -0,0 +1,55 @@ +# Testnet Limitations — MM_EDGE_EXP_001 + +Observed on **Binance USDT-M Futures Testnet** during Data Collection (Aug 2026). + +## userTrades history cutoff + +``` +Endpoint: GET /fapi/v1/userTrades +Observed max trade time (UTC): 2026-08-17T03:08:23 +Behavior: Pagination returns 3890 rows; no further trades via time/fromId + after cutoff, even while probe continues to produce fills until + 2026-08-18. +``` + +**Impact:** Local jsonl fill count can exceed paginated `userTrades` count. +This is **not** evidence of duplicate local logging or fake fills. + +## Order API remains available + +``` +Endpoint: GET /fapi/v1/order?orderId= +Behavior: Post-cutoff orders return status=FILLED, executedQty, avgPrice + while userTrades?orderId= returns 0 rows for the same orderId. +``` + +RECON-03 classifies these as: + +``` +VENUE_CONFIRMED_NO_TRADE_HISTORY +``` + +Evidence grade: **Order only** (not dual Order+Trade). + +## Income ledger continues + +`GET /fapi/v1/income` continues to record COMMISSION / REALIZED_PNL after +the userTrades cutoff. Account reconciliation (RECON-01) uses income, not +userTrades alone. + +## Implications for future runs + +1. **Real-time immutable ledger** — persist on every `OrderFilled`: + `venue_trade_id`, `venue_order_id`, `liquidity_side`, `commission`, + `exchange_ts`, `local_ts`. Do not rely on post-hoc userTrades backfill. + +2. **Reports must use evidence taxonomy** — never compare raw fill count to + userTrades count without cutoff annotation. + +3. **Strict trade-level closure** may remain FAIL on Testnet while + **order-level closure** can still PASS. + +## Maker-only constraint + +Post-only orders rejected with `-5022` when they would take. Verified: +`TAKER_FILLED_COUNT = 0` on all 3890 trades inside userTrades window. diff --git a/configs/event_state_probe.yaml b/configs/event_state_probe.yaml new file mode 100644 index 0000000..2a4ae92 --- /dev/null +++ b/configs/event_state_probe.yaml @@ -0,0 +1,31 @@ +# MM_EDGE_EXP_002 — Event-State Observability Probe +# Data collection only. NO trading. + +experiment_id: MM_EDGE_EXP_002 +probe_version: event_state_v0.1 +experiment_type: Event-State Observability Probe +strategy: NONE +execution_trading: false +depends_on: MM_EDGE_EXP_001 + +quote: frozen +fee: frozen +exchange: frozen + +trader_id: EVENT-STATE-002 +symbol: BTCUSDT-PERP +account_type: USDT_FUTURES +environment: TESTNET + +book_depth: 10 +prefill_window_sec: 5.0 +prefill_margin_sec: 0.25 +large_trade_qty: 0.1 +log_every_book_delta: true + +log_dir: logs/event_state + +# Gate 3 thresholds (frozen) +gate3_trade_coverage_min: 0.99 +gate3_book_coverage_min: 0.99 +gate3_timestamp_valid_min: 0.99 diff --git a/configs/maker_edge_probe.yaml b/configs/maker_edge_probe.yaml new file mode 100644 index 0000000..4aaa14b --- /dev/null +++ b/configs/maker_edge_probe.yaml @@ -0,0 +1,24 @@ +# Reference config (env vars in .env take precedence for secrets) +# Experiment identity — do not change mid-run to "make report look better" + +experiment_id: MM_EDGE_EXP_001 +probe_version: probe_v0.1 +quote: frozen +fee: frozen +exchange: frozen + +trader_id: MAKER-EDGE-001 +symbol: BTCUSDT-PERP +account_type: USDT_FUTURES +environment: TESTNET # TESTNET | LIVE + +order_qty: "0.001" +book_depth: 10 +quote_offset_ticks: 1 +max_quotes: 1 +cooldown_secs: 60 +obi_enter: 0.25 +enable_trading: true + +proxy: http://127.0.0.1:7897 +log_dir: logs/maker_edge diff --git a/deploy/event-state-probe.service b/deploy/event-state-probe.service new file mode 100644 index 0000000..e29f83d --- /dev/null +++ b/deploy/event-state-probe.service @@ -0,0 +1,32 @@ +[Unit] +Description=MM_EDGE_EXP_002 Event-State Observability Probe (no trading) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=jack +Group=jack +WorkingDirectory=/www/Project/nautilus_mm +Environment=PYTHONPATH=/www/Project/nautilus_mm/src +EnvironmentFile=-/www/Project/nautilus_mm/.env +Environment=ENABLE_TRADING=false +Environment=EXPERIMENT_ID=MM_EDGE_EXP_002 +Environment=PROBE_VERSION=event_state_v0.1 +Environment=LEDGER_RUN_ID=EXP-002-RUN-002 +Environment=EVENT_STATE_LOG_DIR=/www/Project/nautilus_mm/logs/event_state/EXP-002-RUN-002 +Environment=HTTP_PROXY= +Environment=HTTPS_PROXY= +Environment=http_proxy= +Environment=https_proxy= +ExecStart=/www/Project/nautilus_mm/scripts/run_event_state.sh +Restart=always +RestartSec=10 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=event-state-probe +NoNewPrivileges=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/deploy/event-state-probe.user.service b/deploy/event-state-probe.user.service new file mode 100644 index 0000000..1b654f4 --- /dev/null +++ b/deploy/event-state-probe.user.service @@ -0,0 +1,30 @@ +# 用户级 systemd — ~/.config/systemd/user/event-state-probe.service +# MM_EDGE_EXP_002 Phase 1 long-term Event-State collection. Trading = NO. +[Unit] +Description=MM_EDGE_EXP_002 Event-State Observability Probe (no trading) +After=network-online.target + +[Service] +Type=simple +WorkingDirectory=/www/Project/nautilus_mm +Environment=PYTHONPATH=/www/Project/nautilus_mm/src +EnvironmentFile=-/www/Project/nautilus_mm/.env +# Layer 0 (systemd): override .env — EXP_002 never trades +Environment=ENABLE_TRADING=false +Environment=EXPERIMENT_ID=MM_EDGE_EXP_002 +Environment=PROBE_VERSION=event_state_v0.1 +Environment=LEDGER_RUN_ID=EXP-002-RUN-002 +Environment=EVENT_STATE_LOG_DIR=/www/Project/nautilus_mm/logs/event_state/EXP-002-RUN-002 +Environment=HTTP_PROXY= +Environment=HTTPS_PROXY= +Environment=http_proxy= +Environment=https_proxy= +ExecStart=/www/Project/nautilus_mm/scripts/run_event_state.sh +Restart=always +RestartSec=10 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=event-state-probe + +[Install] +WantedBy=default.target diff --git a/deploy/mm-edge-probe.service b/deploy/mm-edge-probe.service new file mode 100644 index 0000000..c6a375f --- /dev/null +++ b/deploy/mm-edge-probe.service @@ -0,0 +1,28 @@ +[Unit] +Description=Maker Edge Probe MM_EDGE_EXP_001 (Research Freeze / Data Collection) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=jack +Group=jack +WorkingDirectory=/www/Project/nautilus_mm +Environment=PYTHONPATH=/www/Project/nautilus_mm/src +EnvironmentFile=-/www/Project/nautilus_mm/.env +# 服务器直连交易所 +Environment=HTTP_PROXY= +Environment=HTTPS_PROXY= +Environment=http_proxy= +Environment=https_proxy= +ExecStart=/www/Project/nautilus_mm/.venv/bin/python -m nautilus_mm.run_live +Restart=always +RestartSec=10 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=mm-edge-probe +NoNewPrivileges=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/deploy/mm-edge-probe.user.service b/deploy/mm-edge-probe.user.service new file mode 100644 index 0000000..5da9b71 --- /dev/null +++ b/deploy/mm-edge-probe.user.service @@ -0,0 +1,21 @@ +# 用户级 systemd（无需 sudo）— 安装到 ~/.config/systemd/user/ +[Unit] +Description=Maker Edge Probe MM_EDGE_EXP_001 +After=network-online.target + +[Service] +Type=simple +WorkingDirectory=/www/Project/nautilus_mm +Environment=PYTHONPATH=/www/Project/nautilus_mm/src +EnvironmentFile=-/www/Project/nautilus_mm/.env +Environment=HTTP_PROXY= +Environment=HTTPS_PROXY= +ExecStart=/www/Project/nautilus_mm/.venv/bin/python -m nautilus_mm.run_live +Restart=always +RestartSec=10 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=mm-edge-probe + +[Install] +WantedBy=default.target diff --git a/deploy_server.sh b/deploy_server.sh new file mode 100755 index 0000000..faa57e2 --- /dev/null +++ b/deploy_server.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# 部署 MM_EDGE_EXP_001 → jack@jackyu66.com:/www/Project/nautilus_mm +# +# 默认： +# SSH_HOST=jack@jackyu66.com +# SSH_KEY=~/Project/deploy/zun_hk/id_ed25519_hk +# REMOTE_DIR=/www/Project/nautilus_mm +# +# 覆盖：export SSH_HOST=... SSH_KEY=... REMOTE_DIR=... +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SSH_HOST="${SSH_HOST:-jack@jackyu66.com}" +SSH_KEY="${SSH_KEY:-${HOME}/Project/deploy/zun_hk/id_ed25519_hk}" +REMOTE_DIR="${REMOTE_DIR:-/www/Project/nautilus_mm}" + +if [[ ! -f "$SSH_KEY" ]]; then + echo "SSH key not found: $SSH_KEY" + exit 1 +fi +chmod 400 "$SSH_KEY" 2>/dev/null || true + +SSH_OPTS=(-i "$SSH_KEY" -o StrictHostKeyChecking=accept-new) +SSH=(ssh "${SSH_OPTS[@]}" "$SSH_HOST") +RSYNC_E="ssh ${SSH_OPTS[*]}" + +echo "==> stop remote probe before sync (if running)" +"${SSH[@]}" "systemctl --user stop mm-edge-probe 2>/dev/null || true" + +echo "==> sync $ROOT → $SSH_HOST:$REMOTE_DIR" +"${SSH[@]}" "mkdir -p '$REMOTE_DIR' '$REMOTE_DIR/logs/maker_edge'" +rsync -avz --delete \ + -e "$RSYNC_E" \ + --exclude '.venv' \ + --exclude '__pycache__' \ + --exclude '*.pyc' \ + --exclude 'logs/maker_edge/*.jsonl' \ + --exclude 'logs/maker_edge/*.txt' \ + --exclude 'logs/maker_edge_smoke' \ + --exclude '.env' \ + "$ROOT/" "$SSH_HOST:$REMOTE_DIR/" + +echo "==> remote setup (uv venv + user systemd)" +"${SSH[@]}" "REMOTE_DIR='$REMOTE_DIR' bash -s" <<'REMOTE' +set -euo pipefail +export PATH="$HOME/.local/bin:$PATH" +cd "$REMOTE_DIR" +if [[ ! -f .env ]]; then + cp .env.example .env + { + echo "" + echo "# Server Data Collection — MM_EDGE_EXP_001" + echo "EXPERIMENT_ID=MM_EDGE_EXP_001" + echo "PROBE_VERSION=probe_v0.1" + echo "EXCHANGE_NAME=binance_usdm" + echo "BINANCE_ENVIRONMENT=TESTNET" + echo "ENABLE_TRADING=false" + echo "QUOTE_TTL_SECS=30" + echo "MAX_ABS_INVENTORY=0.005" + echo "HTTP_PROXY=" + echo "HTTPS_PROXY=" + echo "MAKER_EDGE_LOG_DIR=${REMOTE_DIR}/logs/maker_edge" + } >> .env + echo "CREATED .env — fill BINANCE_API_KEY / BINANCE_API_SECRET" +else + echo ".env exists — left untouched" +fi + +if [[ ! -x "$HOME/.local/bin/uv" ]]; then + curl -LsSf https://astral.sh/uv/install.sh | sh +fi +uv python install 3.12 +rm -rf .venv +uv venv .venv --python 3.12 +uv pip install -r requirements.txt --python .venv/bin/python + +mkdir -p "$HOME/.config/systemd/user" +sed -e "s|/www/Project/nautilus_mm|${REMOTE_DIR}|g" \ + deploy/mm-edge-probe.user.service > "$HOME/.config/systemd/user/mm-edge-probe.service" +systemctl --user daemon-reload +systemctl --user enable mm-edge-probe.service +loginctl enable-linger "$(whoami)" 2>/dev/null || true +echo "User systemd installed (not started — fill keys first)." +echo " nano $REMOTE_DIR/.env" +echo " systemctl --user start mm-edge-probe" +echo " journalctl --user -u mm-edge-probe -f" +REMOTE + +echo "" +echo "==> done" +echo "1) ssh -i $SSH_KEY $SSH_HOST" +echo "2) nano $REMOTE_DIR/.env # TESTNET keys" +echo "3) systemctl --user start mm-edge-probe" +echo "4) ./scripts/probe_status.sh" +echo "5) ./scripts/pull_report.sh" diff --git a/economic_attribution_v0_1.py b/economic_attribution_v0_1.py new file mode 100644 index 0000000..416fed4 --- /dev/null +++ b/economic_attribution_v0_1.py @@ -0,0 +1,533 @@ +#!/usr/bin/env python3 +""" +Economic Attribution v0.1 + +Hard Evidence Population only: + MATCHED = Local Fill ↔ Venue Trade dual evidence + +Purpose: + Economic Attribution only. + No strategy modification. + No live execution. + No economic simulation. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from collections import defaultdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +from reconcile_fills import load_local_fills, match, normalize_local, normalize_venue # noqa: E402 + + +def _load_jsonl_df(log_dir: Path) -> pd.DataFrame: + rows: list[dict[str, Any]] = [] + for f in sorted(log_dir.glob("*.jsonl")): + if f.name.startswith(("Account_", "Maker_", "RECON")): + continue + for line in f.open(): + try: + e = json.loads(line) + except Exception: + continue + if isinstance(e, dict): + rows.append(e) + return pd.DataFrame(rows) + + +def _parse_fill_context(df: pd.DataFrame) -> pd.DataFrame: + if df.empty or "fill_context" not in df.columns: + return pd.DataFrame(columns=["fill_id"]) + rows = [] + for _, r in df.iterrows(): + ctx = r.get("fill_context") + if not isinstance(ctx, dict): + continue + rows.append( + { + "fill_id": r.get("fill_id"), + "market_event_before_fill": ctx.get("market_event_before_fill"), + "trade_imbalance_5s": ctx.get("trade_imbalance_5s"), + "price_velocity_5s": ctx.get("price_velocity_5s"), + "fill_type": ctx.get("fill_type"), + } + ) + return pd.DataFrame(rows) + + +def _fav_ret(side: pd.Series, fill: pd.Series, px: pd.Series) -> pd.Series: + raw = (pd.to_numeric(px, errors="coerce") - pd.to_numeric(fill, errors="coerce")) / pd.to_numeric( + fill, errors="coerce" + ) + return pd.Series(np.where(side == "long", raw, -raw), index=side.index) + + +def _cluster_weight(frame: pd.DataFrame) -> pd.Series: + cnt = frame.groupby("event_cluster_id")["event_cluster_id"].transform("count") + return 1.0 / cnt.clip(lower=1) + + +def _pct(v: float | None) -> str: + if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))): + return "n/a" + return f"{v * 100:.4f}%" + + +def _num(v: float | None, digits: int = 4) -> str: + if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))): + return "n/a" + return f"{v:.{digits}f}" + + +def _mean(s: pd.Series) -> float | None: + s = pd.to_numeric(s, errors="coerce").dropna() + return None if s.empty else float(s.mean()) + + +def _sum(s: pd.Series) -> float: + s = pd.to_numeric(s, errors="coerce").fillna(0.0) + return float(s.sum()) + + +def _weighted_mean(v: pd.Series, w: pd.Series) -> float | None: + vv = pd.to_numeric(v, errors="coerce") + ww = pd.to_numeric(w, errors="coerce").fillna(0.0) + mask = vv.notna() & ww.notna() + vv = vv[mask] + ww = ww[mask] + if vv.empty or float(ww.sum()) == 0.0: + return None + return float((vv * ww).sum() / ww.sum()) + + +def _prepare_paths(paths: pd.DataFrame) -> pd.DataFrame: + paths = paths.copy() + if "max_price" in paths.columns and "min_price" in paths.columns and "fill_price" in paths.columns: + rng = (pd.to_numeric(paths["max_price"], errors="coerce") - pd.to_numeric(paths["min_price"], errors="coerce")) / pd.to_numeric( + paths["fill_price"], errors="coerce" + ) + med = float(rng.dropna().median()) if rng.notna().any() else 0.0 + paths["vol_bucket"] = np.where(rng >= med, "high_vol", "low_vol") + if "price_velocity_5s" in paths.columns and pd.to_numeric(paths["price_velocity_5s"], errors="coerce").notna().any(): + v = pd.to_numeric(paths["price_velocity_5s"], errors="coerce") + thr = float(v.abs().median()) * 0.5 + paths["trend_bucket"] = np.where(v > thr, "trend_up", np.where(v < -thr, "trend_down", "range")) + if "spread" in paths.columns and "fill_price" in paths.columns and pd.to_numeric(paths["spread"], errors="coerce").notna().any(): + sp = pd.to_numeric(paths["spread"], errors="coerce") / pd.to_numeric(paths["fill_price"], errors="coerce") + med = float(sp.dropna().median()) if sp.notna().any() else 0.0 + paths["liq_bucket"] = np.where(sp <= med, "tight_spread", "wide_spread") + paths["toxicity_bucket"] = np.where(paths["path_type"].astype(str).str.startswith("C"), "toxic", "non_toxic") + return paths + + +def _inventory_metrics(matched: pd.DataFrame) -> dict[str, float | None]: + if matched.empty: + return {} + g = matched.sort_values("venue_time_ms").copy() + g["signed_qty"] = np.where(g["side"] == "long", g["qty"], -g["qty"]) + g["net_btc"] = g["signed_qty"].cumsum() + g["abs_net_btc"] = g["net_btc"].abs() + times = pd.to_numeric(g["venue_time_ms"], errors="coerce").astype("float64") / 1000.0 + dt = times.shift(-1) - times + dt = dt.fillna(0.0).clip(lower=0.0) + total_t = float(dt.sum()) + tw_abs = float((g["abs_net_btc"] * dt).sum() / total_t) if total_t > 0 else None + tw_signed = float((g["net_btc"] * dt).sum() / total_t) if total_t > 0 else None + return { + "max_net_btc": float(g["net_btc"].max()), + "min_net_btc": float(g["net_btc"].min()), + "max_abs_net_btc": float(g["abs_net_btc"].max()), + "avg_abs_net_btc_per_fill": float(g["abs_net_btc"].mean()), + "time_weighted_abs_net_btc": tw_abs, + "time_weighted_signed_net_btc": tw_signed, + "long_qty": float(g.loc[g["signed_qty"] > 0, "signed_qty"].sum()), + "short_qty": float((-g.loc[g["signed_qty"] < 0, "signed_qty"]).sum()), + "turnover_btc": float(g["qty"].sum()), + } + + +def _bucket_table(paths: pd.DataFrame, bucket: str, title: str) -> list[dict[str, Any]]: + if bucket not in paths.columns or paths.empty: + return [] + rows = [] + for key, grp in paths.groupby(bucket): + notional = grp["notional_usdt"].sum() + clusters = grp["event_cluster_id"].nunique() + rows.append( + { + "dimension": title, + "bucket": str(key), + "fills": int(len(grp)), + "clusters": int(clusters), + "btc_qty": float(grp["qty"].sum()), + "notional_usdt": float(notional), + "fee_usdt": float(grp["commission_usdt"].sum()), + "fee_per_fill": float(grp["commission_usdt"].mean()) if len(grp) else None, + "fee_per_btc": float(grp["commission_usdt"].sum() / grp["qty"].sum()) if grp["qty"].sum() else None, + "markout_1s": _weighted_mean(grp["markout_1s"], grp["notional_usdt"]), + "markout_5s": _weighted_mean(grp["markout_5s"], grp["notional_usdt"]), + "markout_10s": _weighted_mean(grp["markout_10s"], grp["notional_usdt"]), + "markout_30s": _weighted_mean(grp["markout_30s"], grp["notional_usdt"]), + "markout_300s": _weighted_mean(grp["markout_300s"], grp["notional_usdt"]), + "gross_markout_30s_usdt": float(grp["gross_markout_30s_usdt"].sum()), + "realized_pnl_usdt": float(grp["realized_pnl_usdt"].sum()), + "net_attr_30s_usdt": float(grp["net_attr_30s_usdt"].sum()), + } + ) + rows.sort(key=lambda x: (-x["fills"], x["bucket"])) + return rows + + +def _counterfactual(base: pd.DataFrame, exclude_col: str, exclude_values: set[str], label: str) -> dict[str, Any]: + kept = base[~base[exclude_col].astype(str).isin(exclude_values)].copy() + return { + "name": label, + "fills": int(len(kept)), + "clusters": int(kept["event_cluster_id"].nunique()) if not kept.empty else 0, + "btc_qty": float(kept["qty"].sum()) if not kept.empty else 0.0, + "fee_usdt": float(kept["commission_usdt"].sum()) if not kept.empty else 0.0, + "gross_markout_30s_usdt": float(kept["gross_markout_30s_usdt"].sum()) if not kept.empty else 0.0, + "realized_pnl_usdt": float(kept["realized_pnl_usdt"].sum()) if not kept.empty else 0.0, + "net_attr_30s_usdt": float(kept["net_attr_30s_usdt"].sum()) if not kept.empty else 0.0, + "markout_30s": _weighted_mean(kept["markout_30s"], kept["notional_usdt"]), + } + + +def main() -> int: + ap = argparse.ArgumentParser(description="Economic Attribution v0.1 (MATCHED only)") + ap.add_argument("--dir", default=str(ROOT / "logs" / "maker_edge")) + ap.add_argument("--out", default=str(ROOT / "logs" / "maker_edge" / "Economic_Attribution_v0_1.txt")) + ap.add_argument("--recon03", default=str(ROOT / "logs" / "maker_edge" / "RECONCILIATION_03.json")) + ap.add_argument("--account", default=str(ROOT / "logs" / "maker_edge" / "Account_Reconciliation.json")) + ap.add_argument("--venue-trades", default=str(ROOT / "logs" / "maker_edge" / "venue_trades.json")) + args = ap.parse_args() + + log_dir = Path(args.dir) + df = _load_jsonl_df(log_dir) + fills = df[df["event"] == "fill"].copy() + paths = df[df["event"] == "fill_path"].copy() + inv = df[df["event"] == "inventory_tick"].copy() + + venue_trades = json.loads(Path(args.venue_trades).read_text()) + local_fills_raw = load_local_fills(log_dir) + locals_norm = [normalize_local(e, i) for i, e in enumerate(local_fills_raw)] + venues_norm = [normalize_venue(t, i) for i, t in enumerate(venue_trades)] + recon = match(locals_norm, venues_norm) + matched_fill_ids = {m["local"]["fill_id"] for m in recon["matched"]} + matched_trade_ids = {m["venue"]["venue_trade_id"] for m in recon["matched"]} + + fills = fills[fills["fill_id"].isin(matched_fill_ids)].copy() + paths = paths[paths["fill_id"].isin(matched_fill_ids)].copy() + + fc = _parse_fill_context(fills) + meta_cols = [ + c + for c in [ + "fill_id", + "side", + "fill_price", + "spread", + "spread_capture_pct", + "obi", + "trade_imbalance", + "bid_depth_5", + "ask_depth_5", + "book_age_ms", + "inventory", + "inventory_time", + "inventory_skew", + "pre_5s_deteriorated", + "mid", + "event_cluster_id", + "pair", + ] + if c in fills.columns + ] + meta = fills.drop_duplicates("fill_id")[meta_cols] + paths = paths.merge(meta, on="fill_id", how="left", suffixes=("", "_f")) + for col in ["event_cluster_id", "side", "fill_price", "spread_capture_pct", "mid"]: + alt = f"{col}_f" + if alt in paths.columns: + if col not in paths.columns: + paths[col] = paths[alt] + else: + paths[col] = paths[col].fillna(paths[alt]) + if not fc.empty: + paths = paths.merge(fc, on="fill_id", how="left") + paths = _prepare_paths(paths) + + venue = pd.DataFrame(venues_norm) + venue = venue[venue["venue_trade_id"].isin(matched_trade_ids)].copy() + venue = venue.rename( + columns={ + "venue_trade_id": "trade_id_link", + "venue_order_id": "venue_order_id", + "qty": "qty", + "px": "venue_price", + "ts": "venue_ts", + } + ) + raw_v = pd.DataFrame(venue_trades) + raw_v["trade_id_link"] = raw_v["id"].astype(str) + raw_v["venue_order_id"] = raw_v["orderId"].astype(str) + raw_v["commission_usdt"] = pd.to_numeric(raw_v["commission"], errors="coerce") + raw_v["realized_pnl_usdt"] = pd.to_numeric(raw_v["realizedPnl"], errors="coerce").fillna(0.0) + raw_v["venue_time_ms"] = pd.to_numeric(raw_v["time"], errors="coerce") + raw_v["qty"] = pd.to_numeric(raw_v["qty"], errors="coerce") + raw_v["venue_price"] = pd.to_numeric(raw_v["price"], errors="coerce") + raw_v["side"] = np.where(raw_v["buyer"].astype(bool), "long", "short") + raw_v = raw_v[raw_v["trade_id_link"].isin(matched_trade_ids)].copy() + + matched_map = pd.DataFrame( + [ + { + "fill_id": m["local"]["fill_id"], + "trade_id_link": m["venue"]["venue_trade_id"], + "venue_order_id": m["venue"]["venue_order_id"], + } + for m in recon["matched"] + ] + ) + + paths = paths.merge( + matched_map.merge( + raw_v[ + [ + "trade_id_link", + "venue_order_id", + "commission_usdt", + "realized_pnl_usdt", + "venue_time_ms", + "qty", + "venue_price", + "side", + ] + ], + on=["trade_id_link", "venue_order_id"], + how="left", + ), + on="fill_id", + how="left", + suffixes=("", "_venue"), + ) + + paths["fill_price"] = pd.to_numeric(paths["fill_price"], errors="coerce") + paths["qty"] = pd.to_numeric(paths["qty"], errors="coerce") + paths["notional_usdt"] = paths["fill_price"] * paths["qty"] + for sec, col in [(1, "after_1s_price"), (5, "after_5s_price"), (10, "after_10s_price"), (30, "after_30s_price"), (300, "after_5m_price")]: + paths[f"markout_{sec}s"] = _fav_ret(paths["side"], paths["fill_price"], paths[col]) + paths["gross_markout_30s_usdt"] = paths["notional_usdt"] * paths["markout_30s"] + paths["net_attr_30s_usdt"] = ( + paths["gross_markout_30s_usdt"] + - pd.to_numeric(paths["commission_usdt"], errors="coerce").fillna(0.0) + + pd.to_numeric(paths["realized_pnl_usdt"], errors="coerce").fillna(0.0) + ) + + inventory_metrics = _inventory_metrics( + raw_v[ + ["venue_time_ms", "side", "qty", "commission_usdt", "realized_pnl_usdt", "venue_order_id", "trade_id_link"] + ].copy() + ) + + n_matched_paths = len(paths) + n_matched_fills = len(fills) + n_matched_clusters = int(fills["event_cluster_id"].nunique()) if not fills.empty else 0 + cluster_w = _cluster_weight(paths) if not paths.empty and "event_cluster_id" in paths.columns else pd.Series(dtype=float) + + horizon_rows = [] + for sec in (1, 5, 10, 30, 300): + col = f"markout_{sec}s" + valid = paths[col].notna() + sub = paths[valid] + w = sub["notional_usdt"] + horizon_rows.append( + { + "horizon": f"{sec}s", + "n": int(len(sub)), + "fill_w": _weighted_mean(sub[col], w), + "cluster_w": _weighted_mean(sub[col], _cluster_weight(sub) if not sub.empty else pd.Series(dtype=float)), + "gross_usdt": float((sub["notional_usdt"] * sub[col]).sum()) if not sub.empty else 0.0, + } + ) + + fee_total = float(paths["commission_usdt"].sum()) + realized_total = float(paths["realized_pnl_usdt"].sum()) + gross_30_total = float(paths["gross_markout_30s_usdt"].sum()) + net_attr_30_total = float(paths["net_attr_30s_usdt"].sum()) + total_qty = float(paths["qty"].sum()) + total_notional = float(paths["notional_usdt"].sum()) + + bucket_rows: list[dict[str, Any]] = [] + for col, title in [ + ("path_type", "PathType"), + ("toxicity_bucket", "Toxicity"), + ("vol_bucket", "Volatility"), + ("liq_bucket", "Spread"), + ("trend_bucket", "Trend"), + ("market_event_before_fill", "FillContext"), + ]: + bucket_rows.extend(_bucket_table(paths, col, title)) + bucket_df = pd.DataFrame(bucket_rows) + + negative_states: set[str] = set() + if not bucket_df.empty: + neg = bucket_df[(bucket_df["dimension"] != "PathType") & (bucket_df["markout_30s"] < 0)] + negative_states = set(neg["bucket"].astype(str)) + + counterfactuals = [ + { + "name": "BASELINE", + "fills": int(len(paths)), + "clusters": int(paths["event_cluster_id"].nunique()) if not paths.empty else 0, + "btc_qty": total_qty, + "fee_usdt": fee_total, + "gross_markout_30s_usdt": gross_30_total, + "realized_pnl_usdt": realized_total, + "net_attr_30s_usdt": net_attr_30_total, + "markout_30s": _weighted_mean(paths["markout_30s"], paths["notional_usdt"]), + }, + _counterfactual(paths, "path_type", {"C_toxic"}, "EXCLUDE_PATH_C"), + _counterfactual(paths, "toxicity_bucket", {"toxic"}, "EXCLUDE_TOXIC"), + _counterfactual(paths, "market_event_before_fill", negative_states, "EXCLUDE_NEGATIVE_STATE"), + ] + + account = json.loads(Path(args.account).read_text()) if Path(args.account).exists() else {} + recon03 = json.loads(Path(args.recon03).read_text()) if Path(args.recon03).exists() else {} + + out_txt = Path(args.out) + out_json = out_txt.with_suffix(".json") + lines: list[str] = [] + + def p(s: str = "") -> None: + lines.append(s) + print(s) + + p("=" * 72) + p("Economic Attribution v0.1") + p("=" * 72) + p("Experiment: MM_EDGE_EXP_001") + p("Population: MATCHED=3890") + p("Strategy: v0.1 FROZEN") + p("Execution: STOPPED") + p("Stage3: LOCKED") + p("Purpose: Economic Attribution only.") + p("No strategy modification. No live execution. No economic simulation.") + p() + p("Layer 1 — Hard Economic Evidence") + p("-" * 40) + p(f"Matched fills: {n_matched_fills}") + p(f"Matched paths: {n_matched_paths}") + p(f"Matched clusters: {n_matched_clusters}") + p(f"Fee total: {_num(fee_total, 6)} USDT") + p(f"Fee / fill: {_num(fee_total / max(n_matched_paths, 1), 6)} USDT") + p(f"Fee / BTC: {_num(fee_total / max(total_qty, 1e-12), 6)} USDT") + p(f"Fee / cluster: {_num(fee_total / max(n_matched_clusters, 1), 6)} USDT") + p(f"Realized component: {_num(realized_total, 6)} USDT") + p(f"Gross markout @30s: {_num(gross_30_total, 6)} USDT") + p(f"Net attributable @30s: {_num(net_attr_30_total, 6)} USDT") + p() + p("Markout by horizon (MATCHED only)") + p("-" * 40) + for row in horizon_rows: + p( + f"{row['horizon']:>5} n={row['n']:4d} fill-w={_pct(row['fill_w'])} " + f"cluster-w={_pct(row['cluster_w'])} gross={_num(row['gross_usdt'], 6)} USDT" + ) + p() + p("Inventory carry / exposure") + p("-" * 40) + p(f"Max net BTC: {_num(inventory_metrics.get('max_net_btc'), 6)}") + p(f"Min net BTC: {_num(inventory_metrics.get('min_net_btc'), 6)}") + p(f"Max |net BTC|: {_num(inventory_metrics.get('max_abs_net_btc'), 6)}") + p(f"Average |net BTC|: {_num(inventory_metrics.get('avg_abs_net_btc_per_fill'), 6)}") + p(f"TW |net BTC|: {_num(inventory_metrics.get('time_weighted_abs_net_btc'), 6)}") + p(f"TW signed net BTC: {_num(inventory_metrics.get('time_weighted_signed_net_btc'), 6)}") + p(f"Long qty / Short qty: {_num(inventory_metrics.get('long_qty'), 6)} / {_num(inventory_metrics.get('short_qty'), 6)} BTC") + p(f"Inventory turnover: {_num(inventory_metrics.get('turnover_btc'), 6)} BTC") + p() + p("Slices (weighted by notional, MATCHED only)") + p("-" * 40) + for dim in ["PathType", "Toxicity", "Volatility", "Spread", "Trend", "FillContext"]: + sub = bucket_df[bucket_df["dimension"] == dim].copy() + if sub.empty: + continue + p(dim) + for _, r in sub.sort_values(["fills", "bucket"], ascending=[False, True]).iterrows(): + p( + f" {r['bucket']}: n={int(r['fills'])} clusters={int(r['clusters'])} " + f"fee={_num(r['fee_usdt'], 4)} gross30={_num(r['gross_markout_30s_usdt'], 4)} " + f"realized={_num(r['realized_pnl_usdt'], 4)} net30={_num(r['net_attr_30s_usdt'], 4)} " + f"m30={_pct(r['markout_30s'])}" + ) + p() + p("Layer 2 — Evidence Extension (excluded from core conclusion)") + p("-" * 40) + p(f"VENUE_CONFIRMED_NO_TRADE_HISTORY: {recon03.get('venue_confirmed_no_trade_history', 'n/a')}") + p(f"VENUE_PARTIAL_ORDER_CANCELED: {recon03.get('venue_partial_order_canceled', 'n/a')}") + p("These rows are order-confirmed, but not part of the Hard Evidence Population.") + p() + p("Layer 3 — Counterfactual Attribution (NOT backtest)") + p("-" * 40) + p("Observed vs Exclude-Bucket Attribution. These are contribution decompositions only.") + for row in counterfactuals: + p( + f"{row['name']}: fills={row['fills']} clusters={row['clusters']} " + f"fee={_num(row['fee_usdt'], 4)} gross30={_num(row['gross_markout_30s_usdt'], 4)} " + f"realized={_num(row['realized_pnl_usdt'], 4)} net30={_num(row['net_attr_30s_usdt'], 4)} " + f"m30={_pct(row['markout_30s'])}" + ) + p() + p("Interpretation") + p("-" * 40) + p("Core conclusion is based on 3890 fully matched fills.") + p("Economic Attribution asks why MakerAlpha did not convert to money.") + p("It does NOT change quote logic, does NOT restart v0.1, and does NOT unlock Stage 3.") + p("=" * 72) + + out_txt.write_text("\n".join(lines) + "\n", encoding="utf-8") + sidecar = { + "experiment_id": "MM_EDGE_EXP_001", + "population": { + "name": "MATCHED", + "fills": n_matched_fills, + "paths": n_matched_paths, + "clusters": n_matched_clusters, + }, + "strategy": "v0.1 FROZEN", + "execution": "STOPPED", + "stage3": "LOCKED", + "fee_total_usdt": fee_total, + "fee_per_fill_usdt": fee_total / max(n_matched_paths, 1), + "fee_per_btc_usdt": fee_total / max(total_qty, 1e-12), + "fee_per_cluster_usdt": fee_total / max(n_matched_clusters, 1), + "realized_component_usdt": realized_total, + "gross_markout_30s_usdt": gross_30_total, + "net_attr_30s_usdt": net_attr_30_total, + "markout_by_horizon": horizon_rows, + "inventory_metrics": inventory_metrics, + "bucket_rows": bucket_rows, + "counterfactuals": counterfactuals, + "recon03_extension": { + "venue_confirmed_no_trade_history": recon03.get("venue_confirmed_no_trade_history"), + "venue_partial_order_canceled": recon03.get("venue_partial_order_canceled"), + }, + "account_recon_ref": account, + } + out_json.write_text(json.dumps(sidecar, indent=2) + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d40b3b5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +nautilus_trader==1.231.0 +python-dotenv>=1.0.0 +PyYAML>=6.0 +ccxt>=4.0.0 +pandas>=2.2,<3 +numpy>=1.26,<2.1 diff --git a/scripts/Economic_Fee_Sensitivity_v0_1.txt b/scripts/Economic_Fee_Sensitivity_v0_1.txt new file mode 100644 index 0000000..f9c1d7d --- /dev/null +++ b/scripts/Economic_Fee_Sensitivity_v0_1.txt @@ -0,0 +1,20 @@ +======================================================================== +Economic Fee Sensitivity v0.1 (MATCHED=3890) +======================================================================== +gross_markout_30s_usdt: -1.730780 USDT +fee_total_usdt: +42.422146 USDT +realized_component_usdt:-8.521730 USDT + +Fee assumption → Net attributable @30s +------------------------------------------ +fee_factor | fee_usdt_assumed | net_attr_30s_usdt +1.00 | +42.422146 | -52.674655 +0.50 | +21.211073 | -31.463582 +0.25 | +10.605536 | -20.858046 +0.10 | +4.242215 | -14.494724 +0.00 | +0.000000 | -10.252510 + +Interpretation: +- If net remains < 0 at fee_factor=0 → economics not salvageable by fee reduction alone. +- If fee reduction flips net > 0 → current venue/fee tier can be the dominant issue. +======================================================================== diff --git a/scripts/Economic_Metric_Reconciliation_v0_1.txt b/scripts/Economic_Metric_Reconciliation_v0_1.txt new file mode 100644 index 0000000..5baae36 --- /dev/null +++ b/scripts/Economic_Metric_Reconciliation_v0_1.txt @@ -0,0 +1,21 @@ +======================================================================== +Economic Metric Reconciliation v0.1 (MATCHED=3890) +======================================================================== +Matched paths: 3886 (expected ~3886) + +Definitions (same math as analyze_maker_edge): +- markout_30s_return = _fav_ret(side, fill_price, after_30s_price) +- gross_markout_usdt = sum(notional_usdt * markout_30s_return) + +Return-space metrics (sign may differ due to weighting): +MakerAlpha fill-weighted mean return: -0.000693% +MakerAlpha notional-weighted mean return: -0.000816% +MakerAlpha cluster-weighted mean return: -0.000830% + +Dollar-space metrics: +gross_markout_usdt (30s): -1.730780 USDT +total_notional_usdt: 212110.750 USDT + +If fill-weighted return is + but gross_markout_usdt is negative, +it means notional weighting flips sign (alpha is conditionally realized). +======================================================================== diff --git a/scripts/analyze.sh b/scripts/analyze.sh new file mode 100755 index 0000000..7f4ba66 --- /dev/null +++ b/scripts/analyze.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +LOG_DIR="${MAKER_EDGE_LOG_DIR:-$ROOT/logs/maker_edge}" +# Prefer project venv python if present +PY="${ROOT}/.venv/bin/python" +if [[ ! -x "$PY" ]]; then + PY=python3 +fi +export PYTHONPATH="${ROOT}/src${PYTHONPATH:+:$PYTHONPATH}" +exec "$PY" "$ROOT/scripts/analyze_maker_edge.py" --dir "$LOG_DIR" --report --min-fills "${1:-2000}" diff --git a/scripts/analyze_maker_edge.py b/scripts/analyze_maker_edge.py new file mode 100644 index 0000000..66eae16 --- /dev/null +++ b/scripts/analyze_maker_edge.py @@ -0,0 +1,978 @@ +#!/usr/bin/env python3 +""" +Maker Edge Report v0.1 — Research Freeze / Data Collection Phase + +固定格式（每次运行必须相同、可比较）： + Executive Summary + Section 1 — Data Integrity + Section 2 — Fill Alpha + Section 3 — Toxicity Profile + Section 4 — Observed Edge Attribution + Section 5 — Decision + +研究对象：可验证的市场现象（不是策略）。 +见 FREEZE.md — 只许数据字段/质量检查/报告解释；禁止新交易规则。 + +用法： + ./scripts/analyze.sh 2000 + python scripts/analyze_maker_edge.py --report --min-fills 2000 +""" + +from __future__ import annotations + +import argparse +import json +import sys +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +_ROOT = Path(__file__).resolve().parents[1] +_SRC = _ROOT / "src" +if str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) + + +FEE = 0.00016 +EXPECTED_SLIPPAGE = 0.00005 +POSITIVE_EDGE_NET = 0.0002 +CLUSTER_GAP_SEC = 30.0 +TOXIC_FAIL_RATIO = 0.60 +PASS_MIN_FILLS_DEFAULT = 2000 + + +def _load_experiment_from_df(df: pd.DataFrame) -> dict[str, Any]: + """优先用 jsonl 中的 experiment_start / 事件戳；否则回退环境默认。""" + try: + from nautilus_mm.experiment import load_experiment_meta + + base = load_experiment_meta() + except Exception: + base = { + "experiment_id": "MM_EDGE_EXP_001", + "probe_version": "probe_v0.1", + "quote_assumption": "frozen", + "fee_model": "frozen", + "exchange_assumption": "frozen", + "exchange": "binance_usdm", + "environment": "TESTNET", + "symbol": "BTCUSDT-PERP", + } + if df.empty or "event" not in df.columns: + return base + starts = df[df["event"] == "experiment_start"] + if not starts.empty: + row = starts.iloc[-1] + for k in ("experiment_id", "probe_version", "exchange", "environment", "symbol"): + if k in row and pd.notna(row[k]): + base[k] = row[k] + return base + # 任意带 experiment_id 的事件 + if "experiment_id" in df.columns and df["experiment_id"].notna().any(): + base["experiment_id"] = df["experiment_id"].dropna().iloc[-1] + if "probe_version" in df.columns and df["probe_version"].notna().any(): + base["probe_version"] = df["probe_version"].dropna().iloc[-1] + return base + + +def load_events(log_dir: Path) -> pd.DataFrame: + rows = [] + files = sorted(log_dir.glob("*.jsonl")) + if not files: + raise FileNotFoundError(f"No jsonl in {log_dir}") + for f in files: + for line in f.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + rows.append(json.loads(line)) + return pd.DataFrame(rows) + + +def _fav_ret(side: pd.Series, fill: pd.Series, px: pd.Series) -> pd.Series: + raw = (px.astype(float) - fill.astype(float)) / fill.astype(float) + return pd.Series(np.where(side == "long", raw, -raw), index=side.index) + + +def _side_label(side: str) -> str: + return "Bid" if side == "long" else "Ask" + + +def _extract_fill_context(fills: pd.DataFrame) -> pd.DataFrame: + if fills.empty or "fill_context" not in fills.columns: + return pd.DataFrame() + rows = [] + for _, r in fills.iterrows(): + ctx = r.get("fill_context") + if not isinstance(ctx, dict): + continue + rows.append( + { + "fill_id": r.get("fill_id"), + "market_event_before_fill": ctx.get("market_event_before_fill"), + "trade_imbalance_5s": ctx.get("trade_imbalance_5s"), + "price_velocity_5s": ctx.get("price_velocity_5s"), + "fill_type": ctx.get("fill_type"), + } + ) + return pd.DataFrame(rows) + + +def _median_safe(s: pd.Series) -> float | None: + s = pd.to_numeric(s, errors="coerce").dropna() + return float(s.median()) if len(s) else None + + +def _fmt_pct(x: float | None, digits: int = 4) -> str: + if x is None or (isinstance(x, float) and np.isnan(x)): + return "n/a" + return f"{x*100:+.{digits}f}%" + + +def _fmt_pp(x: float | None) -> str: + if x is None or (isinstance(x, float) and np.isnan(x)): + return "n/a" + return f"{x*100:+.1f}pp" + + +def _dist_stats(s: pd.Series) -> dict[str, float | None]: + s = pd.to_numeric(s, errors="coerce").dropna() + if s.empty: + return {"mean": None, "median": None, "p25": None, "p75": None, "n": 0} + return { + "mean": float(s.mean()), + "median": float(s.median()), + "p25": float(s.quantile(0.25)), + "p75": float(s.quantile(0.75)), + "n": int(len(s)), + } + + +def _print_dist(p, title: str, d: dict[str, float | None]) -> None: + if not d.get("n"): + p(f"{title}: n/a") + return + p(f"{title} (n={d['n']}):") + p(f" mean: {_fmt_pct(d['mean'])}") + p(f" median: {_fmt_pct(d['median'])}") + p(f" p25: {_fmt_pct(d['p25'])}") + p(f" p75: {_fmt_pct(d['p75'])}") + + +def _observation_window(n_fills: int, n_clusters: int) -> str: + if n_fills < 500: + return "anomaly-check only (<500 fills)" + if n_fills < 2000: + return "early look (500+) — do not over-interpret" + if n_fills < 10000: + return "preliminary judgment (2000+) — clusters still matter more than fills" + return "stability discussion eligible (10000+ fills)" + + +def assign_clusters_offline(fills: pd.DataFrame, gap_sec: float = CLUSTER_GAP_SEC) -> pd.DataFrame: + out = fills.copy() + if out.empty: + return out + if "event_cluster_id" in out.columns and out["event_cluster_id"].notna().any(): + return out + if "ts_epoch" not in out.columns: + out["event_cluster_id"] = [f"na_{i}" for i in range(len(out))] + out["cluster_fill_index"] = 1 + return out + out = out.sort_values("ts_epoch").reset_index(drop=True) + cids: list[str] = [] + idxs: list[int] = [] + cid = None + last_ts = -1e18 + last_side = None + n = 0 + for _, r in out.iterrows(): + ts = float(r["ts_epoch"]) + side = r.get("side") + if cid is None or side != last_side or (ts - last_ts) > gap_sec: + cid = uuid.uuid4().hex[:12] + n = 0 + n += 1 + cids.append(cid) + idxs.append(n) + last_ts = ts + last_side = side + out["event_cluster_id"] = cids + out["cluster_fill_index"] = idxs + return out + + +def classify_space(raw_capture: float, net_edge: float) -> str: + if raw_capture <= 0 or net_edge <= 0: + return "NO_EDGE" + if net_edge < POSITIVE_EDGE_NET: + return "EDGE_AFTER_COST" + return "POSITIVE_EDGE" + + +def build_mid_series(df: pd.DataFrame) -> pd.DataFrame: + parts = [] + for ev in ("mid_tick", "inventory_tick"): + if "event" not in df.columns: + break + sub = df[df["event"] == ev] + if sub.empty or "mid" not in sub.columns or "ts_epoch" not in sub.columns: + continue + parts.append(sub[["ts_epoch", "mid"]].dropna()) + if not parts: + return pd.DataFrame(columns=["ts_epoch", "mid"]) + m = pd.concat(parts, ignore_index=True) + m["ts_epoch"] = pd.to_numeric(m["ts_epoch"], errors="coerce") + m["mid"] = pd.to_numeric(m["mid"], errors="coerce") + return m.dropna().sort_values("ts_epoch").drop_duplicates("ts_epoch").reset_index(drop=True) + + +def _cluster_weight(frame: pd.DataFrame) -> pd.Series: + if "event_cluster_id" not in frame.columns: + return pd.Series(1.0, index=frame.index) + cnt = frame.groupby("event_cluster_id")["event_cluster_id"].transform("count") + return 1.0 / cnt.clip(lower=1) + + +def _period_str(df: pd.DataFrame) -> str: + if df.empty or "ts_epoch" not in df.columns or df["ts_epoch"].isna().all(): + return "n/a" + t0 = float(pd.to_numeric(df["ts_epoch"], errors="coerce").min()) + t1 = float(pd.to_numeric(df["ts_epoch"], errors="coerce").max()) + a = datetime.fromtimestamp(t0, tz=timezone.utc).strftime("%Y-%m-%d") + b = datetime.fromtimestamp(t1, tz=timezone.utc).strftime("%Y-%m-%d") + return f"{a} ~ {b}" + + +def _instrument(fills: pd.DataFrame, df: pd.DataFrame) -> str: + for src in (fills, df): + if not src.empty and "pair" in src.columns and src["pair"].notna().any(): + return str(src["pair"].dropna().iloc[0]) + return "BTCUSDT Perpetual (assumed)" + + +def _maker_alpha_frame(g: pd.DataFrame) -> tuple[pd.Series, pd.Series]: + """Return (fill_ret, mkt_signed) for MakerAlpha = fill − market.""" + mid0 = g["mid"].astype(float) + mid1 = g["after_30s_price"].astype(float) + mkt_ret = (mid1 - mid0) / mid0 + mkt_signed = pd.Series( + np.where(g["side"] == "long", mkt_ret, -mkt_ret), index=g.index + ) + fill_ret = _fav_ret(g["side"], g["fill_price"], g["after_30s_price"]) + return fill_ret, mkt_signed + + +def report(df: pd.DataFrame, min_fills: int = PASS_MIN_FILLS_DEFAULT, out_path: Path | None = None) -> dict[str, Any]: + fills = df[df["event"] == "fill"].copy() if "event" in df.columns else pd.DataFrame() + paths = df[df["event"] == "fill_path"].copy() if "event" in df.columns else pd.DataFrame() + health = df[df["event"] == "health"].copy() if "event" in df.columns else pd.DataFrame() + exp = _load_experiment_from_df(df) + + lines: list[str] = [] + + def p(s: str = "") -> None: + lines.append(s) + print(s) + + if not fills.empty: + fills = assign_clusters_offline(fills) + + if not paths.empty and not fills.empty and "fill_id" in fills.columns: + meta_cols = [ + c + for c in [ + "side", + "fill_price", + "fill_reason", + "spread", + "spread_capture_pct", + "obi", + "trade_imbalance", + "bid_depth_5", + "ask_depth_5", + "book_age_ms", + "inventory", + "pre_5s_deteriorated", + "mid", + "event_cluster_id", + "cluster_fill_index", + "ts_epoch", + "pair", + ] + if c in fills.columns + ] + meta = fills.drop_duplicates("fill_id")[["fill_id"] + meta_cols] + paths = paths.merge(meta, on="fill_id", how="left", suffixes=("", "_f")) + for col in ("side", "fill_price", "mid", "event_cluster_id", "spread", "spread_capture_pct"): + alt = f"{col}_f" + if alt in paths.columns: + if col not in paths.columns: + paths[col] = paths[alt] + else: + paths[col] = paths[col].fillna(paths[alt]) + fc = _extract_fill_context(fills) + if not fc.empty: + paths = paths.merge(fc, on="fill_id", how="left") + + n_fills = len(fills) + n_paths = len(paths) + n_clusters = int(fills["event_cluster_id"].nunique()) if n_fills and "event_cluster_id" in fills.columns else 0 + cluster_fill_ratio = n_clusters / max(n_fills, 1) + + # ---------- compute: integrity ---------- + integrity_ok = True + integrity_notes: list[str] = [] + healthy_ratio = gap_total = gap_win_max = None + lat_p50 = lat_p95 = lat_p99 = ba_med = None + if health.empty: + integrity_ok = False + integrity_notes.append("no health telemetry") + else: + healthy_ratio = float(health["healthy"].astype(bool).mean()) if "healthy" in health.columns else 0.0 + gap_total = int(health["sequence_gap"].iloc[-1]) if "sequence_gap" in health.columns else 0 + gap_win_max = ( + int(pd.to_numeric(health.get("sequence_gap_window"), errors="coerce").fillna(0).max()) + if "sequence_gap_window" in health.columns + else 0 + ) + lat_p50 = health["latency_ms_p50"].iloc[-1] if "latency_ms_p50" in health.columns else None + lat_p95 = health["latency_ms_p95"].iloc[-1] if "latency_ms_p95" in health.columns else None + lat_p99 = health["latency_ms_p99"].iloc[-1] if "latency_ms_p99" in health.columns else None + ba_series = ( + fills["book_age_ms"] + if "book_age_ms" in fills.columns and fills["book_age_ms"].notna().any() + else health.get("book_age_ms") + ) + ba_med = _median_safe(ba_series) if ba_series is not None else None + if healthy_ratio < 0.99: + integrity_ok = False + integrity_notes.append(f"healthy_ratio={healthy_ratio*100:.2f}% < 99%") + # Binance depth update ids are not contiguous — log only, do not INVALID. + if gap_win_max and gap_win_max > 0: + integrity_notes.append( + f"sequence_gap_window_max={gap_win_max} (observe-only; Binance ids skip)" + ) + if ba_med is not None and ba_med > 500: + integrity_ok = False + integrity_notes.append(f"book_age_median={ba_med:.0f}ms > 500ms") + + decision: dict[str, Any] = { + "integrity": integrity_ok, + "verdict": "INSUFFICIENT_DATA", + "reasons": [], + "space_class": None, + "benchmark_alpha": None, + "maker_alpha_mean": None, + "quality": { + "fills": n_fills, + "clusters": n_clusters, + "cluster_fill_ratio": cluster_fill_ratio, + "healthy_ratio": healthy_ratio, + }, + } + + if fills.empty: + p("=" * 72) + p("Maker Edge Report v0.1") + p("Phase: Research Freeze / Data Collection") + p("=" * 72) + p("\nExecutive Summary") + p(f" Experiment: {exp.get('experiment_id')}") + p(f" Version: {exp.get('probe_version')}") + p(" Quote: frozen") + p(" Fee: frozen") + p(" Exchange: frozen") + p(f" Period: {_period_str(df)}") + p(f" Instrument: {_instrument(fills, df)}") + p(" Samples:") + p(" fills: 0") + p(" clusters: 0") + p(" Decision: INSUFFICIENT_DATA") + p(" Reason: no fills yet — run probe") + decision["experiment"] = exp + _finish(lines, out_path, decision) + return decision + + # ---------- compute: fill alpha table + distributions ---------- + alpha_table: dict[str, dict[str, float | None]] = { + "Bid": {"fill_w": None, "cluster_w": None}, + "Ask": {"fill_w": None, "cluster_w": None}, + "Overall": {"fill_w": None, "cluster_w": None}, + } + fill_alpha_dist: dict[str, float | None] = {} + cluster_alpha_dist: dict[str, float | None] = {} + fq_pass = None + bench_alpha = None + maker_alpha_mean = None + agree = None + pct_fills_positive_alpha = None + + if not paths.empty and "after_30s_price" in paths.columns and "mid" in paths.columns and paths["mid"].notna().any(): + for side_name, g in paths.groupby("side"): + label = _side_label(str(side_name)) + fill_ret, mkt_signed = _maker_alpha_frame(g) + alpha = fill_ret - mkt_signed + w = _cluster_weight(g) + alpha_table[label]["fill_w"] = float(alpha.mean()) + alpha_table[label]["cluster_w"] = float((alpha * w).sum() / w.sum()) if w.sum() else float(alpha.mean()) + + fill_ret, mkt_signed = _maker_alpha_frame(paths) + alpha = fill_ret - mkt_signed + w = _cluster_weight(paths) + alpha_table["Overall"]["fill_w"] = float(alpha.mean()) + alpha_table["Overall"]["cluster_w"] = ( + float((alpha * w).sum() / w.sum()) if w.sum() else float(alpha.mean()) + ) + maker_alpha_mean = alpha_table["Overall"]["cluster_w"] + fw, cw = alpha_table["Overall"]["fill_w"], alpha_table["Overall"]["cluster_w"] + agree = (fw > 0 and cw > 0) or (fw <= 0 and cw <= 0) + fill_alpha_dist = _dist_stats(alpha) + pct_fills_positive_alpha = float((alpha > 0).mean()) + + # per-cluster mean MakerAlpha（事件级分布） + if "event_cluster_id" in paths.columns: + tmp = paths.assign(_alpha=alpha) + cluster_means = tmp.groupby("event_cluster_id")["_alpha"].mean() + cluster_alpha_dist = _dist_stats(cluster_means) + + mkt_fav = mkt_signed > 0 + fill_fav = fill_ret > 0 + bench_alpha = float(np.mean(fill_fav) - np.mean(mkt_fav)) + + fav30 = fill_ret + p30_clu = float((fav30 > 0).astype(float).mul(w).sum() / w.sum()) if w.sum() else float((fav30 > 0).mean()) + fq_pass = p30_clu > 0.50 + + decision["fill_quality"] = fq_pass + decision["benchmark_alpha"] = bench_alpha + decision["maker_alpha_mean"] = maker_alpha_mean + + # ---------- compute: toxicity + loss concentration ---------- + toxicity: dict[str, dict[str, float | None]] = {} + toxic_bid_ratio = None + c_share = None + tox_dist: dict[str, Any] = {} + if not paths.empty: + for side_name, g in paths.groupby("side"): + label = _side_label(str(side_name)) + row: dict[str, float | None] = {} + for hz, col in [ + ("1s", "after_1s_price"), + ("5s", "after_5s_price"), + ("10s", "after_10s_price"), + ("30s", "after_30s_price"), + ("300s", "after_5m_price"), + ]: + if col in g.columns: + row[hz] = float(_fav_ret(g["side"], g["fill_price"], g[col]).mean()) + else: + row[hz] = None + toxicity[label] = row + if "path_type" in paths.columns: + c_share = float((paths["path_type"].astype(str).str.startswith("C")).mean()) + bid = paths[paths["side"] == "long"] + if len(bid): + toxic_bid_ratio = float((bid["path_type"].astype(str).str.startswith("C")).mean()) + + # 毒性分布：多少成交在 10s 不利；最差 20% 占总不利损失比例 + if "after_10s_price" in paths.columns: + fav10 = _fav_ret(paths["side"], paths["fill_price"], paths["after_10s_price"]) + adverse = fav10[fav10 < 0] + tox_dist["pct_adverse_10s"] = float((fav10 < 0).mean()) + tox_dist["fav10"] = _dist_stats(fav10) + if len(adverse) >= 5: + worst_n = max(1, int(np.ceil(0.20 * len(fav10)))) + worst = fav10.nsmallest(worst_n) + total_adv = float((-adverse).sum()) + worst_adv = float((-worst.clip(upper=0)).sum()) + tox_dist["worst20_share_of_adverse"] = ( + worst_adv / total_adv if total_adv > 1e-12 else None + ) + else: + tox_dist["worst20_share_of_adverse"] = None + + # ---------- compute: cost / adverse ---------- + space_class = None + adv_pass = None + raw_capture = net_edge = adv_mag = sc_mean = total_cost = None + if not paths.empty and "after_30s_price" in paths.columns: + fav30 = _fav_ret(paths["side"], paths["fill_price"], paths["after_30s_price"]) + w = _cluster_weight(paths) + raw_capture = float((fav30 * w).sum() / w.sum()) if w.sum() else float(fav30.mean()) + adv_mag = ( + float((-fav30.clip(upper=0) * w).sum() / w.sum()) + if w.sum() + else float((-fav30.clip(upper=0)).mean()) + ) + sc_mean = ( + float(fills["spread_capture_pct"].mean()) + if "spread_capture_pct" in fills.columns and fills["spread_capture_pct"].notna().any() + else 0.0 + ) + if "book_age_ms" in fills.columns and fills["book_age_ms"].notna().any(): + latency_cost = float(fills["book_age_ms"].mean()) / 100.0 * 0.00002 + else: + latency_cost = 0.00002 + total_cost = 2 * FEE + EXPECTED_SLIPPAGE + latency_cost + net_edge = raw_capture - total_cost + space_class = classify_space(raw_capture, net_edge) + adv_ok = (adv_mag < abs(sc_mean)) if sc_mean != 0 else False + adv_pass = bool(adv_ok and space_class in ("POSITIVE_EDGE", "EDGE_AFTER_COST")) + + decision["adverse"] = adv_pass + decision["space_class"] = space_class + + # ---------- compute: attribution (facts only) ---------- + attr_rows: list[tuple[str, str, int, float]] = [] + stab_pass = None + state_coverage_ok = None + concentrated = False + positive_envs = 0 + total_envs = 0 + if not paths.empty and "after_30s_price" in paths.columns: + paths = paths.copy() + paths["_fav30"] = _fav_ret(paths["side"], paths["fill_price"], paths["after_30s_price"]) + if "vol_proxy_5m" in paths.columns and paths["vol_proxy_5m"].notna().any(): + med = paths["vol_proxy_5m"].median() + paths["vol_bucket"] = np.where(paths["vol_proxy_5m"] >= med, "high_vol", "low_vol") + elif "max_price" in paths.columns: + rng = (paths["max_price"] - paths["min_price"]) / paths["fill_price"] + paths["vol_bucket"] = np.where(rng >= rng.median(), "high_vol", "low_vol") + if "price_velocity_5s" in paths.columns and paths["price_velocity_5s"].notna().any(): + v = paths["price_velocity_5s"].astype(float) + thr = v.abs().median() * 0.5 + paths["trend_bucket"] = np.where( + v > thr, "trend_up", np.where(v < -thr, "trend_down", "range") + ) + if "spread" in paths.columns and paths["spread"].notna().any(): + sp_pct = paths["spread"] / paths["fill_price"] + paths["liq_bucket"] = np.where(sp_pct <= sp_pct.median(), "tight_spread", "wide_spread") + if "bid_depth_5" in paths.columns and "ask_depth_5" in paths.columns: + depth = paths["bid_depth_5"].fillna(0) + paths["ask_depth_5"].fillna(0) + if depth.gt(0).any(): + paths["depth_bucket"] = np.where(depth >= depth[depth > 0].median(), "deep_book", "thin_book") + + pos_counts: list[int] = [] + for col, title in [ + ("vol_bucket", "Volatility"), + ("trend_bucket", "Trend"), + ("liq_bucket", "Liquidity(spread)"), + ("depth_bucket", "Liquidity(depth)"), + ("market_event_before_fill", "FillContext"), + ("path_type", "PathType"), + ]: + if col not in paths.columns or paths[col].isna().all(): + continue + for idx, row in paths.groupby(col)["_fav30"].agg(["count", "mean"]).iterrows(): + total_envs += 1 + mean = float(row["mean"]) + n = int(row["count"]) + attr_rows.append((title, str(idx), n, mean)) + if mean > 0: + positive_envs += 1 + pos_counts.append(n) + + state_coverage_ok = total_envs >= 4 + if total_envs >= 2: + if pos_counts: + concentrated = (max(pos_counts) / max(sum(pos_counts), 1)) >= 0.70 and len(pos_counts) == 1 + stab_pass = positive_envs >= 2 and not concentrated + else: + state_coverage_ok = False + + decision["stability"] = stab_pass + decision["quality"]["state_buckets"] = len(attr_rows) + + # ---------- decision ---------- + independence_ok = ( + n_clusters >= max(50, min_fills // 20) if n_fills >= min_fills else None + ) + decision["independence"] = independence_ok + reasons: list[str] = [] + + min_paths = max(1, min_fills // 10) + sample_ok = n_fills >= min_fills and n_paths >= min_paths + + gates = { + "integrity": integrity_ok, + "fill_quality": fq_pass, + "adverse": adv_pass, + "stability": stab_pass, + } + + hard_fail = False + if not integrity_ok: + hard_fail = True + reasons.append("data integrity failed — stop interpretation") + if toxic_bid_ratio is not None and toxic_bid_ratio > TOXIC_FAIL_RATIO: + hard_fail = True + reasons.append(f"Bid toxic fill ratio {toxic_bid_ratio*100:.0f}% > {TOXIC_FAIL_RATIO*100:.0f}%") + if space_class == "NO_EDGE" and sample_ok: + hard_fail = True + reasons.append("edge disappears after cost / NO_EDGE") + if bench_alpha is not None and bench_alpha <= 0 and sample_ok: + reasons.append("benchmark-adjusted alpha negative") + if maker_alpha_mean is not None and maker_alpha_mean <= 0 and sample_ok: + reasons.append("MakerAlpha (fill−market) ≤ 0") + if adv_pass is False and sample_ok: + reasons.append("adverse selection ≥ spread capture") + if concentrated: + reasons.append("edge concentrated in single regime") + + pass_extras = True + if bench_alpha is not None and bench_alpha <= 0: + pass_extras = False + if independence_ok is False: + pass_extras = False + reasons.append(f"insufficient independent clusters ({n_clusters})") + if space_class == "NO_EDGE": + pass_extras = False + + all_gates = all(v is True for v in gates.values()) + + # Stage3 unlock checklist（严格） + stage3_unlock = { + "data_integrity": integrity_ok is True, + "cluster_weighted_alpha_gt_0": bool(maker_alpha_mean is not None and maker_alpha_mean > 0), + "benchmark_alpha_gt_0": bool(bench_alpha is not None and bench_alpha > 0), + "not_concentrated": not concentrated, + } + stage3_ready = all(stage3_unlock.values()) and sample_ok and all_gates and pass_extras + + # 局部正 edge：归因桶分化或集中在单一正 regime + local_positive = positive_envs >= 1 and total_envs >= 2 and ( + (positive_envs < total_envs) or concentrated + ) + + if not integrity_ok: + verdict = "INVALID" + reasons = ["Data Integrity FAIL — do not interpret Alpha; discard / keep collecting clean data"] + reasons.extend(integrity_notes) + elif not sample_ok or state_coverage_ok is False: + verdict = "COLLECTING" + reasons = [] + if n_fills < min_fills: + reasons.append(f"fills {n_fills} < {min_fills}") + if n_paths < min_paths: + reasons.append(f"fill_paths {n_paths} < {min_paths}") + if n_clusters < max(50, min_fills // 20) and n_fills >= 500: + reasons.append(f"clusters {n_clusters} insufficient (independent liquidity events)") + if state_coverage_ok is False: + reasons.append("state coverage incomplete") + if not reasons: + reasons.append("Insufficient independent liquidity events") + elif hard_fail and not local_positive: + verdict = "FAIL" + if not reasons: + reasons.append("market hypothesis does not hold under current quote assumption") + elif stage3_ready: + verdict = "PASS" + reasons = [ + "Maker alpha survives: cost", + "Maker alpha survives: benchmark", + "Maker alpha survives: cluster weighting", + "Maker alpha survives: multiple states", + ] + elif local_positive and integrity_ok and sample_ok: + verdict = "PARTIAL_PASS" + reasons = [ + "edge not universal — observed only in subset of states/events", + f"positive attribution buckets: {positive_envs}/{total_envs}", + ] + if concentrated: + reasons.append("edge concentrated in one regime/event class") + if maker_alpha_mean is not None and maker_alpha_mean <= 0: + reasons.append("overall cluster-weighted MakerAlpha ≤ 0") + else: + verdict = "FAIL" + if not reasons: + reasons.append("gates failed under current quote assumption") + if adv_pass is False: + reasons.insert(0, "adverse selection") + if sc_mean is not None and abs(sc_mean) < 1e-8: + reasons.append("insufficient spread") + + decision["verdict"] = verdict + decision["reasons"] = reasons + decision["stage3_unlock"] = stage3_unlock + decision["stage3_ready"] = stage3_ready + decision["experiment"] = exp + + # ================================================================== + # PRINT — fixed format + # ================================================================== + p("=" * 72) + p("Maker Edge Report v0.1") + p("Phase: Research Freeze / Data Collection") + p("Object: verifiable market phenomenon (not a strategy)") + p("=" * 72) + + # ----- Executive Summary ----- + p("\nExecutive Summary") + p("-" * 40) + p(f"Experiment: {exp.get('experiment_id')}") + p(f"Version: {exp.get('probe_version')}") + p("Quote: frozen") + p("Fee: frozen") + p("Exchange: frozen") + p(f"Venue: {exp.get('exchange')} / {exp.get('environment')}") + p(f"Period: {_period_str(df if not df.empty else fills)}") + p(f"Instrument: {_instrument(fills, df)}") + p("Samples:") + p(f" fills: {n_fills}") + p(f" clusters: {n_clusters}") + p(f" paths: {n_paths}") + p(f" cluster/fill: {cluster_fill_ratio*100:.1f}%") + p(f"Observation window: {_observation_window(n_fills, n_clusters)}") + p(" (500=anomaly · 2000=preliminary · 10000=stability; clusters > fills)") + p(f"Decision: {verdict}") + p("Reason:") + for r in reasons: + p(f" - {r}") + p("Hypothesis under test: passive fills produce +MakerAlpha") + p(" under current BTC perp / venue / quote / execution — not strategy PnL.") + p("Read order: Integrity → distributions (not mean) → Cluster → Toxicity → Decision") + + # ----- Section 1 ----- + p("\n" + "=" * 72) + p("Section 1 — Data Integrity") + p("Question: Is the data trustworthy?") + p("=" * 72) + if health.empty: + p("Healthy: n/a (no health events)") + p("Sequence gap: n/a") + p("Latency: n/a") + p("Book freshness:n/a") + else: + p(f"Healthy: {healthy_ratio*100:.2f}%") + p(f"Sequence gap: total={gap_total} window_max={gap_win_max}") + p("Latency:") + p(f" p50: {lat_p50} ms") + p(f" p95: {lat_p95} ms") + p(f" p99: {lat_p99} ms") + p(f"Book freshness: median={ba_med:.1f} ms" if ba_med is not None else "Book freshness: n/a") + p(f"Integrity: [{'PASS' if integrity_ok else 'FAIL'}]") + for n in integrity_notes: + p(f" · {n}") + if not integrity_ok: + p("\n★ STOP — Data Integrity FAIL → Decision=INVALID.") + p(" Do not interpret Alpha. Bad book/latency/gap fills have no research value.") + + # ----- Section 2 ----- + p("\n" + "=" * 72) + p("Section 2 — Fill Alpha") + p("Question: Fill − Matched Market Move (not PnL)") + p("Priority: distribution (median/p25/p75) over mean") + p("=" * 72) + if not integrity_ok: + p("(skipped for decision — integrity INVALID; numbers below are not evidence)") + if alpha_table["Overall"]["fill_w"] is None: + p("(waiting for fill_path with mid + after_30s)") + else: + p(f"{'':12s} {'Fill weighted':>16s} {'Cluster weighted':>18s}") + for lab in ("Bid", "Ask", "Overall"): + fw = alpha_table[lab]["fill_w"] + cw = alpha_table[lab]["cluster_w"] + p(f"{lab+' Alpha':12s} {_fmt_pct(fw):>16s} {_fmt_pct(cw):>18s}") + p(f"Direction agree (fill-w vs cluster-w): {'YES' if agree else 'NO ★'}") + p(f"Benchmark P(+) Δ (fill − matched mid): {_fmt_pp(bench_alpha)}") + p(f"SPACE class: {space_class or 'PENDING'}") + if raw_capture is not None and net_edge is not None and total_cost is not None: + p(f"Raw capture@30s (cluster-w): {_fmt_pct(raw_capture)}") + p(f"Total cost (fee+slip+lat): {_fmt_pct(total_cost)}") + p(f"Net edge: {_fmt_pct(net_edge)}") + p("") + p("Fill Alpha distribution (do not trust mean alone):") + _print_dist(p, " per-fill MakerAlpha", fill_alpha_dist) + if pct_fills_positive_alpha is not None: + p(f" share of fills with +alpha: {pct_fills_positive_alpha*100:.1f}%") + if pct_fills_positive_alpha < 0.35 and (fill_alpha_dist.get("mean") or 0) > 0: + p(" ★ mean>0 but minority of fills — edge likely event-driven / fat tail") + p("") + p("Cluster Alpha distribution (independent liquidity events):") + _print_dist(p, " per-cluster mean MakerAlpha", cluster_alpha_dist) + if ( + alpha_table["Overall"]["fill_w"] is not None + and alpha_table["Overall"]["cluster_w"] is not None + ): + fw, cw = alpha_table["Overall"]["fill_w"], alpha_table["Overall"]["cluster_w"] + if fw > 0 >= cw: + p(" ★ Fill+ but Cluster≤0 — edge from few burst fills; unstable") + elif fw > 0 and cw > 0: + p(" Fill+ and Cluster+ — credibility higher") + + # ----- Section 3 ----- + p("\n" + "=" * 72) + p("Section 3 — Toxicity Profile") + p("Question: Are fills naturally on the wrong side? (record only — no quote changes)") + p("=" * 72) + if not toxicity: + p("(waiting for fill_path)") + else: + for label, row in toxicity.items(): + p(f"\n{label}:") + p(" Immediate toxicity:") + for hz in ("1s", "5s", "10s"): + p(f" {hz}: {_fmt_pct(row.get(hz))}") + p(" Recovery:") + for hz in ("30s", "300s"): + p(f" {hz}: {_fmt_pct(row.get(hz))}") + # factual pattern note only + t10, t300 = row.get("10s"), row.get("300s") + if t10 is not None and t300 is not None: + if t10 < 0 < t300: + p(" Observed pattern: early toxicity + later recovery (fact; not a rule)") + elif t10 < 0 and t300 <= 0: + p(" Observed pattern: sustained adverse (fact; not a rule)") + elif t10 is not None and t10 > 0: + p(" Observed pattern: immediate favorable (fact; not a rule)") + if c_share is not None: + p(f"\nPath C (toxic) share: {c_share*100:.1f}%") + if toxic_bid_ratio is not None: + p(f"Bid toxic fill ratio: {toxic_bid_ratio*100:.1f}%") + if adv_mag is not None and sc_mean is not None: + p(f"mean_adverse vs |spread_capture|: {_fmt_pct(adv_mag)} vs {_fmt_pct(abs(sc_mean))}") + if tox_dist: + p("\nToxicity distribution:") + if tox_dist.get("pct_adverse_10s") is not None: + p(f" fills adverse@10s: {tox_dist['pct_adverse_10s']*100:.1f}%") + if tox_dist.get("fav10"): + _print_dist(p, " fav@10s", tox_dist["fav10"]) + w20 = tox_dist.get("worst20_share_of_adverse") + if w20 is not None: + p(f" worst 20% of fills share of adverse loss: {w20*100:.1f}%") + if w20 >= 0.70: + p(" ★ losses concentrated — future value may be 'which quotes NOT to place'") + p(" (record only; no cancel/filter rules in freeze)") + + # ----- Section 4 ----- + p("\n" + "=" * 72) + p("Section 4 — Observed Edge Attribution") + p("Facts only. Not strategy recommendations. Not filter rules.") + p("=" * 72) + if not attr_rows: + p("(insufficient state slices)") + else: + cur_title = None + for title, idx, n, mean in attr_rows: + if title != cur_title: + p(f"\n{title}:") + cur_title = title + sign = "positive" if mean > 0 else ("negative" if mean < 0 else "flat") + p(f" {idx}: n={n} E[fav30]={_fmt_pct(mean)} ({sign})") + if concentrated: + p("\nObservation: positive mass concentrated in a single bucket (fact).") + + # ----- Section 5 ----- + p("\n" + "=" * 72) + p("Section 5 — Decision") + p("=" * 72) + p(f"Decision: {verdict}") + p("") + if verdict == "INVALID": + p("Reason:") + for r in reasons: + p(f" - {r}") + p("\nKeep collecting only after Data Integrity is clean.") + elif verdict == "COLLECTING": + p("Reason:") + for r in reasons: + p(f" - {r}") + p("\nDo not over-interpret before 2000 fills / adequate clusters.") + p("500 = anomaly check · 2000 = preliminary · 10000 = stability.") + elif verdict == "PASS": + p("Maker alpha survives:") + for r in reasons: + p(f" - {r.replace('Maker alpha survives: ', '')}") + p("\n→ Unlock Stage3 Economic Simulation → Symmetric MM") + elif verdict == "PARTIAL_PASS": + p("Partial: market hypothesis holds only in some states/events.") + for r in reasons: + p(f" - {r}") + p("\n→ Path: Event-driven LP (not all-day Symmetric MM)") + p(" Still locked: no new filters yet — attribution is observation only.") + else: + p("No maker edge under current quote assumption.") + p("Dominant reasons:") + for r in reasons: + p(f" - {r}") + p("\nConclusion = hypothesis false (not 'strategy failed'). Avoid futile tuning.") + + p("\nStage3 Unlock Checklist (Economic Simulation):") + for k, v in stage3_unlock.items(): + p(f" [{'OK' if v else '·'}] {k}") + p(f" Stage3 ready: {'YES' if stage3_ready else 'NO'}") + + p("") + p("State machine:") + p(" FAIL → change hypothesis") + p(" PARTIAL_PASS → Event-driven LP") + p(" PASS → Economic Simulation → Symmetric MM") + p(" COLLECTING → keep collecting") + p("") + p("Action: run probe. Look at distributions first, Decision second.") + p("=" * 72) + + _finish(lines, out_path, decision) + return decision + + +def _finish(lines: list[str], out_path: Path | None, decision: dict[str, Any]) -> None: + if out_path: + out_path.parent.mkdir(parents=True, exist_ok=True) + footer = { + "event": "maker_edge_decision", + "report": "Maker Edge Report v0.1", + "phase": "Research Freeze / Data Collection", + "verdict": decision.get("verdict"), + "experiment": decision.get("experiment"), + "space_class": decision.get("space_class"), + "benchmark_alpha": decision.get("benchmark_alpha"), + "maker_alpha_mean": decision.get("maker_alpha_mean"), + "stage3_ready": decision.get("stage3_ready"), + "stage3_unlock": decision.get("stage3_unlock"), + "quality": decision.get("quality"), + "gates": { + "integrity": decision.get("integrity"), + "independence": decision.get("independence"), + "fill_quality": decision.get("fill_quality"), + "adverse": decision.get("adverse"), + "stability": decision.get("stability"), + }, + "reasons": decision.get("reasons"), + } + text = "\n".join(lines) + "\n\n---\n" + json.dumps(footer, ensure_ascii=False, indent=2) + "\n" + out_path.write_text(text, encoding="utf-8") + print(f"\nReport saved: {out_path}") + + +def main() -> None: + ap = argparse.ArgumentParser(description="Maker Edge Report v0.1 — Research Freeze") + ap.add_argument( + "--dir", + type=str, + default=str(Path(__file__).resolve().parents[1] / "logs" / "maker_edge"), + ) + ap.add_argument("--min-fills", type=int, default=PASS_MIN_FILLS_DEFAULT) + ap.add_argument("--report", action="store_true") + args = ap.parse_args() + log_dir = Path(args.dir) + if not log_dir.exists(): + print(f"日志目录不存在: {log_dir}") + return + try: + df = load_events(log_dir) + except FileNotFoundError as e: + print(e) + return + out = log_dir / "Maker_Edge_Report_v0.1.txt" if args.report else None + report(df, min_fills=args.min_fills, out_path=out) + + +if __name__ == "__main__": + main() diff --git a/scripts/deploy_server.sh b/scripts/deploy_server.sh new file mode 100755 index 0000000..faa57e2 --- /dev/null +++ b/scripts/deploy_server.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# 部署 MM_EDGE_EXP_001 → jack@jackyu66.com:/www/Project/nautilus_mm +# +# 默认： +# SSH_HOST=jack@jackyu66.com +# SSH_KEY=~/Project/deploy/zun_hk/id_ed25519_hk +# REMOTE_DIR=/www/Project/nautilus_mm +# +# 覆盖：export SSH_HOST=... SSH_KEY=... REMOTE_DIR=... +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SSH_HOST="${SSH_HOST:-jack@jackyu66.com}" +SSH_KEY="${SSH_KEY:-${HOME}/Project/deploy/zun_hk/id_ed25519_hk}" +REMOTE_DIR="${REMOTE_DIR:-/www/Project/nautilus_mm}" + +if [[ ! -f "$SSH_KEY" ]]; then + echo "SSH key not found: $SSH_KEY" + exit 1 +fi +chmod 400 "$SSH_KEY" 2>/dev/null || true + +SSH_OPTS=(-i "$SSH_KEY" -o StrictHostKeyChecking=accept-new) +SSH=(ssh "${SSH_OPTS[@]}" "$SSH_HOST") +RSYNC_E="ssh ${SSH_OPTS[*]}" + +echo "==> stop remote probe before sync (if running)" +"${SSH[@]}" "systemctl --user stop mm-edge-probe 2>/dev/null || true" + +echo "==> sync $ROOT → $SSH_HOST:$REMOTE_DIR" +"${SSH[@]}" "mkdir -p '$REMOTE_DIR' '$REMOTE_DIR/logs/maker_edge'" +rsync -avz --delete \ + -e "$RSYNC_E" \ + --exclude '.venv' \ + --exclude '__pycache__' \ + --exclude '*.pyc' \ + --exclude 'logs/maker_edge/*.jsonl' \ + --exclude 'logs/maker_edge/*.txt' \ + --exclude 'logs/maker_edge_smoke' \ + --exclude '.env' \ + "$ROOT/" "$SSH_HOST:$REMOTE_DIR/" + +echo "==> remote setup (uv venv + user systemd)" +"${SSH[@]}" "REMOTE_DIR='$REMOTE_DIR' bash -s" <<'REMOTE' +set -euo pipefail +export PATH="$HOME/.local/bin:$PATH" +cd "$REMOTE_DIR" +if [[ ! -f .env ]]; then + cp .env.example .env + { + echo "" + echo "# Server Data Collection — MM_EDGE_EXP_001" + echo "EXPERIMENT_ID=MM_EDGE_EXP_001" + echo "PROBE_VERSION=probe_v0.1" + echo "EXCHANGE_NAME=binance_usdm" + echo "BINANCE_ENVIRONMENT=TESTNET" + echo "ENABLE_TRADING=false" + echo "QUOTE_TTL_SECS=30" + echo "MAX_ABS_INVENTORY=0.005" + echo "HTTP_PROXY=" + echo "HTTPS_PROXY=" + echo "MAKER_EDGE_LOG_DIR=${REMOTE_DIR}/logs/maker_edge" + } >> .env + echo "CREATED .env — fill BINANCE_API_KEY / BINANCE_API_SECRET" +else + echo ".env exists — left untouched" +fi + +if [[ ! -x "$HOME/.local/bin/uv" ]]; then + curl -LsSf https://astral.sh/uv/install.sh | sh +fi +uv python install 3.12 +rm -rf .venv +uv venv .venv --python 3.12 +uv pip install -r requirements.txt --python .venv/bin/python + +mkdir -p "$HOME/.config/systemd/user" +sed -e "s|/www/Project/nautilus_mm|${REMOTE_DIR}|g" \ + deploy/mm-edge-probe.user.service > "$HOME/.config/systemd/user/mm-edge-probe.service" +systemctl --user daemon-reload +systemctl --user enable mm-edge-probe.service +loginctl enable-linger "$(whoami)" 2>/dev/null || true +echo "User systemd installed (not started — fill keys first)." +echo " nano $REMOTE_DIR/.env" +echo " systemctl --user start mm-edge-probe" +echo " journalctl --user -u mm-edge-probe -f" +REMOTE + +echo "" +echo "==> done" +echo "1) ssh -i $SSH_KEY $SSH_HOST" +echo "2) nano $REMOTE_DIR/.env # TESTNET keys" +echo "3) systemctl --user start mm-edge-probe" +echo "4) ./scripts/probe_status.sh" +echo "5) ./scripts/pull_report.sh" diff --git a/scripts/economic_attribution_v0_1.py b/scripts/economic_attribution_v0_1.py new file mode 100644 index 0000000..416fed4 --- /dev/null +++ b/scripts/economic_attribution_v0_1.py @@ -0,0 +1,533 @@ +#!/usr/bin/env python3 +""" +Economic Attribution v0.1 + +Hard Evidence Population only: + MATCHED = Local Fill ↔ Venue Trade dual evidence + +Purpose: + Economic Attribution only. + No strategy modification. + No live execution. + No economic simulation. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from collections import defaultdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +from reconcile_fills import load_local_fills, match, normalize_local, normalize_venue # noqa: E402 + + +def _load_jsonl_df(log_dir: Path) -> pd.DataFrame: + rows: list[dict[str, Any]] = [] + for f in sorted(log_dir.glob("*.jsonl")): + if f.name.startswith(("Account_", "Maker_", "RECON")): + continue + for line in f.open(): + try: + e = json.loads(line) + except Exception: + continue + if isinstance(e, dict): + rows.append(e) + return pd.DataFrame(rows) + + +def _parse_fill_context(df: pd.DataFrame) -> pd.DataFrame: + if df.empty or "fill_context" not in df.columns: + return pd.DataFrame(columns=["fill_id"]) + rows = [] + for _, r in df.iterrows(): + ctx = r.get("fill_context") + if not isinstance(ctx, dict): + continue + rows.append( + { + "fill_id": r.get("fill_id"), + "market_event_before_fill": ctx.get("market_event_before_fill"), + "trade_imbalance_5s": ctx.get("trade_imbalance_5s"), + "price_velocity_5s": ctx.get("price_velocity_5s"), + "fill_type": ctx.get("fill_type"), + } + ) + return pd.DataFrame(rows) + + +def _fav_ret(side: pd.Series, fill: pd.Series, px: pd.Series) -> pd.Series: + raw = (pd.to_numeric(px, errors="coerce") - pd.to_numeric(fill, errors="coerce")) / pd.to_numeric( + fill, errors="coerce" + ) + return pd.Series(np.where(side == "long", raw, -raw), index=side.index) + + +def _cluster_weight(frame: pd.DataFrame) -> pd.Series: + cnt = frame.groupby("event_cluster_id")["event_cluster_id"].transform("count") + return 1.0 / cnt.clip(lower=1) + + +def _pct(v: float | None) -> str: + if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))): + return "n/a" + return f"{v * 100:.4f}%" + + +def _num(v: float | None, digits: int = 4) -> str: + if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))): + return "n/a" + return f"{v:.{digits}f}" + + +def _mean(s: pd.Series) -> float | None: + s = pd.to_numeric(s, errors="coerce").dropna() + return None if s.empty else float(s.mean()) + + +def _sum(s: pd.Series) -> float: + s = pd.to_numeric(s, errors="coerce").fillna(0.0) + return float(s.sum()) + + +def _weighted_mean(v: pd.Series, w: pd.Series) -> float | None: + vv = pd.to_numeric(v, errors="coerce") + ww = pd.to_numeric(w, errors="coerce").fillna(0.0) + mask = vv.notna() & ww.notna() + vv = vv[mask] + ww = ww[mask] + if vv.empty or float(ww.sum()) == 0.0: + return None + return float((vv * ww).sum() / ww.sum()) + + +def _prepare_paths(paths: pd.DataFrame) -> pd.DataFrame: + paths = paths.copy() + if "max_price" in paths.columns and "min_price" in paths.columns and "fill_price" in paths.columns: + rng = (pd.to_numeric(paths["max_price"], errors="coerce") - pd.to_numeric(paths["min_price"], errors="coerce")) / pd.to_numeric( + paths["fill_price"], errors="coerce" + ) + med = float(rng.dropna().median()) if rng.notna().any() else 0.0 + paths["vol_bucket"] = np.where(rng >= med, "high_vol", "low_vol") + if "price_velocity_5s" in paths.columns and pd.to_numeric(paths["price_velocity_5s"], errors="coerce").notna().any(): + v = pd.to_numeric(paths["price_velocity_5s"], errors="coerce") + thr = float(v.abs().median()) * 0.5 + paths["trend_bucket"] = np.where(v > thr, "trend_up", np.where(v < -thr, "trend_down", "range")) + if "spread" in paths.columns and "fill_price" in paths.columns and pd.to_numeric(paths["spread"], errors="coerce").notna().any(): + sp = pd.to_numeric(paths["spread"], errors="coerce") / pd.to_numeric(paths["fill_price"], errors="coerce") + med = float(sp.dropna().median()) if sp.notna().any() else 0.0 + paths["liq_bucket"] = np.where(sp <= med, "tight_spread", "wide_spread") + paths["toxicity_bucket"] = np.where(paths["path_type"].astype(str).str.startswith("C"), "toxic", "non_toxic") + return paths + + +def _inventory_metrics(matched: pd.DataFrame) -> dict[str, float | None]: + if matched.empty: + return {} + g = matched.sort_values("venue_time_ms").copy() + g["signed_qty"] = np.where(g["side"] == "long", g["qty"], -g["qty"]) + g["net_btc"] = g["signed_qty"].cumsum() + g["abs_net_btc"] = g["net_btc"].abs() + times = pd.to_numeric(g["venue_time_ms"], errors="coerce").astype("float64") / 1000.0 + dt = times.shift(-1) - times + dt = dt.fillna(0.0).clip(lower=0.0) + total_t = float(dt.sum()) + tw_abs = float((g["abs_net_btc"] * dt).sum() / total_t) if total_t > 0 else None + tw_signed = float((g["net_btc"] * dt).sum() / total_t) if total_t > 0 else None + return { + "max_net_btc": float(g["net_btc"].max()), + "min_net_btc": float(g["net_btc"].min()), + "max_abs_net_btc": float(g["abs_net_btc"].max()), + "avg_abs_net_btc_per_fill": float(g["abs_net_btc"].mean()), + "time_weighted_abs_net_btc": tw_abs, + "time_weighted_signed_net_btc": tw_signed, + "long_qty": float(g.loc[g["signed_qty"] > 0, "signed_qty"].sum()), + "short_qty": float((-g.loc[g["signed_qty"] < 0, "signed_qty"]).sum()), + "turnover_btc": float(g["qty"].sum()), + } + + +def _bucket_table(paths: pd.DataFrame, bucket: str, title: str) -> list[dict[str, Any]]: + if bucket not in paths.columns or paths.empty: + return [] + rows = [] + for key, grp in paths.groupby(bucket): + notional = grp["notional_usdt"].sum() + clusters = grp["event_cluster_id"].nunique() + rows.append( + { + "dimension": title, + "bucket": str(key), + "fills": int(len(grp)), + "clusters": int(clusters), + "btc_qty": float(grp["qty"].sum()), + "notional_usdt": float(notional), + "fee_usdt": float(grp["commission_usdt"].sum()), + "fee_per_fill": float(grp["commission_usdt"].mean()) if len(grp) else None, + "fee_per_btc": float(grp["commission_usdt"].sum() / grp["qty"].sum()) if grp["qty"].sum() else None, + "markout_1s": _weighted_mean(grp["markout_1s"], grp["notional_usdt"]), + "markout_5s": _weighted_mean(grp["markout_5s"], grp["notional_usdt"]), + "markout_10s": _weighted_mean(grp["markout_10s"], grp["notional_usdt"]), + "markout_30s": _weighted_mean(grp["markout_30s"], grp["notional_usdt"]), + "markout_300s": _weighted_mean(grp["markout_300s"], grp["notional_usdt"]), + "gross_markout_30s_usdt": float(grp["gross_markout_30s_usdt"].sum()), + "realized_pnl_usdt": float(grp["realized_pnl_usdt"].sum()), + "net_attr_30s_usdt": float(grp["net_attr_30s_usdt"].sum()), + } + ) + rows.sort(key=lambda x: (-x["fills"], x["bucket"])) + return rows + + +def _counterfactual(base: pd.DataFrame, exclude_col: str, exclude_values: set[str], label: str) -> dict[str, Any]: + kept = base[~base[exclude_col].astype(str).isin(exclude_values)].copy() + return { + "name": label, + "fills": int(len(kept)), + "clusters": int(kept["event_cluster_id"].nunique()) if not kept.empty else 0, + "btc_qty": float(kept["qty"].sum()) if not kept.empty else 0.0, + "fee_usdt": float(kept["commission_usdt"].sum()) if not kept.empty else 0.0, + "gross_markout_30s_usdt": float(kept["gross_markout_30s_usdt"].sum()) if not kept.empty else 0.0, + "realized_pnl_usdt": float(kept["realized_pnl_usdt"].sum()) if not kept.empty else 0.0, + "net_attr_30s_usdt": float(kept["net_attr_30s_usdt"].sum()) if not kept.empty else 0.0, + "markout_30s": _weighted_mean(kept["markout_30s"], kept["notional_usdt"]), + } + + +def main() -> int: + ap = argparse.ArgumentParser(description="Economic Attribution v0.1 (MATCHED only)") + ap.add_argument("--dir", default=str(ROOT / "logs" / "maker_edge")) + ap.add_argument("--out", default=str(ROOT / "logs" / "maker_edge" / "Economic_Attribution_v0_1.txt")) + ap.add_argument("--recon03", default=str(ROOT / "logs" / "maker_edge" / "RECONCILIATION_03.json")) + ap.add_argument("--account", default=str(ROOT / "logs" / "maker_edge" / "Account_Reconciliation.json")) + ap.add_argument("--venue-trades", default=str(ROOT / "logs" / "maker_edge" / "venue_trades.json")) + args = ap.parse_args() + + log_dir = Path(args.dir) + df = _load_jsonl_df(log_dir) + fills = df[df["event"] == "fill"].copy() + paths = df[df["event"] == "fill_path"].copy() + inv = df[df["event"] == "inventory_tick"].copy() + + venue_trades = json.loads(Path(args.venue_trades).read_text()) + local_fills_raw = load_local_fills(log_dir) + locals_norm = [normalize_local(e, i) for i, e in enumerate(local_fills_raw)] + venues_norm = [normalize_venue(t, i) for i, t in enumerate(venue_trades)] + recon = match(locals_norm, venues_norm) + matched_fill_ids = {m["local"]["fill_id"] for m in recon["matched"]} + matched_trade_ids = {m["venue"]["venue_trade_id"] for m in recon["matched"]} + + fills = fills[fills["fill_id"].isin(matched_fill_ids)].copy() + paths = paths[paths["fill_id"].isin(matched_fill_ids)].copy() + + fc = _parse_fill_context(fills) + meta_cols = [ + c + for c in [ + "fill_id", + "side", + "fill_price", + "spread", + "spread_capture_pct", + "obi", + "trade_imbalance", + "bid_depth_5", + "ask_depth_5", + "book_age_ms", + "inventory", + "inventory_time", + "inventory_skew", + "pre_5s_deteriorated", + "mid", + "event_cluster_id", + "pair", + ] + if c in fills.columns + ] + meta = fills.drop_duplicates("fill_id")[meta_cols] + paths = paths.merge(meta, on="fill_id", how="left", suffixes=("", "_f")) + for col in ["event_cluster_id", "side", "fill_price", "spread_capture_pct", "mid"]: + alt = f"{col}_f" + if alt in paths.columns: + if col not in paths.columns: + paths[col] = paths[alt] + else: + paths[col] = paths[col].fillna(paths[alt]) + if not fc.empty: + paths = paths.merge(fc, on="fill_id", how="left") + paths = _prepare_paths(paths) + + venue = pd.DataFrame(venues_norm) + venue = venue[venue["venue_trade_id"].isin(matched_trade_ids)].copy() + venue = venue.rename( + columns={ + "venue_trade_id": "trade_id_link", + "venue_order_id": "venue_order_id", + "qty": "qty", + "px": "venue_price", + "ts": "venue_ts", + } + ) + raw_v = pd.DataFrame(venue_trades) + raw_v["trade_id_link"] = raw_v["id"].astype(str) + raw_v["venue_order_id"] = raw_v["orderId"].astype(str) + raw_v["commission_usdt"] = pd.to_numeric(raw_v["commission"], errors="coerce") + raw_v["realized_pnl_usdt"] = pd.to_numeric(raw_v["realizedPnl"], errors="coerce").fillna(0.0) + raw_v["venue_time_ms"] = pd.to_numeric(raw_v["time"], errors="coerce") + raw_v["qty"] = pd.to_numeric(raw_v["qty"], errors="coerce") + raw_v["venue_price"] = pd.to_numeric(raw_v["price"], errors="coerce") + raw_v["side"] = np.where(raw_v["buyer"].astype(bool), "long", "short") + raw_v = raw_v[raw_v["trade_id_link"].isin(matched_trade_ids)].copy() + + matched_map = pd.DataFrame( + [ + { + "fill_id": m["local"]["fill_id"], + "trade_id_link": m["venue"]["venue_trade_id"], + "venue_order_id": m["venue"]["venue_order_id"], + } + for m in recon["matched"] + ] + ) + + paths = paths.merge( + matched_map.merge( + raw_v[ + [ + "trade_id_link", + "venue_order_id", + "commission_usdt", + "realized_pnl_usdt", + "venue_time_ms", + "qty", + "venue_price", + "side", + ] + ], + on=["trade_id_link", "venue_order_id"], + how="left", + ), + on="fill_id", + how="left", + suffixes=("", "_venue"), + ) + + paths["fill_price"] = pd.to_numeric(paths["fill_price"], errors="coerce") + paths["qty"] = pd.to_numeric(paths["qty"], errors="coerce") + paths["notional_usdt"] = paths["fill_price"] * paths["qty"] + for sec, col in [(1, "after_1s_price"), (5, "after_5s_price"), (10, "after_10s_price"), (30, "after_30s_price"), (300, "after_5m_price")]: + paths[f"markout_{sec}s"] = _fav_ret(paths["side"], paths["fill_price"], paths[col]) + paths["gross_markout_30s_usdt"] = paths["notional_usdt"] * paths["markout_30s"] + paths["net_attr_30s_usdt"] = ( + paths["gross_markout_30s_usdt"] + - pd.to_numeric(paths["commission_usdt"], errors="coerce").fillna(0.0) + + pd.to_numeric(paths["realized_pnl_usdt"], errors="coerce").fillna(0.0) + ) + + inventory_metrics = _inventory_metrics( + raw_v[ + ["venue_time_ms", "side", "qty", "commission_usdt", "realized_pnl_usdt", "venue_order_id", "trade_id_link"] + ].copy() + ) + + n_matched_paths = len(paths) + n_matched_fills = len(fills) + n_matched_clusters = int(fills["event_cluster_id"].nunique()) if not fills.empty else 0 + cluster_w = _cluster_weight(paths) if not paths.empty and "event_cluster_id" in paths.columns else pd.Series(dtype=float) + + horizon_rows = [] + for sec in (1, 5, 10, 30, 300): + col = f"markout_{sec}s" + valid = paths[col].notna() + sub = paths[valid] + w = sub["notional_usdt"] + horizon_rows.append( + { + "horizon": f"{sec}s", + "n": int(len(sub)), + "fill_w": _weighted_mean(sub[col], w), + "cluster_w": _weighted_mean(sub[col], _cluster_weight(sub) if not sub.empty else pd.Series(dtype=float)), + "gross_usdt": float((sub["notional_usdt"] * sub[col]).sum()) if not sub.empty else 0.0, + } + ) + + fee_total = float(paths["commission_usdt"].sum()) + realized_total = float(paths["realized_pnl_usdt"].sum()) + gross_30_total = float(paths["gross_markout_30s_usdt"].sum()) + net_attr_30_total = float(paths["net_attr_30s_usdt"].sum()) + total_qty = float(paths["qty"].sum()) + total_notional = float(paths["notional_usdt"].sum()) + + bucket_rows: list[dict[str, Any]] = [] + for col, title in [ + ("path_type", "PathType"), + ("toxicity_bucket", "Toxicity"), + ("vol_bucket", "Volatility"), + ("liq_bucket", "Spread"), + ("trend_bucket", "Trend"), + ("market_event_before_fill", "FillContext"), + ]: + bucket_rows.extend(_bucket_table(paths, col, title)) + bucket_df = pd.DataFrame(bucket_rows) + + negative_states: set[str] = set() + if not bucket_df.empty: + neg = bucket_df[(bucket_df["dimension"] != "PathType") & (bucket_df["markout_30s"] < 0)] + negative_states = set(neg["bucket"].astype(str)) + + counterfactuals = [ + { + "name": "BASELINE", + "fills": int(len(paths)), + "clusters": int(paths["event_cluster_id"].nunique()) if not paths.empty else 0, + "btc_qty": total_qty, + "fee_usdt": fee_total, + "gross_markout_30s_usdt": gross_30_total, + "realized_pnl_usdt": realized_total, + "net_attr_30s_usdt": net_attr_30_total, + "markout_30s": _weighted_mean(paths["markout_30s"], paths["notional_usdt"]), + }, + _counterfactual(paths, "path_type", {"C_toxic"}, "EXCLUDE_PATH_C"), + _counterfactual(paths, "toxicity_bucket", {"toxic"}, "EXCLUDE_TOXIC"), + _counterfactual(paths, "market_event_before_fill", negative_states, "EXCLUDE_NEGATIVE_STATE"), + ] + + account = json.loads(Path(args.account).read_text()) if Path(args.account).exists() else {} + recon03 = json.loads(Path(args.recon03).read_text()) if Path(args.recon03).exists() else {} + + out_txt = Path(args.out) + out_json = out_txt.with_suffix(".json") + lines: list[str] = [] + + def p(s: str = "") -> None: + lines.append(s) + print(s) + + p("=" * 72) + p("Economic Attribution v0.1") + p("=" * 72) + p("Experiment: MM_EDGE_EXP_001") + p("Population: MATCHED=3890") + p("Strategy: v0.1 FROZEN") + p("Execution: STOPPED") + p("Stage3: LOCKED") + p("Purpose: Economic Attribution only.") + p("No strategy modification. No live execution. No economic simulation.") + p() + p("Layer 1 — Hard Economic Evidence") + p("-" * 40) + p(f"Matched fills: {n_matched_fills}") + p(f"Matched paths: {n_matched_paths}") + p(f"Matched clusters: {n_matched_clusters}") + p(f"Fee total: {_num(fee_total, 6)} USDT") + p(f"Fee / fill: {_num(fee_total / max(n_matched_paths, 1), 6)} USDT") + p(f"Fee / BTC: {_num(fee_total / max(total_qty, 1e-12), 6)} USDT") + p(f"Fee / cluster: {_num(fee_total / max(n_matched_clusters, 1), 6)} USDT") + p(f"Realized component: {_num(realized_total, 6)} USDT") + p(f"Gross markout @30s: {_num(gross_30_total, 6)} USDT") + p(f"Net attributable @30s: {_num(net_attr_30_total, 6)} USDT") + p() + p("Markout by horizon (MATCHED only)") + p("-" * 40) + for row in horizon_rows: + p( + f"{row['horizon']:>5} n={row['n']:4d} fill-w={_pct(row['fill_w'])} " + f"cluster-w={_pct(row['cluster_w'])} gross={_num(row['gross_usdt'], 6)} USDT" + ) + p() + p("Inventory carry / exposure") + p("-" * 40) + p(f"Max net BTC: {_num(inventory_metrics.get('max_net_btc'), 6)}") + p(f"Min net BTC: {_num(inventory_metrics.get('min_net_btc'), 6)}") + p(f"Max |net BTC|: {_num(inventory_metrics.get('max_abs_net_btc'), 6)}") + p(f"Average |net BTC|: {_num(inventory_metrics.get('avg_abs_net_btc_per_fill'), 6)}") + p(f"TW |net BTC|: {_num(inventory_metrics.get('time_weighted_abs_net_btc'), 6)}") + p(f"TW signed net BTC: {_num(inventory_metrics.get('time_weighted_signed_net_btc'), 6)}") + p(f"Long qty / Short qty: {_num(inventory_metrics.get('long_qty'), 6)} / {_num(inventory_metrics.get('short_qty'), 6)} BTC") + p(f"Inventory turnover: {_num(inventory_metrics.get('turnover_btc'), 6)} BTC") + p() + p("Slices (weighted by notional, MATCHED only)") + p("-" * 40) + for dim in ["PathType", "Toxicity", "Volatility", "Spread", "Trend", "FillContext"]: + sub = bucket_df[bucket_df["dimension"] == dim].copy() + if sub.empty: + continue + p(dim) + for _, r in sub.sort_values(["fills", "bucket"], ascending=[False, True]).iterrows(): + p( + f" {r['bucket']}: n={int(r['fills'])} clusters={int(r['clusters'])} " + f"fee={_num(r['fee_usdt'], 4)} gross30={_num(r['gross_markout_30s_usdt'], 4)} " + f"realized={_num(r['realized_pnl_usdt'], 4)} net30={_num(r['net_attr_30s_usdt'], 4)} " + f"m30={_pct(r['markout_30s'])}" + ) + p() + p("Layer 2 — Evidence Extension (excluded from core conclusion)") + p("-" * 40) + p(f"VENUE_CONFIRMED_NO_TRADE_HISTORY: {recon03.get('venue_confirmed_no_trade_history', 'n/a')}") + p(f"VENUE_PARTIAL_ORDER_CANCELED: {recon03.get('venue_partial_order_canceled', 'n/a')}") + p("These rows are order-confirmed, but not part of the Hard Evidence Population.") + p() + p("Layer 3 — Counterfactual Attribution (NOT backtest)") + p("-" * 40) + p("Observed vs Exclude-Bucket Attribution. These are contribution decompositions only.") + for row in counterfactuals: + p( + f"{row['name']}: fills={row['fills']} clusters={row['clusters']} " + f"fee={_num(row['fee_usdt'], 4)} gross30={_num(row['gross_markout_30s_usdt'], 4)} " + f"realized={_num(row['realized_pnl_usdt'], 4)} net30={_num(row['net_attr_30s_usdt'], 4)} " + f"m30={_pct(row['markout_30s'])}" + ) + p() + p("Interpretation") + p("-" * 40) + p("Core conclusion is based on 3890 fully matched fills.") + p("Economic Attribution asks why MakerAlpha did not convert to money.") + p("It does NOT change quote logic, does NOT restart v0.1, and does NOT unlock Stage 3.") + p("=" * 72) + + out_txt.write_text("\n".join(lines) + "\n", encoding="utf-8") + sidecar = { + "experiment_id": "MM_EDGE_EXP_001", + "population": { + "name": "MATCHED", + "fills": n_matched_fills, + "paths": n_matched_paths, + "clusters": n_matched_clusters, + }, + "strategy": "v0.1 FROZEN", + "execution": "STOPPED", + "stage3": "LOCKED", + "fee_total_usdt": fee_total, + "fee_per_fill_usdt": fee_total / max(n_matched_paths, 1), + "fee_per_btc_usdt": fee_total / max(total_qty, 1e-12), + "fee_per_cluster_usdt": fee_total / max(n_matched_clusters, 1), + "realized_component_usdt": realized_total, + "gross_markout_30s_usdt": gross_30_total, + "net_attr_30s_usdt": net_attr_30_total, + "markout_by_horizon": horizon_rows, + "inventory_metrics": inventory_metrics, + "bucket_rows": bucket_rows, + "counterfactuals": counterfactuals, + "recon03_extension": { + "venue_confirmed_no_trade_history": recon03.get("venue_confirmed_no_trade_history"), + "venue_partial_order_canceled": recon03.get("venue_partial_order_canceled"), + }, + "account_recon_ref": account, + } + out_json.write_text(json.dumps(sidecar, indent=2) + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/economic_fee_sensitivity_v0_1.py b/scripts/economic_fee_sensitivity_v0_1.py new file mode 100644 index 0000000..a02ea67 --- /dev/null +++ b/scripts/economic_fee_sensitivity_v0_1.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +""" +Economic Fee Sensitivity v0.1 (MATCHED=3890) + +Computes: + net_attr_30s(fee_factor) = gross_markout_30s_usdt - fee_factor * fee_total_usdt + realized_component_usdt + +Assumption: + realized_component_usdt and gross_markout_30s_usdt are fixed (price/path unchanged). + Only fee scaling is applied as a counterfactual sensitivity. + +This is NOT a strategy backtest and does NOT modify any execution logic. +""" + +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def main() -> int: + ap = argparse.ArgumentParser(description="Economic Fee Sensitivity v0.1") + ap.add_argument( + "--attribution", + default=str(ROOT / "logs" / "maker_edge" / "Economic_Attribution_v0_1.json"), + ) + ap.add_argument( + "--out", + default=str(ROOT / "logs" / "maker_edge" / "Economic_Fee_Sensitivity_v0_1.txt"), + ) + args = ap.parse_args() + + data = json.loads(Path(args.attribution).read_text()) + fee_total = float(data["fee_total_usdt"]) + realized_total = float(data["realized_component_usdt"]) + gross_markout = float(data["gross_markout_30s_usdt"]) + + factors = [1.0, 0.5, 0.25, 0.1, 0.0] + + lines: list[str] = [] + + def p(s: str = "") -> None: + lines.append(s) + print(s) + + p("=" * 72) + p("Economic Fee Sensitivity v0.1 (MATCHED=3890)") + p("=" * 72) + p(f"gross_markout_30s_usdt: {gross_markout:+.6f} USDT") + p(f"fee_total_usdt: {fee_total:+.6f} USDT") + p(f"realized_component_usdt:{realized_total:+.6f} USDT") + p() + p("Fee assumption → Net attributable @30s") + p("-" * 42) + + header = ["fee_factor", "fee_usdt_assumed", "net_attr_30s_usdt"] + p(" | ".join(header)) + + for f in factors: + fee_assumed = f * fee_total + net = gross_markout - fee_assumed + realized_total + row = [f"{f:.2f}", f"{fee_assumed:+.6f}", f"{net:+.6f}"] + p(" | ".join(row)) + + p() + p("Interpretation:") + p("- If net remains < 0 at fee_factor=0 → economics not salvageable by fee reduction alone.") + p("- If fee reduction flips net > 0 → current venue/fee tier can be the dominant issue.") + p("=" * 72) + + Path(args.out).write_text("\n".join(lines) + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/scripts/economic_metric_reconciliation_v0_1.py b/scripts/economic_metric_reconciliation_v0_1.py new file mode 100644 index 0000000..b5d615c --- /dev/null +++ b/scripts/economic_metric_reconciliation_v0_1.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +""" +Metric Reconciliation v0.1 (MATCHED only) + +Confirms consistency between: + - "MakerAlpha" reported in v0.1 research (return space) + - "Gross markout @30s" in Economic Attribution (dollar space) + - realized component used in Economic Attribution + +Key point: + Same definition may flip sign depending on weighting: + fill-weighted mean return vs notional-weighted dollar markout + +This script is read-only: it does NOT change any strategy/execution. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from collections import defaultdict +from pathlib import Path + +import numpy as np +import pandas as pd + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +from reconcile_fills import load_local_fills, match, normalize_local, normalize_venue # noqa: E402 + + +def _load_jsonl_df(log_dir: Path) -> pd.DataFrame: + rows: list[dict] = [] + for f in sorted(log_dir.glob("*.jsonl")): + if f.name.startswith(("Account_", "Maker_", "RECON")): + continue + for line in f.open(): + try: + e = json.loads(line) + except Exception: + continue + if isinstance(e, dict): + rows.append(e) + return pd.DataFrame(rows) + + +def _fav_ret(side: pd.Series, fill: pd.Series, px: pd.Series) -> pd.Series: + # return space, signed by side + fill = pd.to_numeric(fill, errors="coerce") + px = pd.to_numeric(px, errors="coerce") + raw = (px - fill) / fill + return pd.Series(np.where(side == "long", raw, -raw), index=side.index) + + +def _weighted_mean(x: pd.Series, w: pd.Series) -> float | None: + xx = pd.to_numeric(x, errors="coerce") + ww = pd.to_numeric(w, errors="coerce") + mask = xx.notna() & ww.notna() + xx = xx[mask] + ww = ww[mask] + if xx.empty: + return None + sw = float(ww.sum()) + if sw == 0: + return None + return float((xx * ww).sum() / sw) + + +def _cluster_weight(paths: pd.DataFrame) -> pd.Series: + if "event_cluster_id" not in paths.columns: + return pd.Series(1.0, index=paths.index) + cnt = paths.groupby("event_cluster_id")["event_cluster_id"].transform("count") + return 1.0 / cnt.clip(lower=1) + + +def main() -> int: + ap = argparse.ArgumentParser(description="Economic Metric Reconciliation v0.1") + ap.add_argument("--dir", default=str(ROOT / "logs" / "maker_edge")) + ap.add_argument("--out", default=str(ROOT / "logs" / "maker_edge" / "Economic_Metric_Reconciliation_v0_1.txt")) + ap.add_argument("--venue-trades", default=str(ROOT / "logs" / "maker_edge" / "venue_trades.json")) + ap.add_argument("--matched-take", type=int, default=3890) + args = ap.parse_args() + + log_dir = Path(args.dir) + venue_trades_path = Path(args.venue_trades) + + df = _load_jsonl_df(log_dir) + fills = df[df["event"] == "fill"].copy() if "event" in df.columns else pd.DataFrame() + paths = df[df["event"] == "fill_path"].copy() if "event" in df.columns else pd.DataFrame() + + # Hard matched population via RECON-02/03 evidence: use existing matcher logic. + venue_trades = json.loads(venue_trades_path.read_text()) + local_fills_raw = load_local_fills(log_dir) + locals_norm = [normalize_local(e, i) for i, e in enumerate(local_fills_raw)] + venues_norm = [normalize_venue(t, i) for i, t in enumerate(venue_trades)] + recon = match(locals_norm, venues_norm) + + matched_fill_ids = {m["local"]["fill_id"] for m in recon["matched"]} + matched_trade_ids = {m["venue"]["venue_trade_id"] for m in recon["matched"]} + + fills = fills[fills["fill_id"].isin(matched_fill_ids)].copy() + paths = paths[paths["fill_id"].isin(matched_fill_ids)].copy() + + # Build after_30s already present in fill_path fields. + # MakerAlpha in analyze_maker_edge uses after_30s_price and _fav_ret definition. + # We'll recompute: + # return space: + # maker_alpha_fill_weighted = mean(markout_30s) + # maker_alpha_notional_weighted_return = (gross_markout_usdt / total_notional) + # gross_markout_usdt = sum(notional * markout_30s) + # + if paths.empty: + raise SystemExit("No matched paths loaded") + + # Merge meta from fills (side, fill_price, event_cluster_id, notional proxy) + meta_cols = [ + c + for c in [ + "fill_id", + "side", + "fill_price", + "amount", + "event_cluster_id", + "spread_capture_pct", + "pair", + ] + if c in fills.columns + ] + meta = fills.drop_duplicates("fill_id")[meta_cols] + paths = paths.merge(meta, on="fill_id", how="left", suffixes=("", "_m")) + + # If fill_path already had these columns, merge created *_m alternates. + for col in ["side", "fill_price", "amount", "event_cluster_id"]: + alt = f"{col}_m" + if alt in paths.columns: + if col not in paths.columns: + paths[col] = paths[alt] + else: + paths[col] = paths[col].fillna(paths[alt]) + + # Ensure required fields + paths["side"] = paths["side"].astype(str) + paths["fill_price"] = pd.to_numeric(paths["fill_price"], errors="coerce") + paths["qty"] = pd.to_numeric(paths["amount"], errors="coerce") + paths["notional_usdt"] = paths["fill_price"] * paths["qty"] + paths["after_30s_price"] = pd.to_numeric(paths["after_30s_price"], errors="coerce") + + paths["markout_30s_return"] = _fav_ret(paths["side"], paths["fill_price"], paths["after_30s_price"]) + + gross_markout_usdt = float((paths["notional_usdt"] * paths["markout_30s_return"]).sum()) + total_notional = float(paths["notional_usdt"].sum()) + maker_alpha_fill_weighted = float(paths["markout_30s_return"].mean()) + maker_alpha_notional_weighted_return = float(gross_markout_usdt / total_notional) if total_notional else None + + cw = _cluster_weight(paths) + maker_alpha_cluster_weighted_return = _weighted_mean(paths["markout_30s_return"], cw) + + # realized component from userTrades is already in Economic Attribution. + # Here we only validate return space; realized component sign conventions are asserted elsewhere. + out = Path(args.out) + lines: list[str] = [] + + def p(s: str = "") -> None: + lines.append(s) + print(s) + + p("=" * 72) + p("Economic Metric Reconciliation v0.1 (MATCHED=3890)") + p("=" * 72) + p(f"Matched paths: {len(paths)} (expected ~3886)") + p() + p("Definitions (same math as analyze_maker_edge):") + p("- markout_30s_return = _fav_ret(side, fill_price, after_30s_price)") + p("- gross_markout_usdt = sum(notional_usdt * markout_30s_return)") + p() + p("Return-space metrics (sign may differ due to weighting):") + p(f"MakerAlpha fill-weighted mean return: {_pct(maker_alpha_fill_weighted)}") + p(f"MakerAlpha notional-weighted mean return: {_pct(maker_alpha_notional_weighted_return)}") + p(f"MakerAlpha cluster-weighted mean return: {_pct(maker_alpha_cluster_weighted_return)}") + p() + p("Dollar-space metrics:") + p(f"gross_markout_usdt (30s): {gross_markout_usdt:+.6f} USDT") + p(f"total_notional_usdt: {total_notional:.3f} USDT") + p() + p("If fill-weighted return is + but gross_markout_usdt is negative,") + p("it means notional weighting flips sign (alpha is conditionally realized).") + p("=" * 72) + + out.write_text("\n".join(lines) + "\n", encoding="utf-8") + return 0 + + +def _pct(v: float | None) -> str: + if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))): + return "n/a" + return f"{v*100:.6f}%" + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/scripts/event_state_status.sh b/scripts/event_state_status.sh new file mode 100755 index 0000000..c873649 --- /dev/null +++ b/scripts/event_state_status.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Remote EXP_002 long-run status (read-only). Does not analyze Path C. +set -euo pipefail + +SSH_HOST="${SSH_HOST:-jack@jackyu66.com}" +SSH_KEY="${SSH_KEY:-${HOME}/Project/deploy/zun_hk/id_ed25519_hk}" +REMOTE_DIR="${REMOTE_DIR:-/www/Project/nautilus_mm}" + +SSH_OPTS=(-o StrictHostKeyChecking=accept-new) +if [[ -n "$SSH_KEY" ]]; then + chmod 400 "$SSH_KEY" 2>/dev/null || true + SSH_OPTS+=(-i "$SSH_KEY") +fi + +ssh "${SSH_OPTS[@]}" "$SSH_HOST" "REMOTE_DIR='$REMOTE_DIR' bash -s" <<'EOF' +set -euo pipefail +echo "=== systemd --user event-state-probe ===" +systemctl --user is-active event-state-probe || true +systemctl --user show event-state-probe -p Environment --no-pager 2>/dev/null | tr ' ' '\n' | grep -E 'ENABLE_TRADING|EXPERIMENT_ID|LEDGER_RUN_ID' || true +echo "" +echo "=== mm-edge-probe (EXP_001) ===" +systemctl --user is-active mm-edge-probe || true +echo "" +LOG="$REMOTE_DIR/logs/event_state/EXP-002-RUN-002" +echo "=== ledger $LOG ===" +if [[ ! -d "$LOG" ]]; then + echo "no log dir yet" + exit 0 +fi +python3 - < pd.DataFrame: + rows: list[dict[str, Any]] = [] + for f in sorted(log_dir.glob("*.jsonl")): + if f.name.startswith(("Account_", "Maker_", "RECON")): + continue + for line in f.open(): + try: + e = json.loads(line) + except Exception: + continue + if isinstance(e, dict): + rows.append(e) + return pd.DataFrame(rows) + + +def _pct(v: float | None) -> str: + if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))): + return "n/a" + return f"{v*100:.2f}%" + + +def _num(v: float | None, digits: int = 4) -> str: + if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))): + return "n/a" + return f"{v:.{digits}f}" + + +def _fav_ret(side: pd.Series, fill: pd.Series, px: pd.Series) -> pd.Series: + fill = pd.to_numeric(fill, errors="coerce") + px = pd.to_numeric(px, errors="coerce") + raw = (px - fill) / fill + return pd.Series(np.where(side == "long", raw, -raw), index=side.index) + + +def _weighted_mean(x: pd.Series, w: pd.Series) -> float | None: + xx = pd.to_numeric(x, errors="coerce") + ww = pd.to_numeric(w, errors="coerce") + mask = xx.notna() & ww.notna() + xx = xx[mask] + ww = ww[mask] + if xx.empty: + return None + sw = float(ww.sum()) + if sw == 0.0: + return None + return float((xx * ww).sum() / sw) + + +def _sample_grade(n: int) -> str: + if n < 30: + return "LOW_N" + if n < 100: + return "WEAK_EVIDENCE" + return "USABLE" + + +def _grade_probability(delta_pp: float, n_best: int) -> str: + if n_best < 30: + return "LOW_N" + if delta_pp < 5.0: + return "NO_PREFILL_SIGNAL" + if n_best < 100 or delta_pp < 10.0: + return "STATISTICAL_SIGNAL_ONLY" + return "CANDIDATE_V0_2_SIGNAL" + + +def _grade_economic(delta_usdt_per_fill: float, n_best: int) -> str: + if n_best < 30: + return "LOW_N" + if abs(delta_usdt_per_fill) < 0.003: + return "NO_PREFILL_SIGNAL" + if n_best < 100 or abs(delta_usdt_per_fill) < 0.008: + return "STATISTICAL_SIGNAL_ONLY" + return "CANDIDATE_V0_2_SIGNAL" + + +def _state_table_num(df: pd.DataFrame, feature: str, labels: list[str]) -> tuple[list[dict[str, Any]], dict[str, str]]: + s = pd.to_numeric(df[feature], errors="coerce") + valid = df[s.notna()].copy() + valid[feature] = s[s.notna()] + if valid.empty: + return [], {k: "NO_DATA" for k in labels + ["Economic"]} + q30 = float(valid[feature].quantile(0.30)) + q70 = float(valid[feature].quantile(0.70)) + # if no spread, collapse + if math.isclose(q30, q70): + valid["_state"] = "all" + else: + valid["_state"] = np.where( + valid[feature] <= q30, + "low", + np.where(valid[feature] >= q70, "high", "mid"), + ) + base = { + lab: float(valid[lab].mean()) for lab in labels + } + base["economic_mean"] = float(valid["net_attr_30s_usdt"].mean()) + rows = [] + grades: dict[str, str] = {} + for state, g in valid.groupby("_state"): + row = { + "feature": feature, + "state": str(state), + "n": int(len(g)), + "sample_grade": _sample_grade(int(len(g))), + "median": float(g[feature].median()), + "p25": float(g[feature].quantile(0.25)), + "p75": float(g[feature].quantile(0.75)), + "net_attr_mean": float(g["net_attr_30s_usdt"].mean()), + } + for lab in labels: + row[f"p_{lab}"] = float(g[lab].mean()) + row[f"delta_{lab}_pp"] = (row[f"p_{lab}"] - base[lab]) * 100.0 + row["delta_economic_per_fill"] = row["net_attr_mean"] - base["economic_mean"] + rows.append(row) + + # grade by strongest state-vs-baseline shift + for lab in labels: + best = max(rows, key=lambda r: abs(r[f"delta_{lab}_pp"])) + grades[lab] = _grade_probability(abs(best[f"delta_{lab}_pp"]), int(best["n"])) + best_e = max(rows, key=lambda r: abs(r["delta_economic_per_fill"])) + grades["Economic"] = _grade_economic(abs(best_e["delta_economic_per_fill"]), int(best_e["n"])) + return rows, grades + + +def _state_table_cat(df: pd.DataFrame, feature: str, labels: list[str]) -> tuple[list[dict[str, Any]], dict[str, str]]: + valid = df[df[feature].notna()].copy() + if valid.empty: + return [], {k: "NO_DATA" for k in labels + ["Economic"]} + base = { + lab: float(valid[lab].mean()) for lab in labels + } + base["economic_mean"] = float(valid["net_attr_30s_usdt"].mean()) + rows = [] + grades: dict[str, str] = {} + for state, g in valid.groupby(feature): + n = int(len(g)) + row = { + "feature": feature, + "state": str(state), + "n": n, + "sample_grade": _sample_grade(n), + "median": None, + "p25": None, + "p75": None, + "net_attr_mean": float(g["net_attr_30s_usdt"].mean()), + } + for lab in labels: + row[f"p_{lab}"] = float(g[lab].mean()) + row[f"delta_{lab}_pp"] = (row[f"p_{lab}"] - base[lab]) * 100.0 + row["delta_economic_per_fill"] = row["net_attr_mean"] - base["economic_mean"] + rows.append(row) + for lab in labels: + best = max(rows, key=lambda r: abs(r[f"delta_{lab}_pp"])) + grades[lab] = _grade_probability(abs(best[f"delta_{lab}_pp"]), int(best["n"])) + best_e = max(rows, key=lambda r: abs(r["delta_economic_per_fill"])) + grades["Economic"] = _grade_economic(abs(best_e["delta_economic_per_fill"]), int(best_e["n"])) + return rows, grades + + +def main() -> int: + ap = argparse.ArgumentParser(description="Prefill Adverse-Selection Attribution v0.1") + ap.add_argument("--dir", default=str(ROOT / "logs" / "maker_edge")) + ap.add_argument("--venue-trades", default=str(ROOT / "logs" / "maker_edge" / "venue_trades.json")) + ap.add_argument("--out", default=str(ROOT / "logs" / "maker_edge" / "Prefill_Adverse_Selection_Attribution_v0_1.txt")) + ap.add_argument("--margin-sec", type=float, default=0.25) + args = ap.parse_args() + + log_dir = Path(args.dir) + df = _load_jsonl_df(log_dir) + fills = df[df["event"] == "fill"].copy() + paths = df[df["event"] == "fill_path"].copy() + state_ticks = df[df["event"].isin(["mid_tick", "inventory_tick"])].copy() + + venue_trades = json.loads(Path(args.venue_trades).read_text()) + local_fills_raw = load_local_fills(log_dir) + locals_norm = [normalize_local(e, i) for i, e in enumerate(local_fills_raw)] + venues_norm = [normalize_venue(t, i) for i, t in enumerate(venue_trades)] + recon = match(locals_norm, venues_norm) + matched_fill_ids = {m["local"]["fill_id"] for m in recon["matched"]} + matched_trade_ids = {m["venue"]["venue_trade_id"] for m in recon["matched"]} + + fills = fills[fills["fill_id"].isin(matched_fill_ids)].copy() + paths = paths[paths["fill_id"].isin(matched_fill_ids)].copy() + + # merge labels/path info + fill_meta_cols = [ + c + for c in [ + "fill_id", + "side", + "fill_price", + "amount", + "quote_fill_time", + "ts_epoch", + "event_cluster_id", + "pair", + ] + if c in fills.columns + ] + meta = fills.drop_duplicates("fill_id")[fill_meta_cols] + paths = paths.merge(meta, on="fill_id", how="left", suffixes=("", "_f")) + for col in ["side", "fill_price", "amount", "event_cluster_id", "quote_fill_time", "ts_epoch"]: + alt = f"{col}_f" + if alt in paths.columns: + if col not in paths.columns: + paths[col] = paths[alt] + else: + paths[col] = paths[col].fillna(paths[alt]) + paths["fill_ts"] = pd.to_datetime(paths["quote_fill_time"], utc=True, errors="coerce") + # Prefer fill-event epoch seconds. astype(int64)/1e9 breaks when pandas stores UTC as us. + fill_epoch = pd.to_numeric(paths["ts_epoch"], errors="coerce") + iso_epoch = paths["fill_ts"].map(lambda ts: ts.timestamp() if pd.notna(ts) else np.nan) + paths["fill_ts_epoch"] = fill_epoch.fillna(iso_epoch) + paths["qty"] = pd.to_numeric(paths["amount"], errors="coerce") + paths["fill_price"] = pd.to_numeric(paths["fill_price"], errors="coerce") + paths["after_10s_price"] = pd.to_numeric(paths["after_10s_price"], errors="coerce") + paths["after_30s_price"] = pd.to_numeric(paths["after_30s_price"], errors="coerce") + paths["notional_usdt"] = paths["qty"] * paths["fill_price"] + paths["markout_10s"] = _fav_ret(paths["side"], paths["fill_price"], paths["after_10s_price"]) + paths["markout_30s"] = _fav_ret(paths["side"], paths["fill_price"], paths["after_30s_price"]) + paths["path_c"] = paths["path_type"].astype(str).eq("C_toxic") + paths["toxic"] = (paths["markout_10s"] < 0) & (paths["markout_30s"] < 0) + paths["negative_30s"] = paths["markout_30s"] < 0 + + # attach trade economics + raw_v = pd.DataFrame(venue_trades) + raw_v["trade_id_link"] = raw_v["id"].astype(str) + raw_v["commission_usdt"] = pd.to_numeric(raw_v["commission"], errors="coerce") + raw_v["realized_pnl_usdt"] = pd.to_numeric(raw_v["realizedPnl"], errors="coerce").fillna(0.0) + matched_map = pd.DataFrame( + [ + { + "fill_id": m["local"]["fill_id"], + "trade_id_link": m["venue"]["venue_trade_id"], + } + for m in recon["matched"] + ] + ) + paths = paths.merge( + matched_map.merge(raw_v[["trade_id_link", "commission_usdt", "realized_pnl_usdt"]], on="trade_id_link", how="left"), + on="fill_id", + how="left", + ) + paths["gross_markout_30s_usdt"] = paths["notional_usdt"] * paths["markout_30s"] + paths["net_attr_30s_usdt"] = ( + paths["gross_markout_30s_usdt"] + - pd.to_numeric(paths["commission_usdt"], errors="coerce").fillna(0.0) + + pd.to_numeric(paths["realized_pnl_usdt"], errors="coerce").fillna(0.0) + ) + paths["economic_negative"] = paths["net_attr_30s_usdt"] < 0 + + # strict prefill state from sampled historical ticks only + state_ticks = state_ticks.copy() + state_ticks["ts_epoch"] = pd.to_numeric(state_ticks["ts_epoch"], errors="coerce") + state_ticks = state_ticks.dropna(subset=["ts_epoch"]).sort_values("ts_epoch").drop_duplicates("ts_epoch") + keep_cols = [ + c + for c in [ + "ts_epoch", + "mid", + "spread", + "bid_depth_1", + "ask_depth_1", + "bid_depth_5", + "ask_depth_5", + "obi", + "delta", + "trade_imbalance", + "delta_efficiency", + "inventory", + "inventory_time", + "inventory_skew", + ] + if c in state_ticks.columns + ] + states = state_ticks[keep_cols].copy() + num_cols = [c for c in keep_cols if c != "ts_epoch"] + for col in num_cols: + states[col] = pd.to_numeric(states[col], errors="coerce") + + # 5s lag features using sampled states + lag_df = states[["ts_epoch"] + [c for c in ["mid", "spread", "obi", "bid_depth_5", "ask_depth_5", "trade_imbalance", "delta"] if c in states.columns]].copy() + lag_df["lag_ts"] = lag_df["ts_epoch"] + 5.0 + lag_cols = {c: f"{c}_past5s" for c in lag_df.columns if c not in {"ts_epoch", "lag_ts"}} + lag_df = lag_df.rename(columns=lag_cols) + + paths = paths[paths["fill_ts_epoch"].notna()].copy() + fill_states = paths[["fill_id", "fill_ts_epoch", "side"]].copy().sort_values("fill_ts_epoch") + fill_states["feature_cutoff_ts"] = fill_states["fill_ts_epoch"] - float(args.margin_sec) + + # latest sampled tick strictly before fill-margin + snap = pd.merge_asof( + fill_states.sort_values("feature_cutoff_ts"), + states.sort_values("ts_epoch"), + left_on="feature_cutoff_ts", + right_on="ts_epoch", + direction="backward", + ) + snap = snap[snap["ts_epoch"].notna()].copy() + snap = pd.merge_asof( + snap.sort_values("ts_epoch"), + lag_df.sort_values("lag_ts"), + left_on="ts_epoch", + right_on="lag_ts", + direction="backward", + ) + if "ts_epoch_x" in snap.columns: + snap = snap.rename(columns={"ts_epoch_x": "ts_epoch"}) + + # derived strict-prefill features + snap["spread_pct"] = snap["spread"] / snap["mid"] + snap["depth_total_5"] = snap["bid_depth_5"] + snap["ask_depth_5"] + snap["depth_imbalance_5"] = (snap["bid_depth_5"] - snap["ask_depth_5"]) / snap["depth_total_5"] + snap["price_velocity_5s"] = (snap["mid"] - snap["mid_past5s"]) / snap["mid_past5s"] + snap["spread_change_5s"] = snap["spread_pct"] - (snap["spread_past5s"] / snap["mid_past5s"]) + snap["obi_change_5s"] = snap["obi"] - snap["obi_past5s"] + snap["depth_total_5_past"] = snap["bid_depth_5_past5s"] + snap["ask_depth_5_past5s"] + snap["depth_change_5s"] = snap["depth_total_5"] - snap["depth_total_5_past"] + snap["trade_imbalance_change_5s"] = snap["trade_imbalance"] - snap["trade_imbalance_past5s"] + snap["delta_change_5s"] = snap["delta"] - snap["delta_past5s"] + snap["pre_deteriorated_strict"] = np.where( + snap["side"].eq("long"), + (snap["price_velocity_5s"] < 0) | (snap["depth_change_5s"] < 0), + (snap["price_velocity_5s"] > 0) | (snap["depth_change_5s"] < 0), + ) + snap["feature_age_ms"] = (snap["fill_ts_epoch"] - snap["ts_epoch"]) * 1000.0 + snap = snap.rename(columns={"ts_epoch": "feature_ts_epoch"}) + + snap_feature_cols = [ + "fill_id", + "feature_ts_epoch", + "feature_cutoff_ts", + "mid", + "spread", + "bid_depth_1", + "ask_depth_1", + "bid_depth_5", + "ask_depth_5", + "obi", + "delta", + "trade_imbalance", + "delta_efficiency", + "inventory", + "inventory_time", + "inventory_skew", + "mid_past5s", + "spread_past5s", + "obi_past5s", + "bid_depth_5_past5s", + "ask_depth_5_past5s", + "trade_imbalance_past5s", + "delta_past5s", + "spread_pct", + "depth_total_5", + "depth_imbalance_5", + "price_velocity_5s", + "spread_change_5s", + "depth_total_5_past", + "depth_change_5s", + "obi_change_5s", + "trade_imbalance_change_5s", + "delta_change_5s", + "pre_deteriorated_strict", + "feature_age_ms", + ] + snap_feature_cols = [c for c in snap_feature_cols if c in snap.columns] + rename_map = { + c: f"strict_{c}" + for c in snap_feature_cols + if c not in {"fill_id", "feature_ts_epoch", "feature_cutoff_ts", "feature_age_ms", "pre_deteriorated_strict"} + } + rename_map["feature_ts_epoch"] = "strict_feature_ts_epoch" + rename_map["feature_cutoff_ts"] = "strict_feature_cutoff_ts" + rename_map["feature_age_ms"] = "strict_feature_age_ms" + rename_map["pre_deteriorated_strict"] = "strict_pre_deteriorated" + snap_merge = snap[snap_feature_cols].rename(columns=rename_map) + pref = paths.merge(snap_merge, on="fill_id", how="left") + pref = pref[pref["strict_feature_ts_epoch"].notna()].copy() + + labels = ["path_c", "toxic", "negative_30s"] + numeric_features = [ + "strict_obi", + "strict_delta", + "strict_trade_imbalance", + "strict_spread_pct", + "strict_bid_depth_5", + "strict_ask_depth_5", + "strict_depth_total_5", + "strict_depth_imbalance_5", + "strict_price_velocity_5s", + "strict_spread_change_5s", + "strict_depth_change_5s", + "strict_obi_change_5s", + "strict_trade_imbalance_change_5s", + "strict_delta_change_5s", + "strict_inventory", + "strict_inventory_skew", + "strict_inventory_time", + "strict_feature_age_ms", + ] + cat_features = ["strict_pre_deteriorated"] + + result_rows: list[dict[str, Any]] = [] + matrix_rows: list[dict[str, Any]] = [] + for feat in numeric_features: + if feat not in pref.columns: + continue + rows, grades = _state_table_num(pref, feat, labels) + result_rows.extend(rows) + matrix_rows.append( + { + "feature": feat, + "Path C": grades["path_c"], + "Toxic": grades["toxic"], + "Neg30s": grades["negative_30s"], + "Economic": grades["Economic"], + } + ) + for feat in cat_features: + if feat not in pref.columns: + continue + rows, grades = _state_table_cat(pref, feat, labels) + result_rows.extend(rows) + matrix_rows.append( + { + "feature": feat, + "Path C": grades["path_c"], + "Toxic": grades["toxic"], + "Neg30s": grades["negative_30s"], + "Economic": grades["Economic"], + } + ) + + baseline = { + "path_c": float(pref["path_c"].mean()), + "toxic": float(pref["toxic"].mean()), + "negative_30s": float(pref["negative_30s"].mean()), + "economic_negative": float(pref["economic_negative"].mean()), + "net_attr_30s_usdt_mean": float(pref["net_attr_30s_usdt"].mean()), + "markout_30s_mean": float(pref["markout_30s"].mean()), + } + coverage = { + "matched_paths": int(len(paths)), + "strict_prefill_rows": int(len(pref)), + "strict_prefill_coverage_pct": float(len(pref) / max(len(paths), 1) * 100.0), + "mean_feature_age_ms": float(pref["strict_feature_age_ms"].mean()), + "median_feature_age_ms": float(pref["strict_feature_age_ms"].median()), + } + + out_txt = Path(args.out) + out_json = out_txt.with_suffix(".json") + lines: list[str] = [] + + def p(s: str = "") -> None: + lines.append(s) + print(s) + + p("=" * 72) + p("Prefill Adverse-Selection Attribution v0.1") + p("=" * 72) + p("Experiment: MM_EDGE_EXP_001") + p("Population: frozen historical fills (Hard core = MATCHED only)") + p("Strategy: v0.1 FROZEN") + p("Execution: STOPPED") + p("Purpose: Pre-fill adverse-selection predictability audit") + p("NOT: strategy / backtest / optimization / model training") + p() + p("Time Contract") + p("-" * 40) + p(f"feature_timestamp <= t_fill - {args.margin_sec:.2f}s") + p("Only sampled historical mid_tick / inventory_tick states are used.") + p("Fill-callback contemporaneous fields are intentionally excluded to avoid leakage.") + p() + p("Unavailable under strict contract in v0.1") + p("-" * 40) + p("- event intensity / large trades / time_since_last_market_event") + p("- fill-callback market_event_before_fill") + p("- any future path / realized / cancel-after-fill info as features") + p() + p("Baseline labels (MATCHED only)") + p("-" * 40) + p(f"P(Path C): {_pct(baseline['path_c'])}") + p(f"P(Toxic): {_pct(baseline['toxic'])}") + p(f"P(Neg30s): {_pct(baseline['negative_30s'])}") + p(f"P(Economic<0): {_pct(baseline['economic_negative'])}") + p(f"Mean net_attr_30s: {_num(baseline['net_attr_30s_usdt_mean'], 6)} USDT/fill") + p(f"Mean markout_30s: {_pct(baseline['markout_30s_mean'])}") + p(f"Matched path rows: {coverage['matched_paths']}") + p(f"Strict prefill rows: {coverage['strict_prefill_rows']}") + p(f"Strict coverage: {coverage['strict_prefill_coverage_pct']:.1f}%") + p(f"Feature age ms: mean={coverage['mean_feature_age_ms']:.1f} median={coverage['median_feature_age_ms']:.1f}") + p() + p("Sample-size policy") + p("-" * 40) + p("n < 30 exploratory only (LOW_N)") + p("n < 100 weak evidence (WEAK_EVIDENCE)") + p("n >= 100 usable attribution (USABLE)") + p() + p("Conclusion Matrix") + p("-" * 40) + p("feature | Path C | Toxic | Neg30s | Economic") + for row in matrix_rows: + p(f"{row['feature']} | {row['Path C']} | {row['Toxic']} | {row['Neg30s']} | {row['Economic']}") + p() + p("State tables") + p("-" * 40) + for feat in [r["feature"] for r in matrix_rows]: + sub = [r for r in result_rows if r["feature"] == feat] + if not sub: + continue + p(feat) + for r in sub: + med = _num(r["median"], 6) if r["median"] is not None else "n/a" + p( + f" {r['state']}: n={r['n']} [{r['sample_grade']}] median={med} " + f"P(C)={_pct(r['p_path_c'])} Δ={r['delta_path_c_pp']:+.1f}pp " + f"P(Toxic)={_pct(r['p_toxic'])} Δ={r['delta_toxic_pp']:+.1f}pp " + f"P(Neg30)={_pct(r['p_negative_30s'])} Δ={r['delta_negative_30s_pp']:+.1f}pp " + f"E[net30]={_num(r['net_attr_mean'], 5)} Δ={_num(r['delta_economic_per_fill'], 5)}" + ) + p() + p("Interpretation") + p("-" * 40) + p("Only pre-fill observable states count as candidate signals.") + p("A feature may separate Path C statistically but still fail Economic relevance.") + p("Only rows graded CANDIDATE_V0_2_SIGNAL with usable n should enter v0.2 hypothesis design.") + p("=" * 72) + + out_txt.write_text("\n".join(lines) + "\n", encoding="utf-8") + out_json.write_text( + json.dumps( + { + "experiment_id": "MM_EDGE_EXP_001", + "purpose": "prefill adverse-selection predictability audit", + "population": { + "matched_rows": int(len(pref)), + "margin_sec": float(args.margin_sec), + }, + "baseline": baseline, + "coverage": coverage, + "matrix": matrix_rows, + "states": result_rows, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/scripts/probe_status.sh b/scripts/probe_status.sh new file mode 100755 index 0000000..9283bd8 --- /dev/null +++ b/scripts/probe_status.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# 远程查看探针状态 + fills/clusters 粗计数 +# 用法：export SSH_HOST=user@ip [SSH_KEY=...] ./scripts/probe_status.sh +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SSH_HOST="${SSH_HOST:-jack@jackyu66.com}" +SSH_KEY="${SSH_KEY:-${HOME}/Project/deploy/zun_hk/id_ed25519_hk}" +REMOTE_DIR="${REMOTE_DIR:-/www/Project/nautilus_mm}" + +SSH_OPTS=(-o StrictHostKeyChecking=accept-new) +if [[ -n "$SSH_KEY" ]]; then + chmod 400 "$SSH_KEY" 2>/dev/null || true + SSH_OPTS+=(-i "$SSH_KEY") +fi + +ssh "${SSH_OPTS[@]}" "$SSH_HOST" "REMOTE_DIR='$REMOTE_DIR' bash -s" <<'EOF' +set -euo pipefail +echo "=== systemd --user ===" +systemctl --user is-active mm-edge-probe || true +systemctl --user status mm-edge-probe --no-pager -l | head -20 || true +echo "" +echo "=== experiment (.env) ===" +grep -E '^(EXPERIMENT_ID|PROBE_VERSION|BINANCE_ENVIRONMENT|ENABLE_TRADING)=' "$REMOTE_DIR/.env" 2>/dev/null || true +grep -E '^BINANCE_API_KEY=.+' "$REMOTE_DIR/.env" >/dev/null && echo "API key: SET" || echo "API key: EMPTY" +echo "" +echo "=== fills / clusters (jsonl) ===" +cd "$REMOTE_DIR/logs/maker_edge" 2>/dev/null || { echo "no log dir"; exit 0; } +python3 - <<'PY' +import json +from pathlib import Path +fills=0 +cids=set() +health=0 +for f in sorted(Path('.').glob('*.jsonl')): + for line in f.read_text().splitlines(): + try: + ev=json.loads(line) + except Exception: + continue + if ev.get('event')=='fill': + fills+=1 + if ev.get('event_cluster_id'): + cids.add(ev['event_cluster_id']) + elif ev.get('event')=='health': + health+=1 +print(f"fills={fills} clusters={len(cids)} health_ticks={health}") +print(f"cluster/fill={len(cids)/fills*100:.1f}%" if fills else "cluster/fill=n/a") +PY +echo "" +echo "=== recent journal (--user) ===" +journalctl --user -u mm-edge-probe -n 30 --no-pager || true +EOF diff --git a/scripts/pull_report.sh b/scripts/pull_report.sh new file mode 100755 index 0000000..955ea11 --- /dev/null +++ b/scripts/pull_report.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# 从服务器拉取 jsonl + 本地生成 Maker Edge Report +# 用法：export SSH_HOST=user@ip [SSH_KEY=...] ./scripts/pull_report.sh [min_fills] +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SSH_HOST="${SSH_HOST:-jack@jackyu66.com}" +SSH_KEY="${SSH_KEY:-${HOME}/Project/deploy/zun_hk/id_ed25519_hk}" +REMOTE_DIR="${REMOTE_DIR:-/www/Project/nautilus_mm}" +LOCAL_LOG="${LOCAL_LOG:-$ROOT/logs/maker_edge}" + +SSH_OPTS=(-o StrictHostKeyChecking=accept-new) +if [[ -n "$SSH_KEY" ]]; then + chmod 400 "$SSH_KEY" 2>/dev/null || true + SSH_OPTS+=(-i "$SSH_KEY") +fi + +mkdir -p "$LOCAL_LOG" +echo "==> pull logs from $SSH_HOST" +rsync -avz -e "ssh ${SSH_OPTS[*]}" \ + "$SSH_HOST:$REMOTE_DIR/logs/maker_edge/" "$LOCAL_LOG/" + +echo "==> analyze" +export PYTHONPATH="${ROOT}/src${PYTHONPATH:+:$PYTHONPATH}" +exec "$ROOT/scripts/analyze.sh" "${1:-2000}" diff --git a/scripts/reconcile_account.py b/scripts/reconcile_account.py new file mode 100644 index 0000000..afb70d5 --- /dev/null +++ b/scripts/reconcile_account.py @@ -0,0 +1,402 @@ +#!/usr/bin/env python3 +""" +Account Ledger Reconciliation — MM_EDGE_EXP_001 + +Separates: + MakerAlpha (research markout) ≠ Account Equity (wallet economics) + +Pulls paginated Binance Futures: + - /fapi/v1/userTrades (maker flag, commission per fill) + - /fapi/v1/income (REALIZED_PNL, COMMISSION, FUNDING_FEE, …) + - /fapi/v2/account (wallet + unrealized + position) + +Hard gate: + TAKER_FILLED_COUNT == 0 else Maker-only = INVALID + +Equity identity (target error ≈ 0): + StartWallet + Σincome_types + (EndUnrealized − StartUnrealized*) + + Transfers/Adjustments ≈ EndMarginBalance + +* StartUnrealized often unknown → report EndUnrealized separately. +""" + +from __future__ import annotations + +import argparse +import hashlib +import hmac +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from collections import defaultdict +from datetime import datetime, timezone +from pathlib import Path + +from dotenv import load_dotenv + +_ROOT = Path(__file__).resolve().parents[1] +load_dotenv(_ROOT / ".env") + + +def _env(name: str, default: str = "") -> str: + return os.getenv(name, default).strip() + + +def _base_url() -> str: + env = _env("BINANCE_ENVIRONMENT", "TESTNET").upper() + if env == "TESTNET": + return "https://testnet.binancefuture.com" + if env == "LIVE": + return "https://fapi.binance.com" + raise SystemExit(f"BINANCE_ENVIRONMENT must be TESTNET|LIVE, got {env!r}") + + +def _signed_get(path: str, params: dict | None = None) -> object: + key = _env("BINANCE_API_KEY") + sec = _env("BINANCE_API_SECRET") + if not key or not sec: + raise SystemExit("BINANCE_API_KEY / BINANCE_API_SECRET required") + params = dict(params or {}) + params["timestamp"] = int(time.time() * 1000) + params["recvWindow"] = 60_000 + qs = urllib.parse.urlencode(params) + sig = hmac.new(sec.encode(), qs.encode(), hashlib.sha256).hexdigest() + url = f"{_base_url()}{path}?{qs}&signature={sig}" + req = urllib.request.Request(url, headers={"X-MBX-APIKEY": key}) + try: + with urllib.request.urlopen(req, timeout=60) as r: + return json.loads(r.read().decode()) + except urllib.error.HTTPError as e: + body = e.read().decode(errors="replace") + raise RuntimeError(f"HTTP {e.code} {path} params={params} body={body}") from e + + +def _fetch_user_trades(symbol: str, start_ms: int, end_ms: int) -> list[dict]: + """Paginate userTrades by time windows (dedupe by trade id). + + Note: Testnet userTrades can stop returning rows after ~7d of dense history + even while orders/income continue — RECON-02 must flag that gap separately. + """ + out: dict[int, dict] = {} + cursor = start_ms + safety = 0 + while cursor < end_ms and safety < 2000: + safety += 1 + chunk_end = min(cursor + 7 * 86400_000 - 1, end_ms) + batch = _signed_get( + "/fapi/v1/userTrades", + { + "symbol": symbol, + "startTime": cursor, + "endTime": chunk_end, + "limit": 1000, + }, + ) + assert isinstance(batch, list) + if not batch: + cursor = chunk_end + 1 + continue + for t in batch: + out[int(t["id"])] = t + last_t = int(batch[-1]["time"]) + if len(batch) < 1000: + cursor = max(last_t + 1, chunk_end + 1) + else: + nxt = last_t + 1 + if nxt <= cursor: + nxt = cursor + 1 + cursor = nxt + time.sleep(0.08) + return sorted(out.values(), key=lambda x: (int(x["time"]), int(x["id"]))) + + +def _fetch_income(start_ms: int, end_ms: int) -> list[dict]: + """Paginate income by time only.""" + out: list[dict] = [] + seen: set[tuple] = set() + cursor = start_ms + safety = 0 + while cursor < end_ms and safety < 2000: + safety += 1 + chunk_end = min(cursor + 7 * 86400_000 - 1, end_ms) + batch = _signed_get( + "/fapi/v1/income", + {"startTime": cursor, "endTime": chunk_end, "limit": 1000}, + ) + assert isinstance(batch, list) + if not batch: + cursor = chunk_end + 1 + continue + for row in batch: + key = ( + row.get("tranId"), + row.get("time"), + row.get("incomeType"), + row.get("income"), + row.get("asset"), + row.get("symbol"), + ) + if key in seen: + continue + seen.add(key) + out.append(row) + last_t = int(batch[-1]["time"]) + if len(batch) < 1000: + cursor = max(last_t + 1, chunk_end + 1) + else: + cursor = last_t + 1 + time.sleep(0.08) + return out + + +def _ms_iso(ms: int) -> str: + return datetime.fromtimestamp(ms / 1000, tz=timezone.utc).isoformat() + + +def load_jsonl_fill_count(log_dir: Path) -> int: + n = 0 + if not log_dir.exists(): + return 0 + for f in sorted(log_dir.glob("*.jsonl")): + for line in f.open(): + try: + e = json.loads(line) + except Exception: + continue + if isinstance(e, dict) and e.get("event") == "fill": + n += 1 + return n + + +def main() -> int: + ap = argparse.ArgumentParser(description="Maker Edge account reconciliation") + ap.add_argument( + "--start-wallet", + type=float, + default=float(_env("RECON_START_WALLET", "5000")), + help="Observed starting USDT wallet (default 5000 testnet grant)", + ) + ap.add_argument( + "--symbol", + default=_env("RECON_SYMBOL", "BTCUSDT"), + help="Futures symbol for userTrades (default BTCUSDT)", + ) + ap.add_argument( + "--since-days", + type=float, + default=float(_env("RECON_SINCE_DAYS", "14")), + ) + ap.add_argument( + "--out", + default=str(_ROOT / "logs" / "maker_edge" / "Account_Reconciliation.txt"), + ) + args = ap.parse_args() + + end_ms = int(time.time() * 1000) + start_ms = end_ms - int(args.since_days * 86400 * 1000) + + print(f"[recon] env={_env('BINANCE_ENVIRONMENT','TESTNET')} base={_base_url()}") + print(f"[recon] window {_ms_iso(start_ms)} → {_ms_iso(end_ms)}") + + print("[recon] pulling userTrades (paginated)…") + trades = _fetch_user_trades(args.symbol, start_ms, end_ms) + + print(f"[recon] userTrades={len(trades)}") + print("[recon] pulling income (paginated)…") + income = _fetch_income(start_ms, end_ms) + print(f"[recon] income rows={len(income)}") + + acct = _signed_get("/fapi/v2/account") + assert isinstance(acct, dict) + + # --- Maker-only hard check --- + maker_n = sum(1 for t in trades if t.get("maker") is True) + taker_n = sum(1 for t in trades if t.get("maker") is False) + unknown_n = len(trades) - maker_n - taker_n + maker_only_ok = taker_n == 0 and unknown_n == 0 and len(trades) > 0 + maker_only_status = "PASS" if maker_only_ok else ("INVALID" if taker_n > 0 else "NEED VERIFY") + + fee_by_asset: dict[str, float] = defaultdict(float) + notional = 0.0 + buy_qty = sell_qty = 0.0 + for t in trades: + fee_by_asset[t.get("commissionAsset") or "?"] += float(t.get("commission") or 0) + q = float(t.get("qty") or 0) + px = float(t.get("price") or 0) + notional += abs(q * px) + if t.get("buyer"): + buy_qty += q + else: + sell_qty += q + net_qty = buy_qty - sell_qty + + income_by: dict[str, float] = defaultdict(float) + for row in income: + income_by[str(row.get("incomeType"))] += float(row.get("income") or 0) + + wallet = float(acct.get("totalWalletBalance") or 0) + upnl = float(acct.get("totalUnrealizedProfit") or 0) + margin = float(acct.get("totalMarginBalance") or 0) + avail = float(acct.get("availableBalance") or 0) + + positions = [] + for p in acct.get("positions") or []: + amt = float(p.get("positionAmt") or 0) + if abs(amt) > 1e-12: + positions.append( + { + "symbol": p.get("symbol"), + "amt": amt, + "entry": float(p.get("entryPrice") or 0), + "unrealized": float(p.get("unrealizedProfit") or 0), + } + ) + + start_wallet = float(args.start_wallet) + income_sum = sum(income_by.values()) + # Identity without known start upnl: + # EndWallet ≈ StartWallet + Σ income (transfers included in income types if any) + implied_end_wallet = start_wallet + income_sum + wallet_gap = wallet - implied_end_wallet + equity_now = margin # wallet + upnl + equity_vs_start = equity_now - start_wallet + + jsonl_fills = load_jsonl_fill_count(_ROOT / "logs" / "maker_edge") + + lines: list[str] = [] + def p(s: str = "") -> None: + lines.append(s) + print(s) + + p("=" * 72) + p("Account Reconciliation — MM_EDGE_EXP_001 / probe_v0.1") + p("Research markout (MakerAlpha) ≠ Account equity") + p("=" * 72) + p() + p("Status Snapshot") + p("-" * 40) + p("Maker Phenomenon PARTIAL_PASS") + p("Data Integrity PASS (from Maker Edge Report)") + p(f"Maker-only constraint {maker_only_status}") + p("Account Reconciliation NOT COMPLETE" if abs(wallet_gap) > 0.5 else "Account Reconciliation CLOSE") + p("Economic Edge UNKNOWN") + p("Stage 3 LOCKED") + p("Probe STOPPED (no further volume until ledger closes)") + p() + + p("Section A — Maker-only hard check (exchange userTrades)") + p("-" * 40) + p(f"Symbol: {args.symbol}") + p(f"Exchange trades: {len(trades)}") + p(f"Jsonl fills (local): {jsonl_fills}") + p(f"MAKER fills: {maker_n}") + p(f"TAKER fills: {taker_n}") + p(f"Unknown liquidity: {unknown_n}") + p(f"TAKER_FILLED_COUNT: {taker_n}") + if taker_n > 0: + p("→ INVALID: sample contaminated by taker fills") + elif maker_only_ok: + p("→ PASS: all exchange trades marked maker=true") + else: + p("→ NEED VERIFY") + p(f"Buy qty / Sell qty: {buy_qty:.6f} / {sell_qty:.6f}") + p(f"Net inventory (qty): {net_qty:.6f}") + p(f"Gross notional: {notional:.4f} USDT") + for asset, fee in sorted(fee_by_asset.items()): + p(f"Commission ({asset}): {fee}") + p() + + p("Section B — Income ledger (paginated, full window)") + p("-" * 40) + for k, v in sorted(income_by.items(), key=lambda kv: -abs(kv[1])): + p(f" {k:24s} {v:+.8f}") + p(f" {'Σ income':24s} {income_sum:+.8f}") + p() + + p("Section C — Account snapshot (now)") + p("-" * 40) + p(f"totalWalletBalance: {wallet:.8f}") + p(f"totalUnrealizedProfit: {upnl:.8f}") + p(f"totalMarginBalance: {margin:.8f} ← equity") + p(f"availableBalance: {avail:.8f}") + if positions: + p("Open positions:") + for pos in positions: + p( + f" {pos['symbol']} amt={pos['amt']} entry={pos['entry']} " + f"upnl={pos['unrealized']}" + ) + else: + p("Open positions: (none)") + p() + + p("Section D — Equity bridge (attempt)") + p("-" * 40) + p(f"Start wallet (assumed): {start_wallet:.8f}") + p(f"+ Σ income: {income_sum:+.8f}") + p(f"= Implied end wallet: {implied_end_wallet:.8f}") + p(f"Actual end wallet: {wallet:.8f}") + p(f"Wallet residual gap: {wallet_gap:+.8f}") + p(f"End unrealized: {upnl:+.8f}") + p(f"End equity: {equity_now:.8f}") + p(f"Equity − start wallet: {equity_vs_start:+.8f}") + p() + p("Interpretation:") + p(" - Do NOT equate EquityΔ with MakerAlpha failure/success.") + p(" - Residual gap means incomplete history, wrong start, or missing") + p(" transfer/adjustment types — Account Reconciliation stays open.") + p(" - Inventory drift (net qty / open position) can dominate economics") + p(" even when per-fill markout is slightly positive.") + p() + + p("Section E — Next required chain") + p("-" * 40) + p("QuoteIntent → Submitted → Accepted → Filled") + p(" → fill_px/qty → liquidity=MAKER → fee") + p(" → position Δ → realized → funding → equity") + p("Daily: StartEquity + TradingPnL + Fees + Funding + uPnL + Transfers = EndEquity") + p("Target residual ≈ 0 before any Stage3 unlock / further volume.") + p("=" * 72) + + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text("\n".join(lines) + "\n", encoding="utf-8") + # machine-readable sidecar + sidecar = out.with_suffix(".json") + sidecar.write_text( + json.dumps( + { + "experiment_id": "MM_EDGE_EXP_001", + "maker_only_status": maker_only_status, + "taker_filled_count": taker_n, + "maker_filled_count": maker_n, + "exchange_trades": len(trades), + "jsonl_fills": jsonl_fills, + "income_by_type": dict(income_by), + "income_sum": income_sum, + "start_wallet_assumed": start_wallet, + "end_wallet": wallet, + "end_unrealized": upnl, + "end_equity": equity_now, + "wallet_residual_gap": wallet_gap, + "net_qty": net_qty, + "fee_by_asset": dict(fee_by_asset), + "positions": positions, + "probe": "STOPPED", + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + print(f"[recon] saved {out}") + print(f"[recon] saved {sidecar}") + return 0 if maker_only_ok or taker_n == 0 else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/reconcile_fills.py b/scripts/reconcile_fills.py new file mode 100644 index 0000000..2790bc9 --- /dev/null +++ b/scripts/reconcile_fills.py @@ -0,0 +1,594 @@ +#!/usr/bin/env python3 +""" +RECONCILIATION-02 — Local Fill ↔ Venue Trade 1:1 / quantity-level closure + +Does NOT resume the probe. Does NOT change quote logic. + +Gate: 100% of local fills and venue trades classified into: + MATCHED | DUPLICATE | ORPHAN_LOCAL | ORPHAN_VENUE | MISMATCH | MALFORMED + +Primary link: venue_trade_id when present. +Fallback (historical jsonl has trade_id=None): + venue_order_id + side + qty + price + timestamp window +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from collections import defaultdict +from datetime import datetime, timezone +from pathlib import Path + +_ROOT = Path(__file__).resolve().parents[1] +_SRC = _ROOT / "src" +if str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) + +# reuse pagination from recon-01 +sys.path.insert(0, str(_ROOT / "scripts")) +from reconcile_account import _env, _fetch_user_trades, _signed_get # noqa: E402 + + +PX_TICK = 0.1 # BTCUSDT tick +QTY_EPS = 1e-8 +TIME_MATCH_SEC = 30.0 +TIME_DUP_SEC = 2.0 + + +def _parse_iso(s: str | None) -> float | None: + if not s: + return None + try: + return datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp() + except Exception: + return None + + +def _ms_ts(ms: int | None) -> float | None: + if ms is None: + return None + return int(ms) / 1000.0 + + +def load_local_fills(log_dir: Path) -> list[dict]: + fills: list[dict] = [] + for f in sorted(log_dir.glob("*.jsonl")): + if f.name.startswith("Account_") or f.name.startswith("Maker_") or f.name.startswith("RECON"): + continue + for line in f.open(): + try: + e = json.loads(line) + except Exception: + continue + if not isinstance(e, dict) or e.get("event") != "fill": + continue + fills.append(e) + return fills + + +def normalize_local(e: dict, idx: int) -> dict: + px = float(e.get("fill_price") or 0) + qty = float(e.get("amount") or 0) + side = e.get("side") # long / short + venue_oid = e.get("venue_order_id") + if venue_oid is not None: + venue_oid = str(venue_oid) + trade_id = e.get("venue_trade_id") or e.get("trade_id") + if trade_id in (None, "None", ""): + trade_id = None + else: + trade_id = str(trade_id) + ts = _parse_iso(e.get("quote_fill_time")) + malformed = [] + if px <= 0: + malformed.append("bad_price") + if qty <= 0: + malformed.append("bad_qty") + if side not in ("long", "short"): + malformed.append("bad_side") + if not venue_oid: + malformed.append("missing_venue_order_id") + return { + "idx": idx, + "fill_id": e.get("fill_id"), + "client_order_id": e.get("client_order_id"), + "venue_order_id": venue_oid, + "venue_trade_id": trade_id, + "side": side, + "px": px, + "qty": qty, + "ts": ts, + "ts_iso": e.get("quote_fill_time"), + "commission": e.get("commission"), + "malformed": malformed, + "raw_keys": sorted(e.keys()), + } + + +def normalize_venue(t: dict, idx: int) -> dict: + buyer = bool(t.get("buyer")) + side = "long" if buyer else "short" + return { + "idx": idx, + "venue_trade_id": str(t.get("id")), + "venue_order_id": str(t.get("orderId")), + "side": side, + "px": float(t.get("price") or 0), + "qty": float(t.get("qty") or 0), + "ts": _ms_ts(t.get("time")), + "ts_iso": datetime.fromtimestamp(int(t["time"]) / 1000, tz=timezone.utc).isoformat() + if t.get("time") + else None, + "commission": float(t.get("commission") or 0), + "commission_asset": t.get("commissionAsset"), + "maker": t.get("maker"), + "symbol": t.get("symbol"), + } + + +def _compatible(loc: dict, ven: dict) -> tuple[bool, str]: + if loc["side"] != ven["side"]: + return False, "side" + if abs(loc["qty"] - ven["qty"]) > QTY_EPS: + return False, "qty" + if abs(loc["px"] - ven["px"]) > PX_TICK + 1e-9: + return False, "price" + if loc["ts"] is not None and ven["ts"] is not None: + if abs(loc["ts"] - ven["ts"]) > TIME_MATCH_SEC: + return False, "time" + return True, "ok" + + +def match(locals_: list[dict], venues: list[dict]) -> dict: + """Greedy unique matching. Each venue trade consumed at most once.""" + used_v: set[int] = set() + used_l: set[int] = set() + matched: list[dict] = [] + mismatch: list[dict] = [] + duplicate: list[dict] = [] + + loc_by_tid: dict[str, list[dict]] = defaultdict(list) + ven_by_tid: dict[str, dict] = {} + for v in venues: + ven_by_tid[v["venue_trade_id"]] = v + for loc in locals_: + if loc["venue_trade_id"]: + loc_by_tid[loc["venue_trade_id"]].append(loc) + + # Pass 1: explicit venue_trade_id + for tid, locs in loc_by_tid.items(): + v = ven_by_tid.get(tid) + if v is None: + continue + primary, *rest = locs + ok, why = _compatible(primary, v) + rec = {"local": primary, "venue": v, "link": "venue_trade_id", "compat": why} + if ok: + matched.append(rec) + else: + rec["mismatch_reason"] = why + mismatch.append(rec) + used_v.add(v["idx"]) + used_l.add(primary["idx"]) + for d in rest: + duplicate.append( + {"local": d, "venue": v, "link": "venue_trade_id_dup", "reason": "same venue_trade_id"} + ) + used_l.add(d["idx"]) + + # Pass 2: same venue_order_id, greedy best (qty, px, time) + loc_by_oid: dict[str, list[dict]] = defaultdict(list) + ven_by_oid: dict[str, list[dict]] = defaultdict(list) + for loc in locals_: + if loc["idx"] in used_l or loc["malformed"]: + continue + if loc["venue_order_id"]: + loc_by_oid[loc["venue_order_id"]].append(loc) + for v in venues: + if v["idx"] in used_v: + continue + ven_by_oid[v["venue_order_id"]].append(v) + + def score(loc: dict, v: dict) -> float: + ok, _ = _compatible(loc, v) + if not ok: + return 1e18 + dt = 0.0 + if loc["ts"] is not None and v["ts"] is not None: + dt = abs(loc["ts"] - v["ts"]) + return dt + abs(loc["px"] - v["px"]) * 1e-6 + + for oid, locs in loc_by_oid.items(): + cands = [v for v in ven_by_oid.get(oid, []) if v["idx"] not in used_v] + remaining = [x for x in locs if x["idx"] not in used_l] + for loc in sorted(remaining, key=lambda x: x["ts"] or 0): + best = None + best_s = 1e18 + for v in cands: + if v["idx"] in used_v: + continue + s = score(loc, v) + if s < best_s: + best_s = s + best = v + if best is None or best_s >= 1e17: + continue + matched.append({"local": loc, "venue": best, "link": "order_id+px+qty+time", "compat": "ok"}) + used_l.add(loc["idx"]) + used_v.add(best["idx"]) + + # Pass 3: remaining locals that share (oid, px, qty) with an already-matched + # local → DUPLICATE (restart / double-log of same execution) + matched_sig: dict[tuple, dict] = {} + for m in matched: + loc = m["local"] + v = m["venue"] + matched_sig[(loc["venue_order_id"], round(loc["px"], 2), round(loc["qty"], 8), loc["side"])] = v + + for loc in locals_: + if loc["idx"] in used_l or loc["malformed"]: + continue + key = (loc["venue_order_id"], round(loc["px"], 2), round(loc["qty"], 8), loc["side"]) + v = matched_sig.get(key) + if v is None: + continue + dt_ok = True + if loc["ts"] is not None and v["ts"] is not None: + dt_ok = abs(loc["ts"] - v["ts"]) <= TIME_MATCH_SEC + if not dt_ok: + continue + duplicate.append( + { + "local": loc, + "venue": v, + "link": "dup_of_matched", + "reason": "same order/px/qty/side as a matched fill", + } + ) + used_l.add(loc["idx"]) + + # Pass 4: global leftover by px+qty+side+time (order id mismatch) + leftover_v = [v for v in venues if v["idx"] not in used_v] + leftover_l = [x for x in locals_ if x["idx"] not in used_l and not x["malformed"]] + for loc in leftover_l: + best = None + best_s = 1e18 + for v in leftover_v: + if v["idx"] in used_v: + continue + s = score(loc, v) + if s < best_s: + best_s = s + best = v + if best is None or best_s >= 1e17: + continue + matched.append({"local": loc, "venue": best, "link": "global_px_qty_time", "compat": "ok"}) + used_l.add(loc["idx"]) + used_v.add(best["idx"]) + + malformed = [x for x in locals_ if x["malformed"]] + for x in malformed: + used_l.add(x["idx"]) + + orphan_local = [x for x in locals_ if x["idx"] not in used_l] + orphan_venue = [v for v in venues if v["idx"] not in used_v] + + return { + "matched": matched, + "duplicate": duplicate, + "mismatch": mismatch, + "malformed": malformed, + "orphan_local": orphan_local, + "orphan_venue": orphan_venue, + } + + +def _qty(xs, key="qty") -> float: + return sum(float(x[key]) for x in xs) + + +def audit_orphan_orders(orphans: list[dict], symbol: str, max_checks: int = 40) -> dict: + """Cross-check orphan locals against /fapi/v1/order and /userTrades?orderId=.""" + stats = { + "checked": 0, + "order_filled_no_trades": 0, + "order_missing": 0, + "order_other": 0, + "trades_found": 0, + } + samples: list[dict] = [] + for loc in orphans[:max_checks]: + oid = loc["venue_order_id"] + if not oid: + continue + stats["checked"] += 1 + try: + order = _signed_get("/fapi/v1/order", {"symbol": symbol, "orderId": oid}) + except Exception as exc: + stats["order_missing"] += 1 + samples.append({"oid": oid, "fill_id": loc["fill_id"], "order": "ERR", "detail": str(exc)}) + continue + st = order.get("status") + try: + tr = _signed_get("/fapi/v1/userTrades", {"symbol": symbol, "orderId": oid}) + except Exception: + tr = [] + ntr = len(tr) if isinstance(tr, list) else 0 + if st == "FILLED" and ntr == 0: + stats["order_filled_no_trades"] += 1 + elif ntr > 0: + stats["trades_found"] += 1 + else: + stats["order_other"] += 1 + if len(samples) < 8: + samples.append( + { + "oid": oid, + "fill_id": loc["fill_id"], + "status": st, + "execQty": order.get("executedQty"), + "avgPrice": order.get("avgPrice"), + "userTrades_n": ntr, + } + ) + stats["samples"] = samples + return stats + + +def write_report( + out: Path, + result: dict, + n_local: int, + n_venue: int, + *, + venue_t_max: str | None = None, + orphan_audit: dict | None = None, +) -> None: + m = result["matched"] + d = result["duplicate"] + mm = result["mismatch"] + mal = result["malformed"] + ol = result["orphan_local"] + ov = result["orphan_venue"] + + loc_explained = len(m) + len(d) + len(mm) + len(mal) + len(ol) + ven_explained = len(m) + len(mm) + len(ov) # dups share venue; orphans leftover + # every local in exactly one bucket + # every venue in matched, mismatch, or orphan_venue (dups don't extra-count venue) + + m_qty_l = sum(x["local"]["qty"] for x in m) + m_qty_v = sum(x["venue"]["qty"] for x in m) + m_fee_v = sum(x["venue"]["commission"] for x in m) + dt = [ + abs(x["local"]["ts"] - x["venue"]["ts"]) + for x in m + if x["local"]["ts"] is not None and x["venue"]["ts"] is not None + ] + dt.sort() + + def pctile(a, q): + if not a: + return None + i = min(len(a) - 1, max(0, int(round(q * (len(a) - 1))))) + return a[i] + + unexplained_local = n_local - (len(m) + len(d) + len(mm) + len(mal)) + # orphan_local IS unexplained in the sense of no venue link, but classified + classified_local = len(m) + len(d) + len(mm) + len(mal) + len(ol) + classified_venue = len({x["venue"]["idx"] for x in m + mm} | {x["idx"] for x in ov}) + + gate = ( + classified_local == n_local + and classified_venue == n_venue + and len(ol) == 0 + and len(ov) == 0 + and len(mm) == 0 + and len(mal) == 0 + ) + # 100% explainable ≠ zero orphans. User asked 100% explainable. + # We treat orphans as classified. Gate PASS if all rows classified (always if logic sound) + # Strict gate: no orphans/mismatch/malformed + explainable = classified_local == n_local and classified_venue == n_venue + + lines = [] + + def p(s: str = "") -> None: + lines.append(s) + p("=" * 72) + p("RECONCILIATION-02 — Local Fill ↔ Venue Trade") + p("MM_EDGE_EXP_001 / probe_v0.1 / TESTNET BTCUSDT") + p("Probe remains STOPPED") + p("=" * 72) + p() + p("Counts") + p("-" * 40) + p(f"Local JSONL fills: {n_local}") + p(f"Venue userTrades: {n_venue}") + p(f" MATCHED: {len(m)}") + p(f" DUPLICATE (local): {len(d)}") + p(f" MISMATCH: {len(mm)}") + p(f" MALFORMED (local): {len(mal)}") + p(f" ORPHAN_LOCAL: {len(ol)}") + p(f" ORPHAN_VENUE: {len(ov)}") + p(f"Local classified: {classified_local}/{n_local}") + p(f"Venue classified: {classified_venue}/{n_venue}") + venue_t_max_ts = None + if venue_t_max: + p(f"Venue history max (UTC): {venue_t_max}") + try: + venue_t_max_ts = datetime.fromisoformat(venue_t_max).timestamp() + except Exception: + venue_t_max_ts = None + if ol and venue_t_max_ts: + orphan_after = sum(1 for x in ol if x["ts"] is not None and x["ts"] > venue_t_max_ts) + orphan_before = len(ol) - orphan_after + p(f"Orphan after venue cutoff: {orphan_after} (userTrades history gap on testnet)") + p(f"Orphan before cutoff: {orphan_before}") + if orphan_audit: + p() + p("Orphan order audit (sample)") + p("-" * 40) + p(f" checked: {orphan_audit.get('checked')}") + p(f" order FILLED, 0 trades: {orphan_audit.get('order_filled_no_trades')}") + p(f" userTrades found: {orphan_audit.get('trades_found')}") + for s in orphan_audit.get("samples") or []: + p(f" oid={s.get('oid')} status={s.get('status')} exec={s.get('execQty')} trades={s.get('userTrades_n')}") + p() + p("Quantity (BTC)") + p("-" * 40) + p(f"Matched local qty: {m_qty_l:.6f}") + p(f"Matched venue qty: {m_qty_v:.6f}") + p(f"Qty residual: {m_qty_l - m_qty_v:+.8f}") + p(f"Orphan local qty: {sum(x['qty'] for x in ol):.6f}") + p(f"Orphan venue qty: {sum(x['qty'] for x in ov):.6f}") + p(f"Duplicate local qty: {sum(x['local']['qty'] for x in d):.6f}") + p() + p("Fee / time (matched only)") + p("-" * 40) + p(f"Venue commission sum: {m_fee_v:.8f} USDT") + if dt: + p(f"|Δt| n={len(dt)} p50={pctile(dt,0.5):.3f}s p95={pctile(dt,0.95):.3f}s max={dt[-1]:.3f}s") + p() + p("Link methods (matched)") + p("-" * 40) + by = defaultdict(int) + for x in m: + by[x["link"]] += 1 + for k, v in sorted(by.items(), key=lambda kv: -kv[1]): + p(f" {k:28s} {v}") + p() + p("Gate") + p("-" * 40) + p(f"100% classified: {'PASS' if explainable else 'FAIL'}") + p(f"Strict (no orphan/mismatch/malformed): {'PASS' if gate else 'FAIL'}") + p("Do not resume probe until strict gate PASS or leftovers 100% attributed.") + p() + + def dump_sample(title: str, rows: list, kind: str, n: int = 8) -> None: + if not rows: + return + p(f"Samples — {title} (showing {min(n, len(rows))}/{len(rows)})") + p("-" * 40) + for row in rows[:n]: + if kind == "match": + loc, v = row["local"], row["venue"] + p( + f" fill={loc['fill_id']} oid={loc['venue_order_id']} " + f"tid={v['venue_trade_id']} px={loc['px']}/{v['px']} " + f"qty={loc['qty']}/{v['qty']} link={row['link']}" + ) + elif kind == "dup": + loc, v = row["local"], row["venue"] + p( + f" fill={loc['fill_id']} oid={loc['venue_order_id']} " + f"tid={v['venue_trade_id']} reason={row.get('reason')}" + ) + elif kind == "local": + p( + f" fill={row['fill_id']} oid={row['venue_order_id']} " + f"px={row['px']} qty={row['qty']} side={row['side']} ts={row['ts_iso']}" + ) + elif kind == "venue": + p( + f" tid={row['venue_trade_id']} oid={row['venue_order_id']} " + f"px={row['px']} qty={row['qty']} side={row['side']} ts={row['ts_iso']}" + ) + p() + + dump_sample("ORPHAN_LOCAL", ol, "local") + dump_sample("ORPHAN_VENUE", ov, "venue") + dump_sample("DUPLICATE", d, "dup") + dump_sample("MISMATCH", mm, "match") + p("=" * 72) + + out.write_text("\n".join(lines) + "\n", encoding="utf-8") + print("\n".join(lines)) + + sidecar = { + "experiment_id": "MM_EDGE_EXP_001", + "recon": "RECONCILIATION-02", + "n_local": n_local, + "n_venue": n_venue, + "matched": len(m), + "duplicate": len(d), + "mismatch": len(mm), + "malformed": len(mal), + "orphan_local": len(ol), + "orphan_venue": len(ov), + "classified_local": classified_local, + "classified_venue": classified_venue, + "qty_matched_local": m_qty_l, + "qty_matched_venue": m_qty_v, + "qty_orphan_local": sum(x["qty"] for x in ol), + "qty_orphan_venue": sum(x["qty"] for x in ov), + "qty_duplicate_local": sum(x["local"]["qty"] for x in d), + "fee_matched_venue": m_fee_v, + "strict_gate": gate, + "classified_gate": explainable, + "dt_p50_sec": pctile(dt, 0.5), + "dt_p95_sec": pctile(dt, 0.95), + "orphan_local_oids": [x["venue_order_id"] for x in ol[:50]], + "orphan_venue_tids": [x["venue_trade_id"] for x in ov[:50]], + "venue_history_max": venue_t_max, + "orphan_audit": orphan_audit, + "probe": "STOPPED", + } + out.with_suffix(".json").write_text(json.dumps(sidecar, indent=2) + "\n") + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--dir", default=str(_ROOT / "logs" / "maker_edge")) + ap.add_argument("--symbol", default=_env("RECON_SYMBOL", "BTCUSDT")) + ap.add_argument("--since-days", type=float, default=20.0) + ap.add_argument("--trades-cache", default="") + ap.add_argument("--fetch", action="store_true", help="Fetch userTrades from exchange") + ap.add_argument("--out", default="") + args = ap.parse_args() + + log_dir = Path(args.dir) + cache = Path(args.trades_cache) if args.trades_cache else log_dir / "venue_trades.json" + + if args.fetch or not cache.exists(): + import time + + end_ms = int(time.time() * 1000) + start_ms = end_ms - int(args.since_days * 86400 * 1000) + print(f"[recon-02] fetching userTrades {args.symbol} …") + trades = _fetch_user_trades(args.symbol, start_ms, end_ms) + cache.write_text(json.dumps(trades)) + print(f"[recon-02] cached {len(trades)} trades → {cache}") + else: + trades = json.loads(cache.read_text()) + print(f"[recon-02] loaded {len(trades)} trades from {cache}") + + raw_fills = load_local_fills(log_dir) + locals_ = [normalize_local(e, i) for i, e in enumerate(raw_fills)] + venues = [normalize_venue(t, i) for i, t in enumerate(trades)] + print(f"[recon-02] local fills={len(locals_)} venue={len(venues)}") + + result = match(locals_, venues) + venue_t_max = None + if venues: + venue_t_max = datetime.fromtimestamp( + max(int(t["time"]) for t in trades) / 1000, tz=timezone.utc + ).isoformat() + orphan_audit = audit_orphan_orders(result["orphan_local"], args.symbol) + out = Path(args.out) if args.out else log_dir / "RECONCILIATION_02.txt" + write_report( + out, + result, + len(locals_), + len(venues), + venue_t_max=venue_t_max, + orphan_audit=orphan_audit, + ) + print(f"[recon-02] saved {out}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/reconcile_order_evidence.py b/scripts/reconcile_order_evidence.py new file mode 100644 index 0000000..26edd25 --- /dev/null +++ b/scripts/reconcile_order_evidence.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +""" +RECONCILIATION-03 — Order-level evidence for ORPHAN_LOCAL (post userTrades cutoff) + +Does NOT resume probe. Does NOT reclassify as MATCHED. + +For each ORPHAN_LOCAL from RECON-02, query /fapi/v1/order and validate: + status == FILLED + executedQty ~= sum(local qty per order) + avgPrice ~= local weighted avg + side consistent + +Reclassify passing rows as: + VENUE_CONFIRMED_NO_TRADE_HISTORY + (Order evidence only — no userTrades row on Testnet after cutoff) + +See TESTNET_LIMITATIONS.md +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from collections import defaultdict +from datetime import datetime, timezone +from pathlib import Path + +_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(_ROOT / "scripts")) +sys.path.insert(0, str(_ROOT / "src")) + +from reconcile_fills import ( # noqa: E402 + load_local_fills, + match, + normalize_local, + normalize_venue, +) +from reconcile_account import _env, _fetch_user_trades, _signed_get # noqa: E402 + +PX_TICK = 0.1 +QTY_EPS = 1e-8 + + +def _order_side_to_local(side: str) -> str: + return "long" if side.upper() == "BUY" else "short" + + +def fetch_order(symbol: str, order_id: str, cache: dict) -> dict | None: + if order_id in cache: + return cache[order_id] + try: + o = _signed_get("/fapi/v1/order", {"symbol": symbol, "orderId": order_id}) + except Exception as exc: + cache[order_id] = {"_error": str(exc)} + return cache[order_id] + cache[order_id] = o if isinstance(o, dict) else {"_error": "bad_response"} + time.sleep(0.05) + return cache[order_id] + + +def validate_order_group(fills: list[dict], order: dict) -> tuple[str, list[str]]: + """Return (classification, reasons).""" + reasons: list[str] = [] + if order.get("_error"): + return "ORPHAN_LOCAL_UNCONFIRMED", [f"order_api_error:{order['_error']}"] + st = order.get("status") + exec_qty = float(order.get("executedQty") or 0) + avg_px = float(order.get("avgPrice") or 0) + local_qty = sum(f["qty"] for f in fills) + if exec_qty <= 0: + return "ORPHAN_LOCAL_UNCONFIRMED", [f"status={st} executedQty=0"] + # Partial fill then TTL cancel: status=CANCELED but executedQty>0 + if st not in ("FILLED", "CANCELED"): + return "ORPHAN_LOCAL_UNCONFIRMED", [f"status={st}"] + if abs(local_qty - exec_qty) > QTY_EPS: + reasons.append(f"qty local={local_qty} order={exec_qty}") + wavg = sum(f["px"] * f["qty"] for f in fills) / local_qty if local_qty else 0 + if avg_px > 0 and abs(wavg - avg_px) > PX_TICK + 1e-6: + reasons.append(f"px local_wavg={wavg:.2f} order_avg={avg_px:.2f}") + order_side = _order_side_to_local(str(order.get("side", ""))) + for f in fills: + if f["side"] != order_side: + reasons.append(f"side local={f['side']} order={order_side}") + break + if reasons: + return "ORDER_MISMATCH", reasons + if st == "CANCELED": + return "VENUE_PARTIAL_ORDER_CANCELED", [] + return "VENUE_CONFIRMED_NO_TRADE_HISTORY", [] + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--dir", default=str(_ROOT / "logs" / "maker_edge")) + ap.add_argument("--symbol", default=_env("RECON_SYMBOL", "BTCUSDT")) + ap.add_argument("--trades-cache", default="") + ap.add_argument("--out", default="") + args = ap.parse_args() + + log_dir = Path(args.dir) + cache_path = Path(args.trades_cache) if args.trades_cache else log_dir / "venue_trades.json" + trades = json.loads(cache_path.read_text()) if cache_path.exists() else [] + + raw = load_local_fills(log_dir) + locals_ = [normalize_local(e, i) for i, e in enumerate(raw)] + venues = [normalize_venue(t, i) for i, t in enumerate(trades)] + r02 = match(locals_, venues) + orphans = r02["orphan_local"] + + by_oid: dict[str, list[dict]] = defaultdict(list) + for f in orphans: + if f.get("venue_order_id"): + by_oid[f["venue_order_id"]].append(f) + + order_cache: dict[str, dict] = {} + fill_class: dict[str, tuple[str, list[str], dict | None]] = {} + counts = defaultdict(int) + order_rows: list[dict] = [] + + for oid, fills in sorted(by_oid.items()): + order = fetch_order(args.symbol, oid, order_cache) + cls, reasons = validate_order_group(fills, order or {}) + counts[cls] += len(fills) + order_rows.append( + { + "venue_order_id": oid, + "classification": cls, + "n_local_fills": len(fills), + "local_qty": sum(f["qty"] for f in fills), + "order_executedQty": order.get("executedQty") if order else None, + "order_avgPrice": order.get("avgPrice") if order else None, + "order_status": order.get("status") if order else None, + "order_updateTime": order.get("updateTime") if order else None, + "reasons": reasons, + } + ) + for f in fills: + fill_class[f["fill_id"]] = (cls, reasons, order) + + # Summary from RECON-02 matched + n_matched = len(r02["matched"]) + n_dup = len(r02["duplicate"]) + n_mismatch = len(r02["mismatch"]) + n_mal = len(r02["malformed"]) + n_confirmed = counts["VENUE_CONFIRMED_NO_TRADE_HISTORY"] + n_partial_canceled = counts["VENUE_PARTIAL_ORDER_CANCELED"] + n_order_mismatch = counts["ORDER_MISMATCH"] + n_unconfirmed = counts["ORPHAN_LOCAL_UNCONFIRMED"] + n_local = len(locals_) + + venue_t_max = None + if trades: + venue_t_max = datetime.fromtimestamp( + max(int(t["time"]) for t in trades) / 1000, tz=timezone.utc + ).isoformat() + + order_evidence_ok = ( + n_unconfirmed == 0 + and n_order_mismatch == 0 + and (n_confirmed + n_partial_canceled) == len(orphans) + ) + classified = ( + n_matched + n_dup + n_mismatch + n_mal + + n_confirmed + n_partial_canceled + n_order_mismatch + n_unconfirmed + ) + + lines: list[str] = [] + + def p(s: str = "") -> None: + lines.append(s) + print(s) + + p("=" * 72) + p("RECONCILIATION-03 — Order-level evidence (ORPHAN_LOCAL backfill)") + p("MM_EDGE_EXP_001 / probe_v0.1 / TESTNET BTCUSDT") + p("Probe remains STOPPED") + p("=" * 72) + p() + p("Prior RECON-02 (trade-level)") + p("-" * 40) + p(f"MATCHED (Order+Trade): {n_matched}") + p(f"DUPLICATE: {n_dup}") + p(f"MISMATCH: {n_mismatch}") + p(f"MALFORMED: {n_mal}") + p(f"ORPHAN_LOCAL (pre-03): {len(orphans)}") + if venue_t_max: + p(f"userTrades history max (UTC): {venue_t_max}") + p() + p("RECON-03 order-level reclassification") + p("-" * 40) + p(f"VENUE_CONFIRMED_NO_TRADE_HISTORY: {n_confirmed}") + p(f"VENUE_PARTIAL_ORDER_CANCELED: {n_partial_canceled}") + p(f"ORDER_MISMATCH: {n_order_mismatch}") + p(f"ORPHAN_LOCAL_UNCONFIRMED: {n_unconfirmed}") + p(f"Unique orders checked: {len(by_oid)}") + p() + p("Evidence grades (permanent taxonomy)") + p("-" * 40) + p("MATCHED = Order + Trade row (dual evidence)") + p("VENUE_CONFIRMED_NO_TRADE_HISTORY = Order FILLED, no userTrades row") + p("VENUE_PARTIAL_ORDER_CANCELED = Partial fill, order later CANCELED (TTL)") + p("ORDER_MISMATCH = Order exists but qty/px/side disagree") + p("ORPHAN_LOCAL_UNCONFIRMED = No reliable order evidence") + p() + p("Gates") + p("-" * 40) + p(f"RECON-02 classification (all buckets): {'PASS' if classified == n_local else 'FAIL'}") + p(f"Order-level closure (887 backfill): {'PASS' if order_evidence_ok else 'FAIL'}") + p(f"Strict trade-level closure: FAIL (by design until live trade_id ledger)") + p() + p("Testnet limitation") + p("-" * 40) + p("userTrades history is NOT guaranteed complete after observed cutoff.") + p("Order-level FILLED status remains queryable via /fapi/v1/order.") + p("Do NOT treat VENUE_CONFIRMED fills as fake or duplicate.") + p() + + fails = [r for r in order_rows if r["classification"] in ("ORDER_MISMATCH", "ORPHAN_LOCAL_UNCONFIRMED")] + if fails: + p(f"Non-confirmed orders (showing {min(8, len(fails))}/{len(fails)})") + p("-" * 40) + for r in fails[:8]: + p( + f" oid={r['venue_order_id']} cls={r['classification']} " + f"local_qty={r['local_qty']} exec={r['order_executedQty']} reasons={r['reasons']}" + ) + p() + + p("=" * 72) + + out = Path(args.out) if args.out else log_dir / "RECONCILIATION_03.txt" + out.write_text("\n".join(lines) + "\n", encoding="utf-8") + + sidecar = { + "experiment_id": "MM_EDGE_EXP_001", + "recon": "RECONCILIATION-03", + "n_local": n_local, + "matched_trade_level": n_matched, + "orphan_local_pre03": len(orphans), + "venue_confirmed_no_trade_history": n_confirmed, + "venue_partial_order_canceled": n_partial_canceled, + "order_mismatch": n_order_mismatch, + "orphan_local_unconfirmed": n_unconfirmed, + "unique_orders_checked": len(by_oid), + "userTrades_cutoff_utc": venue_t_max, + "recon02_classification_pass": classified == n_local, + "order_level_closure_pass": order_evidence_ok, + "strict_trade_level_pass": False, + "probe": "STOPPED", + "order_rows": order_rows, + } + out.with_suffix(".json").write_text(json.dumps(sidecar, indent=2) + "\n") + print(f"[recon-03] saved {out}") + return 0 if order_evidence_ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/record_l2_ccxt.py b/scripts/record_l2_ccxt.py new file mode 100644 index 0000000..973b7db --- /dev/null +++ b/scripts/record_l2_ccxt.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +""" +CCXT 轻量 L2 录音机（不依赖 Nautilus） + +用途：在 Nautilus 节点未就绪时，先用代理拉 Binance USDT-M 盘口 + trades， +写入与 Maker Edge 相同的 jsonl schema（book history + 模拟 quote 心跳）。 + +用法： + cd nautilus_mm + source .venv/bin/activate + export PYTHONPATH=src + export HTTPS_PROXY=http://127.0.0.1:7897 + python scripts/record_l2_ccxt.py +""" + +from __future__ import annotations + +import os +import sys +import time +from pathlib import Path + +_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(_ROOT / "src")) + +import ccxt # type: ignore + +from nautilus_mm.recorder import MakerEdgeLogger + + +def main() -> None: + proxy = os.getenv("HTTPS_PROXY") or os.getenv("HTTP_PROXY") or "http://127.0.0.1:7897" + symbol = os.getenv("CCXT_SYMBOL", "BTC/USDT:USDT") + poll = float(os.getenv("POLL_SECS", "2")) + log_dir = os.getenv("MAKER_EDGE_LOG_DIR", str(_ROOT / "logs" / "maker_edge")) + + ex = ccxt.binanceusdm( + { + "enableRateLimit": True, + "proxies": {"http": proxy, "https": proxy}, + "options": {"defaultType": "future"}, + } + ) + lg = MakerEdgeLogger(log_dir=log_dir, levels=10) + last_mid = None + print(f"[ccxt-recorder] {symbol} proxy={proxy} log={log_dir}") + print("Ctrl+C to stop. This mode records book only (no live orders).") + + while True: + try: + ob = ex.fetch_order_book(symbol, limit=10) + trades = ex.fetch_trades(symbol, limit=100) + snap = MakerEdgeLogger.snapshot_from_orderbook( + ob, levels=10, recent_trades=trades, last_mid=last_mid + ) + if snap.mid: + last_mid = snap.mid + now = time.time() + lg.record_book(snap, now=now) + # 心跳 quote（不挂单，仅记录可报价位置） + if snap.best_bid: + lg.write( + { + "event": "book_tick", + "pair": symbol, + "inventory": 0, + **snap.to_book_fields(), + } + ) + lg.update_paths(symbol, snap.mid or 0, now=now) + print( + f"\r mid={snap.mid:.1f} spread={snap.spread:.2f} obi={snap.obi:+.3f} " + f"timb={snap.trade_imbalance:+.3f} pending_fills={lg.pending_count}", + end="", + flush=True, + ) + except Exception as e: + print(f"\nerror: {e}") + time.sleep(poll) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_event_state.sh b/scripts/run_event_state.sh new file mode 100755 index 0000000..383d8c7 --- /dev/null +++ b/scripts/run_event_state.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +# Preserve systemd/caller identity before .env (which still belongs to EXP_001) +PRESERVE_RUN_ID="${LEDGER_RUN_ID:-}" +PRESERVE_LOG_DIR="${EVENT_STATE_LOG_DIR:-}" + +if [[ -f .env ]]; then + set -a + # shellcheck disable=SC1091 + source .env + set +a +fi + +# Layer 1 (script): force EXP_002 contract after .env +export EXPERIMENT_ID=MM_EDGE_EXP_002 +export PROBE_VERSION=event_state_v0.1 +export ENABLE_TRADING=false +export LEDGER_RUN_ID="${PRESERVE_RUN_ID:-${LEDGER_RUN_ID:-EXP-002-RUN-002}}" +export EVENT_STATE_LOG_DIR="${PRESERVE_LOG_DIR:-$ROOT/logs/event_state/$LEDGER_RUN_ID}" +export PYTHONPATH="${PYTHONPATH:-$ROOT/src}" +mkdir -p "$EVENT_STATE_LOG_DIR" + +echo "[run_event_state] EXP_002 observability | trading=NO | run=$LEDGER_RUN_ID | log=$EVENT_STATE_LOG_DIR" +exec "$ROOT/.venv/bin/python" -m nautilus_mm.run_event_state diff --git a/scripts/run_probe.sh b/scripts/run_probe.sh new file mode 100755 index 0000000..8216201 --- /dev/null +++ b/scripts/run_probe.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +if [[ ! -d .venv ]]; then + python3 -m venv .venv + .venv/bin/pip install -U pip + .venv/bin/pip install -r requirements.txt +fi + +# shellcheck disable=SC1091 +source .venv/bin/activate +export PYTHONPATH="${ROOT}/src:${PYTHONPATH:-}" + +if [[ -f .env ]]; then + set -a + # shellcheck disable=SC1091 + source .env + set +a +fi + +# 本地可开代理；服务器 systemd 直连，勿强制 7897 +if [[ "${USE_PROXY:-}" == "1" || "${USE_PROXY:-}" == "true" ]]; then + export HTTP_PROXY="${HTTP_PROXY:-http://127.0.0.1:7897}" + export HTTPS_PROXY="${HTTPS_PROXY:-http://127.0.0.1:7897}" + echo "[run_probe] proxy=$HTTPS_PROXY" +elif [[ -n "${HTTPS_PROXY:-}${HTTP_PROXY:-}" ]]; then + echo "[run_probe] proxy=${HTTPS_PROXY:-$HTTP_PROXY}" +else + echo "[run_probe] direct (no proxy)" +fi + +exec python -m nautilus_mm.run_live diff --git a/scripts/smoke_test_event_state.sh b/scripts/smoke_test_event_state.sh new file mode 100755 index 0000000..e8f75eb --- /dev/null +++ b/scripts/smoke_test_event_state.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# MM_EDGE_EXP_002 smoke test — 10–15 min, restart in the middle, NO trading. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +RUN_ID="${LEDGER_RUN_ID:-EXP-002-RUN-001}" +SESSION_SECS="${SESSION_SECS:-360}" # 6 min × 2 = 12 min collect +LOG_DIR="${EVENT_STATE_LOG_DIR:-$ROOT/logs/event_state/$RUN_ID}" +PYTHON="${ROOT}/.venv/bin/python" + +if [[ -f .env ]]; then + set -a + # shellcheck disable=SC1091 + source .env + set +a +fi + +export EXPERIMENT_ID=MM_EDGE_EXP_002 +export PROBE_VERSION=event_state_v0.1 +export ENABLE_TRADING=false +export LEDGER_RUN_ID="$RUN_ID" +export EVENT_STATE_LOG_DIR="$LOG_DIR" +export PYTHONPATH="$ROOT/src" + +mkdir -p "$LOG_DIR" + +run_session() { + local label="$1" + export LEDGER_SESSION_ID="$(python3 -c 'import uuid; print(uuid.uuid4().hex[:12])')" + echo "[smoke] session ${label} start session_id=${LEDGER_SESSION_ID} secs=${SESSION_SECS}" + "$PYTHON" -m nautilus_mm.run_event_state & + local pid=$! + echo "[smoke] pid=${pid}" + sleep "$SESSION_SECS" + echo "[smoke] session ${label} stopping pid=${pid}" + kill -INT "$pid" 2>/dev/null || true + # allow experiment_stop flush + local i=0 + while kill -0 "$pid" 2>/dev/null && [[ $i -lt 30 ]]; do + sleep 1 + i=$((i + 1)) + done + if kill -0 "$pid" 2>/dev/null; then + echo "[smoke] SIGINT timeout — SIGTERM" + kill -TERM "$pid" 2>/dev/null || true + sleep 3 + fi + if kill -0 "$pid" 2>/dev/null; then + echo "[smoke] SIGTERM timeout — SIGKILL" + kill -KILL "$pid" 2>/dev/null || true + fi + wait "$pid" 2>/dev/null || true + echo "[smoke] session ${label} stopped" +} + +echo "[smoke] RUN_ID=${RUN_ID} log=${LOG_DIR} trading=NO" +run_session A +echo "[smoke] restart gap 5s" +sleep 5 +run_session B + +echo "[smoke] validating ledger" +"$PYTHON" "$ROOT/scripts/validate_event_ledger.py" \ + --dir "$LOG_DIR" \ + --run-id "$RUN_ID" \ + --out "$LOG_DIR/Event_Ledger_Validation.json" +echo "[smoke] done" diff --git a/scripts/validate_event_ledger.py b/scripts/validate_event_ledger.py new file mode 100644 index 0000000..11c0f97 --- /dev/null +++ b/scripts/validate_event_ledger.py @@ -0,0 +1,453 @@ +#!/usr/bin/env python3 +""" +Validate MM_EDGE_EXP_002 Immutable Event Ledger. + +Phase 1 smoke: Gates 1–3 plus ledger engineering contract. +Gate 4 (predictability) is blocked until fill anchors exist. +""" + +from __future__ import annotations + +import argparse +import json +import math +import random +from collections import Counter +from pathlib import Path +from typing import Any + + +TRADE_REQUIRED = [ + "event_type", + "exchange_ts_ns", + "local_ts_epoch", + "local_ts_ns", + "trade_side", + "trade_qty", + "trade_price", + "best_bid", # optional on trade; counted separately +] +TRADE_CORE = [ + "event_type", + "exchange_ts_ns", + "local_ts_epoch", + "local_ts_ns", + "trade_side", + "trade_qty", + "trade_price", + "price", + "quantity", + "best_bid", + "best_ask", + "mid", + "spread", +] +BOOK_CORE = [ + "event_type", + "exchange_ts_ns", + "local_ts_epoch", + "local_ts_ns", + "best_bid", + "best_ask", + "mid", + "spread", + "bid_depth_1", + "ask_depth_1", + "bid_depth_5", + "ask_depth_5", +] +BOOK_DELTA_KEYS = [ + "bid_depth_delta_1", + "ask_depth_delta_1", + "bid_move", + "ask_move", + "spread_change", +] + + +def _pctile(xs: list[float], q: float) -> float | None: + if not xs: + return None + ys = sorted(xs) + if len(ys) == 1: + return ys[0] + i = (len(ys) - 1) * q + lo = math.floor(i) + hi = math.ceil(i) + if lo == hi: + return ys[lo] + return ys[lo] * (hi - i) + ys[hi] * (i - lo) + + +def _num(v: float | None, digits: int = 3) -> str: + if v is None or (isinstance(v, float) and (math.isnan(v) or math.isinf(v))): + return "n/a" + return f"{v:.{digits}f}" + + +def load_jsonl(log_dir: Path) -> tuple[list[dict[str, Any]], int, int]: + rows: list[dict[str, Any]] = [] + parse_fail = 0 + empty = 0 + for f in sorted(log_dir.glob("*.jsonl")): + for line in f.open(): + s = line.strip() + if not s: + empty += 1 + continue + try: + e = json.loads(s) + except Exception: + parse_fail += 1 + continue + if isinstance(e, dict): + rows.append(e) + else: + parse_fail += 1 + return rows, parse_fail, empty + + +def _present(ev: dict[str, Any], key: str) -> bool: + v = ev.get(key) + return v is not None and v != "" + + +def _hollow_book(ev: dict[str, Any]) -> bool: + depths = [ + ev.get("bid_depth_1"), + ev.get("ask_depth_1"), + ev.get("bid_depth_5"), + ev.get("ask_depth_5"), + ev.get("mid"), + ] + nums = [] + for d in depths: + try: + nums.append(float(d)) + except (TypeError, ValueError): + nums.append(0.0) + return all(abs(x) < 1e-12 for x in nums) + + +def main() -> int: + ap = argparse.ArgumentParser(description="Validate EXP_002 event ledger / smoke contract") + ap.add_argument("--dir", default="logs/event_state") + ap.add_argument("--out", default="") + ap.add_argument("--run-id", default="") + ap.add_argument("--sample", type=int, default=200) + ap.add_argument("--latency-tolerance-ms", type=float, default=50.0) + ap.add_argument("--seed", type=int, default=42) + args = ap.parse_args() + + log_dir = Path(args.dir) + rows, parse_fail, empty_lines = load_jsonl(log_dir) + if args.run_id: + rows = [r for r in rows if r.get("run_id") == args.run_id] + + market = [r for r in rows if r.get("event") == "market_event"] + trades = [r for r in market if r.get("event_type") == "aggressive_trade"] + books = [r for r in market if r.get("event_type") == "book_update"] + starts = [r for r in rows if r.get("event") == "experiment_start"] + stops = [r for r in rows if r.get("event") == "experiment_stop"] + anchors = [r for r in rows if r.get("event") == "fill_anchor"] + + run_ids = sorted({r.get("run_id") for r in rows if r.get("run_id")}) + sessions = [r.get("session_id") for r in starts] + + # Duration from first/last local_ts + local_epochs = [float(r["local_ts_epoch"]) for r in market if r.get("local_ts_epoch") is not None] + duration_s = (max(local_epochs) - min(local_epochs)) if len(local_epochs) >= 2 else 0.0 + if duration_s <= 0: + duration_s = 1.0 + + rates = { + "aggressive_trade_per_sec": len(trades) / duration_s, + "book_update_per_sec": len(books) / duration_s, + "total_market_events_per_sec": len(market) / duration_s, + "duration_sec": duration_s, + } + + # Timestamp quality + ex_ok = sum(1 for r in market if r.get("exchange_ts_ns") is not None) + loc_ok = sum(1 for r in market if r.get("local_ts_epoch") is not None and r.get("local_ts_ns") is not None) + latencies_ms: list[float] = [] + skew_violations = 0 + for r in market: + ex = r.get("exchange_ts_ns") + loc = r.get("local_ts_ns") + if ex is None or loc is None: + continue + lag_ms = (float(loc) - float(ex)) / 1e6 + latencies_ms.append(lag_ms) + if float(ex) > float(loc) + args.latency_tolerance_ms * 1e6: + skew_violations += 1 + + ts_quality = { + "exchange_ts_ns_pct": (ex_ok / len(market)) if market else 0.0, + "local_ts_pct": (loc_ok / len(market)) if market else 0.0, + "latency_n": len(latencies_ms), + "latency_ms_p50": _pctile(latencies_ms, 0.50), + "latency_ms_p95": _pctile(latencies_ms, 0.95), + "latency_ms_p99": _pctile(latencies_ms, 0.99), + "latency_ms_max": max(latencies_ms) if latencies_ms else None, + "latency_ms_min": min(latencies_ms) if latencies_ms else None, + "exchange_after_local_violations": skew_violations, + "tolerance_ms": args.latency_tolerance_ms, + } + + # Event order: exchange_ts regression (do not silently sort) + regressions = 0 + max_back_ns = 0 + prev_ex = None + for r in market: + ex = r.get("exchange_ts_ns") + if ex is None: + continue + ex = int(ex) + if prev_ex is not None and ex < prev_ex: + regressions += 1 + max_back_ns = max(max_back_ns, prev_ex - ex) + prev_ex = ex + + # Schema completeness (sample) + rng = random.Random(args.seed) + n_trade_s = min(args.sample, len(trades)) + n_book_s = min(args.sample, len(books)) + trade_sample = rng.sample(trades, n_trade_s) if n_trade_s else [] + book_sample = rng.sample(books, n_book_s) if n_book_s else [] + + def missing_rate(sample: list[dict], keys: list[str]) -> dict[str, float]: + if not sample: + return {k: 1.0 for k in keys} + out = {} + for k in keys: + miss = sum(1 for e in sample if not _present(e, k)) + out[k] = miss / len(sample) + return out + + trade_missing = missing_rate(trade_sample, TRADE_CORE) + book_missing = missing_rate(book_sample, BOOK_CORE) + book_delta_key_miss = 0.0 + if book_sample: + book_delta_key_miss = sum( + 1 for e in book_sample if any(k not in e for k in BOOK_DELTA_KEYS) + ) / len(book_sample) + hollow = sum(1 for e in book_sample if _hollow_book(e)) + + # Restart / integrity + event_ids = [r.get("event_id") for r in market if r.get("event_id")] + dup_ids = [k for k, v in Counter(event_ids).items() if v > 1] + + seq_ok = True + seq_notes = [] + by_session: dict[str, list[int]] = {} + for r in rows: + sid = r.get("session_id") + seq = r.get("event_seq") + if sid is None or seq is None: + continue + by_session.setdefault(str(sid), []).append(int(seq)) + for sid, seqs in by_session.items(): + if seqs != list(range(1, len(seqs) + 1)) and seqs != sorted(seqs): + # allow gaps only if we filtered; within session expect 1..n + expected = list(range(min(seqs), max(seqs) + 1)) + if seqs != expected: + seq_ok = False + seq_notes.append(f"{sid}: not contiguous {seqs[:5]}...{seqs[-3:]}") + if seqs and seqs[0] != 1: + seq_notes.append(f"{sid}: seq starts at {seqs[0]} (expected 1 after restart)") + + seq_reset_expected = len(sessions) >= 2 and all( + (by_session.get(str(s), [None])[0] == 1) for s in sessions if s + ) + + # Gates + gate1_pass: bool | None + if anchors: + reconstruct_fail = 0 + for anc in anchors: + fill_ts = float(anc["fill_ts_epoch"]) + start = float(anc.get("window_start_epoch", fill_ts - 5.0)) + cutoff = float(anc.get("feature_cutoff_epoch", fill_ts - 0.25)) + window = [] + for r in market: + ex = r.get("exchange_ts_ns") + ts = float(ex) / 1e9 if ex is not None else r.get("local_ts_epoch") + if ts is None: + continue + if start <= float(ts) < cutoff: + window.append(r) + if not window: + reconstruct_fail += 1 + gate1_pass = reconstruct_fail == 0 + gate1_status = "PASS" if gate1_pass else "FAIL" + else: + # Phase 1: stream completeness stands in for fill reconstruction + stream_ok = parse_fail == 0 and len(market) > 0 and loc_ok == len(market) + gate1_pass = stream_ok + gate1_status = ( + "PASS (Phase 1 stream completeness; no fill_anchor — expected)" + if stream_ok + else "FAIL (stream incomplete)" + ) + + gate2_ok = ( + ts_quality["exchange_ts_ns_pct"] >= 0.99 + and ts_quality["local_ts_pct"] >= 0.99 + and skew_violations == 0 + ) + gate2_status = "PASS" if gate2_ok else "FAIL" + + schema_ok = ( + all(v == 0.0 for v in trade_missing.values()) + and all(v == 0.0 for v in book_missing.values()) + and book_delta_key_miss == 0.0 + and hollow == 0 + and len(trades) > 0 + and len(books) > 0 + ) + gate3_ok = schema_ok and ts_quality["exchange_ts_ns_pct"] >= 0.99 + gate3_status = "PASS" if gate3_ok else "FAIL" + + restart_ok = ( + parse_fail == 0 + and len(dup_ids) == 0 + and len(starts) >= 1 + and (len(starts) == 1 or (len(stops) >= len(starts) - 1 and seq_reset_expected)) + ) + + integrity = { + "parse_fail_lines": parse_fail, + "empty_lines": empty_lines, + "duplicate_event_ids": len(dup_ids), + "experiment_start_count": len(starts), + "experiment_stop_count": len(stops), + "sessions": sessions, + "seq_contiguous_ok": seq_ok, + "seq_reset_expected": seq_reset_expected, + "seq_notes": seq_notes[:8], + "restart_contract": "PASS" if restart_ok else "FAIL", + } + + run_id = args.run_id or (run_ids[0] if len(run_ids) == 1 else ",".join(run_ids) or "UNSET") + start0 = starts[0] if starts else {} + manifest = { + "run_id": run_id, + "start_ts": start0.get("local_ts"), + "end_ts": stops[-1].get("local_ts") if stops else (rows[-1].get("local_ts") if rows else None), + "host": start0.get("host"), + "commit": start0.get("commit"), + "config_hash": start0.get("config_hash"), + "schema_version": start0.get("schema_version"), + "event_count": len(rows), + "trade_event_count": len(trades), + "book_event_count": len(books), + "session_count": len(sessions), + } + + report = { + "experiment_id": start0.get("experiment_id", "MM_EDGE_EXP_002"), + "run_id": run_id, + "purpose": "ledger smoke / Gates 1-3", + "gate4_predictability": "BLOCKED", + "gates": { + "gate1_event_completeness": gate1_status, + "gate2_temporal_integrity": gate2_status, + "gate3_event_coverage": gate3_status, + }, + "manifest": manifest, + "rates": rates, + "timestamp_quality": ts_quality, + "order": { + "exchange_ts_regressions": regressions, + "max_regression_ns": max_back_ns, + "max_regression_ms": max_back_ns / 1e6 if regressions else 0.0, + "note": "regressions recorded, not silently sorted", + }, + "schema": { + "trade_sample_n": n_trade_s, + "book_sample_n": n_book_s, + "trade_missing_rate": trade_missing, + "book_missing_rate": book_missing, + "hollow_book_in_sample": hollow, + }, + "integrity": integrity, + "counts": { + "total_rows": len(rows), + "market_events": len(market), + "aggressive_trades": len(trades), + "book_updates": len(books), + "fill_anchors": len(anchors), + }, + } + + lines = [ + "=" * 68, + "MM_EDGE_EXP_002 Ledger Smoke / Gates 1–3", + "=" * 68, + f"run_id: {run_id}", + f"sessions: {len(sessions)} {sessions}", + f"host/commit:{start0.get('host')} / {str(start0.get('commit') or '')[:12]}", + f"config_hash:{start0.get('config_hash')}", + f"schema: {start0.get('schema_version')}", + "", + "Gate 1 Event Completeness: " + gate1_status, + "Gate 2 Temporal Integrity: " + gate2_status, + "Gate 3 Event Coverage: " + gate3_status, + "Gate 4 Predictability: BLOCKED", + "", + "1. Event write rates", + "-" * 40, + f"duration_sec: {_num(duration_s, 1)}", + f"aggressive_trade / sec: {_num(rates['aggressive_trade_per_sec'], 3)}", + f"book_update / sec: {_num(rates['book_update_per_sec'], 3)}", + f"total market events / sec: {_num(rates['total_market_events_per_sec'], 3)}", + f"counts: trades={len(trades)} books={len(books)} total={len(market)}", + "", + "2. Timestamp quality", + "-" * 40, + f"exchange_ts_ns != null: {ts_quality['exchange_ts_ns_pct']*100:.2f}%", + f"local_ts_ns != null: {ts_quality['local_ts_pct']*100:.2f}%", + f"exchange > local+tol: {skew_violations} (tol={args.latency_tolerance_ms}ms)", + f"local-exchange lag ms: p50={_num(ts_quality['latency_ms_p50'])} " + f"p95={_num(ts_quality['latency_ms_p95'])} p99={_num(ts_quality['latency_ms_p99'])} " + f"max={_num(ts_quality['latency_ms_max'])}", + "", + "3. Event order (exchange_ts_ns regression, not sorted)", + "-" * 40, + f"regressions: {regressions} max_back_ms={_num(max_back_ns/1e6 if regressions else 0.0)}", + "", + "4. Raw event completeness (sample)", + "-" * 40, + f"trade sample={n_trade_s} missing={trade_missing}", + f"book sample={n_book_s} missing={book_missing}", + f"hollow book_update (all depth/mid empty): {hollow}", + "", + "5. Restart / immutable integrity", + "-" * 40, + f"parse_fail_lines={parse_fail} empty_lines={empty_lines}", + f"duplicate_event_ids={len(dup_ids)}", + f"start={len(starts)} stop={len(stops)} seq_ok={seq_ok} seq_reset_expected={seq_reset_expected}", + f"restart_contract={integrity['restart_contract']}", + "", + "Gate 4 remains BLOCKED until fill_anchor exists. Do not resume trading.", + "=" * 68, + ] + text = "\n".join(lines) + "\n" + print(text) + + out_json = Path(args.out) if args.out else log_dir / "Event_Ledger_Validation.json" + out_txt = out_json.with_suffix(".txt") + out_json.parent.mkdir(parents=True, exist_ok=True) + out_json.write_text(json.dumps(report, indent=2, default=str) + "\n", encoding="utf-8") + out_txt.write_text(text, encoding="utf-8") + (log_dir / f"{run_id.replace('/', '_')}.manifest.json").write_text( + json.dumps(manifest, indent=2, default=str) + "\n", encoding="utf-8" + ) + + ok = gate1_pass is not False and gate2_ok and gate3_ok and restart_ok and parse_fail == 0 + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/nautilus_mm/__init__.py b/src/nautilus_mm/__init__.py new file mode 100644 index 0000000..e8cceb0 --- /dev/null +++ b/src/nautilus_mm/__init__.py @@ -0,0 +1,3 @@ +"""Nautilus MM — Trading OS Execution Reality Layer (Maker Edge).""" + +__version__ = "0.1.0" diff --git a/src/nautilus_mm/book_utils.py b/src/nautilus_mm/book_utils.py new file mode 100644 index 0000000..c08653e --- /dev/null +++ b/src/nautilus_mm/book_utils.py @@ -0,0 +1,124 @@ +"""Order book → MicroSnapshot helpers (Nautilus OrderBook / dict).""" + +from __future__ import annotations + +from typing import Any, Optional + +from nautilus_mm.recorder import MicroSnapshot + + +def snapshot_from_nautilus_book( + book, + levels: int = 10, + recent_buy_qty: float = 0.0, + recent_sell_qty: float = 0.0, + last_mid: float | None = None, + liq_low: float | None = None, + liq_high: float | None = None, +) -> MicroSnapshot: + """Convert nautilus_trader OrderBook to MicroSnapshot.""" + try: + bids = list(book.bids())[:levels] if callable(getattr(book, "bids", None)) else [] + asks = list(book.asks())[:levels] if callable(getattr(book, "asks", None)) else [] + except Exception: + # Some versions expose bid/ask sequences differently + bids = getattr(book, "bids", [])[:levels] or [] + asks = getattr(book, "asks", [])[:levels] or [] + + def _px_qty(level) -> tuple[float, float]: + # Nautilus BookLevel: price is attribute, size() is method + if hasattr(level, "price") and hasattr(level, "size"): + size = level.size() if callable(level.size) else level.size + return float(level.price), float(size) + if isinstance(level, (list, tuple)) and len(level) >= 2: + return float(level[0]), float(level[1]) + return 0.0, 0.0 + + if not bids or not asks: + # try best bid/ask API + try: + bb = float(book.best_bid_price()) if book.best_bid_price() is not None else 0.0 + ba = float(book.best_ask_price()) if book.best_ask_price() is not None else 0.0 + bs = float(book.best_bid_size() or 0) + az = float(book.best_ask_size() or 0) + if bb and ba: + mid = (bb + ba) / 2 + return MicroSnapshot( + best_bid=bb, + best_ask=ba, + mid=mid, + spread=ba - bb, + bid_depth_1=bs, + ask_depth_1=az, + bid_depth_5=bs, + ask_depth_5=az, + bid_depth=bs, + ask_depth=az, + obi=((bs - az) / (bs + az)) if (bs + az) else 0.0, + ) + except Exception: + return MicroSnapshot() + return MicroSnapshot() + + bid_levels = [_px_qty(x) for x in bids] + ask_levels = [_px_qty(x) for x in asks] + best_bid, bid1 = bid_levels[0] + best_ask, ask1 = ask_levels[0] + mid = (best_bid + best_ask) / 2.0 + spread = best_ask - best_bid + + def depth(lvls, n): + return sum(q for _, q in lvls[:n]) + + bid_depth_5 = depth(bid_levels, 5) + ask_depth_5 = depth(ask_levels, 5) + bid_depth = depth(bid_levels, levels) + ask_depth = depth(ask_levels, levels) + tot = bid_depth + ask_depth + obi = ((bid_depth - ask_depth) / tot) if tot else 0.0 + + delta = recent_buy_qty - recent_sell_qty + timb_den = recent_buy_qty + recent_sell_qty + trade_imbalance = (delta / timb_den) if timb_den else 0.0 + + de = 0.0 + if last_mid and mid and abs(delta) > 1e-12: + de = ((mid - last_mid) / last_mid) / delta + + liq_dist = 0.0 + if liq_low and liq_high and mid and (liq_high - liq_low) > 0: + liq_dist = ((mid - liq_low) / (liq_high - liq_low)) * 2 - 1 + + return MicroSnapshot( + best_bid=best_bid, + best_ask=best_ask, + mid=mid, + spread=spread, + bid_depth_1=bid1, + ask_depth_1=ask1, + bid_depth_5=bid_depth_5, + ask_depth_5=ask_depth_5, + bid_depth=bid_depth, + ask_depth=ask_depth, + obi=obi, + delta=delta, + trade_imbalance=trade_imbalance, + delta_efficiency=de, + liquidation_distance=liq_dist, + ) + + +def snapshot_from_ccxt_ob( + ob: dict[str, Any], + levels: int = 10, + recent_trades: list | None = None, + last_mid: float | None = None, +) -> MicroSnapshot: + from nautilus_mm.recorder import MakerEdgeLogger + + return MakerEdgeLogger.snapshot_from_orderbook( + ob, + levels=levels, + recent_trades=recent_trades, + last_mid=last_mid, + ) diff --git a/src/nautilus_mm/event_ledger.py b/src/nautilus_mm/event_ledger.py new file mode 100644 index 0000000..0bd8e5f --- /dev/null +++ b/src/nautilus_mm/event_ledger.py @@ -0,0 +1,403 @@ +""" +Immutable Event Ledger — MM_EDGE_EXP_002 + +Raw Event > Derived Feature + +Stores immutable market events for later reconstruction of pre-fill windows. +Features are computed offline; this module only persists observability data. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import socket +import subprocess +import time +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from nautilus_mm.experiment import load_experiment_meta, stamp_event +from nautilus_mm.recorder import MicroSnapshot + +LEDGER_SCHEMA_VERSION = "immutable_event_v1" + + +def _git_commit(root: Path) -> str | None: + try: + r = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=root, + capture_output=True, + text=True, + timeout=3, + check=False, + ) + if r.returncode == 0: + return r.stdout.strip() or None + except Exception: + return None + return None + + +def load_run_identity(*, extra_config: dict[str, Any] | None = None) -> dict[str, Any]: + """Freeze one Ledger Integrity ID per collection run (sessions share the same run_id).""" + root = Path(__file__).resolve().parents[2] + cfg = { + "prefill_window_sec": os.getenv("PREFILL_WINDOW_SEC", "5.0"), + "prefill_margin_sec": os.getenv("PREFILL_MARGIN_SEC", "0.25"), + "large_trade_qty": os.getenv("LARGE_TRADE_QTY", "0.1"), + "book_depth": os.getenv("BOOK_DEPTH", "10"), + "symbol": os.getenv("SYMBOL", "BTCUSDT-PERP"), + "environment": os.getenv("BINANCE_ENVIRONMENT", "TESTNET"), + "schema_version": LEDGER_SCHEMA_VERSION, + } + if extra_config: + cfg.update({k: str(v) for k, v in extra_config.items()}) + payload = json.dumps(cfg, sort_keys=True, default=str) + return { + "run_id": os.getenv("LEDGER_RUN_ID", "EXP-002-RUN-UNSET"), + "session_id": os.getenv("LEDGER_SESSION_ID") or uuid.uuid4().hex[:12], + "schema_version": LEDGER_SCHEMA_VERSION, + "host": socket.gethostname(), + "commit": _git_commit(root), + "config_hash": hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16], + "config_snapshot": cfg, + } + + +def _utc_iso(ts: float | None = None) -> str: + t = datetime.fromtimestamp(ts or time.time(), tz=timezone.utc) + return t.isoformat() + + +@dataclass +class EventTimingState: + last_trade_ts: float | None = None + last_large_trade_ts: float | None = None + last_book_event_ts: float | None = None + last_tob_change_ts: float | None = None + last_spread_change_ts: float | None = None + last_mid_change_ts: float | None = None + last_depth_change_ts: float | None = None + + +class ImmutableEventLedger: + """Append-only JSONL ledger for market events.""" + + def __init__( + self, + log_dir: str | Path | None = None, + *, + prefill_window_sec: float = 5.0, + prefill_margin_sec: float = 0.25, + large_trade_qty: float = 0.1, + book_levels: int = 10, + ) -> None: + root = Path(__file__).resolve().parents[2] + self.log_dir = Path(log_dir) if log_dir else root / "logs" / "event_state" + self.log_dir.mkdir(parents=True, exist_ok=True) + self.prefill_window_sec = float(prefill_window_sec) + self.prefill_margin_sec = float(prefill_margin_sec) + self.large_trade_qty = float(large_trade_qty) + self.book_levels = int(book_levels) + self.experiment = load_experiment_meta() + self.run_identity = load_run_identity( + extra_config={ + "prefill_window_sec": self.prefill_window_sec, + "prefill_margin_sec": self.prefill_margin_sec, + "large_trade_qty": self.large_trade_qty, + "book_levels": self.book_levels, + } + ) + self._timing = EventTimingState() + self._prev_snap: MicroSnapshot | None = None + self._event_seq = 0 + self._session_event_count = 0 + + def _file(self) -> Path: + return self.log_dir / f"{datetime.now(timezone.utc).strftime('%Y%m%d')}.jsonl" + + def _next_seq(self) -> int: + self._event_seq += 1 + return self._event_seq + + def _timing_fields(self, now: float, *, is_trade: bool = False, is_large_trade: bool = False) -> dict[str, Any]: + def _since(last: float | None) -> float | None: + if last is None: + return None + return (now - last) * 1000.0 + + fields = { + "time_since_last_trade_ms": _since(self._timing.last_trade_ts), + "time_since_last_large_trade_ms": _since(self._timing.last_large_trade_ts), + "time_since_last_book_event_ms": _since(self._timing.last_book_event_ts), + "time_since_last_tob_change_ms": _since(self._timing.last_tob_change_ts), + "time_since_last_spread_change_ms": _since(self._timing.last_spread_change_ts), + "time_since_last_mid_change_ms": _since(self._timing.last_mid_change_ts), + "time_since_last_depth_change_ms": _since(self._timing.last_depth_change_ts), + } + self._timing.last_book_event_ts = now + if is_trade: + self._timing.last_trade_ts = now + if is_large_trade: + self._timing.last_large_trade_ts = now + return fields + + def write(self, event: dict[str, Any]) -> None: + # Receive time is always local wall clock. Never copy exchange_ts into local_ts. + now = time.time() + now_ns = time.time_ns() + event["ledger"] = LEDGER_SCHEMA_VERSION + event["schema_version"] = LEDGER_SCHEMA_VERSION + event["run_id"] = self.run_identity["run_id"] + event["session_id"] = self.run_identity["session_id"] + event["local_ts_epoch"] = now + event["local_ts_ns"] = now_ns + event["local_ts"] = _utc_iso(now) + event["event_seq"] = self._next_seq() + self._session_event_count += 1 + stamp_event(event, self.experiment) + with self._file().open("a", encoding="utf-8") as f: + f.write(json.dumps(event, ensure_ascii=False, default=str) + "\n") + f.flush() + + def write_experiment_start(self, extra: dict | None = None) -> None: + ev = { + "event": "experiment_start", + "experiment_type": "Event-State Observability Probe", + "strategy": "NONE", + "execution_trading": False, + "prefill_window_sec": self.prefill_window_sec, + "prefill_margin_sec": self.prefill_margin_sec, + "large_trade_qty": self.large_trade_qty, + **self.experiment, + **{k: v for k, v in self.run_identity.items() if k != "config_snapshot"}, + "config_snapshot": self.run_identity["config_snapshot"], + } + if extra: + ev.update(extra) + self.write(ev) + + def write_experiment_stop(self, extra: dict | None = None) -> None: + ev = { + "event": "experiment_stop", + "session_event_count": self._session_event_count, + "run_id": self.run_identity["run_id"], + "session_id": self.run_identity["session_id"], + "execution_trading": False, + } + if extra: + ev.update(extra) + self.write(ev) + + def log_aggressive_trade( + self, + *, + pair: str, + price: float, + qty: float, + trade_side: str, + exchange_ts_ns: int | None, + local_ts_epoch: float | None = None, + aggressor_side: str | None = None, + trade_id: str | None = None, + snap: MicroSnapshot | None = None, + ) -> str: + now = local_ts_epoch or time.time() + notional = price * qty + is_large = qty >= self.large_trade_qty + event_id = uuid.uuid4().hex[:16] + ev = { + "event": "market_event", + "event_id": event_id, + "event_type": "aggressive_trade", + "pair": pair, + "price": price, + "quantity": qty, + "trade_qty": qty, + "trade_price": price, + "trade_notional": notional, + "trade_side": trade_side, + "aggressor_side": aggressor_side or trade_side, + "large_trade_flag": is_large, + "trade_id": trade_id, + "exchange_ts_ns": exchange_ts_ns, + } + if snap is not None: + ev.update(self._snap_book_fields(snap)) + ev.update(self._depth_deltas(self._prev_snap, snap)) + ev.update(self._timing_fields(now, is_trade=True, is_large_trade=is_large)) + self.write(ev) + return event_id + + def _snap_book_fields(self, snap: MicroSnapshot) -> dict[str, Any]: + return { + "best_bid": snap.best_bid, + "best_ask": snap.best_ask, + "mid": snap.mid, + "spread": snap.spread, + "bid_depth_1": snap.bid_depth_1, + "ask_depth_1": snap.ask_depth_1, + "bid_depth_5": snap.bid_depth_5, + "ask_depth_5": snap.ask_depth_5, + "bid_depth": snap.bid_depth, + "ask_depth": snap.ask_depth, + "obi": snap.obi, + } + + def _depth_deltas(self, prev: MicroSnapshot | None, cur: MicroSnapshot) -> dict[str, Any]: + if prev is None: + return { + "bid_depth_delta_1": None, + "ask_depth_delta_1": None, + "bid_depth_delta_5": None, + "ask_depth_delta_5": None, + "bid_depth_delta": None, + "ask_depth_delta": None, + } + return { + "bid_depth_delta_1": cur.bid_depth_1 - prev.bid_depth_1, + "ask_depth_delta_1": cur.ask_depth_1 - prev.ask_depth_1, + "bid_depth_delta_5": cur.bid_depth_5 - prev.bid_depth_5, + "ask_depth_delta_5": cur.ask_depth_5 - prev.ask_depth_5, + "bid_depth_delta": cur.bid_depth - prev.bid_depth, + "ask_depth_delta": cur.ask_depth - prev.ask_depth, + } + + def log_book_state( + self, + *, + pair: str, + snap: MicroSnapshot, + exchange_ts_ns: int | None, + local_ts_epoch: float | None = None, + sequence: int | None = None, + delta_count: int | None = None, + event_type: str = "book_update", + ) -> str: + now = local_ts_epoch or time.time() + prev = self._prev_snap + event_id = uuid.uuid4().hex[:16] + + bid_move = None + ask_move = None + mid_move = None + spread_change = None + if prev and prev.mid > 0: + bid_move = snap.best_bid - prev.best_bid + ask_move = snap.best_ask - prev.best_ask + mid_move = snap.mid - prev.mid + spread_change = snap.spread - prev.spread + + depth_deltas = self._depth_deltas(prev, snap) + ev = { + "event": "market_event", + "event_id": event_id, + "event_type": event_type, + "pair": pair, + "exchange_ts_ns": exchange_ts_ns, + "sequence": sequence, + "delta_count": delta_count, + **self._snap_book_fields(snap), + **depth_deltas, + "bid_move": bid_move, + "ask_move": ask_move, + "mid_move": mid_move, + "spread_change": spread_change, + } + ev.update(self._timing_fields(now)) + + if prev is not None: + if bid_move not in (None, 0.0) or ask_move not in (None, 0.0): + self._timing.last_tob_change_ts = now + if spread_change not in (None, 0.0): + self._timing.last_spread_change_ts = now + if mid_move not in (None, 0.0): + self._timing.last_mid_change_ts = now + if any( + depth_deltas[k] not in (None, 0.0) + for k in ( + "bid_depth_delta_1", + "ask_depth_delta_1", + "bid_depth_delta_5", + "ask_depth_delta_5", + ) + ): + self._timing.last_depth_change_ts = now + + self._prev_snap = snap + self.write(ev) + return event_id + + def log_fill_anchor( + self, + *, + fill_id: str, + fill_ts_epoch: float, + exchange_ts_ns: int | None, + side: str, + fill_price: float, + fill_qty: float, + order_id: str | None = None, + venue_order_id: str | None = None, + venue_trade_id: str | None = None, + pair: str | None = None, + snap: MicroSnapshot | None = None, + extra: dict | None = None, + ) -> str: + """ + Anchor for offline [-prefill_window_sec, fill) reconstruction. + + EXP_002 Phase 1 may not emit these (no trading). Schema is frozen for + future fill-anchored analysis (Gate 4). + """ + window_id = uuid.uuid4().hex[:16] + window_start = fill_ts_epoch - self.prefill_window_sec + feature_cutoff = fill_ts_epoch - self.prefill_margin_sec + ev = { + "event": "fill_anchor", + "window_id": window_id, + "fill_id": fill_id, + "fill_ts_epoch": fill_ts_epoch, + "fill_ts": _utc_iso(fill_ts_epoch), + "exchange_ts_ns": exchange_ts_ns, + "window_start_epoch": window_start, + "feature_cutoff_epoch": feature_cutoff, + "prefill_window_sec": self.prefill_window_sec, + "prefill_margin_sec": self.prefill_margin_sec, + "side": side, + "fill_price": fill_price, + "fill_qty": fill_qty, + "order_id": order_id, + "venue_order_id": venue_order_id, + "venue_trade_id": venue_trade_id, + "pair": pair, + } + if snap is not None: + ev.update(self._snap_book_fields(snap)) + if extra: + ev.update(extra) + self.write(ev) + return window_id + + def events_in_window(self, events: list[dict], fill_ts_epoch: float) -> list[dict]: + """Offline helper: filter market_event rows in [-window, fill-margin).""" + start = fill_ts_epoch - self.prefill_window_sec + cutoff = fill_ts_epoch - self.prefill_margin_sec + out = [] + for ev in events: + if ev.get("event") != "market_event": + continue + ts = ev.get("exchange_ts_ns") + if ts is not None: + ts_epoch = float(ts) / 1e9 + else: + ts_epoch = float(ev.get("local_ts_epoch", 0.0)) + if start <= ts_epoch < cutoff: + out.append(ev) + return out diff --git a/src/nautilus_mm/experiment.py b/src/nautilus_mm/experiment.py new file mode 100644 index 0000000..290c56d --- /dev/null +++ b/src/nautilus_mm/experiment.py @@ -0,0 +1,63 @@ +""" +实验身份绑定 — Research Freeze / Data Collection + +每条 jsonl 与每份 Maker Edge Report 必须绑定同一 Experiment ID， +避免 v2/v3 混淆「哪个实验验证出了什么」。 + +冻结字段（运行期不可为「结果好看」而改）： + quote / fee / exchange / probe version +""" + +from __future__ import annotations + +import os +from typing import Any + + +# 默认实验身份（可用环境变量覆盖 ID，其余保持 frozen 语义） +DEFAULT_EXPERIMENT_ID = "MM_EDGE_EXP_001" +DEFAULT_PROBE_VERSION = "probe_v0.1" + +EXP_002_ID = "MM_EDGE_EXP_002" +EXP_002_PROBE_VERSION = "event_state_v0.1" + + +def load_experiment_meta() -> dict[str, Any]: + """从环境变量加载实验元数据；冻结维度固定为 frozen。""" + exp_id = os.getenv("EXPERIMENT_ID", DEFAULT_EXPERIMENT_ID) + probe_version = os.getenv("PROBE_VERSION", DEFAULT_PROBE_VERSION) + if exp_id == EXP_002_ID: + phase = "Event-State Observability / Data Collection" + experiment_type = "Event-State Observability Probe" + else: + phase = "Research Freeze / Data Collection" + experiment_type = "Maker Edge Phenomenon Probe" + return { + "experiment_id": exp_id, + "probe_version": probe_version, + "experiment_type": experiment_type, + "quote_assumption": "frozen", + "fee_model": "frozen", + "exchange_assumption": "frozen", + "exchange": os.getenv("EXCHANGE_NAME", "binance_usdm"), + "environment": os.getenv("BINANCE_ENVIRONMENT", "TESTNET").upper(), + "symbol": os.getenv("SYMBOL", "BTCUSDT-PERP"), + "phase": phase, + "depends_on": "MM_EDGE_EXP_001" if exp_id == EXP_002_ID else None, + } + + +def stamp_event(event: dict[str, Any], meta: dict[str, Any] | None = None) -> dict[str, Any]: + """给单条事件打上实验身份（不覆盖已有显式字段）。""" + m = meta or load_experiment_meta() + event.setdefault("experiment_id", m["experiment_id"]) + event.setdefault("probe_version", m["probe_version"]) + event.setdefault("experiment", { + "quote": m["quote_assumption"], + "fee": m["fee_model"], + "exchange": m["exchange_assumption"], + "venue": m["exchange"], + "environment": m["environment"], + "symbol": m["symbol"], + }) + return event diff --git a/src/nautilus_mm/health.py b/src/nautilus_mm/health.py new file mode 100644 index 0000000..1fd5434 --- /dev/null +++ b/src/nautilus_mm/health.py @@ -0,0 +1,147 @@ +""" +Phase 0 — 连接 / 数据健康度 + +关键指标： + sequence_gap — order book 失真信号（>0 需警惕） + latency_ms — p50 / p95 / p99 / max（做市看尾部） + book_age_ms — quote/fill 使用盘口时的新鲜度 + book/trade update rate +""" + +from __future__ import annotations + +import time +from collections import deque +from dataclasses import dataclass, field +from typing import Optional + + +def empty_market_state_snapshot() -> dict: + """预留给 Market Pulse；探针阶段全部为 null，不做预测/下单决策。""" + return { + "regime": None, + "trend_state": None, + "liquidity_state": None, + "volatility_state": None, + } + + +def _percentile(sorted_vals: list[float], q: float) -> Optional[float]: + if not sorted_vals: + return None + if len(sorted_vals) == 1: + return sorted_vals[0] + idx = min(len(sorted_vals) - 1, max(0, int(round(q * (len(sorted_vals) - 1))))) + return sorted_vals[idx] + + +@dataclass +class ConnectionHealth: + window_sec: float = 60.0 + report_every_sec: float = 30.0 + _book_ts: deque = field(default_factory=lambda: deque(maxlen=50_000)) + _trade_ts: deque = field(default_factory=lambda: deque(maxlen=50_000)) + _latencies_ms: deque = field(default_factory=lambda: deque(maxlen=5_000)) + _seq_gaps: int = 0 + _seq_gaps_window: deque = field(default_factory=lambda: deque(maxlen=10_000)) + _last_seq: Optional[int] = None + _last_report: float = 0.0 + _book_count: int = 0 + _trade_count: int = 0 + _last_book_wall: float = 0.0 # 本地收到最新 book 的时间 + + def on_book(self, seq: int | None = None, event_ts_ns: int | None = None) -> None: + now = time.time() + self._book_ts.append(now) + self._last_book_wall = now + self._book_count += 1 + if event_ts_ns is not None and event_ts_ns > 0: + lat = (now * 1e9 - event_ts_ns) / 1e6 + if -1000 < lat < 60_000: + self._latencies_ms.append(lat) + if seq is not None: + if self._last_seq is not None and seq > self._last_seq + 1: + gap = seq - self._last_seq - 1 + # Binance L2 update ids often jump across snapshot/reconnect; + # only count modest gaps as packet loss. Huge jumps → reset. + if gap <= 1000: + self._seq_gaps += gap + self._seq_gaps_window.append((now, gap)) + self._last_seq = seq + self._trim(now) + + def on_trade(self, event_ts_ns: int | None = None) -> None: + now = time.time() + self._trade_ts.append(now) + self._trade_count += 1 + if event_ts_ns is not None and event_ts_ns > 0: + lat = (now * 1e9 - event_ts_ns) / 1e6 + if -1000 < lat < 60_000: + self._latencies_ms.append(lat) + self._trim(now) + + def book_age_ms(self, now: float | None = None) -> Optional[float]: + """当前时刻距离最近一次 book 更新的年龄（ms）。""" + if self._last_book_wall <= 0: + return None + now = now or time.time() + return max(0.0, (now - self._last_book_wall) * 1000.0) + + def _trim(self, now: float) -> None: + cut = now - self.window_sec + while self._book_ts and self._book_ts[0] < cut: + self._book_ts.popleft() + while self._trade_ts and self._trade_ts[0] < cut: + self._trade_ts.popleft() + while self._seq_gaps_window and self._seq_gaps_window[0][0] < cut: + self._seq_gaps_window.popleft() + + def snapshot(self) -> dict: + now = time.time() + self._trim(now) + w = max(self.window_sec, 1e-6) + lat = sorted(self._latencies_ms) + gaps_in_window = sum(g for _, g in self._seq_gaps_window) + book_age = self.book_age_ms(now) + # Binance depth update ids are not contiguous; gap is observe-only. + # Healthy = sufficient book rate + fresh book. + healthy = ( + len(self._book_ts) / w >= 0.5 + and (book_age is None or book_age < 500.0) + ) + return { + "event": "health", + "window_sec": self.window_sec, + "book_update_rate": len(self._book_ts) / w, + "trade_update_rate": len(self._trade_ts) / w, + "latency_ms_mean": (sum(lat) / len(lat)) if lat else None, + "latency_ms_p50": _percentile(lat, 0.50), + "latency_ms_p95": _percentile(lat, 0.95), + "latency_ms_p99": _percentile(lat, 0.99), + "latency_ms_max": lat[-1] if lat else None, + "sequence_gap": self._seq_gaps, # 累计 + "sequence_gap_window": gaps_in_window, # 近窗 + "book_age_ms": book_age, + "book_total": self._book_count, + "trade_total": self._trade_count, + "healthy": healthy, + } + + def maybe_report(self) -> Optional[dict]: + now = time.time() + if now - self._last_report < self.report_every_sec: + return None + self._last_report = now + return self.snapshot() + + def allow_quoting(self, max_book_age_ms: float = 500.0) -> bool: + """Gate new quotes on freshness + update rate (not Binance seq jumps).""" + age = self.book_age_ms() + if age is None or age >= max_book_age_ms: + return False + now = time.time() + self._trim(now) + w = max(self.window_sec, 1e-6) + if len(self._book_ts) / w < 0.5: + return False + return True diff --git a/src/nautilus_mm/recorder.py b/src/nautilus_mm/recorder.py new file mode 100644 index 0000000..eaed5ef --- /dev/null +++ b/src/nautilus_mm/recorder.py @@ -0,0 +1,861 @@ +""" +Maker Edge 事件记录器 — Execution Reality Layer（Nautilus / CCXT 共用） + +事件： +- quote_created / quote_canceled / quote_filled +- fill / fill_path / fill_exit + +默认输出：nautilus_mm/logs/maker_edge/YYYYMMDD.jsonl +（与 Freqtrade MakerEdgeProbe schema 对齐，可用同一 analyze 脚本） +""" + +from __future__ import annotations + +import json +import logging +import time +import uuid +from collections import deque +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Optional + +from nautilus_mm.experiment import load_experiment_meta, stamp_event + +logger = logging.getLogger(__name__) + + +def empty_market_state_snapshot() -> dict: + """Market Pulse 预留位；探针阶段保持 null，不做交易决策。""" + return { + "regime": None, + "trend_state": None, + "liquidity_state": None, + "volatility_state": None, + } + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def _iso(ts: datetime | float | None = None) -> str: + if ts is None: + t = _utc_now() + elif isinstance(ts, (int, float)): + t = datetime.fromtimestamp(ts, tz=timezone.utc) + else: + t = ts if ts.tzinfo else ts.replace(tzinfo=timezone.utc) + return t.isoformat() + + +@dataclass +class MicroSnapshot: + best_bid: float = 0.0 + best_ask: float = 0.0 + mid: float = 0.0 + spread: float = 0.0 + bid_depth_1: float = 0.0 + ask_depth_1: float = 0.0 + bid_depth_5: float = 0.0 + ask_depth_5: float = 0.0 + bid_depth: float = 0.0 # top-N + ask_depth: float = 0.0 + obi: float = 0.0 + delta: float = 0.0 + trade_imbalance: float = 0.0 # (buy-sell)/(buy+sell) on recent trades + delta_efficiency: float = 0.0 + liquidation_distance: float = 0.0 + + def to_book_fields(self) -> dict[str, float]: + return { + "bid_price": self.best_bid, + "ask_price": self.best_ask, + "mid": self.mid, + "spread": self.spread, + "bid_depth_1": self.bid_depth_1, + "ask_depth_1": self.ask_depth_1, + "bid_depth_5": self.bid_depth_5, + "ask_depth_5": self.ask_depth_5, + "bid_depth": self.bid_depth, + "ask_depth": self.ask_depth, + "obi": self.obi, + "delta": self.delta, + "trade_imbalance": self.trade_imbalance, + "delta_efficiency": self.delta_efficiency, + "liquidation_distance": self.liquidation_distance, + # 兼容旧字段 + "buy1_depth": self.bid_depth_1, + "sell1_depth": self.ask_depth_1, + } + + +@dataclass +class ActiveQuote: + quote_id: str + pair: str + side: str # bid / ask + quote_price: float + created_ts: float + reason: str = "" + trade_id: Optional[int] = None + status: str = "open" # open / filled / canceled + + +@dataclass +class PendingFillPath: + fill_id: str + pair: str + side: str + fill_price: float + fill_ts: float + quote_id: Optional[str] = None + exit_reason: Optional[str] = None + # horizon prices: +1s +5s +10s +30s +60s +300s + after_1s_price: Optional[float] = None + after_5s_price: Optional[float] = None + after_10s_price: Optional[float] = None + after_30s_price: Optional[float] = None + after_1m_price: Optional[float] = None + after_5m_price: Optional[float] = None + # running extrema + min_price: float = 0.0 + max_price: float = 0.0 + # time-MAE / MFE at horizons + mae_1s: Optional[float] = None + mae_5s: Optional[float] = None + mae_10s: Optional[float] = None + mae_30s: Optional[float] = None + mae_1m: Optional[float] = None + mae_5m: Optional[float] = None + mfe_1s: Optional[float] = None + mfe_5s: Optional[float] = None + mfe_10s: Optional[float] = None + mfe_30s: Optional[float] = None + mfe_1m: Optional[float] = None + mfe_5m: Optional[float] = None + done: bool = False + + def __post_init__(self): + self.min_price = self.fill_price + self.max_price = self.fill_price + + def signed_excursions(self) -> tuple[float, float]: + """Return (mae, mfe) at current min/max. mae<=0 adverse, mfe>=0 favorable.""" + if self.side == "long": + mae = (self.min_price - self.fill_price) / self.fill_price + mfe = (self.max_price - self.fill_price) / self.fill_price + else: + mae = (self.fill_price - self.max_price) / self.fill_price + mfe = (self.fill_price - self.min_price) / self.fill_price + return mae, mfe + + def fav_ret_at(self, px: Optional[float]) -> Optional[float]: + if px is None or self.fill_price <= 0: + return None + raw = (px - self.fill_price) / self.fill_price + return raw if self.side == "long" else -raw + + +def classify_path_type(p: "PendingFillPath") -> str: + """ + 成交后路径形态（决定未来 Quote Logic）： + A_immediate_edge — 立即有利（1s/5s 已正，30s 仍正） + B_drawdown_then_recover — 先亏后赚（早期 MAE，末期有利） + C_toxic — 成交即错误（持续不利） + D_mixed — 其它 + """ + r1 = p.fav_ret_at(p.after_1s_price) + r5 = p.fav_ret_at(p.after_5s_price) + r30 = p.fav_ret_at(p.after_30s_price) + r300 = p.fav_ret_at(p.after_5m_price) + mae30 = p.mae_30s or 0.0 + + if r30 is not None and r30 > 0 and (r1 or 0) >= 0 and (r5 or 0) >= 0: + return "A_immediate_edge" + if mae30 < -1e-6 and r300 is not None and r300 > 0: + return "B_drawdown_then_recover" + if r30 is not None and r30 < 0 and (r300 is None or r300 <= 0): + return "C_toxic" + return "D_mixed" + + +class MakerEdgeLogger: + def __init__( + self, + log_dir: str | Path | None = None, + levels: int = 10, + book_history_sec: float = 30.0, + cluster_gap_sec: float = 30.0, + mid_tick_every_sec: float = 1.0, + ): + # .../nautilus_mm/src/nautilus_mm/recorder.py → parents[2] = nautilus_mm + root = Path(__file__).resolve().parents[2] + self.log_dir = Path(log_dir) if log_dir else root / "logs" / "maker_edge" + self.log_dir.mkdir(parents=True, exist_ok=True) + self.levels = levels + self.book_history_sec = book_history_sec + self.cluster_gap_sec = cluster_gap_sec + self.mid_tick_every_sec = mid_tick_every_sec + self._pending: dict[str, PendingFillPath] = {} + self._quotes: dict[str, ActiveQuote] = {} # quote_id -> ActiveQuote + self._quotes_by_trade: dict[int, str] = {} # trade_id -> quote_id + self._book_hist: deque[tuple[float, MicroSnapshot]] = deque(maxlen=2000) + # inventory tracking for future quote engine + self._inv: float = 0.0 + self._inv_nonzero_since: Optional[float] = None + self._inv_target: float = 0.0 # flat target; skew = inv - target + # liquidity-event clustering (样本独立性) + self._cluster_id: Optional[str] = None + self._cluster_side: Optional[str] = None + self._cluster_last_ts: float = 0.0 + self._cluster_start_mid: Optional[float] = None + self._cluster_n: int = 0 + self._last_mid_tick_ts: float = 0.0 + self.experiment = load_experiment_meta() + + def _file(self) -> Path: + return self.log_dir / f"{_utc_now().strftime('%Y%m%d')}.jsonl" + + def update_inventory(self, inventory: float, now: float | None = None) -> dict: + """更新库存并返回 inventory / inventory_time / inventory_skew。""" + now = now or time.time() + self._inv = float(inventory) + if abs(self._inv) < 1e-12: + self._inv_nonzero_since = None + inv_time = 0.0 + else: + if self._inv_nonzero_since is None: + self._inv_nonzero_since = now + inv_time = now - self._inv_nonzero_since + skew = self._inv - self._inv_target + return { + "inventory": self._inv, + "inventory_time": inv_time, + "inventory_skew": skew, + } + + def _attach_common( + self, + ev: dict[str, Any], + inventory: float | None = None, + state: dict | None = None, + now: float | None = None, + ) -> dict[str, Any]: + now = now or time.time() + if inventory is not None: + ev.update(self.update_inventory(inventory, now=now)) + # 始终带 market_state_snapshot（可被 state 覆盖内部字段） + mss = empty_market_state_snapshot() + if state: + for k in mss: + if k in state and state[k] is not None: + mss[k] = state[k] + # 兼容旧扁平字段 + for k, v in state.items(): + if k not in mss and k != "market_state_snapshot": + ev.setdefault(k, v) + ev["market_state_snapshot"] = mss + return ev + + def write(self, event: dict[str, Any]) -> None: + event.setdefault("ts", _iso()) + event.setdefault("ts_epoch", time.time()) + if "market_state_snapshot" not in event: + event["market_state_snapshot"] = empty_market_state_snapshot() + stamp_event(event, self.experiment) + with self._file().open("a", encoding="utf-8") as f: + f.write(json.dumps(event, ensure_ascii=False, default=str) + "\n") + + def write_experiment_start(self, extra: dict | None = None) -> None: + """探针启动时写入一次，绑定本轮 Data Collection。""" + ev = { + "event": "experiment_start", + **self.experiment, + } + if extra: + ev.update(extra) + self.write(ev) + + # ------------------------------------------------------------------ # + # Snapshot + # ------------------------------------------------------------------ # + @staticmethod + def snapshot_from_orderbook( + ob: dict, + levels: int = 10, + recent_trades: list | None = None, + last_mid: float | None = None, + liq_proxy_low: float | None = None, + liq_proxy_high: float | None = None, + ) -> MicroSnapshot: + bids = (ob.get("bids") or [])[:levels] + asks = (ob.get("asks") or [])[:levels] + if not bids or not asks: + return MicroSnapshot() + + best_bid = float(bids[0][0]) + best_ask = float(asks[0][0]) + mid = (best_bid + best_ask) / 2.0 + spread = best_ask - best_bid + + def depth(levels_side, n): + return sum(float(x[1]) for x in levels_side[:n]) + + bid_depth_1 = depth(bids, 1) + ask_depth_1 = depth(asks, 1) + bid_depth_5 = depth(bids, 5) + ask_depth_5 = depth(asks, 5) + bid_depth = depth(bids, levels) + ask_depth = depth(asks, levels) + tot = bid_depth + ask_depth + obi = ((bid_depth - ask_depth) / tot) if tot > 0 else 0.0 + + buy_v = sell_v = 0.0 + if recent_trades: + for t in recent_trades: + amt = float(t.get("amount") or t.get("qty") or 0.0) + side = (t.get("side") or "").lower() + if side in ("buy", "b"): + buy_v += amt + elif side in ("sell", "s"): + sell_v += amt + delta = buy_v - sell_v + timb_den = buy_v + sell_v + trade_imbalance = ((buy_v - sell_v) / timb_den) if timb_den > 0 else 0.0 + + de = 0.0 + if last_mid and mid and abs(delta) > 1e-12: + de = ((mid - last_mid) / last_mid) / delta + + liq_dist = 0.0 + if liq_proxy_low and liq_proxy_high and mid: + rng = liq_proxy_high - liq_proxy_low + if rng > 0: + liq_dist = ((mid - liq_proxy_low) / rng) * 2 - 1 + + return MicroSnapshot( + best_bid=best_bid, + best_ask=best_ask, + mid=mid, + spread=spread, + bid_depth_1=bid_depth_1, + ask_depth_1=ask_depth_1, + bid_depth_5=bid_depth_5, + ask_depth_5=ask_depth_5, + bid_depth=bid_depth, + ask_depth=ask_depth, + obi=obi, + delta=delta, + trade_imbalance=trade_imbalance, + delta_efficiency=de, + liquidation_distance=liq_dist, + ) + + def record_book( + self, + snap: MicroSnapshot, + now: float | None = None, + *, + emit_mid_tick: bool = True, + pair: str | None = None, + ) -> None: + now = now or time.time() + self._book_hist.append((now, snap)) + # trim old + cutoff = now - self.book_history_sec + while self._book_hist and self._book_hist[0][0] < cutoff: + self._book_hist.popleft() + # mid 时间序列：供 Fill vs Random Benchmark（研究保护栏） + if ( + emit_mid_tick + and snap.mid > 0 + and (now - self._last_mid_tick_ts) >= self.mid_tick_every_sec + ): + self._last_mid_tick_ts = now + self.write( + { + "event": "mid_tick", + "pair": pair, + "mid": snap.mid, + "best_bid": snap.best_bid, + "best_ask": snap.best_ask, + "spread": snap.spread, + "ts_epoch": now, + } + ) + + def assign_event_cluster( + self, + side: str, + mid: float, + now: float | None = None, + ) -> dict[str, Any]: + """ + 将连续同侧成交归入同一流动性事件（event_cluster_id）。 + + 规则（研究保护栏，非策略）： + - 同 side + - 与上一笔间隔 < cluster_gap_sec + → 同一 cluster；否则新开 cluster。 + + 统计时应用 cluster 加权，避免「暴跌连续 50 笔 Bid」当成 50 个独立样本。 + """ + now = now or time.time() + new_cluster = ( + self._cluster_id is None + or self._cluster_side != side + or (now - self._cluster_last_ts) > self.cluster_gap_sec + ) + if new_cluster: + self._cluster_id = uuid.uuid4().hex[:12] + self._cluster_side = side + self._cluster_start_mid = mid if mid > 0 else None + self._cluster_n = 0 + self._cluster_n += 1 + self._cluster_last_ts = now + mid_move = None + if self._cluster_start_mid and mid > 0: + mid_move = (mid - self._cluster_start_mid) / self._cluster_start_mid + return { + "event_cluster_id": self._cluster_id, + "cluster_fill_index": self._cluster_n, + "cluster_mid_move_from_start": mid_move, + } + + def book_at(self, target_ts: float) -> Optional[MicroSnapshot]: + """取最接近 target_ts 的历史盘口（用于成交前5s）。""" + if not self._book_hist: + return None + best = min(self._book_hist, key=lambda x: abs(x[0] - target_ts)) + return best[1] + + def build_fill_context(self, side: str, now: float | None = None) -> dict: + """ + 成交主动性上下文：区分「砸盘后吸收」vs「下跌接刀」。 + 不接 Market Pulse，仅用本地 book history + trade imbalance。 + """ + now = now or time.time() + cur = self.book_at(now) + past = self.book_at(now - 5.0) + fill_type = "bid" if side == "long" else "ask" + ctx: dict[str, Any] = { + "fill_type": fill_type, + "market_event_before_fill": "unknown", + "trade_imbalance_5s": None, + "price_velocity_5s": None, + } + if not cur or not past or past.mid <= 0: + return {"fill_context": ctx} + + vel = (cur.mid - past.mid) / past.mid + # 用当前与 5s 前 imbalance 的平均作代理 + timb = (cur.trade_imbalance + past.trade_imbalance) / 2.0 + ctx["trade_imbalance_5s"] = timb + ctx["price_velocity_5s"] = vel + + if fill_type == "bid": + # 卖压后吸收：价格下行/企稳 + 卖向 imbalance，但盘口未继续恶化太狠 + if timb < -0.2 and vel < 0: + if abs(vel) < 0.0003: + ctx["market_event_before_fill"] = "sell_pressure_absorbing" + else: + ctx["market_event_before_fill"] = "sell_pressure_falling" + elif vel < -0.0005: + ctx["market_event_before_fill"] = "momentum_down_catching_knife" + elif timb > 0.15: + ctx["market_event_before_fill"] = "buy_support" + else: + ctx["market_event_before_fill"] = "neutral" + else: + if timb > 0.2 and vel > 0: + if abs(vel) < 0.0003: + ctx["market_event_before_fill"] = "buy_pressure_absorbing" + else: + ctx["market_event_before_fill"] = "buy_pressure_rising" + elif vel > 0.0005: + ctx["market_event_before_fill"] = "momentum_up_chasing" + elif timb < -0.15: + ctx["market_event_before_fill"] = "sell_resistance" + else: + ctx["market_event_before_fill"] = "neutral" + return {"fill_context": ctx} + + def book_deterioration(self, side: str, now: float | None = None, lookback: float = 5.0) -> dict: + """ + 成交前 lookback 秒盘口是否恶化。 + long: bid_depth 下降 / ask_depth 上升 / mid 下跌 → 恶化 + """ + now = now or time.time() + cur = self.book_at(now) + past = self.book_at(now - lookback) + if not cur or not past or past.mid <= 0: + return {"book_ok": False} + mid_chg = (cur.mid - past.mid) / past.mid + bid5_chg = (cur.bid_depth_5 - past.bid_depth_5) / past.bid_depth_5 if past.bid_depth_5 else 0.0 + ask5_chg = (cur.ask_depth_5 - past.ask_depth_5) / past.ask_depth_5 if past.ask_depth_5 else 0.0 + obi_chg = cur.obi - past.obi + if side == "long": + deteriorated = (mid_chg < -0.00005) or (bid5_chg < -0.15) or (obi_chg < -0.1) + else: + deteriorated = (mid_chg > 0.00005) or (ask5_chg < -0.15) or (obi_chg > 0.1) + return { + "book_ok": True, + "pre_5s_mid_chg": mid_chg, + "pre_5s_bid_depth_5_chg": bid5_chg, + "pre_5s_ask_depth_5_chg": ask5_chg, + "pre_5s_obi_chg": obi_chg, + "pre_5s_deteriorated": bool(deteriorated), + "pre_5s_bid_depth_1": past.bid_depth_1, + "pre_5s_ask_depth_1": past.ask_depth_1, + "pre_5s_bid_depth_5": past.bid_depth_5, + "pre_5s_ask_depth_5": past.ask_depth_5, + "pre_5s_obi": past.obi, + "pre_5s_spread": past.spread, + "pre_5s_trade_imbalance": past.trade_imbalance, + } + + # ------------------------------------------------------------------ # + # Quote lifecycle + # ------------------------------------------------------------------ # + def create_quote( + self, + pair: str, + side: str, + quote_price: float, + inventory: float, + snap: MicroSnapshot, + reason: str = "", + trade_id: Optional[int] = None, + state: dict | None = None, + extra: dict | None = None, + ) -> str: + qid = uuid.uuid4().hex[:16] + now = time.time() + q = ActiveQuote( + quote_id=qid, + pair=pair, + side=side, + quote_price=quote_price, + created_ts=now, + reason=reason, + trade_id=trade_id, + status="open", + ) + self._quotes[qid] = q + if trade_id is not None: + self._quotes_by_trade[trade_id] = qid + + ev = { + "event": "quote_created", + "quote_id": qid, + "pair": pair, + "side": side, + "quote_price": quote_price, + "quote_created_time": _iso(now), + "quote_created_epoch": now, + "reason": reason, + "trade_id": trade_id, + "status": "open", + "filled": False, + } + ev.update(snap.to_book_fields()) + self._attach_common(ev, inventory=inventory, state=state, now=now) + if extra: + ev.update(extra) + self.write(ev) + return qid + + def cancel_quote( + self, + quote_id: str | None = None, + trade_id: Optional[int] = None, + reason: str = "timeout", + snap: MicroSnapshot | None = None, + ) -> None: + q = None + if quote_id and quote_id in self._quotes: + q = self._quotes[quote_id] + elif trade_id is not None and trade_id in self._quotes_by_trade: + q = self._quotes.get(self._quotes_by_trade[trade_id]) + if q is None or q.status != "open": + return + + now = time.time() + q.status = "canceled" + ev = { + "event": "quote_canceled", + "quote_id": q.quote_id, + "pair": q.pair, + "side": q.side, + "quote_price": q.quote_price, + "quote_created_time": _iso(q.created_ts), + "quote_cancel_time": _iso(now), + "quote_cancel_epoch": now, + "time_alive_sec": now - q.created_ts, + "cancel_reason": reason, + "filled": False, + "status": "canceled", + "trade_id": q.trade_id, + } + if snap: + ev.update(snap.to_book_fields()) + self.write(ev) + + def bind_trade(self, quote_id: str, trade_id: int) -> None: + if quote_id in self._quotes: + self._quotes[quote_id].trade_id = trade_id + self._quotes_by_trade[trade_id] = quote_id + + # ------------------------------------------------------------------ # + # Fill + path + # ------------------------------------------------------------------ # + def log_fill( + self, + pair: str, + side: str, + fill_price: float, + amount: float, + inventory: float, + snap: MicroSnapshot | None, + order_type: str = "limit", + quote_id: str | None = None, + trade_id: Optional[int] = None, + fill_reason: str = "maker_hit", + state: dict | None = None, + extra: dict | None = None, + quote_terminal: bool = True, + ) -> str: + """Record a fill. Always writes even if snap is None (book unavailable). + + quote_terminal=False keeps quote open for partial fills so later slices + retain quote_id linkage until the order closes. + """ + now = time.time() + fill_id = uuid.uuid4().hex[:16] + snap = snap or MicroSnapshot() + + # resolve quote lifecycle + q: Optional[ActiveQuote] = None + if quote_id and quote_id in self._quotes: + q = self._quotes[quote_id] + elif trade_id is not None and trade_id in self._quotes_by_trade: + q = self._quotes.get(self._quotes_by_trade[trade_id]) + + time_to_fill = None + quote_created_time = None + quote_price = fill_price + if q is not None: + if quote_terminal: + q.status = "filled" + time_to_fill = now - q.created_ts + quote_created_time = _iso(q.created_ts) + quote_price = q.quote_price + quote_id = q.quote_id + + det = self.book_deterioration(side, now=now, lookback=5.0) + fctx = self.build_fill_context(side, now=now) + mid_for_cluster = snap.mid if snap.mid > 0 else fill_price + cluster = self.assign_event_cluster(side, mid_for_cluster, now=now) + + ev = { + "event": "fill", + "fill_id": fill_id, + "quote_id": quote_id, + "pair": pair, + "side": side, + "fill_price": fill_price, + "quote_price": quote_price, + "amount": amount, + "order_type": order_type, + "fill_reason": fill_reason, + "quote_created_time": quote_created_time, + "quote_fill_time": _iso(now), + "time_to_fill": time_to_fill, + "trade_id": trade_id, + "filled": True, + "quote_terminal": quote_terminal, + "book_available": bool(snap.mid > 0), + } + ev.update(snap.to_book_fields()) + ev.update(det) + ev.update(fctx) + ev.update(cluster) + # Effective spread capture proxy: 相对 mid 的被动成交优势 + if snap.mid > 0: + if side == "long": + ev["spread_capture_pct"] = (snap.mid - fill_price) / snap.mid + else: + ev["spread_capture_pct"] = (fill_price - snap.mid) / snap.mid + self._attach_common(ev, inventory=inventory, state=state, now=now) + if extra: + ev.update(extra) + self.write(ev) + + # also emit quote_filled lifecycle event (only when order fully done) + if q is not None and quote_terminal: + self.write( + { + "event": "quote_filled", + "quote_id": q.quote_id, + "fill_id": fill_id, + "pair": pair, + "side": q.side, + "quote_price": q.quote_price, + "quote_created_time": _iso(q.created_ts), + "quote_fill_time": _iso(now), + "time_to_fill": time_to_fill, + "fill_reason": fill_reason, + "filled": True, + "status": "filled", + "trade_id": trade_id, + **snap.to_book_fields(), + **det, + } + ) + + self._pending[fill_id] = PendingFillPath( + fill_id=fill_id, + pair=pair, + side=side, + fill_price=fill_price, + fill_ts=now, + quote_id=quote_id, + ) + return fill_id + + def attach_exit_reason(self, fill_id: str, exit_reason: str) -> None: + if fill_id in self._pending: + self._pending[fill_id].exit_reason = exit_reason + # also write lightweight annotation + self.write( + { + "event": "fill_exit", + "fill_id": fill_id, + "exit_reason": exit_reason, + } + ) + + def update_paths(self, pair: str, last_price: float, now: float | None = None) -> None: + now = now or time.time() + finished = [] + for fid, p in self._pending.items(): + if p.pair != pair or p.done: + continue + p.min_price = min(p.min_price, last_price) + p.max_price = max(p.max_price, last_price) + mae, mfe = p.signed_excursions() + age = now - p.fill_ts + + def mark(horizon_attr_price, horizon_mae, horizon_mfe, sec, price_val): + if getattr(p, horizon_attr_price) is None and age >= sec: + setattr(p, horizon_attr_price, price_val) + setattr(p, horizon_mae, mae) + setattr(p, horizon_mfe, mfe) + + mark("after_1s_price", "mae_1s", "mfe_1s", 1, last_price) + mark("after_5s_price", "mae_5s", "mfe_5s", 5, last_price) + mark("after_10s_price", "mae_10s", "mfe_10s", 10, last_price) + mark("after_30s_price", "mae_30s", "mfe_30s", 30, last_price) + mark("after_1m_price", "mae_1m", "mfe_1m", 60, last_price) + + if p.after_5m_price is None and age >= 300: + p.after_5m_price = last_price + p.mae_5m = mae + p.mfe_5m = mfe + p.done = True + # Price MAE absolute + if p.side == "long": + price_mae = p.min_price - p.fill_price + price_mfe = p.max_price - p.fill_price + else: + price_mae = p.fill_price - p.max_price + price_mfe = p.fill_price - p.min_price + + fav_30 = p.fav_ret_at(p.after_30s_price) or 0.0 + fav_1 = p.fav_ret_at(p.after_1s_price) + fav_5 = p.fav_ret_at(p.after_5s_price) + fav_10 = p.fav_ret_at(p.after_10s_price) + fav_60 = p.fav_ret_at(p.after_1m_price) + fav_300 = p.fav_ret_at(p.after_5m_price) or 0.0 + + vol_proxy = abs(p.max_price - p.min_price) / p.fill_price if p.fill_price else 0.0 + toxicity_score = max(0.0, -fav_30) / max(vol_proxy, 1e-8) + mfe_gt_mae_30 = (p.mfe_30s or 0.0) > abs(p.mae_30s or 0.0) + path_type = classify_path_type(p) + + # 路径点（供形态分析 / 复现） + price_path = { + "t0": p.fill_price, + "t1s": p.after_1s_price, + "t5s": p.after_5s_price, + "t10s": p.after_10s_price, + "t30s": p.after_30s_price, + "t60s": p.after_1m_price, + "t300s": p.after_5m_price, + } + ret_path = { + "t1s": fav_1, + "t5s": fav_5, + "t10s": fav_10, + "t30s": fav_30, + "t60s": fav_60, + "t300s": fav_300, + } + + self.write( + { + "event": "fill_path", + "fill_id": p.fill_id, + "quote_id": p.quote_id, + "pair": p.pair, + "side": p.side, + "fill_price": p.fill_price, + "exit_reason": p.exit_reason, + "after_1s_price": p.after_1s_price, + "after_5s_price": p.after_5s_price, + "after_10s_price": p.after_10s_price, + "after_30s_price": p.after_30s_price, + "after_1m_price": p.after_1m_price, + "after_5m_price": p.after_5m_price, + "price_path": price_path, + "ret_path": ret_path, + "path_type": path_type, + "min_price": p.min_price, + "max_price": p.max_price, + "mae_1s": p.mae_1s, + "mae_5s": p.mae_5s, + "mae_10s": p.mae_10s, + "mae_30s": p.mae_30s, + "mae_1m": p.mae_1m, + "mae_5m": p.mae_5m, + "mfe_1s": p.mfe_1s, + "mfe_5s": p.mfe_5s, + "mfe_10s": p.mfe_10s, + "mfe_30s": p.mfe_30s, + "mfe_1m": p.mfe_1m, + "mfe_5m": p.mfe_5m, + "price_mae": price_mae, + "price_mfe": price_mfe, + "price_mae_pct": mae, + "price_mfe_pct": mfe, + "fav_ret_30s": fav_30, + "vol_proxy_5m": vol_proxy, + "toxicity_score": toxicity_score, + "mfe_gt_mae_30s": mfe_gt_mae_30, + } + ) + finished.append(fid) + + for fid in finished: + self._pending.pop(fid, None) + + @property + def pending_count(self) -> int: + return len(self._pending) + + # 兼容旧 API + def log_quote(self, *args, **kwargs): + """Deprecated wrapper → create_quote for live quotes; heartbeat uses book only.""" + return self.create_quote(*args, **kwargs) diff --git a/src/nautilus_mm/run_event_state.py b/src/nautilus_mm/run_event_state.py new file mode 100644 index 0000000..5d71584 --- /dev/null +++ b/src/nautilus_mm/run_event_state.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +""" +Launch Event-State Observability Probe (MM_EDGE_EXP_002) + +Data collection only — NO trading, NO strategy, NO Stage 3 unlock. + +Environment: + EXPERIMENT_ID=MM_EDGE_EXP_002 + PROBE_VERSION=event_state_v0.1 + ENABLE_TRADING=false (hard-enforced; any true value is ignored) + +Usage: + cd nautilus_mm + source .venv/bin/activate + export PYTHONPATH=src + python -m nautilus_mm.run_event_state +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +from dotenv import load_dotenv + +_ROOT = Path(__file__).resolve().parents[2] +_SRC = _ROOT / "src" +if str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) + +load_dotenv(_ROOT / ".env") + +from nautilus_trader.adapters.binance import BINANCE +from nautilus_trader.adapters.binance import BinanceAccountType +from nautilus_trader.adapters.binance import BinanceDataClientConfig +from nautilus_trader.adapters.binance import BinanceExecClientConfig +from nautilus_trader.adapters.binance import BinanceInstrumentProviderConfig +from nautilus_trader.adapters.binance import BinanceLiveDataClientFactory +from nautilus_trader.adapters.binance import BinanceLiveExecClientFactory +from nautilus_trader.adapters.binance.common.enums import BinanceEnvironment +from nautilus_trader.config import LiveDataEngineConfig +from nautilus_trader.config import LiveExecEngineConfig +from nautilus_trader.config import LoggingConfig +from nautilus_trader.config import TradingNodeConfig +from nautilus_trader.live.node import TradingNode +from nautilus_trader.model.identifiers import ClientId +from nautilus_trader.model.identifiers import InstrumentId +from nautilus_trader.model.identifiers import TraderId + +from nautilus_mm.experiment import load_experiment_meta +from nautilus_mm.strategies.event_state_probe import EventStateProbe +from nautilus_mm.strategies.event_state_probe import EventStateProbeConfig + + +def _env_bool(name: str, default: bool = False) -> bool: + v = os.getenv(name) + if v is None: + return default + return v.strip().lower() in ("1", "true", "yes", "y") + + +def _resolve_environment() -> BinanceEnvironment: + raw = os.getenv("BINANCE_ENVIRONMENT", "TESTNET") + env_name = raw.strip().upper() + if env_name not in ("TESTNET", "LIVE"): + print(f"ERROR: BINANCE_ENVIRONMENT must be TESTNET or LIVE, got {raw!r}") + sys.exit(1) + if env_name == "LIVE" and not _env_bool("I_UNDERSTAND_LIVE", False): + print("ERROR: LIVE blocked for EXP_002 unless I_UNDERSTAND_LIVE=yes") + sys.exit(1) + return BinanceEnvironment.LIVE if env_name == "LIVE" else BinanceEnvironment.TESTNET + + +def main() -> None: + # Layer 2: runner hard-forces trading off even if .env / systemd is wrong + if _env_bool("ENABLE_TRADING", False): + print( + "WARNING: ENABLE_TRADING=true ignored — MM_EDGE_EXP_002 is observability-only" + ) + os.environ["ENABLE_TRADING"] = "false" + os.environ["EXPERIMENT_ID"] = "MM_EDGE_EXP_002" + os.environ.setdefault("PROBE_VERSION", "event_state_v0.1") + + exp_id = os.getenv("EXPERIMENT_ID", "MM_EDGE_EXP_002") + if exp_id != "MM_EDGE_EXP_002": + print( + f"WARNING: EXPERIMENT_ID={exp_id!r} — expected MM_EDGE_EXP_002 for this runner" + ) + + api_key = os.getenv("BINANCE_API_KEY", "") + api_secret = os.getenv("BINANCE_API_SECRET", "") + environment = _resolve_environment() + symbol = os.getenv("SYMBOL", "BTCUSDT-PERP") + instrument_id = InstrumentId.from_str(f"{symbol}.{BINANCE}") + log_dir = os.getenv("EVENT_STATE_LOG_DIR", str(_ROOT / "logs" / "event_state")) + + if not api_key or not api_secret: + print("ERROR: set BINANCE_API_KEY / BINANCE_API_SECRET in nautilus_mm/.env") + sys.exit(1) + + config_node = TradingNodeConfig( + trader_id=TraderId("EVENT-STATE-002"), + logging=LoggingConfig(log_level="INFO", log_colors=True, use_pyo3=True), + data_engine=LiveDataEngineConfig(external_clients=[ClientId(BINANCE)]), + exec_engine=LiveExecEngineConfig( + reconciliation=False, + open_check_interval_secs=0.0, + graceful_shutdown_on_exception=True, + ), + data_clients={ + BINANCE: BinanceDataClientConfig( + api_key=api_key, + api_secret=api_secret, + account_type=BinanceAccountType.USDT_FUTURES, + environment=environment, + instrument_provider=BinanceInstrumentProviderConfig( + load_ids=frozenset([instrument_id]), + ), + ), + }, + exec_clients={ + BINANCE: BinanceExecClientConfig( + api_key=api_key, + api_secret=api_secret, + account_type=BinanceAccountType.USDT_FUTURES, + environment=environment, + instrument_provider=BinanceInstrumentProviderConfig( + load_ids=frozenset([instrument_id]), + ), + max_retries=3, + ), + }, + timeout_connection=30.0, + timeout_reconciliation=10.0, + timeout_portfolio=10.0, + timeout_disconnection=10.0, + timeout_post_stop=5.0, + ) + + node = TradingNode(config=config_node) + strat_config = EventStateProbeConfig( + instrument_id=instrument_id, + book_depth=int(os.getenv("BOOK_DEPTH", "10")), + log_dir=log_dir, + prefill_window_sec=float(os.getenv("PREFILL_WINDOW_SEC", "5.0")), + prefill_margin_sec=float(os.getenv("PREFILL_MARGIN_SEC", "0.25")), + large_trade_qty=float(os.getenv("LARGE_TRADE_QTY", "0.1")), + log_every_book_delta=_env_bool("LOG_EVERY_BOOK_DELTA", True), + ) + node.trader.add_strategy(EventStateProbe(config=strat_config)) + node.add_data_client_factory(BINANCE, BinanceLiveDataClientFactory) + node.add_exec_client_factory(BINANCE, BinanceLiveExecClientFactory) + node.build() + + exp = load_experiment_meta() + print( + f"[event_state] Experiment={exp['experiment_id']} {exp['probe_version']} | " + f"type=Event-State Observability | trading=NO | {symbol} env={environment} | " + f"run={os.getenv('LEDGER_RUN_ID', 'EXP-002-RUN-UNSET')} | log={log_dir}" + ) + try: + node.run() + finally: + node.dispose() + + +if __name__ == "__main__": + main() diff --git a/src/nautilus_mm/run_live.py b/src/nautilus_mm/run_live.py new file mode 100644 index 0000000..6db55bf --- /dev/null +++ b/src/nautilus_mm/run_live.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +""" +启动 Nautilus TradingNode + MakerEdgeProbe（Binance USDT-M） + +环境变量（或 .env）： + BINANCE_API_KEY + BINANCE_API_SECRET + BINANCE_ENVIRONMENT=TESTNET|LIVE (仅允许这两个值；默认 TESTNET) + I_UNDERSTAND_LIVE=yes (LIVE 必填) + ENABLE_TRADING=false (默认关闭；显式 true 才挂单) + HTTP_PROXY / HTTPS_PROXY (可选) + +用法： + cd nautilus_mm + source .venv/bin/activate + export PYTHONPATH=src + python -m nautilus_mm.run_live +""" + +from __future__ import annotations + +import os +import sys +from decimal import Decimal +from pathlib import Path + +from dotenv import load_dotenv + +# ensure src on path when run as script +_ROOT = Path(__file__).resolve().parents[2] +_SRC = _ROOT / "src" +if str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) + +load_dotenv(_ROOT / ".env") + +from nautilus_trader.adapters.binance import BINANCE +from nautilus_trader.adapters.binance import BinanceAccountType +from nautilus_trader.adapters.binance import BinanceDataClientConfig +from nautilus_trader.adapters.binance import BinanceExecClientConfig +from nautilus_trader.adapters.binance import BinanceInstrumentProviderConfig +from nautilus_trader.adapters.binance import BinanceLiveDataClientFactory +from nautilus_trader.adapters.binance import BinanceLiveExecClientFactory +from nautilus_trader.adapters.binance.common.enums import BinanceEnvironment +from nautilus_trader.config import LiveDataEngineConfig +from nautilus_trader.config import LiveExecEngineConfig +from nautilus_trader.config import LoggingConfig +from nautilus_trader.config import TradingNodeConfig +from nautilus_trader.live.node import TradingNode +from nautilus_trader.model.identifiers import ClientId +from nautilus_trader.model.identifiers import InstrumentId +from nautilus_trader.model.identifiers import TraderId + +from nautilus_mm.experiment import load_experiment_meta +from nautilus_mm.strategies.maker_edge_probe import MakerEdgeProbe +from nautilus_mm.strategies.maker_edge_probe import MakerEdgeProbeConfig + + +def _env_bool(name: str, default: bool = False) -> bool: + v = os.getenv(name) + if v is None: + return default + return v.strip().lower() in ("1", "true", "yes", "y") + + +def _resolve_environment() -> BinanceEnvironment: + raw = os.getenv("BINANCE_ENVIRONMENT", "TESTNET") + env_name = raw.strip().upper() + if env_name not in ("TESTNET", "LIVE"): + print( + f"ERROR: BINANCE_ENVIRONMENT must be exactly TESTNET or LIVE, got {raw!r}" + ) + sys.exit(1) + if env_name == "LIVE": + if not _env_bool("I_UNDERSTAND_LIVE", False): + print( + "ERROR: LIVE trading blocked. Set I_UNDERSTAND_LIVE=yes " + "only after you accept real-money risk." + ) + sys.exit(1) + return BinanceEnvironment.LIVE + return BinanceEnvironment.TESTNET + + +def main() -> None: + api_key = os.getenv("BINANCE_API_KEY", "") + api_secret = os.getenv("BINANCE_API_SECRET", "") + environment = _resolve_environment() + + symbol = os.getenv("SYMBOL", "BTCUSDT-PERP") + instrument_id = InstrumentId.from_str(f"{symbol}.{BINANCE}") + order_qty = Decimal(os.getenv("ORDER_QTY", "0.001")) + enable_trading = _env_bool("ENABLE_TRADING", False) + max_abs_inventory = Decimal(os.getenv("MAX_ABS_INVENTORY", "0.005")) + quote_ttl_secs = float(os.getenv("QUOTE_TTL_SECS", "30")) + log_dir = os.getenv("MAKER_EDGE_LOG_DIR", str(_ROOT / "logs" / "maker_edge")) + + # 代理：Nautilus/httpx 会读 HTTP(S)_PROXY；这里仅提示 + proxy = os.getenv("HTTPS_PROXY") or os.getenv("HTTP_PROXY") or "" + if proxy: + print(f"[nautilus_mm] using proxy: {proxy}") + + if not api_key or not api_secret: + print("ERROR: set BINANCE_API_KEY / BINANCE_API_SECRET in nautilus_mm/.env") + print("For TESTNET keys: https://testnet.binancefuture.com/") + sys.exit(1) + + config_node = TradingNodeConfig( + trader_id=TraderId("MAKER-EDGE-001"), + logging=LoggingConfig(log_level="INFO", log_colors=True, use_pyo3=True), + data_engine=LiveDataEngineConfig(external_clients=[ClientId(BINANCE)]), + exec_engine=LiveExecEngineConfig( + reconciliation=True, + open_check_interval_secs=5.0, + graceful_shutdown_on_exception=True, + ), + data_clients={ + BINANCE: BinanceDataClientConfig( + api_key=api_key, + api_secret=api_secret, + account_type=BinanceAccountType.USDT_FUTURES, + environment=environment, + instrument_provider=BinanceInstrumentProviderConfig( + load_ids=frozenset([instrument_id]), + ), + ), + }, + exec_clients={ + BINANCE: BinanceExecClientConfig( + api_key=api_key, + api_secret=api_secret, + account_type=BinanceAccountType.USDT_FUTURES, + environment=environment, + instrument_provider=BinanceInstrumentProviderConfig( + load_ids=frozenset([instrument_id]), + ), + max_retries=3, + ), + }, + timeout_connection=30.0, + timeout_reconciliation=10.0, + timeout_portfolio=10.0, + timeout_disconnection=10.0, + timeout_post_stop=5.0, + ) + + node = TradingNode(config=config_node) + + strat_config = MakerEdgeProbeConfig( + instrument_id=instrument_id, + order_qty=order_qty, + book_depth=10, + quote_offset_ticks=1, + max_quotes=1, + quote_ttl_secs=quote_ttl_secs, + cooldown_secs=float(os.getenv("COOLDOWN_SECS", "60")), + book_sample_secs=2.0, + log_dir=log_dir, + obi_enter=float(os.getenv("OBI_ENTER", "0.25")), + enable_trading=enable_trading, + max_abs_inventory=max_abs_inventory, + ) + strategy = MakerEdgeProbe(config=strat_config) + node.trader.add_strategy(strategy) + + node.add_data_client_factory(BINANCE, BinanceLiveDataClientFactory) + node.add_exec_client_factory(BINANCE, BinanceLiveExecClientFactory) + node.build() + + exp = load_experiment_meta() + print( + f"[nautilus_mm] Experiment={exp['experiment_id']} {exp['probe_version']} " + f"quote/fee/exchange=frozen | {symbol} env={environment} " + f"trading={enable_trading} ttl={quote_ttl_secs}s max_inv={max_abs_inventory} " + f"log={log_dir}" + ) + try: + node.run() + finally: + node.dispose() + + +if __name__ == "__main__": + main() diff --git a/src/nautilus_mm/strategies/__init__.py b/src/nautilus_mm/strategies/__init__.py new file mode 100644 index 0000000..cc655f2 --- /dev/null +++ b/src/nautilus_mm/strategies/__init__.py @@ -0,0 +1,9 @@ +from nautilus_mm.strategies.event_state_probe import EventStateProbe, EventStateProbeConfig +from nautilus_mm.strategies.maker_edge_probe import MakerEdgeProbe, MakerEdgeProbeConfig + +__all__ = [ + "MakerEdgeProbe", + "MakerEdgeProbeConfig", + "EventStateProbe", + "EventStateProbeConfig", +] diff --git a/src/nautilus_mm/strategies/event_state_probe.py b/src/nautilus_mm/strategies/event_state_probe.py new file mode 100644 index 0000000..a13796f --- /dev/null +++ b/src/nautilus_mm/strategies/event_state_probe.py @@ -0,0 +1,187 @@ +""" +Event-State Observability Probe — MM_EDGE_EXP_002 + +Type: Data Collection / Observability Experiment +Strategy: NONE (no quotes, no orders, no trading) +Purpose: Capture immutable pre-fill Event State + +EXP_001 remains FROZEN. This probe never submits orders. +""" + +from __future__ import annotations + +import time +from collections import deque + +from nautilus_trader.common.enums import LogColor +from nautilus_trader.config import PositiveInt, StrategyConfig +from nautilus_trader.model.data import OrderBookDeltas, TradeTick +from nautilus_trader.model.enums import AggressorSide, BookType +from nautilus_trader.model.identifiers import InstrumentId +from nautilus_trader.model.instruments import Instrument +from nautilus_trader.trading.strategy import Strategy + +from nautilus_mm.book_utils import snapshot_from_nautilus_book +from nautilus_mm.event_ledger import ImmutableEventLedger +from nautilus_mm.health import ConnectionHealth + + +class EventStateProbeConfig(StrategyConfig, frozen=True): + instrument_id: InstrumentId + book_depth: PositiveInt = 10 + log_dir: str = "" + prefill_window_sec: float = 5.0 + prefill_margin_sec: float = 0.25 + large_trade_qty: float = 0.1 + # Log every book delta batch (raw). Do not downsample. + log_every_book_delta: bool = True + + +class EventStateProbe(Strategy): + """Read-only market observability — immutable event ledger only.""" + + def __init__(self, config: EventStateProbeConfig) -> None: + super().__init__(config) + self.instrument: Instrument | None = None + self._ledger = ImmutableEventLedger( + log_dir=config.log_dir or None, + prefill_window_sec=float(config.prefill_window_sec), + prefill_margin_sec=float(config.prefill_margin_sec), + large_trade_qty=float(config.large_trade_qty), + book_levels=int(config.book_depth), + ) + self._health = ConnectionHealth(window_sec=60.0, report_every_sec=30.0) + self._last_mid: float | None = None + self._recent_buys: deque[tuple[float, float]] = deque(maxlen=500) + self._recent_sells: deque[tuple[float, float]] = deque(maxlen=500) + + def on_start(self) -> None: + self.instrument = self.cache.instrument(self.config.instrument_id) + if self.instrument is None: + self.log.error(f"Instrument not found: {self.config.instrument_id}") + self.stop() + return + + self.subscribe_order_book_deltas( + instrument_id=self.config.instrument_id, + book_type=BookType.L2_MBP, + depth=int(self.config.book_depth), + ) + self.subscribe_trade_ticks(self.config.instrument_id) + + exp = self._ledger.experiment + ident = self._ledger.run_identity + self._ledger.write_experiment_start( + extra={ + "instrument_id": str(self.config.instrument_id), + "log_dir": str(self._ledger.log_dir), + "log_every_book_delta": bool(self.config.log_every_book_delta), + "depends_on": "MM_EDGE_EXP_001 / v0.1 FROZEN", + } + ) + self.log.info( + f"EXP_002 Event-State Observability | {exp['experiment_id']} | " + f"{exp['probe_version']} | run={ident['run_id']} session={ident['session_id']} | " + f"trading=NO | log={self._ledger.log_dir}", + LogColor.GREEN, + ) + + def submit_order(self, *args, **kwargs): # noqa: ANN002 + raise RuntimeError( + "MM_EDGE_EXP_002 forbids submit_order — observability probe, trading=NO" + ) + + def submit_order_list(self, *args, **kwargs): # noqa: ANN002 + raise RuntimeError( + "MM_EDGE_EXP_002 forbids submit_order_list — observability probe, trading=NO" + ) + + def on_stop(self) -> None: + try: + self._ledger.write_experiment_stop() + except Exception as exc: + self.log.warning(f"experiment_stop write failed: {exc}") + self.log.info("EventStateProbe stopped (no orders were submitted)") + + def _trade_qty_window(self, window_sec: float = 20.0) -> tuple[float, float]: + now = time.time() + buy = sum(q for t, q in self._recent_buys if now - t <= window_sec) + sell = sum(q for t, q in self._recent_sells if now - t <= window_sec) + return buy, sell + + def _current_snap(self): + book = self.cache.order_book(self.config.instrument_id) + if book is None: + return None + buy, sell = self._trade_qty_window() + snap = snapshot_from_nautilus_book( + book, + levels=int(self.config.book_depth), + recent_buy_qty=buy, + recent_sell_qty=sell, + last_mid=self._last_mid, + ) + if snap.mid: + self._last_mid = snap.mid + return snap + + def on_order_book_deltas(self, deltas: OrderBookDeltas) -> None: + seq = getattr(deltas, "sequence", None) + ts_event = getattr(deltas, "ts_event", None) + self._health.on_book(seq=int(seq) if seq is not None else None, event_ts_ns=ts_event) + + report = self._health.maybe_report() + if report: + self._ledger.write({**report, "event": "phase0_health"}) + + if not self.config.log_every_book_delta: + return + + snap = self._current_snap() + if snap is None or snap.mid <= 0: + return + + delta_count = len(getattr(deltas, "deltas", []) or []) + exchange_ts = int(ts_event) if ts_event is not None else None + self._ledger.log_book_state( + pair=str(self.config.instrument_id), + snap=snap, + exchange_ts_ns=exchange_ts, + local_ts_epoch=time.time(), + sequence=int(seq) if seq is not None else None, + delta_count=delta_count, + event_type="book_update", + ) + + def on_trade_tick(self, tick: TradeTick) -> None: + ts_event = getattr(tick, "ts_event", None) + self._health.on_trade(event_ts_ns=ts_event) + + qty = float(tick.size) + price = float(tick.price) + now = time.time() + trade_side = "unknown" + aggressor = str(getattr(tick, "aggressor_side", "")) + try: + if tick.aggressor_side == AggressorSide.BUYER: + trade_side = "buy" + self._recent_buys.append((now, qty)) + elif tick.aggressor_side == AggressorSide.SELLER: + trade_side = "sell" + self._recent_sells.append((now, qty)) + except Exception: + trade_side = "unknown" + + exchange_ts = int(ts_event) if ts_event is not None else None + trade_id = str(getattr(tick, "trade_id", "") or getattr(tick, "id", "") or "") + self._ledger.log_aggressive_trade( + pair=str(self.config.instrument_id), + price=price, + qty=qty, + trade_side=trade_side, + exchange_ts_ns=exchange_ts, + local_ts_epoch=now, + aggressor_side=aggressor, + trade_id=trade_id or None, + snap=self._current_snap(), + ) diff --git a/src/nautilus_mm/strategies/maker_edge_probe.py b/src/nautilus_mm/strategies/maker_edge_probe.py new file mode 100644 index 0000000..dc9d104 --- /dev/null +++ b/src/nautilus_mm/strategies/maker_edge_probe.py @@ -0,0 +1,430 @@ +""" +MakerEdgeProbe — Nautilus 事件驱动探针（v0） + +实验冻结见 nautilus_mm/FREEZE.md — 三不动： + 1. 不动 Quote Logic（无动态 spread / inv skew / AI / Pulse） + 2. 不动成本模型 + 3. 不动 PASS/COLLECTING/FAIL 定义 + +只记录：quote / fill / outcome + Phase0 健康度。 +market_state_snapshot 必须保持 null，禁止注入交易决策。 +Stage3+ 未解锁前禁止进化为本文件的「聪明报价」。 + +安全闸（非报价进化）：TTL 撤单、健康度 gate、库存上限、fill 必记。 +""" + +from __future__ import annotations + +import time +from collections import deque +from decimal import Decimal +from typing import Optional + +from nautilus_trader.common.enums import LogColor +from nautilus_trader.config import PositiveInt, StrategyConfig +from nautilus_trader.model.data import OrderBookDeltas, TradeTick +from nautilus_trader.model.enums import BookType, OrderSide, TimeInForce +from nautilus_trader.model.events import OrderCanceled, OrderDenied, OrderFilled, OrderRejected +from nautilus_trader.model.identifiers import InstrumentId +from nautilus_trader.model.instruments import Instrument +from nautilus_trader.model.objects import Price, Quantity +from nautilus_trader.trading.strategy import Strategy + +from nautilus_mm.book_utils import snapshot_from_nautilus_book +from nautilus_mm.health import ConnectionHealth, empty_market_state_snapshot +from nautilus_mm.recorder import MakerEdgeLogger + + +class MakerEdgeProbeConfig(StrategyConfig, frozen=True): + instrument_id: InstrumentId + order_qty: Decimal = Decimal("0.001") + book_depth: PositiveInt = 10 + quote_offset_ticks: PositiveInt = 1 + max_quotes: PositiveInt = 1 + quote_ttl_secs: float = 30.0 + cooldown_secs: float = 60.0 + book_sample_secs: float = 2.0 + log_dir: str = "" + # 探针：仅在 OBI 极端时挂一侧（吸收叙事），避免噪音 + obi_enter: float = 0.25 + enable_trading: bool = False # False = 只录盘口不挂单 + # 风险熔断：|inventory| 达上限后只允许减仓方向挂单 + max_abs_inventory: Decimal = Decimal("0.005") + + +class MakerEdgeProbe(Strategy): + def __init__(self, config: MakerEdgeProbeConfig) -> None: + super().__init__(config) + self.instrument: Instrument | None = None + self._logger = MakerEdgeLogger( + log_dir=config.log_dir or None, + levels=int(config.book_depth), + ) + self._last_mid: float | None = None + self._last_book_sample = 0.0 + self._last_quote_ts = 0.0 + self._recent_buys = deque(maxlen=200) + self._recent_sells = deque(maxlen=200) + self._quote_id_by_client: dict[str, str] = {} + self._quote_submit_ts: dict[str, float] = {} + self._fill_id_by_client: dict[str, str] = {} + self._liq_high = 0.0 + self._liq_low = 0.0 + self._health = ConnectionHealth(window_sec=60.0, report_every_sec=30.0) + self._quoting_halted = False + + def on_start(self) -> None: + self.instrument = self.cache.instrument(self.config.instrument_id) + if self.instrument is None: + self.log.error(f"Instrument not found: {self.config.instrument_id}") + self.stop() + return + + # 启动清场：避免上次硬杀残留挂单污染实验 + try: + self.cancel_all_orders(self.config.instrument_id) + self.log.info("Startup cancel_all_orders issued", LogColor.BLUE) + except Exception as exc: + self.log.warning(f"Startup cancel_all failed: {exc}") + + self.subscribe_order_book_deltas( + instrument_id=self.config.instrument_id, + book_type=BookType.L2_MBP, + depth=int(self.config.book_depth), + ) + self.subscribe_trade_ticks(self.config.instrument_id) + exp = self._logger.experiment + self._logger.write_experiment_start( + extra={ + "instrument_id": str(self.config.instrument_id), + "enable_trading": bool(self.config.enable_trading), + "quote_ttl_secs": float(self.config.quote_ttl_secs), + "max_abs_inventory": str(self.config.max_abs_inventory), + "log_dir": str(self._logger.log_dir), + } + ) + self.log.info( + f"Experiment {exp['experiment_id']} | {exp['probe_version']} | " + f"quote/fee/exchange=frozen | log={self._logger.log_dir} | " + f"trading={self.config.enable_trading} ttl={self.config.quote_ttl_secs}s " + f"max_inv={self.config.max_abs_inventory}", + LogColor.GREEN, + ) + self.log.info( + "Research Freeze: Data Collection only — no Pulse / no quote evolution", + LogColor.BLUE, + ) + + def on_stop(self) -> None: + try: + self.cancel_all_orders(self.config.instrument_id) + except Exception as exc: + self.log.warning(f"Stop cancel_all failed: {exc}") + self.log.info("MakerEdgeProbe stopped") + + # ------------------------------------------------------------------ # + def _trade_qty_window(self, window_sec: float = 20.0) -> tuple[float, float]: + now = time.time() + buy = sum(q for t, q in self._recent_buys if now - t <= window_sec) + sell = sum(q for t, q in self._recent_sells if now - t <= window_sec) + return buy, sell + + def _current_snap(self): + book = self.cache.order_book(self.config.instrument_id) + if book is None: + return None + buy, sell = self._trade_qty_window() + snap = snapshot_from_nautilus_book( + book, + levels=int(self.config.book_depth), + recent_buy_qty=buy, + recent_sell_qty=sell, + last_mid=self._last_mid, + liq_low=self._liq_low or None, + liq_high=self._liq_high or None, + ) + if snap.mid: + self._last_mid = snap.mid + self._liq_high = max(self._liq_high or snap.mid, snap.mid) + self._liq_low = min(self._liq_low or snap.mid, snap.mid) if self._liq_low else snap.mid + return snap + + def _inventory(self) -> float: + try: + positions = self.cache.positions_open(instrument_id=self.config.instrument_id) + except TypeError: + positions = [ + p + for p in self.cache.positions_open() + if p.instrument_id == self.config.instrument_id + ] + if not positions: + return 0.0 + inv = 0.0 + for pos in positions: + qty = float(pos.quantity) + inv += -qty if pos.is_short else qty + return inv + + def _open_orders(self): + try: + return list(self.cache.orders_open(instrument_id=self.config.instrument_id)) + except TypeError: + return [ + o + for o in self.cache.orders_open() + if o.instrument_id == self.config.instrument_id + ] + + def _expire_stale_quotes(self, now: float) -> None: + """Cancel GTC quotes older than quote_ttl_secs.""" + ttl = float(self.config.quote_ttl_secs) + if ttl <= 0: + return + for order in self._open_orders(): + cid = order.client_order_id.value + submitted = self._quote_submit_ts.get(cid) + if submitted is None: + # 非本进程跟踪的单（启动残留等）— 一并撤掉 + self.log.warning(f"TTL cancel untracked open order {cid}") + self.cancel_order(order) + continue + if now - submitted >= ttl: + self.log.info(f"TTL cancel {cid} age={now - submitted:.1f}s", LogColor.YELLOW) + self.cancel_order(order) + + def _inventory_allows(self, side: OrderSide, inv: float) -> bool: + max_abs = float(self.config.max_abs_inventory) + if max_abs <= 0: + return True + if abs(inv) < max_abs: + return True + # 超限：只允许减仓方向 + if inv >= max_abs and side == OrderSide.SELL: + return True + if inv <= -max_abs and side == OrderSide.BUY: + return True + return False + + def _release_quote_client(self, cid: str, reason: str, snap=None) -> None: + qid = self._quote_id_by_client.pop(cid, None) + self._quote_submit_ts.pop(cid, None) + if qid is not None: + self._logger.cancel_quote(quote_id=qid, reason=reason, snap=snap) + + def on_order_book_deltas(self, deltas: OrderBookDeltas) -> None: + now = time.time() + # Phase 0 health + seq = getattr(deltas, "sequence", None) + ts_event = getattr(deltas, "ts_event", None) + self._health.on_book(seq=int(seq) if seq is not None else None, event_ts_ns=ts_event) + report = self._health.maybe_report() + if report: + self._logger.write(report) + gap_w = report.get("sequence_gap_window", 0) + color = LogColor.RED if gap_w or not report.get("healthy") else LogColor.CYAN + self.log.info( + f"Phase0 book/s={report['book_update_rate']:.1f} " + f"trade/s={report['trade_update_rate']:.1f} " + f"lat_p50/p99/max={report['latency_ms_p50']}/" + f"{report['latency_ms_p99']}/{report['latency_ms_max']} " + f"gap_win={gap_w} book_age_ms={report.get('book_age_ms')}", + color, + ) + + # TTL 撤单：与报价逻辑无关的生命周期闭环 + self._expire_stale_quotes(now) + + snap = self._current_snap() + if snap is None or snap.mid <= 0: + return + + if now - self._last_book_sample >= float(self.config.book_sample_secs): + self._last_book_sample = now + self._logger.record_book( + snap, + now=now, + emit_mid_tick=True, + pair=str(self.config.instrument_id), + ) + inv_fields = self._logger.update_inventory(self._inventory(), now=now) + self._logger.write( + { + "event": "inventory_tick", + "pair": str(self.config.instrument_id), + **inv_fields, + "market_state_snapshot": empty_market_state_snapshot(), + **snap.to_book_fields(), + } + ) + + # 推进 fill path + self._logger.update_paths(str(self.config.instrument_id), snap.mid, now=now) + + if not self.config.enable_trading: + return + if not self._health.allow_quoting(): + if not self._quoting_halted: + self._quoting_halted = True + self.log.warning("Quoting halted: health gate (stale book / low update rate)") + return + if self._quoting_halted: + self._quoting_halted = False + self.log.info("Quoting resumed: health OK", LogColor.GREEN) + + if now - self._last_quote_ts < float(self.config.cooldown_secs): + return + if len(self._open_orders()) >= int(self.config.max_quotes): + return + + if self.instrument is None: + return + + # 简单吸收探针：OBI 极端 → 挂被动单 + tick = float(self.instrument.price_increment) + offset = int(self.config.quote_offset_ticks) * tick + qty = self.instrument.make_qty(self.config.order_qty) + inv = self._inventory() + + if snap.obi >= float(self.config.obi_enter): + side = OrderSide.SELL + if not self._inventory_allows(side, inv): + return + price = self.instrument.make_price(snap.best_ask + offset) + self._submit_quote(side, price, qty, snap, reason="obi_bid_thick") + elif snap.obi <= -float(self.config.obi_enter): + side = OrderSide.BUY + if not self._inventory_allows(side, inv): + return + price = self.instrument.make_price(snap.best_bid - offset) + self._submit_quote(side, price, qty, snap, reason="obi_ask_thick") + + def on_trade_tick(self, tick: TradeTick) -> None: + now = time.time() + self._health.on_trade(event_ts_ns=getattr(tick, "ts_event", None)) + qty = float(tick.size) + # Aggressor side + try: + from nautilus_trader.model.enums import AggressorSide + + if tick.aggressor_side == AggressorSide.BUYER: + self._recent_buys.append((now, qty)) + elif tick.aggressor_side == AggressorSide.SELLER: + self._recent_sells.append((now, qty)) + except Exception: + self._recent_buys.append((now, qty * 0.5)) + self._recent_sells.append((now, qty * 0.5)) + + snap_mid = self._last_mid or float(tick.price) + self._logger.update_paths(str(self.config.instrument_id), snap_mid, now=now) + + def _submit_quote(self, side: OrderSide, price: Price, qty: Quantity, snap, reason: str) -> None: + assert self.instrument is not None + order = self.order_factory.limit( + instrument_id=self.config.instrument_id, + order_side=side, + quantity=qty, + price=price, + time_in_force=TimeInForce.GTC, + post_only=True, + ) + qside = "bid" if side == OrderSide.BUY else "ask" + cid = order.client_order_id.value + # 先 submit，成功后再记 quote（避免幽灵 quote_created） + try: + self.submit_order(order) + except Exception as exc: + self.log.error(f"submit_order failed: {exc}") + return + + qid = self._logger.create_quote( + pair=str(self.config.instrument_id), + side=qside, + quote_price=float(price), + inventory=self._inventory(), + snap=snap, + reason=reason, + state=empty_market_state_snapshot(), # 故意不接 Market Pulse + extra={"book_age_ms": self._health.book_age_ms()}, + ) + self._quote_id_by_client[cid] = qid + self._quote_submit_ts[cid] = time.time() + self._last_quote_ts = time.time() + self.log.info(f"QUOTE {qside} {price} qty={qty} reason={reason}", LogColor.BLUE) + + def on_order_canceled(self, event: OrderCanceled) -> None: + cid = event.client_order_id.value + snap = self._current_snap() + self._release_quote_client(cid, reason="canceled", snap=snap) + + def on_order_rejected(self, event: OrderRejected) -> None: + cid = event.client_order_id.value + reason = getattr(event, "reason", None) or "rejected" + self.log.warning(f"OrderRejected {cid}: {reason}") + snap = self._current_snap() + self._release_quote_client(cid, reason=f"rejected:{reason}", snap=snap) + + def on_order_denied(self, event: OrderDenied) -> None: + cid = event.client_order_id.value + reason = getattr(event, "reason", None) or "denied" + self.log.warning(f"OrderDenied {cid}: {reason}") + snap = self._current_snap() + self._release_quote_client(cid, reason=f"denied:{reason}", snap=snap) + + def on_order_filled(self, event: OrderFilled) -> None: + cid = event.client_order_id.value + qid = self._quote_id_by_client.get(cid) + snap = self._current_snap() + # snap 缺失仍必须记 fill（book 字段可空） + side = "long" if event.order_side == OrderSide.BUY else "short" + det = self._logger.book_deterioration(side) + fill_reason = "toxic_passive" if det.get("pre_5s_deteriorated") else "maker_hit" + + order = self.cache.order(event.client_order_id) + terminal = True + if order is not None: + terminal = bool(order.is_closed) or float(order.leaves_qty) <= 0 + + fill_id = self._logger.log_fill( + pair=str(self.config.instrument_id), + side=side, + fill_price=float(event.last_px), + amount=float(event.last_qty), + inventory=self._inventory(), + snap=snap, + order_type="limit", + quote_id=qid, + fill_reason=fill_reason, + state=empty_market_state_snapshot(), + quote_terminal=terminal, + extra={ + "client_order_id": cid, + "venue_order_id": str(event.venue_order_id), + "trade_id": str(event.trade_id), + "venue_trade_id": str(event.trade_id), + "exchange_ts_ns": int(event.ts_event) if getattr(event, "ts_event", None) else None, + "local_ts": time.time(), + "book_age_ms": self._health.book_age_ms(), + "leaves_qty": float(order.leaves_qty) if order is not None else None, + # Maker-only hard evidence (do not trust post_only param alone) + "liquidity_side": str(event.liquidity_side), + "is_maker": event.liquidity_side.name == "MAKER" + if hasattr(event.liquidity_side, "name") + else str(event.liquidity_side) == "MAKER", + "commission": float(event.commission) if event.commission is not None else None, + "commission_currency": ( + str(event.commission.currency) if event.commission is not None else None + ), + "post_only": True, + "execution_type": "TRADE", + }, + ) + self._fill_id_by_client[cid] = fill_id + if terminal: + self._quote_id_by_client.pop(cid, None) + self._quote_submit_ts.pop(cid, None) + self.log.info( + f"FILL {side} px={event.last_px} qty={event.last_qty} " + f"reason={fill_reason} terminal={terminal} book={'ok' if snap else 'none'}", + LogColor.YELLOW, + )