Snapshot nautilus_mm after EXP_002 Phase 1 stop.
Keep frozen research conclusions and code; raw ledgers and secrets stay out of git. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
jackyu66git
@@ -0,0 +1,402 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Account Ledger Reconciliation — MM_EDGE_EXP_001
|
||||
|
||||
Separates:
|
||||
MakerAlpha (research markout) ≠ Account Equity (wallet economics)
|
||||
|
||||
Pulls paginated Binance Futures:
|
||||
- /fapi/v1/userTrades (maker flag, commission per fill)
|
||||
- /fapi/v1/income (REALIZED_PNL, COMMISSION, FUNDING_FEE, …)
|
||||
- /fapi/v2/account (wallet + unrealized + position)
|
||||
|
||||
Hard gate:
|
||||
TAKER_FILLED_COUNT == 0 else Maker-only = INVALID
|
||||
|
||||
Equity identity (target error ≈ 0):
|
||||
StartWallet + Σincome_types + (EndUnrealized − StartUnrealized*)
|
||||
+ Transfers/Adjustments ≈ EndMarginBalance
|
||||
|
||||
* StartUnrealized often unknown → report EndUnrealized separately.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
_ROOT = Path(__file__).resolve().parents[1]
|
||||
load_dotenv(_ROOT / ".env")
|
||||
|
||||
|
||||
def _env(name: str, default: str = "") -> str:
|
||||
return os.getenv(name, default).strip()
|
||||
|
||||
|
||||
def _base_url() -> str:
|
||||
env = _env("BINANCE_ENVIRONMENT", "TESTNET").upper()
|
||||
if env == "TESTNET":
|
||||
return "https://testnet.binancefuture.com"
|
||||
if env == "LIVE":
|
||||
return "https://fapi.binance.com"
|
||||
raise SystemExit(f"BINANCE_ENVIRONMENT must be TESTNET|LIVE, got {env!r}")
|
||||
|
||||
|
||||
def _signed_get(path: str, params: dict | None = None) -> object:
|
||||
key = _env("BINANCE_API_KEY")
|
||||
sec = _env("BINANCE_API_SECRET")
|
||||
if not key or not sec:
|
||||
raise SystemExit("BINANCE_API_KEY / BINANCE_API_SECRET required")
|
||||
params = dict(params or {})
|
||||
params["timestamp"] = int(time.time() * 1000)
|
||||
params["recvWindow"] = 60_000
|
||||
qs = urllib.parse.urlencode(params)
|
||||
sig = hmac.new(sec.encode(), qs.encode(), hashlib.sha256).hexdigest()
|
||||
url = f"{_base_url()}{path}?{qs}&signature={sig}"
|
||||
req = urllib.request.Request(url, headers={"X-MBX-APIKEY": key})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
return json.loads(r.read().decode())
|
||||
except urllib.error.HTTPError as e:
|
||||
body = e.read().decode(errors="replace")
|
||||
raise RuntimeError(f"HTTP {e.code} {path} params={params} body={body}") from e
|
||||
|
||||
|
||||
def _fetch_user_trades(symbol: str, start_ms: int, end_ms: int) -> list[dict]:
|
||||
"""Paginate userTrades by time windows (dedupe by trade id).
|
||||
|
||||
Note: Testnet userTrades can stop returning rows after ~7d of dense history
|
||||
even while orders/income continue — RECON-02 must flag that gap separately.
|
||||
"""
|
||||
out: dict[int, dict] = {}
|
||||
cursor = start_ms
|
||||
safety = 0
|
||||
while cursor < end_ms and safety < 2000:
|
||||
safety += 1
|
||||
chunk_end = min(cursor + 7 * 86400_000 - 1, end_ms)
|
||||
batch = _signed_get(
|
||||
"/fapi/v1/userTrades",
|
||||
{
|
||||
"symbol": symbol,
|
||||
"startTime": cursor,
|
||||
"endTime": chunk_end,
|
||||
"limit": 1000,
|
||||
},
|
||||
)
|
||||
assert isinstance(batch, list)
|
||||
if not batch:
|
||||
cursor = chunk_end + 1
|
||||
continue
|
||||
for t in batch:
|
||||
out[int(t["id"])] = t
|
||||
last_t = int(batch[-1]["time"])
|
||||
if len(batch) < 1000:
|
||||
cursor = max(last_t + 1, chunk_end + 1)
|
||||
else:
|
||||
nxt = last_t + 1
|
||||
if nxt <= cursor:
|
||||
nxt = cursor + 1
|
||||
cursor = nxt
|
||||
time.sleep(0.08)
|
||||
return sorted(out.values(), key=lambda x: (int(x["time"]), int(x["id"])))
|
||||
|
||||
|
||||
def _fetch_income(start_ms: int, end_ms: int) -> list[dict]:
|
||||
"""Paginate income by time only."""
|
||||
out: list[dict] = []
|
||||
seen: set[tuple] = set()
|
||||
cursor = start_ms
|
||||
safety = 0
|
||||
while cursor < end_ms and safety < 2000:
|
||||
safety += 1
|
||||
chunk_end = min(cursor + 7 * 86400_000 - 1, end_ms)
|
||||
batch = _signed_get(
|
||||
"/fapi/v1/income",
|
||||
{"startTime": cursor, "endTime": chunk_end, "limit": 1000},
|
||||
)
|
||||
assert isinstance(batch, list)
|
||||
if not batch:
|
||||
cursor = chunk_end + 1
|
||||
continue
|
||||
for row in batch:
|
||||
key = (
|
||||
row.get("tranId"),
|
||||
row.get("time"),
|
||||
row.get("incomeType"),
|
||||
row.get("income"),
|
||||
row.get("asset"),
|
||||
row.get("symbol"),
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(row)
|
||||
last_t = int(batch[-1]["time"])
|
||||
if len(batch) < 1000:
|
||||
cursor = max(last_t + 1, chunk_end + 1)
|
||||
else:
|
||||
cursor = last_t + 1
|
||||
time.sleep(0.08)
|
||||
return out
|
||||
|
||||
|
||||
def _ms_iso(ms: int) -> str:
|
||||
return datetime.fromtimestamp(ms / 1000, tz=timezone.utc).isoformat()
|
||||
|
||||
|
||||
def load_jsonl_fill_count(log_dir: Path) -> int:
|
||||
n = 0
|
||||
if not log_dir.exists():
|
||||
return 0
|
||||
for f in sorted(log_dir.glob("*.jsonl")):
|
||||
for line in f.open():
|
||||
try:
|
||||
e = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(e, dict) and e.get("event") == "fill":
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="Maker Edge account reconciliation")
|
||||
ap.add_argument(
|
||||
"--start-wallet",
|
||||
type=float,
|
||||
default=float(_env("RECON_START_WALLET", "5000")),
|
||||
help="Observed starting USDT wallet (default 5000 testnet grant)",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--symbol",
|
||||
default=_env("RECON_SYMBOL", "BTCUSDT"),
|
||||
help="Futures symbol for userTrades (default BTCUSDT)",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--since-days",
|
||||
type=float,
|
||||
default=float(_env("RECON_SINCE_DAYS", "14")),
|
||||
)
|
||||
ap.add_argument(
|
||||
"--out",
|
||||
default=str(_ROOT / "logs" / "maker_edge" / "Account_Reconciliation.txt"),
|
||||
)
|
||||
args = ap.parse_args()
|
||||
|
||||
end_ms = int(time.time() * 1000)
|
||||
start_ms = end_ms - int(args.since_days * 86400 * 1000)
|
||||
|
||||
print(f"[recon] env={_env('BINANCE_ENVIRONMENT','TESTNET')} base={_base_url()}")
|
||||
print(f"[recon] window {_ms_iso(start_ms)} → {_ms_iso(end_ms)}")
|
||||
|
||||
print("[recon] pulling userTrades (paginated)…")
|
||||
trades = _fetch_user_trades(args.symbol, start_ms, end_ms)
|
||||
|
||||
print(f"[recon] userTrades={len(trades)}")
|
||||
print("[recon] pulling income (paginated)…")
|
||||
income = _fetch_income(start_ms, end_ms)
|
||||
print(f"[recon] income rows={len(income)}")
|
||||
|
||||
acct = _signed_get("/fapi/v2/account")
|
||||
assert isinstance(acct, dict)
|
||||
|
||||
# --- Maker-only hard check ---
|
||||
maker_n = sum(1 for t in trades if t.get("maker") is True)
|
||||
taker_n = sum(1 for t in trades if t.get("maker") is False)
|
||||
unknown_n = len(trades) - maker_n - taker_n
|
||||
maker_only_ok = taker_n == 0 and unknown_n == 0 and len(trades) > 0
|
||||
maker_only_status = "PASS" if maker_only_ok else ("INVALID" if taker_n > 0 else "NEED VERIFY")
|
||||
|
||||
fee_by_asset: dict[str, float] = defaultdict(float)
|
||||
notional = 0.0
|
||||
buy_qty = sell_qty = 0.0
|
||||
for t in trades:
|
||||
fee_by_asset[t.get("commissionAsset") or "?"] += float(t.get("commission") or 0)
|
||||
q = float(t.get("qty") or 0)
|
||||
px = float(t.get("price") or 0)
|
||||
notional += abs(q * px)
|
||||
if t.get("buyer"):
|
||||
buy_qty += q
|
||||
else:
|
||||
sell_qty += q
|
||||
net_qty = buy_qty - sell_qty
|
||||
|
||||
income_by: dict[str, float] = defaultdict(float)
|
||||
for row in income:
|
||||
income_by[str(row.get("incomeType"))] += float(row.get("income") or 0)
|
||||
|
||||
wallet = float(acct.get("totalWalletBalance") or 0)
|
||||
upnl = float(acct.get("totalUnrealizedProfit") or 0)
|
||||
margin = float(acct.get("totalMarginBalance") or 0)
|
||||
avail = float(acct.get("availableBalance") or 0)
|
||||
|
||||
positions = []
|
||||
for p in acct.get("positions") or []:
|
||||
amt = float(p.get("positionAmt") or 0)
|
||||
if abs(amt) > 1e-12:
|
||||
positions.append(
|
||||
{
|
||||
"symbol": p.get("symbol"),
|
||||
"amt": amt,
|
||||
"entry": float(p.get("entryPrice") or 0),
|
||||
"unrealized": float(p.get("unrealizedProfit") or 0),
|
||||
}
|
||||
)
|
||||
|
||||
start_wallet = float(args.start_wallet)
|
||||
income_sum = sum(income_by.values())
|
||||
# Identity without known start upnl:
|
||||
# EndWallet ≈ StartWallet + Σ income (transfers included in income types if any)
|
||||
implied_end_wallet = start_wallet + income_sum
|
||||
wallet_gap = wallet - implied_end_wallet
|
||||
equity_now = margin # wallet + upnl
|
||||
equity_vs_start = equity_now - start_wallet
|
||||
|
||||
jsonl_fills = load_jsonl_fill_count(_ROOT / "logs" / "maker_edge")
|
||||
|
||||
lines: list[str] = []
|
||||
def p(s: str = "") -> None:
|
||||
lines.append(s)
|
||||
print(s)
|
||||
|
||||
p("=" * 72)
|
||||
p("Account Reconciliation — MM_EDGE_EXP_001 / probe_v0.1")
|
||||
p("Research markout (MakerAlpha) ≠ Account equity")
|
||||
p("=" * 72)
|
||||
p()
|
||||
p("Status Snapshot")
|
||||
p("-" * 40)
|
||||
p("Maker Phenomenon PARTIAL_PASS")
|
||||
p("Data Integrity PASS (from Maker Edge Report)")
|
||||
p(f"Maker-only constraint {maker_only_status}")
|
||||
p("Account Reconciliation NOT COMPLETE" if abs(wallet_gap) > 0.5 else "Account Reconciliation CLOSE")
|
||||
p("Economic Edge UNKNOWN")
|
||||
p("Stage 3 LOCKED")
|
||||
p("Probe STOPPED (no further volume until ledger closes)")
|
||||
p()
|
||||
|
||||
p("Section A — Maker-only hard check (exchange userTrades)")
|
||||
p("-" * 40)
|
||||
p(f"Symbol: {args.symbol}")
|
||||
p(f"Exchange trades: {len(trades)}")
|
||||
p(f"Jsonl fills (local): {jsonl_fills}")
|
||||
p(f"MAKER fills: {maker_n}")
|
||||
p(f"TAKER fills: {taker_n}")
|
||||
p(f"Unknown liquidity: {unknown_n}")
|
||||
p(f"TAKER_FILLED_COUNT: {taker_n}")
|
||||
if taker_n > 0:
|
||||
p("→ INVALID: sample contaminated by taker fills")
|
||||
elif maker_only_ok:
|
||||
p("→ PASS: all exchange trades marked maker=true")
|
||||
else:
|
||||
p("→ NEED VERIFY")
|
||||
p(f"Buy qty / Sell qty: {buy_qty:.6f} / {sell_qty:.6f}")
|
||||
p(f"Net inventory (qty): {net_qty:.6f}")
|
||||
p(f"Gross notional: {notional:.4f} USDT")
|
||||
for asset, fee in sorted(fee_by_asset.items()):
|
||||
p(f"Commission ({asset}): {fee}")
|
||||
p()
|
||||
|
||||
p("Section B — Income ledger (paginated, full window)")
|
||||
p("-" * 40)
|
||||
for k, v in sorted(income_by.items(), key=lambda kv: -abs(kv[1])):
|
||||
p(f" {k:24s} {v:+.8f}")
|
||||
p(f" {'Σ income':24s} {income_sum:+.8f}")
|
||||
p()
|
||||
|
||||
p("Section C — Account snapshot (now)")
|
||||
p("-" * 40)
|
||||
p(f"totalWalletBalance: {wallet:.8f}")
|
||||
p(f"totalUnrealizedProfit: {upnl:.8f}")
|
||||
p(f"totalMarginBalance: {margin:.8f} ← equity")
|
||||
p(f"availableBalance: {avail:.8f}")
|
||||
if positions:
|
||||
p("Open positions:")
|
||||
for pos in positions:
|
||||
p(
|
||||
f" {pos['symbol']} amt={pos['amt']} entry={pos['entry']} "
|
||||
f"upnl={pos['unrealized']}"
|
||||
)
|
||||
else:
|
||||
p("Open positions: (none)")
|
||||
p()
|
||||
|
||||
p("Section D — Equity bridge (attempt)")
|
||||
p("-" * 40)
|
||||
p(f"Start wallet (assumed): {start_wallet:.8f}")
|
||||
p(f"+ Σ income: {income_sum:+.8f}")
|
||||
p(f"= Implied end wallet: {implied_end_wallet:.8f}")
|
||||
p(f"Actual end wallet: {wallet:.8f}")
|
||||
p(f"Wallet residual gap: {wallet_gap:+.8f}")
|
||||
p(f"End unrealized: {upnl:+.8f}")
|
||||
p(f"End equity: {equity_now:.8f}")
|
||||
p(f"Equity − start wallet: {equity_vs_start:+.8f}")
|
||||
p()
|
||||
p("Interpretation:")
|
||||
p(" - Do NOT equate EquityΔ with MakerAlpha failure/success.")
|
||||
p(" - Residual gap means incomplete history, wrong start, or missing")
|
||||
p(" transfer/adjustment types — Account Reconciliation stays open.")
|
||||
p(" - Inventory drift (net qty / open position) can dominate economics")
|
||||
p(" even when per-fill markout is slightly positive.")
|
||||
p()
|
||||
|
||||
p("Section E — Next required chain")
|
||||
p("-" * 40)
|
||||
p("QuoteIntent → Submitted → Accepted → Filled")
|
||||
p(" → fill_px/qty → liquidity=MAKER → fee")
|
||||
p(" → position Δ → realized → funding → equity")
|
||||
p("Daily: StartEquity + TradingPnL + Fees + Funding + uPnL + Transfers = EndEquity")
|
||||
p("Target residual ≈ 0 before any Stage3 unlock / further volume.")
|
||||
p("=" * 72)
|
||||
|
||||
out = Path(args.out)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
# machine-readable sidecar
|
||||
sidecar = out.with_suffix(".json")
|
||||
sidecar.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"experiment_id": "MM_EDGE_EXP_001",
|
||||
"maker_only_status": maker_only_status,
|
||||
"taker_filled_count": taker_n,
|
||||
"maker_filled_count": maker_n,
|
||||
"exchange_trades": len(trades),
|
||||
"jsonl_fills": jsonl_fills,
|
||||
"income_by_type": dict(income_by),
|
||||
"income_sum": income_sum,
|
||||
"start_wallet_assumed": start_wallet,
|
||||
"end_wallet": wallet,
|
||||
"end_unrealized": upnl,
|
||||
"end_equity": equity_now,
|
||||
"wallet_residual_gap": wallet_gap,
|
||||
"net_qty": net_qty,
|
||||
"fee_by_asset": dict(fee_by_asset),
|
||||
"positions": positions,
|
||||
"probe": "STOPPED",
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"[recon] saved {out}")
|
||||
print(f"[recon] saved {sidecar}")
|
||||
return 0 if maker_only_ok or taker_n == 0 else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user