360 lines
12 KiB
Python
360 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Generate HMM regime chart as single-file HTML with pure Canvas rendering.
|
|
No external dependencies — works everywhere.
|
|
"""
|
|
import json, sys, os, glob
|
|
import numpy as np
|
|
import pandas as pd
|
|
from sklearn.preprocessing import StandardScaler
|
|
from hmmlearn.hmm import GaussianHMM
|
|
import warnings
|
|
warnings.filterwarnings("ignore")
|
|
|
|
SEED = 42
|
|
N_REGIMES = 3
|
|
STEP = 5
|
|
|
|
def load_snapshots(filepath):
|
|
snaps = []
|
|
with open(filepath) as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
snaps.append(json.loads(line))
|
|
return snaps
|
|
|
|
def extract_features(snapshots):
|
|
records = []
|
|
for s in snapshots:
|
|
ts = s["_collect_ts"]
|
|
bids = sorted(s["bids"], key=lambda x: x[0], reverse=True)
|
|
asks = sorted(s["asks"], key=lambda x: x[0])
|
|
bb = bids[0][0] if bids else 0
|
|
ba = asks[0][0] if asks else 0
|
|
mid = (bb + ba) / 2 if bb and ba else 0
|
|
spread = ba - bb if bb and ba else 0
|
|
bd = sum(b[1] for b in bids[:10])
|
|
ad = sum(a[1] for a in asks[:10])
|
|
depth = bd + ad
|
|
bv = sum(b[0] * b[1] for b in bids[:10])
|
|
av = sum(a[0] * a[1] for a in asks[:10])
|
|
imbalance = (bv - av) / (bv + av + 1e-12)
|
|
records.append({"ts": ts, "mid": mid, "spread": spread, "depth": depth, "imbalance": imbalance})
|
|
return pd.DataFrame(records)
|
|
|
|
def engineer_features(df, win=20):
|
|
s, d, imb = df["spread"].values, df["depth"].values, df["imbalance"].values
|
|
rv = pd.Series(df["mid"]).pct_change().rolling(win, min_periods=1).std().fillna(0).values
|
|
ofi = pd.Series(df["mid"]).diff().rolling(win, min_periods=1).mean().fillna(0).values
|
|
return np.column_stack([s, d, imb, rv, ofi])
|
|
|
|
def fit_hmm(X):
|
|
best_score, best_model = -np.inf, None
|
|
for k in range(12):
|
|
m = GaussianHMM(n_components=N_REGIMES, covariance_type="full",
|
|
n_iter=400, tol=1e-7, random_state=SEED + k,
|
|
init_params="stmc", params="stmc")
|
|
try:
|
|
m.fit(X)
|
|
sc = m.score(X)
|
|
if sc > best_score:
|
|
best_score, best_model = sc, m
|
|
except Exception:
|
|
continue
|
|
if best_model is None:
|
|
raise RuntimeError("HMM fitting failed.")
|
|
return best_model
|
|
|
|
HTML = """<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<!-- Google tag (gtag.js) -->
|
|
<script async src="https://www.googletagmanager.com/gtag/js?id=G-LVVXH3TL04"></script>
|
|
<script>
|
|
window.dataLayer = window.dataLayer || [];
|
|
function gtag(){dataLayer.push(arguments);}
|
|
gtag('js', new Date());
|
|
gtag('config', 'G-LVVXH3TL04');
|
|
</script>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width,initial-scale=1.0">
|
|
<title>LINK HMM Regime — Canvas Chart</title>
|
|
<style>
|
|
*{margin:0;padding:0;box-sizing:border-box}
|
|
body{background:#0d1117;color:#c9d1d9;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,sans-serif;overflow:hidden}
|
|
#topbar{background:#161b22;padding:10px 20px;display:flex;justify-content:space-between;align-items:center;border-bottom:1px solid #30363d;flex-wrap:wrap;gap:8px}
|
|
#topbar h1{font-size:15px;font-weight:600;color:#f0f6fc}
|
|
.legend{display:flex;gap:14px;font-size:12px;flex-wrap:wrap}
|
|
.legend span{display:flex;align-items:center;gap:5px}
|
|
.legend i{display:inline-block;width:10px;height:10px;border-radius:2px}
|
|
#chart{width:100%;height:calc(100vh - 46px);display:block}
|
|
#loading{position:fixed;top:50%;left:50%;transform:translate(-50%,-50%);color:#8b949e;font-size:14px;z-index:10}
|
|
#tooltip{position:fixed;background:#21262d;color:#c9d1d9;padding:6px 10px;border-radius:6px;font-size:12px;border:1px solid #30363d;pointer-events:none;display:none;white-space:nowrap;z-index:20}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div id="topbar">
|
|
<h1>LINK/USD — HMM Regime Detection (12h)</h1>
|
|
<div class="legend">
|
|
<span><i style="background:rgba(46,204,113,0.4);border:1px solid #2ecc71"></i> Low Spread</span>
|
|
<span><i style="background:rgba(241,196,15,0.3);border:1px solid #f1c40f"></i> Mid Spread</span>
|
|
<span><i style="background:rgba(231,76,60,0.3);border:1px solid #e74c3c"></i> High Spread</span>
|
|
<span style="color:#8b949e">▽ transition</span>
|
|
</div>
|
|
</div>
|
|
<canvas id="chart"></canvas>
|
|
<div id="tooltip"></div>
|
|
<div id="loading">Loading data...</div>
|
|
<script>
|
|
(function(){
|
|
var canvas = document.getElementById('chart');
|
|
var ctx = canvas.getContext('2d');
|
|
var tooltip = document.getElementById('tooltip');
|
|
|
|
// Colors
|
|
var REGIME_COLORS = ['rgba(231,76,60,0.35)', 'rgba(241,196,15,0.2)', 'rgba(46,204,113,0.25)'];
|
|
var REGIME_LINES = ['#e74c3c', '#f1c40f', '#2ecc71'];
|
|
var PRICE_COLOR = '#58a6ff';
|
|
var GRID_COLOR = '#21262d';
|
|
var AXIS_COLOR = '#8b949e';
|
|
var BG = '#0d1117';
|
|
var MARGIN = {top:20, right:70, bottom:30, left:10};
|
|
|
|
var DATA = null;
|
|
var PADDING = 60; // px padding for price axis
|
|
|
|
function init() {
|
|
canvas.width = window.innerWidth;
|
|
canvas.height = window.innerHeight - 46;
|
|
draw();
|
|
}
|
|
|
|
function draw() {
|
|
if (!DATA) return;
|
|
var W = canvas.width, H = canvas.height;
|
|
var pw = W - MARGIN.left - MARGIN.right;
|
|
var ph = H - MARGIN.top - MARGIN.bottom;
|
|
|
|
ctx.clearRect(0, 0, W, H);
|
|
ctx.fillStyle = BG;
|
|
ctx.fillRect(0, 0, W, H);
|
|
|
|
// Find price range
|
|
var pMin = Infinity, pMax = -Infinity;
|
|
for (var i = 0; i < DATA.length; i++) {
|
|
var v = DATA[i].v;
|
|
if (v < pMin) pMin = v;
|
|
if (v > pMax) pMax = v;
|
|
}
|
|
var pRange = pMax - pMin;
|
|
pMin -= pRange * 0.05;
|
|
pMax += pRange * 0.05;
|
|
pRange = pMax - pMin;
|
|
|
|
function toX(i) { return MARGIN.left + (i / (DATA.length - 1)) * pw; }
|
|
function toY(v) { return MARGIN.top + (1 - (v - pMin) / pRange) * ph; }
|
|
|
|
// Grid lines
|
|
ctx.strokeStyle = GRID_COLOR;
|
|
ctx.lineWidth = 0.5;
|
|
var nY = 8;
|
|
for (var gy = 0; gy <= nY; gy++) {
|
|
var y = MARGIN.top + (gy / nY) * ph;
|
|
var price = pMax - (gy / nY) * pRange;
|
|
ctx.beginPath();
|
|
ctx.moveTo(MARGIN.left, y);
|
|
ctx.lineTo(MARGIN.left + pw, y);
|
|
ctx.stroke();
|
|
|
|
// Y-axis labels
|
|
ctx.fillStyle = AXIS_COLOR;
|
|
ctx.font = '11px -apple-system,BlinkMacSystemFont,sans-serif';
|
|
ctx.textAlign = 'right';
|
|
ctx.fillText(price.toFixed(4), W - 5, y + 4);
|
|
}
|
|
|
|
// X-axis time labels
|
|
var nX = 6;
|
|
ctx.textAlign = 'center';
|
|
ctx.fillStyle = AXIS_COLOR;
|
|
ctx.font = '10px -apple-system,BlinkMacSystemFont,sans-serif';
|
|
for (var gx = 0; gx <= nX; gx++) {
|
|
var idx = Math.floor((gx / nX) * (DATA.length - 1));
|
|
var x = toX(idx);
|
|
var d = new Date(DATA[idx].t * 1000);
|
|
var label = d.toISOString().substring(11, 19); // HH:MM:SS
|
|
ctx.fillText(label, x, H - MARGIN.bottom + 16);
|
|
|
|
// Vertical grid
|
|
ctx.strokeStyle = GRID_COLOR;
|
|
ctx.lineWidth = 0.5;
|
|
ctx.beginPath();
|
|
ctx.moveTo(x, MARGIN.top);
|
|
ctx.lineTo(x, MARGIN.top + ph);
|
|
ctx.stroke();
|
|
}
|
|
|
|
// Regime bands (draw as vertical strips)
|
|
var prevRegime = -1;
|
|
var bandStart = 0;
|
|
for (var i = 0; i < DATA.length; i++) {
|
|
var r = DATA[i].r;
|
|
if (r !== prevRegime) {
|
|
if (prevRegime >= 0) {
|
|
ctx.fillStyle = REGIME_COLORS[prevRegime];
|
|
ctx.fillRect(toX(bandStart), MARGIN.top, toX(i) - toX(bandStart), ph);
|
|
}
|
|
bandStart = i;
|
|
prevRegime = r;
|
|
}
|
|
}
|
|
// Last band
|
|
if (prevRegime >= 0) {
|
|
ctx.fillStyle = REGIME_COLORS[prevRegime];
|
|
ctx.fillRect(toX(bandStart), MARGIN.top, toX(DATA.length - 1) - toX(bandStart) + 1, ph);
|
|
}
|
|
|
|
// Regime transition markers (thin vertical lines)
|
|
prevRegime = DATA[0].r;
|
|
for (var i = 1; i < DATA.length; i++) {
|
|
if (DATA[i].r !== prevRegime) {
|
|
var mx = toX(i);
|
|
ctx.strokeStyle = REGIME_LINES[DATA[i].r];
|
|
ctx.lineWidth = 1;
|
|
ctx.setLineDash([3, 4]);
|
|
ctx.beginPath();
|
|
ctx.moveTo(mx, MARGIN.top);
|
|
ctx.lineTo(mx, MARGIN.top + ph);
|
|
ctx.stroke();
|
|
ctx.setLineDash([]);
|
|
|
|
// Small arrow marker at bottom
|
|
ctx.fillStyle = REGIME_LINES[DATA[i].r];
|
|
ctx.beginPath();
|
|
ctx.moveTo(mx, MARGIN.top + ph + 2);
|
|
ctx.lineTo(mx - 4, MARGIN.top + ph - 4);
|
|
ctx.lineTo(mx + 4, MARGIN.top + ph - 4);
|
|
ctx.closePath();
|
|
ctx.fill();
|
|
|
|
prevRegime = DATA[i].r;
|
|
}
|
|
}
|
|
|
|
// Price line
|
|
ctx.strokeStyle = PRICE_COLOR;
|
|
ctx.lineWidth = 1.5;
|
|
ctx.beginPath();
|
|
for (var i = 0; i < DATA.length; i++) {
|
|
var px = toX(i), py = toY(DATA[i].v);
|
|
if (i === 0) ctx.moveTo(px, py);
|
|
else ctx.lineTo(px, py);
|
|
}
|
|
ctx.stroke();
|
|
|
|
// Current price label
|
|
var lastV = DATA[DATA.length-1].v;
|
|
var lastY = toY(lastV);
|
|
ctx.fillStyle = PRICE_COLOR;
|
|
ctx.font = 'bold 12px -apple-system,BlinkMacSystemFont,sans-serif';
|
|
ctx.textAlign = 'left';
|
|
ctx.fillText(lastV.toFixed(4), MARGIN.left + pw + 4, lastY + 4);
|
|
}
|
|
|
|
// Mouse/touch tooltip
|
|
canvas.addEventListener('mousemove', function(e) {
|
|
if (!DATA) return;
|
|
var rect = canvas.getBoundingClientRect();
|
|
var mx = e.clientX - rect.left;
|
|
var my = e.clientY - rect.top;
|
|
var W = canvas.width, H = canvas.height;
|
|
var pw = W - MARGIN.left - MARGIN.right;
|
|
|
|
if (mx < MARGIN.left || mx > MARGIN.left + pw) {
|
|
tooltip.style.display = 'none';
|
|
return;
|
|
}
|
|
var idx = Math.round(((mx - MARGIN.left) / pw) * (DATA.length - 1));
|
|
idx = Math.max(0, Math.min(DATA.length - 1, idx));
|
|
var pt = DATA[idx];
|
|
var d = new Date(pt.t * 1000);
|
|
tooltip.innerHTML = '<b>' + pt.v.toFixed(4) + '</b> Regime: ' + pt.r +
|
|
'<br><span style="color:#8b949e">' + d.toISOString().substring(11, 19) + '</span>';
|
|
tooltip.style.display = 'block';
|
|
tooltip.style.left = (e.clientX + 15) + 'px';
|
|
tooltip.style.top = (e.clientY - 40) + 'px';
|
|
});
|
|
canvas.addEventListener('mouseleave', function() { tooltip.style.display = 'none'; });
|
|
|
|
// Touch
|
|
canvas.addEventListener('touchmove', function(e) {
|
|
var t = e.touches[0];
|
|
var ev = {clientX:t.clientX, clientY:t.clientY};
|
|
canvas.dispatchEvent(new MouseEvent('mousemove', ev));
|
|
});
|
|
canvas.addEventListener('touchend', function() { tooltip.style.display = 'none'; });
|
|
|
|
window.addEventListener('resize', init);
|
|
|
|
// Load data
|
|
fetch('DATA_URL_PLACEHOLDER')
|
|
.then(function(r) { return r.json(); })
|
|
.then(function(d) {
|
|
DATA = d;
|
|
var ld = document.getElementById('loading');
|
|
if (ld) ld.remove();
|
|
init();
|
|
})
|
|
.catch(function(e) {
|
|
document.getElementById('loading').textContent = 'Error: ' + e;
|
|
});
|
|
})();
|
|
</script>
|
|
</body>
|
|
</html>"""
|
|
|
|
|
|
def main(datafile):
|
|
print(f"[1/3] Loading {datafile}...")
|
|
snaps = load_snapshots(datafile)
|
|
df = extract_features(snaps)
|
|
print(f" {len(snaps)} snapshots — price range: {df.mid.min():.4f} ~ {df.mid.max():.4f}")
|
|
|
|
print(f"[2/3] Fitting HMM...")
|
|
X_raw = engineer_features(df)
|
|
X_scaled = StandardScaler().fit_transform(X_raw)
|
|
model = fit_hmm(X_scaled[::STEP])
|
|
Z = model.predict(X_scaled)
|
|
for k in range(N_REGIMES):
|
|
mask = Z == k
|
|
print(f" Regime {k}: {mask.mean()*100:.1f}% spread={X_raw[mask,0].mean():.6f}")
|
|
|
|
pts = [{"t": int(df.iloc[i]["ts"]), "v": round(df.iloc[i]["mid"], 6), "r": int(Z[i])} for i in range(len(df))]
|
|
|
|
base = os.path.splitext(datafile)[0]
|
|
json_file = base + "_data.json"
|
|
html_file = base + "_canvas_chart.html"
|
|
|
|
with open(json_file, "w") as f:
|
|
json.dump(pts, f, separators=(',', ':'))
|
|
|
|
html = HTML.replace("DATA_URL_PLACEHOLDER", os.path.basename(json_file))
|
|
with open(html_file, "w") as f:
|
|
f.write(html)
|
|
|
|
print(f"[3/3] Done: {html_file} ({os.path.getsize(html_file)/1024:.0f} KB)")
|
|
print(f" {json_file} ({os.path.getsize(json_file)/1024:.0f} KB)")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) < 2:
|
|
files = sorted(glob.glob("data/l2_LINK_*.jsonl"))
|
|
if not files:
|
|
print("No data files found in data/")
|
|
sys.exit(1)
|
|
datafile = files[-1]
|
|
else:
|
|
datafile = sys.argv[1]
|
|
main(datafile)
|