jackyu66gitandCursor
74dec4e50b
将根目录引擎迁入 chanlun/ 并保留兼容 shim；拆分 TF_DF 与 web 服务； 前端模块化；strategies 改用 chanlun 导入；补充 ESS 文档与 golden 回归。 Co-authored-by: Cursor <cursoragent@cursor.com>
273 lines
10 KiB
Python
273 lines
10 KiB
Python
"""
|
|
Phase 2: Run ChanPivotClassifier on real data, compute bi_out for each pivot,
|
|
export the dataset, and run single-variable statistics.
|
|
|
|
Usage: python test_classifier.py
|
|
"""
|
|
|
|
import csv
|
|
import sys
|
|
import os
|
|
|
|
# Ensure Chan module is importable
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
from chanlun import TF_DF
|
|
from chanlun.analysis.ChanPivotClassifier import ChanPivotClassifier
|
|
from chanlun.core.ChanEnum import Chan_BI_DIR
|
|
|
|
# Monkey-patch: TF_DF.init_TF_DF calls self.get_zs_list() which was removed.
|
|
# Add it back as an alias for get_bi_zs_list.
|
|
if not hasattr(TF_DF, 'get_zs_list'):
|
|
TF_DF.get_zs_list = lambda self, bi_list, seg_list: self.get_bi_zs_list(bi_list)
|
|
|
|
|
|
def load_csv(path: str) -> list[dict]:
|
|
"""Load OHLCV CSV into list of dicts expected by TF_DF."""
|
|
import pandas as pd
|
|
df = pd.read_csv(path)
|
|
df.columns = [c.lower() for c in df.columns]
|
|
# TF_DF expects 'date' column
|
|
if 'timestamp' in df.columns:
|
|
df.rename(columns={'timestamp': 'date'}, inplace=True)
|
|
df['date'] = pd.to_datetime(df['date'])
|
|
return df
|
|
|
|
|
|
def compute_bi_out(zs, bi_list: list) -> object:
|
|
"""
|
|
Determine the first bi after the pivot's end_bi that breaks out of the pivot range.
|
|
A breakout is: bi.high > zs.gg (up) or bi.low < zs.dd (down).
|
|
"""
|
|
if zs.end_bi is None or not zs.is_sure:
|
|
return None
|
|
|
|
# Find end_bi position in bi_list
|
|
end_idx = None
|
|
for i, bi in enumerate(bi_list):
|
|
if bi is zs.end_bi or bi.index == zs.end_bi.index:
|
|
end_idx = i
|
|
break
|
|
|
|
if end_idx is None:
|
|
return None
|
|
|
|
# Look for the first bi after end_bi that breaks the pivot range
|
|
for i in range(end_idx + 1, len(bi_list)):
|
|
bi = bi_list[i]
|
|
if not bi.is_sure:
|
|
continue
|
|
# A breakout: goes above gg or below dd
|
|
if bi.high > zs.gg or bi.low < zs.dd:
|
|
return bi
|
|
|
|
return None
|
|
|
|
|
|
def run_pipeline(csv_path: str, symbol: str, timeframe: str, interval: int = 1):
|
|
"""Full pipeline: CSV → TF_DF → compute bi_out → ChanPivotClassifier."""
|
|
print(f"\n{'='*60}")
|
|
print(f"Processing: {symbol} {timeframe}")
|
|
print(f"{'='*60}")
|
|
|
|
# Step 1: Load data
|
|
df = load_csv(csv_path)
|
|
print(f"Loaded {len(df)} rows")
|
|
|
|
# Step 2: Run TF_DF pipeline
|
|
tf_df = TF_DF(df, interval, timeframe)
|
|
print(f"KLC count: {len(tf_df.klc_list)}")
|
|
print(f"BI count: {len(tf_df.bi_list)}")
|
|
|
|
# Get bi_zs_list via the seg-based method (matching find_all_bsp)
|
|
bi_zs_list = tf_df.cal_bi_zs(tf_df.seg_list)
|
|
print(f"Pivot count (raw): {len(bi_zs_list)}")
|
|
|
|
# Filter to sure pivots with enough internal strokes
|
|
sure_pivots = [zs for zs in bi_zs_list if zs.is_sure and len(zs.bi_list) >= 3]
|
|
print(f"Pivot count (sure, >=3 strokes): {len(sure_pivots)}")
|
|
|
|
# Step 3: Compute bi_out for each pivot
|
|
for zs in sure_pivots:
|
|
zs.bi_out = compute_bi_out(zs, tf_df.bi_list)
|
|
|
|
bi_out_count = sum(1 for zs in sure_pivots if zs.bi_out is not None)
|
|
print(f"Pivots with bi_out: {bi_out_count}/{len(sure_pivots)}")
|
|
|
|
# Step 4: Run ChanPivotClassifier
|
|
classifier = ChanPivotClassifier(sure_pivots, symbol=symbol, timeframe=timeframe)
|
|
dataset = classifier.extract()
|
|
print(f"Dataset samples: {len(dataset)}")
|
|
|
|
# Step 5: Export
|
|
output_path = f"/tmp/chan_dataset_{symbol.replace('/', '_')}_{timeframe}.json"
|
|
count = classifier.export_json(output_path)
|
|
print(f"Exported {count} samples to {output_path}")
|
|
|
|
return dataset
|
|
|
|
|
|
def run_statistics(dataset: list[dict]):
|
|
"""Phase 2 statistics: single-variable analysis."""
|
|
print(f"\n{'='*60}")
|
|
print("Phase 2 — Single-Variable Statistics")
|
|
print(f"{'='*60}\n")
|
|
|
|
if not dataset:
|
|
print("No data to analyze.")
|
|
return
|
|
|
|
total = len(dataset)
|
|
up = [d for d in dataset if d["label"] == "up"]
|
|
down = [d for d in dataset if d["label"] == "down"]
|
|
none_ = [d for d in dataset if d["label"] == "none"]
|
|
|
|
print(f"Total samples: {total}")
|
|
print(f" Up: {len(up)} ({len(up)/total*100:.1f}%)")
|
|
print(f" Down: {len(down)} ({len(down)/total*100:.1f}%)")
|
|
print(f" None: {len(none_)} ({len(none_)/total*100:.1f}%)")
|
|
|
|
# ================================================================
|
|
# Feature 1: contraction vs break direction
|
|
# ================================================================
|
|
print(f"\n--- Feature: contraction (convergence rate) ---")
|
|
for label, subset in [("up", up), ("down", down), ("none", none_)]:
|
|
if not subset:
|
|
continue
|
|
contractions = [d["contraction"] for d in subset]
|
|
avg = sum(contractions) / len(contractions)
|
|
print(f" {label}: mean contraction = {avg:.4f}")
|
|
|
|
# Contraction < 0.7 → P(up)?
|
|
high_contraction = [d for d in dataset if d["contraction"] < 0.7]
|
|
if high_contraction:
|
|
up_in_hc = len([d for d in high_contraction if d["label"] == "up"])
|
|
down_in_hc = len([d for d in high_contraction if d["label"] == "down"])
|
|
print(f"\n Contraction < 0.7 (converging): {len(high_contraction)} samples")
|
|
print(f" P(up) = {up_in_hc/len(high_contraction)*100:.1f}%")
|
|
print(f" P(down) = {down_in_hc/len(high_contraction)*100:.1f}%")
|
|
|
|
# Contraction > 1.2 → P(down)?
|
|
low_contraction = [d for d in dataset if d["contraction"] > 1.2]
|
|
if low_contraction:
|
|
up_in_lc = len([d for d in low_contraction if d["label"] == "up"])
|
|
down_in_lc = len([d for d in low_contraction if d["label"] == "down"])
|
|
print(f"\n Contraction > 1.2 (expanding): {len(low_contraction)} samples")
|
|
print(f" P(up) = {up_in_lc/len(low_contraction)*100:.1f}%")
|
|
print(f" P(down) = {down_in_lc/len(low_contraction)*100:.1f}%")
|
|
|
|
# ================================================================
|
|
# Feature 2: shift_norm vs break direction
|
|
# ================================================================
|
|
print(f"\n--- Feature: shift_norm (center drift) ---")
|
|
for label, subset in [("up", up), ("down", down), ("none", none_)]:
|
|
if not subset:
|
|
continue
|
|
shifts = [d["shift_norm"] for d in subset]
|
|
avg = sum(shifts) / len(shifts)
|
|
print(f" {label}: mean shift_norm = {avg:.4f}")
|
|
|
|
# shift > 0 → P(up)?
|
|
shift_up = [d for d in dataset if d["shift_norm"] > 0]
|
|
if shift_up:
|
|
up_in_su = len([d for d in shift_up if d["label"] == "up"])
|
|
down_in_su = len([d for d in shift_up if d["label"] == "down"])
|
|
print(f"\n shift_norm > 0 (drifting up): {len(shift_up)} samples")
|
|
print(f" P(up) = {up_in_su/len(shift_up)*100:.1f}%")
|
|
print(f" P(down) = {down_in_su/len(shift_up)*100:.1f}%")
|
|
|
|
# shift < 0 → P(down)?
|
|
shift_down = [d for d in dataset if d["shift_norm"] < 0]
|
|
if shift_down:
|
|
up_in_sd = len([d for d in shift_down if d["label"] == "up"])
|
|
down_in_sd = len([d for d in shift_down if d["label"] == "down"])
|
|
print(f"\n shift_norm < 0 (drifting down): {len(shift_down)} samples")
|
|
print(f" P(up) = {up_in_sd/len(shift_down)*100:.1f}%")
|
|
print(f" P(down) = {down_in_sd/len(shift_down)*100:.1f}%")
|
|
|
|
# ================================================================
|
|
# Feature 3: duration_norm vs break direction
|
|
# ================================================================
|
|
print(f"\n--- Feature: duration_norm (relative duration) ---")
|
|
for label, subset in [("up", up), ("down", down), ("none", none_)]:
|
|
if not subset:
|
|
continue
|
|
durations = [d["duration_norm"] for d in subset]
|
|
avg = sum(durations) / len(durations)
|
|
print(f" {label}: mean duration_norm = {avg:.4f}")
|
|
|
|
# ================================================================
|
|
# Combined: contraction < 0.7 AND shift_norm > 0 → P(up)?
|
|
# ================================================================
|
|
print(f"\n--- Combined signals ---")
|
|
converging_up = [d for d in dataset if d["contraction"] < 0.7 and d["shift_norm"] > 0]
|
|
if converging_up:
|
|
up_in_cu = len([d for d in converging_up if d["label"] == "up"])
|
|
down_in_cu = len([d for d in converging_up if d["label"] == "down"])
|
|
print(f" Contraction < 0.7 AND shift_norm > 0: {len(converging_up)} samples")
|
|
print(f" P(up) = {up_in_cu/len(converging_up)*100:.1f}%")
|
|
print(f" P(down) = {down_in_cu/len(converging_up)*100:.1f}%")
|
|
|
|
converging_down = [d for d in dataset if d["contraction"] < 0.7 and d["shift_norm"] < 0]
|
|
if converging_down:
|
|
up_in_cd = len([d for d in converging_down if d["label"] == "up"])
|
|
down_in_cd = len([d for d in converging_down if d["label"] == "down"])
|
|
print(f" Contraction < 0.7 AND shift_norm < 0: {len(converging_down)} samples")
|
|
print(f" P(up) = {up_in_cd/len(converging_down)*100:.1f}%")
|
|
print(f" P(down) = {down_in_cd/len(converging_down)*100:.1f}%")
|
|
|
|
return dataset
|
|
|
|
|
|
def extract_symbol(csv_name: str) -> str:
|
|
"""Extract symbol from filename like 'BTC_USDT_1d.csv'."""
|
|
parts = csv_name.replace(".csv", "").split("_")
|
|
if len(parts) >= 2:
|
|
return f"{parts[0]}/{parts[1]}"
|
|
return csv_name
|
|
|
|
|
|
def extract_timeframe(csv_name: str) -> str:
|
|
"""Extract timeframe from filename like 'BTC_USDT_1d.csv'."""
|
|
parts = csv_name.replace(".csv", "").split("_")
|
|
if len(parts) >= 3:
|
|
return parts[2]
|
|
return "1d"
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import glob
|
|
import json
|
|
|
|
data_dir = "/Users/jack/Project/freqtrade/binance_data"
|
|
csv_files = sorted(glob.glob(f"{data_dir}/*_USDT_1h.csv"))
|
|
|
|
if not csv_files:
|
|
print("No data files found.")
|
|
sys.exit(1)
|
|
|
|
print(f"Found {len(csv_files)} data files:")
|
|
for f in csv_files:
|
|
print(f" {os.path.basename(f)}")
|
|
|
|
# Batch process all coins
|
|
all_data = []
|
|
for csv_path in csv_files:
|
|
basename = os.path.basename(csv_path)
|
|
symbol = extract_symbol(basename)
|
|
timeframe = extract_timeframe(basename)
|
|
try:
|
|
dataset = run_pipeline(csv_path, symbol, timeframe)
|
|
all_data.extend(dataset)
|
|
except Exception as e:
|
|
print(f" ERROR: {symbol} — {e}")
|
|
|
|
# Export combined dataset
|
|
combined_path = "/tmp/chan_dataset_all_coins.json"
|
|
with open(combined_path, "w", encoding="utf-8") as f:
|
|
json.dump(all_data, f, indent=2, ensure_ascii=False, default=str)
|
|
print(f"\nCombined dataset: {len(all_data)} samples → {combined_path}")
|
|
|
|
# Run statistics on combined dataset
|
|
run_statistics(all_data)
|