缠论引擎提速 2.6x，瓶颈是逐行 Series 查找而非指标计算
原以为浪费在 add_indicators 算了太多用不到的指标，实测它只占全量构建的 1.3%——talib 是向量化 C 代码，便宜。真正的两处： cal_kl_data 占 96%：每根 K 线 df.iloc[i] 新建一个 40 列 Series，再在其上做 几十次逐键查找。改为预取 ndarray 后 2 万根 1946ms → 824ms。 ChanKLC.cal_all_ema_status 占 25%：每次合并 KLU 都立即重算，而它产出的 ema_status / ema52_pos / ema52_status 全仓无任何读取方（含前端）。改为惰性 求值，保留属性形式以防将来有人读。顺带删掉 get_klc_list 里累加一整轮后直接 丢弃的 ema_up_list / ema_down_list。 另加 TF_DF(lean=True)：只构建到中枢，跳过线段/走势中枢/MACD 状态机——这些 只服务 bsp_list 与 web 展示，笔和中枢不依赖。研究与实盘走这条快 3.6x。 结果 2 万根 5m：full 1946 → 754ms，lean → 543ms。 step46_engine_parity.py 是配套的安全网，改引擎前先跑一次 --save。它对 KLC 端点与分型、笔起止价与 is_sure、中枢 zg/zd/available_ts/阶梯、信号全部输出列， 以及 26 个被下游消费的 dataframe 列取哈希。本次三处改动逐步验证，另用 git stash 切回改动前代码在 20 万根 × 5 用例上做了跨版本逐位对拍，全部一致； 增量路径与 web API 也各验一遍。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
jackyu66git
+32
-8
@@ -63,10 +63,11 @@ class ChanKLC():
|
||||
self.bsp = False
|
||||
self.bsp_type = Chan_BSP_TYPE.NONE
|
||||
# EMA状态字典：key为EMA名称，value为 {'pos': Chan_EMA_POS, 'semantic': Chan_EMA_SEMANTIC}
|
||||
self.ema_status = {}
|
||||
self._ema_status = {}
|
||||
self._ema_status_dirty = False
|
||||
# 向后兼容：保留 ema52_status 和 ema52_pos
|
||||
self.ema52_status = 0
|
||||
self.ema52_pos = Chan_EMA_POS.UNKNOWN
|
||||
self._ema52_status = 0
|
||||
self._ema52_pos = Chan_EMA_POS.UNKNOWN
|
||||
self.bb2633upper = klu.bb2633upper
|
||||
self.bb2633lower = klu.bb2633lower
|
||||
self.bb2633middle = klu.bb2633middle
|
||||
@@ -283,21 +284,44 @@ class ChanKLC():
|
||||
'ema156': self.ema156,
|
||||
'ema208': self.ema208,
|
||||
}
|
||||
self.ema_status = {}
|
||||
self._ema_status_dirty = False
|
||||
self._ema_status = {}
|
||||
for name, value in ema_configs.items():
|
||||
# 按 EMA 值的百分比自动计算阈值
|
||||
threshold = abs(value) * self.threshold_pct if value and self.threshold_pct > 0 else 0
|
||||
pos = ChanKLC.cal_ema_pos(self.high, self.low, self.close, value, threshold)
|
||||
semantic = ChanKLC.cal_ema_semantic(pos, self.dir, self.ema_dir)
|
||||
self.ema_status[name] = {
|
||||
self._ema_status[name] = {
|
||||
'pos': pos,
|
||||
'semantic': semantic,
|
||||
'value': value,
|
||||
'threshold': threshold,
|
||||
}
|
||||
# 向后兼容
|
||||
self.ema52_pos = self.ema_status['ema52']['pos']
|
||||
self.ema52_status = ChanKLC.semantic_to_int(self.ema_status['ema52']['semantic'])
|
||||
self._ema52_pos = self._ema_status['ema52']['pos']
|
||||
self._ema52_status = ChanKLC.semantic_to_int(self._ema_status['ema52']['semantic'])
|
||||
|
||||
# 以下三个改成惰性求值。原来 set_end_klu 每次合并 KLU 都会立刻重算一遍，
|
||||
# 实测占 TF_DF 构建的约 25%，而全仓（含前端）没有任何地方读取它的产出。
|
||||
# 保留属性形式是为了任何外部读取仍拿到正确值，只是推迟到真被读时才算。
|
||||
@property
|
||||
def ema_status(self):
|
||||
if self._ema_status_dirty:
|
||||
self.cal_all_ema_status()
|
||||
return self._ema_status
|
||||
|
||||
@property
|
||||
def ema52_pos(self):
|
||||
if self._ema_status_dirty:
|
||||
self.cal_all_ema_status()
|
||||
return self._ema52_pos
|
||||
|
||||
@property
|
||||
def ema52_status(self):
|
||||
if self._ema_status_dirty:
|
||||
self.cal_all_ema_status()
|
||||
return self._ema52_status
|
||||
|
||||
def get_ema_pos(self, ema_name):
|
||||
"""获取指定EMA的客观位置，如 klc.get_ema_pos('ema24')"""
|
||||
if ema_name in self.ema_status:
|
||||
@@ -448,7 +472,7 @@ class ChanKLC():
|
||||
klu.set_klc(self)
|
||||
self.klc_dir = Chan_KLINE_DIR.UP if self.close > self.open else Chan_KLINE_DIR.DOWN
|
||||
self.cal_indicators()
|
||||
self.cal_all_ema_status()
|
||||
self._ema_status_dirty = True
|
||||
if self.open > self.high:
|
||||
self.open = self.high
|
||||
if self.close > self.high:
|
||||
|
||||
+33
-20
@@ -160,27 +160,40 @@ class ChanKLU:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
# 属性名 ← dataframe 列名。两条赋值路径（逐行 Series / 预取 ndarray）共用这张表，
|
||||
# 免得将来加指标时只改一处、另一处静默漏掉。
|
||||
INDICATOR_FIELDS = (
|
||||
('macd', 'macd'), ('signal', 'macdsignal'), ('macdhist', 'macdhist'),
|
||||
('ema26', 'ema26'), ('ema52', 'ema52'), ('ema24', 'ema24'),
|
||||
('ema104', 'ema104'), ('ema156', 'ema156'), ('ema208', 'ema208'),
|
||||
('ema13', 'ema13'), ('ema7', 'ema7'), ('ema5', 'ema5'),
|
||||
('rsi', 'rsi'), ('volume_ratio', 'volume_ratio'),
|
||||
('bb52upper', 'bb52upper'), ('bb52lower', 'bb52lower'),
|
||||
('bb2633upper', 'bb2633upper'), ('bb2633lower', 'bb2633lower'),
|
||||
('bb2633middle', 'bb2633middle'), ('ma5', 'ma5'),
|
||||
)
|
||||
|
||||
def set_indicators(self, item):
|
||||
self.macd = float(item['macd']) if 'macd' in item and item['macd'] else 0
|
||||
self.signal = float(item['macdsignal']) if 'macdsignal' in item and item['macdsignal'] else 0
|
||||
self.macdhist = float(item['macdhist']) if 'macdhist' in item and item['macdhist'] else 0
|
||||
self.ema26 = float(item['ema26']) if 'ema26' in item and item['ema26'] else 0
|
||||
self.ema52 = float(item['ema52']) if 'ema52' in item and item['ema52'] else 0
|
||||
self.ema24 = float(item['ema24']) if 'ema24' in item and item['ema24'] else 0
|
||||
self.ema104 = float(item['ema104']) if 'ema104' in item and item['ema104'] else 0
|
||||
self.ema156 = float(item['ema156']) if 'ema156' in item and item['ema156'] else 0
|
||||
self.ema208 = float(item['ema208']) if 'ema208' in item and item['ema208'] else 0
|
||||
self.ema13 = float(item['ema13']) if 'ema13' in item and item['ema13'] else 0
|
||||
self.ema7 = float(item['ema7']) if 'ema7' in item and item['ema7'] else 0
|
||||
self.rsi = float(item['rsi']) if 'rsi' in item and item['rsi'] else 0
|
||||
self.volume_ratio = float(item['volume_ratio']) if 'volume_ratio' in item and item['volume_ratio'] else 0
|
||||
self.bb52upper = float(item['bb52upper']) if 'bb52upper' in item and item['bb52upper'] else 0
|
||||
self.bb52lower = float(item['bb52lower']) if 'bb52lower' in item and item['bb52lower'] else 0
|
||||
self.bb2633upper = float(item['bb2633upper']) if 'bb2633upper' in item and item['bb2633upper'] else 0
|
||||
self.bb2633lower = float(item['bb2633lower']) if 'bb2633lower' in item and item['bb2633lower'] else 0
|
||||
self.bb2633middle = float(item['bb2633middle']) if 'bb2633middle' in item and item['bb2633middle'] else 0
|
||||
self.ma5 = float(item['ma5']) if 'ma5' in item and item['ma5'] else 0
|
||||
self.ema5 = float(item['ema5']) if 'ema5' in item and item['ema5'] else 0
|
||||
"""单根赋值。增量追加时每次只有一根，走这条即可。
|
||||
|
||||
原写法是 `float(item[c]) if c in item and item[c] else 0`。其中的真值判断
|
||||
是空转：值为 0.0 时 float(0.0) 仍是 0，值为 NaN 时 NaN 为真值、照样透传。
|
||||
唯一起作用的是「列不存在则填 0」，所以这里只保留那一层。
|
||||
"""
|
||||
for attr, col in self.INDICATOR_FIELDS:
|
||||
v = item[col] if col in item else 0
|
||||
setattr(self, attr, float(v) if v else 0)
|
||||
|
||||
def set_indicators_from(self, cols, i):
|
||||
"""从预取的 {列名: ndarray} 按下标赋值，语义与 set_indicators 相同。
|
||||
|
||||
全量构建时用这条：避免每根 `df.iloc[i]` 构造一个 Series，再在其上做
|
||||
几十次逐键查找——那是 TF_DF 构建 96% 的耗时所在。
|
||||
"""
|
||||
for attr, col in self.INDICATOR_FIELDS:
|
||||
arr = cols.get(col)
|
||||
v = arr[i] if arr is not None else 0
|
||||
setattr(self, attr, float(v) if v else 0)
|
||||
def cal_macd_state(self):
|
||||
# 按定义精简实现：优先级 CROSS0 > 位置(HIGH/HE/RETURN_ZERO) > NEAR0 > UNKNOWN
|
||||
# 首条或缺前一根
|
||||
|
||||
@@ -147,41 +147,53 @@ class KlineBuilderMixin:
|
||||
return df['volume_ratio']
|
||||
|
||||
def cal_kl_data(self, dataframe:DataFrame):
|
||||
fields = "time,open,high,low,close,volume"
|
||||
"""按行构造 KLU 链。
|
||||
|
||||
这里刻意不用 `dataframe.iloc[i]`：那会为每一根新建一个几十列的 Series，
|
||||
随后 set_indicators 再在其上做几十次逐键查找。实测这两件事合计占 TF_DF
|
||||
构建耗时的 96%。改为先把用到的列取成 ndarray，循环里只做整数下标访问。
|
||||
"""
|
||||
n = len(dataframe)
|
||||
if n == 0:
|
||||
return []
|
||||
|
||||
times = self._format_times(dataframe['date'])
|
||||
o_a = dataframe['open'].to_numpy(dtype=float)
|
||||
h_a = dataframe['high'].to_numpy(dtype=float)
|
||||
l_a = dataframe['low'].to_numpy(dtype=float)
|
||||
c_a = dataframe['close'].to_numpy(dtype=float)
|
||||
v_a = dataframe['volume'].to_numpy(dtype=float)
|
||||
|
||||
has_ind = 'macd' in dataframe.columns
|
||||
ind_cols = {}
|
||||
if has_ind:
|
||||
for _attr, col in ChanKLU.INDICATOR_FIELDS:
|
||||
if col in dataframe.columns:
|
||||
ind_cols[col] = dataframe[col].to_numpy(dtype=float)
|
||||
|
||||
klu_list = []
|
||||
last_klu = None
|
||||
for i in range(0, len(dataframe)):
|
||||
item = dataframe.iloc[i]
|
||||
date = item['date']
|
||||
o = item['open']
|
||||
h = item['high']
|
||||
l = item['low']
|
||||
c = item['close']
|
||||
v = item['volume']
|
||||
# time_obj = date.fromtimestamp(date)
|
||||
# date = date + timedelta(hours=8)
|
||||
time_str = date.strftime('%Y-%m-%d %H:%M:%S')
|
||||
item_data = [
|
||||
time_str,
|
||||
o,
|
||||
h,
|
||||
l,
|
||||
c,
|
||||
v
|
||||
]
|
||||
# klu = KLU(self.create_item_dict(item_data, GetColumnNameFromFieldList(fields)))
|
||||
klu = ChanKLU(time_str, o, h, l, c, v)
|
||||
# print(klu.time, klu.open, klu.high, klu.low, klu.close, klu.volume)
|
||||
for i in range(n):
|
||||
klu = ChanKLU(times[i], o_a[i], h_a[i], l_a[i], c_a[i], v_a[i])
|
||||
klu.set_idx(i)
|
||||
klu_list.append(klu)
|
||||
if last_klu:
|
||||
last_klu.set_next(klu)
|
||||
klu.set_pre(last_klu)
|
||||
last_klu = klu
|
||||
if 'macd' in item:
|
||||
klu.set_indicators(item)
|
||||
if has_ind:
|
||||
klu.set_indicators_from(ind_cols, i)
|
||||
return klu_list
|
||||
|
||||
@staticmethod
|
||||
def _format_times(col):
|
||||
"""向量化 strftime。非 datetime 列（少见）退回逐个格式化。"""
|
||||
fmt = '%Y-%m-%d %H:%M:%S'
|
||||
try:
|
||||
return col.dt.strftime(fmt).to_numpy()
|
||||
except AttributeError:
|
||||
return np.array([d.strftime(fmt) for d in col], dtype=object)
|
||||
|
||||
def get_kl_data(self, dataframe:DataFrame):
|
||||
return self.cal_kl_data(dataframe)
|
||||
|
||||
@@ -224,35 +236,20 @@ class KlineBuilderMixin:
|
||||
|
||||
def get_klc_list(self, klu_list):
|
||||
klc_list = []
|
||||
last_klu = None
|
||||
# ChanMACD.__init__ 已调用 cal_macd_state，切勿再调一次（会重复堆积 seg/unittf）
|
||||
macd = ChanMACD(klu_list)
|
||||
klu_list = macd.klu_list
|
||||
self._last_chan_macd = macd
|
||||
ema_up_list = []
|
||||
ema_down_list = []
|
||||
ema_up_count = 0
|
||||
ema_down_count = 0
|
||||
# ChanMACD.__init__ 已调用 cal_macd_state，切勿再调一次（会重复堆积 seg/unittf）。
|
||||
# lean 模式跳过整套 MACD 状态机：它只服务于 bsp/背驰/web 展示，笔与中枢不依赖它。
|
||||
if getattr(self, 'lean', False):
|
||||
self._last_chan_macd = None
|
||||
else:
|
||||
macd = ChanMACD(klu_list)
|
||||
klu_list = macd.klu_list
|
||||
self._last_chan_macd = macd
|
||||
|
||||
last_klu = None
|
||||
for klu in klu_list:
|
||||
ema = klu.ema52
|
||||
last_ema = last_klu.ema52 if last_klu else 0
|
||||
if klu.close >= ema:
|
||||
ema_up_count += 1
|
||||
elif klu.close < ema:
|
||||
ema_down_count += 1
|
||||
if last_klu and last_klu.close >= last_ema and klu.close < ema:
|
||||
ema_up_list.append(ema_up_count)
|
||||
#print(last_klu.time, ema_up_count, "UP END")
|
||||
ema_up_count = 0
|
||||
elif last_klu and last_klu.close < last_ema and klu.close >= ema:
|
||||
ema_down_list.append(ema_down_count)
|
||||
#print(last_klu.time, ema_down_count, "DOWN END")
|
||||
ema_down_count = 0
|
||||
self._push_klu_into_klc_list(klc_list, klu, last_klu)
|
||||
last_klu = klu
|
||||
klc_list = self.cal_trend(klc_list)
|
||||
#print(ema52_up_list, ema52_down_list)
|
||||
return klc_list
|
||||
|
||||
|
||||
|
||||
@@ -38,10 +38,18 @@ from chanlun.pipeline.builders.seg import SegBuilderMixin
|
||||
from chanlun.pipeline.builders.zs import ZsBuilderMixin
|
||||
|
||||
class TF_DF(IndicatorsBuilderMixin, KlineBuilderMixin, BiBuilderMixin, SegBuilderMixin, ZsBuilderMixin, BspBuilderMixin, FastBspBuilderMixin, IncrementalBuilderMixin):
|
||||
def __init__(self, df=None, interval=0, timeframe=None):
|
||||
def __init__(self, df=None, interval=0, timeframe=None, lean=False):
|
||||
"""lean=True 只构建到中枢，跳过线段/走势中枢/MACD 状态机。
|
||||
|
||||
研究与实盘只吃 bi_list → 中枢 → fast_bsp 这条链；线段、zs、big_zs 和整套
|
||||
MACD 背驰状态机是 web 展示与 bsp_list 才用的。实测这些占全量构建的约四成。
|
||||
注意 lean 下 bsp_list/seg_list/chanmacd 均为空，**不要给 web 用**。
|
||||
"""
|
||||
self.lean = lean
|
||||
if df is not None:
|
||||
self.init_TF_DF(df, interval, timeframe)
|
||||
def init_TF_DF(self, df, interval, timeframe):
|
||||
self.init_TF_DF(df, interval, timeframe, lean=lean)
|
||||
def init_TF_DF(self, df, interval, timeframe, lean=False):
|
||||
self.lean = lean
|
||||
self.timeframe = timeframe
|
||||
self.interval = interval
|
||||
# 检查 DataFrame 是否为空或没有 date 列
|
||||
@@ -69,6 +77,10 @@ class TF_DF(IndicatorsBuilderMixin, KlineBuilderMixin, BiBuilderMixin, SegBuilde
|
||||
self.klc_list = self.get_klc_list(self.klu_list)
|
||||
self.bi_list = self.cal_bi_list(self.klc_list)
|
||||
self.bi_zs_list = self.cal_bi_zs_list_pure(self.bi_list)
|
||||
if self.lean:
|
||||
self.big_zs_list = []
|
||||
self.chanmacd = None
|
||||
return
|
||||
self.seg_list = self.get_seg_list(self.bi_list)
|
||||
self.zs_list = self.get_zs_list(self.bi_list, self.seg_list)
|
||||
self.big_zs_list = self.get_big_zs_list(self.zs_list)
|
||||
|
||||
Reference in New Issue
Block a user