Add files to chanlun_1

data/__init__.py

@@ -0,0 +1,8 @@
+"""
+数据模块：负责获取和处理K线数据
+"""
+
+from .data_fetcher import DataFetcher
+from .data_processor import DataProcessor
+
+__all__ = ['DataFetcher', 'DataProcessor'] 

data/data_fetcher.py

@@ -0,0 +1,160 @@
+"""
+数据获取模块：使用ccxt库拉取数字货币市场数据
+"""
+
+import ccxt
+import pandas as pd
+import numpy as np
+from datetime import datetime, timedelta
+import time
+from typing import List, Dict, Optional, Tuple
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class DataFetcher:
+    """数据获取器：负责从交易所获取K线数据"""
+    
+    def __init__(self, exchange_name: str = 'binance'):
+        """
+        初始化数据获取器
+        
+        Args:
+            exchange_name: 交易所名称，默认为binance
+        """
+        self.exchange_name = exchange_name
+        self.exchange = self._init_exchange()
+        
+    def _init_exchange(self) -> ccxt.Exchange:
+        """初始化交易所连接"""
+        try:
+            exchange_class = getattr(ccxt, self.exchange_name)
+            exchange = exchange_class({
+                'apiKey': '',  # 对于公开数据不需要API密钥
+                'secret': '',
+                'timeout': 30000,
+                'enableRateLimit': True,
+            })
+            return exchange
+        except Exception as e:
+            logger.error(f"初始化交易所失败: {e}")
+            raise
+            
+    def fetch_klines(self, 
+                    symbol: str, 
+                    timeframe: str = '1h', 
+                    limit: int = 1000,
+                    since: Optional[int] = None) -> pd.DataFrame:
+        """
+        获取K线数据
+        
+        Args:
+            symbol: 交易对符号，如'BTC/USDT'
+            timeframe: 时间周期，如'1h', '4h', '1d'
+            limit: 获取的K线数量
+            since: 开始时间戳（毫秒）
+            
+        Returns:
+            包含K线数据的DataFrame
+        """
+        try:
+            logger.info(f"正在获取 {symbol} {timeframe} 数据，数量: {limit}")
+            
+            # 获取原始数据
+            ohlcv = self.exchange.fetch_ohlcv(
+                symbol=symbol,
+                timeframe=timeframe,
+                limit=limit,
+                since=since
+            )
+            
+            if not ohlcv:
+                raise ValueError("未获取到数据")
+                
+            # 转换为DataFrame
+            df = pd.DataFrame(ohlcv, columns=[
+                'timestamp', 'open', 'high', 'low', 'close', 'volume'
+            ])
+            
+            # 转换时间戳为datetime
+            df['datetime'] = pd.to_datetime(df['timestamp'], unit='ms')
+            df.set_index('datetime', inplace=True)
+            
+            # 确保数据类型正确
+            price_columns = ['open', 'high', 'low', 'close']
+            df[price_columns] = df[price_columns].astype(float)
+            df['volume'] = df['volume'].astype(float)
+            
+            logger.info(f"成功获取 {len(df)} 条K线数据")
+            return df
+            
+        except Exception as e:
+            logger.error(f"获取K线数据失败: {e}")
+            raise
+            
+    def fetch_multiple_timeframes(self, 
+                                 symbol: str, 
+                                 timeframes: List[str],
+                                 limit: int = 1000) -> Dict[str, pd.DataFrame]:
+        """
+        获取多个时间周期的数据
+        
+        Args:
+            symbol: 交易对符号
+            timeframes: 时间周期列表
+            limit: 每个周期获取的数量
+            
+        Returns:
+            字典，键为时间周期，值为对应的DataFrame
+        """
+        result = {}
+        
+        for timeframe in timeframes:
+            try:
+                df = self.fetch_klines(symbol, timeframe, limit)
+                result[timeframe] = df
+                
+                # 避免请求过于频繁
+                time.sleep(self.exchange.rateLimit / 1000)
+                
+            except Exception as e:
+                logger.error(f"获取 {timeframe} 数据失败: {e}")
+                continue
+                
+        return result
+        
+    def get_latest_price(self, symbol: str) -> float:
+        """
+        获取最新价格
+        
+        Args:
+            symbol: 交易对符号
+            
+        Returns:
+            最新价格
+        """
+        try:
+            ticker = self.exchange.fetch_ticker(symbol)
+            return float(ticker['last'])
+        except Exception as e:
+            logger.error(f"获取最新价格失败: {e}")
+            raise
+            
+    def validate_symbol(self, symbol: str) -> bool:
+        """
+        验证交易对是否有效
+        
+        Args:
+            symbol: 交易对符号
+            
+        Returns:
+            是否有效
+        """
+        try:
+            markets = self.exchange.load_markets()
+            return symbol in markets
+        except Exception as e:
+            logger.error(f"验证交易对失败: {e}")
+            return False 

data/data_processor.py

@@ -0,0 +1,224 @@
+"""
+数据处理模块：负责K线数据的清理、格式化和预处理
+"""
+
+import pandas as pd
+import numpy as np
+from typing import Optional, Tuple, List
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class DataProcessor:
+    """数据处理器：负责K线数据的处理和验证"""
+    
+    @staticmethod
+    def validate_klines(df: pd.DataFrame) -> bool:
+        """
+        验证K线数据的完整性和正确性
+        
+        Args:
+            df: K线数据DataFrame
+            
+        Returns:
+            是否通过验证
+        """
+        required_columns = ['open', 'high', 'low', 'close', 'volume']
+        
+        # 检查必需列是否存在
+        if not all(col in df.columns for col in required_columns):
+            logger.error("缺少必需的列")
+            return False
+            
+        # 检查数据是否为空
+        if df.empty:
+            logger.error("数据为空")
+            return False
+            
+        # 检查价格关系是否正确
+        invalid_rows = (
+            (df['high'] < df['low']) |
+            (df['high'] < df['open']) |
+            (df['high'] < df['close']) |
+            (df['low'] > df['open']) |
+            (df['low'] > df['close'])
+        )
+        
+        if invalid_rows.any():
+            logger.warning(f"发现 {invalid_rows.sum()} 行无效的价格关系")
+            
+        # 检查是否有NaN值
+        if df[required_columns].isnull().any().any():
+            logger.warning("数据中包含NaN值")
+            
+        return True
+        
+    @staticmethod
+    def clean_klines(df: pd.DataFrame) -> pd.DataFrame:
+        """
+        清理K线数据
+        
+        Args:
+            df: 原始K线数据
+            
+        Returns:
+            清理后的K线数据
+        """
+        df_clean = df.copy()
+        
+        # 移除NaN值
+        df_clean = df_clean.dropna()
+        
+        # 修正无效的价格关系
+        # 如果high < max(open, close)，则设置high = max(open, close, low)
+        df_clean['high'] = np.maximum.reduce([
+            df_clean['high'], 
+            df_clean['open'], 
+            df_clean['close'],
+            df_clean['low']
+        ])
+        
+        # 如果low > min(open, close)，则设置low = min(open, close, high)
+        df_clean['low'] = np.minimum.reduce([
+            df_clean['low'], 
+            df_clean['open'], 
+            df_clean['close'],
+            df_clean['high']
+        ])
+        
+        # 确保volume非负
+        df_clean['volume'] = np.maximum(df_clean['volume'], 0)
+        
+        # 按时间排序
+        df_clean = df_clean.sort_index()
+        
+        logger.info(f"数据清理完成，剩余 {len(df_clean)} 条记录")
+        return df_clean
+        
+    @staticmethod
+    def add_technical_indicators(df: pd.DataFrame) -> pd.DataFrame:
+        """
+        添加技术指标
+        
+        Args:
+            df: K线数据
+            
+        Returns:
+            包含技术指标的数据
+        """
+        df_with_indicators = df.copy()
+        
+        # 添加价格范围
+        df_with_indicators['range'] = df_with_indicators['high'] - df_with_indicators['low']
+        
+        # 添加实体大小
+        df_with_indicators['body'] = abs(df_with_indicators['close'] - df_with_indicators['open'])
+        
+        # 添加上影线长度
+        df_with_indicators['upper_shadow'] = df_with_indicators['high'] - np.maximum(
+            df_with_indicators['open'], 
+            df_with_indicators['close']
+        )
+        
+        # 添加下影线长度
+        df_with_indicators['lower_shadow'] = np.minimum(
+            df_with_indicators['open'], 
+            df_with_indicators['close']
+        ) - df_with_indicators['low']
+        
+        # 添加K线方向
+        df_with_indicators['direction'] = np.where(
+            df_with_indicators['close'] > df_with_indicators['open'], 1, -1
+        )
+        
+        return df_with_indicators
+        
+    @staticmethod
+    def resample_klines(df: pd.DataFrame, new_timeframe: str) -> pd.DataFrame:
+        """
+        重采样K线数据到新的时间周期
+        
+        Args:
+            df: 原始K线数据
+            new_timeframe: 新的时间周期，如'4H', '1D'
+            
+        Returns:
+            重采样后的K线数据
+        """
+        try:
+            # 重采样规则
+            agg_dict = {
+                'open': 'first',
+                'high': 'max',
+                'low': 'min',
+                'close': 'last',
+                'volume': 'sum'
+            }
+            
+            # 执行重采样
+            resampled = df.resample(new_timeframe).agg(agg_dict)
+            
+            # 移除空值
+            resampled = resampled.dropna()
+            
+            logger.info(f"重采样到 {new_timeframe}，得到 {len(resampled)} 条记录")
+            return resampled
+            
+        except Exception as e:
+            logger.error(f"重采样失败: {e}")
+            raise
+            
+    @staticmethod
+    def calculate_returns(df: pd.DataFrame) -> pd.DataFrame:
+        """
+        计算收益率
+        
+        Args:
+            df: K线数据
+            
+        Returns:
+            包含收益率的数据
+        """
+        df_with_returns = df.copy()
+        
+        # 计算收盘价收益率
+        df_with_returns['returns'] = df_with_returns['close'].pct_change()
+        
+        # 计算对数收益率
+        df_with_returns['log_returns'] = np.log(df_with_returns['close'] / df_with_returns['close'].shift(1))
+        
+        return df_with_returns
+        
+    @staticmethod
+    def get_data_summary(df: pd.DataFrame) -> dict:
+        """
+        获取数据摘要信息
+        
+        Args:
+            df: K线数据
+            
+        Returns:
+            数据摘要字典
+        """
+        summary = {
+            'total_records': len(df),
+            'date_range': {
+                'start': df.index.min(),
+                'end': df.index.max()
+            },
+            'price_range': {
+                'min': df['low'].min(),
+                'max': df['high'].max()
+            },
+            'volume_stats': {
+                'total': df['volume'].sum(),
+                'avg': df['volume'].mean(),
+                'max': df['volume'].max()
+            }
+        }
+        
+        if len(df) > 0:
+            summary['latest_price'] = df['close'].iloc[-1]
+            
+        return summary