Initial commit: A-Share Data Platform v0.1.0
Parquet + DuckDB storage with REST/WebSocket APIs for Chinese A-share market data. Supports 9 K-line frequencies with dual backend (East Money / Sina) and auto-fallback. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
a11e1ceee1
commit
21378c4f6d
@@ -0,0 +1,58 @@
|
||||
"""Parquet partition path management.
|
||||
|
||||
Handles Hive-style partitioning paths: kline_{freq}/year=YYYY/month=MM/day=DD/
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
from ashare_dp.config import Settings
|
||||
from ashare_dp.core.models import Freq
|
||||
|
||||
settings = Settings()
|
||||
|
||||
|
||||
def partition_path(freq: Freq, d: date) -> Path:
|
||||
"""Build the Hive-partitioned directory path for a given frequency and date.
|
||||
|
||||
Returns: data/parquet/kline_{freq}/year=YYYY/month=MM/day=DD/
|
||||
"""
|
||||
return (
|
||||
Path(settings.parquet_dir)
|
||||
/ f"kline_{freq.storage_dir}"
|
||||
/ f"year={d.year}"
|
||||
/ f"month={d.month:02d}"
|
||||
/ f"day={d.day:02d}"
|
||||
)
|
||||
|
||||
|
||||
def partition_glob(freq: Freq) -> str:
|
||||
"""Glob pattern to match all Parquet files for a given frequency.
|
||||
|
||||
Returns: data/parquet/kline_{freq}/**/*.parquet
|
||||
"""
|
||||
return f"{settings.parquet_dir}/kline_{freq.storage_dir}/**/*.parquet"
|
||||
|
||||
|
||||
def ensure_partition_dir(freq: Freq, d: date) -> Path:
|
||||
"""Create the partition directory if it doesn't exist and return the path."""
|
||||
p = partition_path(freq, d)
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
def parse_partition_from_path(path: str) -> dict:
|
||||
"""Parse Hive partition keys from a file path.
|
||||
|
||||
E.g., 'data/parquet/kline_1d/year=2025/month=01/day=15/data.parquet'
|
||||
returns {'year': 2025, 'month': 1, 'day': 15}
|
||||
"""
|
||||
parts = Path(path).parts
|
||||
result = {}
|
||||
for part in parts:
|
||||
if "=" in part:
|
||||
key, val = part.split("=", 1)
|
||||
result[key] = int(val)
|
||||
return result
|
||||
Reference in New Issue
Block a user