jackyu66gitandClaude Opus 4.6
21378c4f6d
Parquet + DuckDB storage with REST/WebSocket APIs for Chinese A-share market data. Supports 9 K-line frequencies with dual backend (East Money / Sina) and auto-fallback. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
59 lines
1.6 KiB
Python
59 lines
1.6 KiB
Python
"""Parquet partition path management.
|
|
|
|
Handles Hive-style partitioning paths: kline_{freq}/year=YYYY/month=MM/day=DD/
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import date
|
|
from pathlib import Path
|
|
|
|
from ashare_dp.config import Settings
|
|
from ashare_dp.core.models import Freq
|
|
|
|
settings = Settings()
|
|
|
|
|
|
def partition_path(freq: Freq, d: date) -> Path:
|
|
"""Build the Hive-partitioned directory path for a given frequency and date.
|
|
|
|
Returns: data/parquet/kline_{freq}/year=YYYY/month=MM/day=DD/
|
|
"""
|
|
return (
|
|
Path(settings.parquet_dir)
|
|
/ f"kline_{freq.storage_dir}"
|
|
/ f"year={d.year}"
|
|
/ f"month={d.month:02d}"
|
|
/ f"day={d.day:02d}"
|
|
)
|
|
|
|
|
|
def partition_glob(freq: Freq) -> str:
|
|
"""Glob pattern to match all Parquet files for a given frequency.
|
|
|
|
Returns: data/parquet/kline_{freq}/**/*.parquet
|
|
"""
|
|
return f"{settings.parquet_dir}/kline_{freq.storage_dir}/**/*.parquet"
|
|
|
|
|
|
def ensure_partition_dir(freq: Freq, d: date) -> Path:
|
|
"""Create the partition directory if it doesn't exist and return the path."""
|
|
p = partition_path(freq, d)
|
|
p.mkdir(parents=True, exist_ok=True)
|
|
return p
|
|
|
|
|
|
def parse_partition_from_path(path: str) -> dict:
|
|
"""Parse Hive partition keys from a file path.
|
|
|
|
E.g., 'data/parquet/kline_1d/year=2025/month=01/day=15/data.parquet'
|
|
returns {'year': 2025, 'month': 1, 'day': 15}
|
|
"""
|
|
parts = Path(path).parts
|
|
result = {}
|
|
for part in parts:
|
|
if "=" in part:
|
|
key, val = part.split("=", 1)
|
|
result[key] = int(val)
|
|
return result
|