Fix build: Bundle schwab_scraper source and use local dependencies
All checks were successful
Build and Push Docker Image / build (push) Successful in 34s
All checks were successful
Build and Push Docker Image / build (push) Successful in 34s
This commit is contained in:
0
schwab_scraper/storage/__init__.py
Normal file
0
schwab_scraper/storage/__init__.py
Normal file
74
schwab_scraper/storage/cache.py
Normal file
74
schwab_scraper/storage/cache.py
Normal file
@@ -0,0 +1,74 @@
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
CACHE_DIR = "data/morningstar_pdfs"
|
||||
TRANSACTION_CACHE_DIR = "data/transaction_csvs"
|
||||
|
||||
|
||||
def ensure_cache_dir() -> str:
|
||||
os.makedirs(CACHE_DIR, exist_ok=True)
|
||||
return CACHE_DIR
|
||||
|
||||
|
||||
def ensure_transaction_cache_dir() -> str:
|
||||
os.makedirs(TRANSACTION_CACHE_DIR, exist_ok=True)
|
||||
return TRANSACTION_CACHE_DIR
|
||||
|
||||
|
||||
def cache_filename(ticker: str, formatted_date: str) -> str:
|
||||
ensure_cache_dir()
|
||||
# Sanitize date string to remove slashes that would create subdirectories
|
||||
safe_date = formatted_date.replace('/', '_').replace('\\', '_')
|
||||
return os.path.join(CACHE_DIR, f"{ticker.upper()}_{safe_date}.pdf")
|
||||
|
||||
|
||||
def transaction_cache_filename(account_label: str, timestamp_str: str) -> str:
|
||||
"""Return a path like data/transaction_csvs/<account_label>/<account_label>_Transactions_<timestamp>.csv
|
||||
|
||||
account_label examples: "Joint_XXX604", "IRA_XXX873". Timestamp is usually YYYYMMDD-HHMMSS.
|
||||
"""
|
||||
ensure_transaction_cache_dir()
|
||||
safe_label = account_label.replace("/", "_")
|
||||
account_dir = os.path.join(TRANSACTION_CACHE_DIR, safe_label)
|
||||
os.makedirs(account_dir, exist_ok=True)
|
||||
return os.path.join(account_dir, f"{safe_label}_Transactions_{timestamp_str}.csv")
|
||||
|
||||
|
||||
def read_cached_pdf(ticker: str) -> Optional[bytes]:
|
||||
ensure_cache_dir()
|
||||
files = [f for f in os.listdir(CACHE_DIR) if f.startswith(ticker.upper()) and f.endswith(".pdf")]
|
||||
if not files:
|
||||
return None
|
||||
with open(os.path.join(CACHE_DIR, files[0]), "rb") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def read_cached_transaction_csv(account_label: str) -> Optional[bytes]:
|
||||
"""Return latest cached CSV bytes for an account label, if any."""
|
||||
ensure_transaction_cache_dir()
|
||||
safe_label = account_label.replace("/", "_")
|
||||
account_dir = os.path.join(TRANSACTION_CACHE_DIR, safe_label)
|
||||
if not os.path.isdir(account_dir):
|
||||
return None
|
||||
files = [f for f in os.listdir(account_dir) if f.endswith('.csv')]
|
||||
if not files:
|
||||
return None
|
||||
# Pick most recent by name (timestamp in filename)
|
||||
files.sort(reverse=True)
|
||||
with open(os.path.join(account_dir, files[0]), 'rb') as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def write_cached_pdf(ticker: str, formatted_date: str, pdf_bytes: bytes) -> str:
|
||||
ensure_cache_dir()
|
||||
path = cache_filename(ticker, formatted_date)
|
||||
with open(path, "wb") as f:
|
||||
f.write(pdf_bytes)
|
||||
return path
|
||||
|
||||
|
||||
def write_cached_transaction_csv(account_label: str, timestamp_str: str, csv_bytes: bytes) -> str:
|
||||
path = transaction_cache_filename(account_label, timestamp_str)
|
||||
with open(path, 'wb') as f:
|
||||
f.write(csv_bytes)
|
||||
return path
|
||||
Reference in New Issue
Block a user