From cc3a9b54e32113c4e7f3616d7e6350e432b1aa1c Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 4 May 2026 23:08:14 +0100 Subject: add logic to cache data download in get_latest_data() --- fnme/data.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) (limited to 'fnme/data.py') diff --git a/fnme/data.py b/fnme/data.py index bdec6df..17f30c5 100644 --- a/fnme/data.py +++ b/fnme/data.py @@ -1,13 +1,35 @@ -from io import StringIO +import os +from pathlib import Path from typing import Optional import pandas as pd import requests +from platformdirs import user_cache_path from fnme.constants import ENDPOINT, HEADERS -def get_latest_data() -> tuple[pd.DataFrame, Optional[str]]: - response = requests.get(ENDPOINT, headers=HEADERS, timeout=10) - response.raise_for_status() - return pd.read_csv(StringIO(response.text)), response.headers.get("Last-Modified") +def get_latest_data() -> tuple[pd.DataFrame, str | None]: + cache_dir = Path(user_cache_path(appname="fnme", appauthor=False)) + csv_path = cache_dir / "latest_data.csv" + timestamp_path = cache_dir / "timestamp.txt" + + cache_dir.mkdir(parents=True, exist_ok=True) + + remote_last_modified = requests.head( + ENDPOINT, headers=HEADERS, timeout=10 + ).headers.get("Last-Modified") + + cached_last_modified = ( + timestamp_path.read_text() if timestamp_path.exists() else None + ) + + if not csv_path.exists() or remote_last_modified != cached_last_modified: + response = requests.get(ENDPOINT, headers=HEADERS, timeout=10) + response.raise_for_status() + last_modified = response.headers.get("Last-Modified") + csv_path.write_text(response.text, encoding="utf-8") + timestamp_path.write_text(last_modified or "") + return pd.read_csv(csv_path), last_modified + + return pd.read_csv(csv_path), cached_last_modified -- cgit v1.2.3