diff --git a/httpxthrottlecache/filecache/transport.py b/httpxthrottlecache/filecache/transport.py index acba781..e395ced 100644 --- a/httpxthrottlecache/filecache/transport.py +++ b/httpxthrottlecache/filecache/transport.py @@ -7,6 +7,7 @@ import json import logging import os +import re import time from email.utils import formatdate, parsedate_to_datetime from pathlib import Path @@ -24,6 +25,8 @@ # Held for the cache rule's TTL, like a 200. CACHED_REDIRECTS = frozenset({301, 302, 307, 308}) +MAX_READABLE_NAME = 120 + class AlreadyLockedError(Exception): pass @@ -91,7 +94,9 @@ def to_path(self, host: str, path: str, query: str) -> Path: if query: name += "-" + unquote(query).replace("&", "-").replace("=", "-") digest = hashlib.blake2b(f"{path}?{query}".encode(), digest_size=4).hexdigest() - return self.cache_dir / site / (quote(name, safe="._-~") + "-" + digest) + # filesystem name limits; digest disambiguates + readable = re.sub(r"%[0-9A-Fa-f]?$", "", quote(name, safe="._-~")[:MAX_READABLE_NAME]) + return self.cache_dir / site / (readable + "-" + digest) def get_if_fresh( self, host: str, path: str, query: str, cache_rules: dict[str, dict[str, Union[bool, int]]] diff --git a/tests/test_long_url_cache.py b/tests/test_long_url_cache.py new file mode 100644 index 0000000..0f7ec33 --- /dev/null +++ b/tests/test_long_url_cache.py @@ -0,0 +1,33 @@ +import email.utils +import time + +from httpxthrottlecache import httpx +from httpxthrottlecache.filecache.transport import CachingTransport, FileCache + +LONG_QUERY = "&".join(f"param_{i}=value_{i}_" + "x" * 20 for i in range(15)) + + +def test_long_url_name_is_bounded(tmp_path): + p = FileCache(tmp_path, locking=False).to_path("a.test", "/feed", LONG_QUERY) + assert len(p.name) <= 150 + + +def test_long_urls_stay_distinct(tmp_path): + fc = FileCache(tmp_path, locking=False) + assert fc.to_path("a.test", "/feed", LONG_QUERY + "1") != fc.to_path("a.test", "/feed", LONG_QUERY + "2") + + +def test_long_url_is_cached(tmp_path): + calls = [] + + def handler(req): + calls.append(req.url) + headers = {"Date": email.utils.formatdate(time.time(), usegmt=True)} + return httpx.Response(200, headers=headers, stream=httpx.ByteStream(b"body"), request=req) + + transport = CachingTransport(tmp_path, {".*": {".*": 3600}}, transport=httpx.MockTransport(handler)) + url = f"https://a.test/feed?{LONG_QUERY}" + with httpx.Client(transport=transport) as c: + assert c.get(url).content == b"body" + assert c.get(url).content == b"body" + assert len(calls) == 1