shorten names due to filesystem length

httpxthrottlecache/filecache/transport.py

@@ -7,6 +7,7 @@
 import json
 import logging
 import os
+import re
 import time
 from email.utils import formatdate, parsedate_to_datetime
 from pathlib import Path
@@ -24,6 +25,8 @@
 # Held for the cache rule's TTL, like a 200.
 CACHED_REDIRECTS = frozenset({301, 302, 307, 308})
 
+MAX_READABLE_NAME = 120
+
 
 class AlreadyLockedError(Exception):
     pass
@@ -91,7 +94,9 @@ def to_path(self, host: str, path: str, query: str) -> Path:
         if query:
             name += "-" + unquote(query).replace("&", "-").replace("=", "-")
         digest = hashlib.blake2b(f"{path}?{query}".encode(), digest_size=4).hexdigest()
-        return self.cache_dir / site / (quote(name, safe="._-~") + "-" + digest)
+        # filesystem name limits; digest disambiguates
+        readable = re.sub(r"%[0-9A-Fa-f]?$", "", quote(name, safe="._-~")[:MAX_READABLE_NAME])
+        return self.cache_dir / site / (readable + "-" + digest)
 
     def get_if_fresh(
         self, host: str, path: str, query: str, cache_rules: dict[str, dict[str, Union[bool, int]]]

tests/test_long_url_cache.py

@@ -0,0 +1,33 @@
+import email.utils
+import time
+
+from httpxthrottlecache import httpx
+from httpxthrottlecache.filecache.transport import CachingTransport, FileCache
+
+LONG_QUERY = "&".join(f"param_{i}=value_{i}_" + "x" * 20 for i in range(15))
+
+
+def test_long_url_name_is_bounded(tmp_path):
+    p = FileCache(tmp_path, locking=False).to_path("a.test", "/feed", LONG_QUERY)
+    assert len(p.name) <= 150
+
+
+def test_long_urls_stay_distinct(tmp_path):
+    fc = FileCache(tmp_path, locking=False)
+    assert fc.to_path("a.test", "/feed", LONG_QUERY + "1") != fc.to_path("a.test", "/feed", LONG_QUERY + "2")
+
+
+def test_long_url_is_cached(tmp_path):
+    calls = []
+
+    def handler(req):
+        calls.append(req.url)
+        headers = {"Date": email.utils.formatdate(time.time(), usegmt=True)}
+        return httpx.Response(200, headers=headers, stream=httpx.ByteStream(b"body"), request=req)
+
+    transport = CachingTransport(tmp_path, {".*": {".*": 3600}}, transport=httpx.MockTransport(handler))
+    url = f"https://a.test/feed?{LONG_QUERY}"
+    with httpx.Client(transport=transport) as c:
+        assert c.get(url).content == b"body"
+        assert c.get(url).content == b"body"
+    assert len(calls) == 1