From ff969e0e189542b93304146677e452d33fc59d04 Mon Sep 17 00:00:00 2001 From: sebastianMindee <130448732+sebastianMindee@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:03:17 +0200 Subject: [PATCH] :rewind: revert URL safety checks --- mindee/mindee_http/response_validation.py | 52 +------------ tests/v1/input/test_url_validation.py | 92 ----------------------- 2 files changed, 1 insertion(+), 143 deletions(-) diff --git a/mindee/mindee_http/response_validation.py b/mindee/mindee_http/response_validation.py index 20061001..3a3eefba 100644 --- a/mindee/mindee_http/response_validation.py +++ b/mindee/mindee_http/response_validation.py @@ -1,4 +1,3 @@ -import ipaddress import json from urllib.parse import urlparse @@ -7,25 +6,12 @@ from mindee.error.mindee_error import MindeeSourceError from mindee.parsing.common.string_dict import StringDict -_CGNAT_BLOCK = ipaddress.IPv4Network("100.64.0.0/10") -_IPV6_UNIQUE_LOCAL = ipaddress.IPv6Network("fc00::/7") - def validate_url_for_source(url: str) -> None: """ Validates that a URL is safe to send to the Mindee server. - Rejects any URL that could be used for Server-Side Request Forgery (SSRF): - - - non-HTTPS schemes, - - embedded userinfo (e.g. ``https://user:pass@host``), - - loopback hostnames (``localhost``, ``*.localhost``), - - literal IP addresses that are loopback, link-local, private (RFC 1918), - any-local (``0.0.0.0``), multicast, IPv6 unique-local (``fc00::/7``), - or carrier-grade NAT (``100.64.0.0/10``). - - Note: DNS resolution is not performed. A hostname that resolves to a - private IP will not be caught here. + Rejects any URL that follow non-HTTPS schemes. :param url: The URL string to validate. :raises MindeeSourceError: If the URL fails any security check. @@ -38,42 +24,6 @@ def validate_url_for_source(url: str) -> None: if parsed.scheme.lower() != "https": raise MindeeSourceError("URL must be HTTPS") - if parsed.username or parsed.password: - raise MindeeSourceError("Source URLs must not embed user credentials") - - host = parsed.hostname - if not host: - raise MindeeSourceError("Source URL is missing a host") - - lower_host = host.lower() - if ( - lower_host == "localhost" - or lower_host.endswith(".localhost") - or lower_host == "ip6-localhost" - or lower_host == "ip6-loopback" - ): - raise MindeeSourceError(f"Loopback hostnames are not allowed: {host}") - - try: - addr = ipaddress.ip_address(lower_host) - except ValueError: - return - - if ( - addr.is_loopback - or addr.is_link_local - or addr.is_private - or addr.is_unspecified - or addr.is_multicast - ): - raise MindeeSourceError(f"URL host resolves to a disallowed address: {addr}") - - if isinstance(addr, ipaddress.IPv4Address) and addr in _CGNAT_BLOCK: - raise MindeeSourceError(f"URL host resolves to a disallowed address: {addr}") - - if isinstance(addr, ipaddress.IPv6Address) and addr in _IPV6_UNIQUE_LOCAL: - raise MindeeSourceError(f"URL host resolves to a disallowed address: {addr}") - def is_valid_sync_response(response: httpx.Response) -> bool: """ diff --git a/tests/v1/input/test_url_validation.py b/tests/v1/input/test_url_validation.py index ac87eb72..bb1fbb02 100644 --- a/tests/v1/input/test_url_validation.py +++ b/tests/v1/input/test_url_validation.py @@ -15,95 +15,3 @@ def test_rejects_ftp(self): def test_accepts_https(self): validate_url_for_source("https://example.com/file.pdf") - - -class TestValidateUrlUserinfo: - def test_rejects_username_and_password(self): - with pytest.raises(MindeeSourceError, match="credentials"): - validate_url_for_source("https://user:pass@example.com/file.pdf") - - def test_rejects_username_only(self): - with pytest.raises(MindeeSourceError, match="credentials"): - validate_url_for_source("https://user@example.com/file.pdf") - - -class TestValidateUrlLoopbackHostnames: - def test_rejects_localhost(self): - with pytest.raises(MindeeSourceError, match="Loopback"): - validate_url_for_source("https://localhost/file.pdf") - - def test_rejects_localhost_subdomain(self): - with pytest.raises(MindeeSourceError, match="Loopback"): - validate_url_for_source("https://myapp.localhost/file.pdf") - - def test_rejects_ip6_localhost(self): - with pytest.raises(MindeeSourceError, match="Loopback"): - validate_url_for_source("https://ip6-localhost/file.pdf") - - def test_rejects_ip6_loopback(self): - with pytest.raises(MindeeSourceError, match="Loopback"): - validate_url_for_source("https://ip6-loopback/file.pdf") - - -class TestValidateUrlLoopbackIPs: - def test_rejects_ipv4_loopback(self): - with pytest.raises(MindeeSourceError, match="disallowed"): - validate_url_for_source("https://127.0.0.1/file.pdf") - - def test_rejects_ipv4_loopback_other(self): - with pytest.raises(MindeeSourceError, match="disallowed"): - validate_url_for_source("https://127.0.0.2/file.pdf") - - def test_rejects_ipv6_loopback(self): - with pytest.raises(MindeeSourceError, match="disallowed"): - validate_url_for_source("https://[::1]/file.pdf") - - -class TestValidateUrlPrivateIPs: - def test_rejects_rfc1918_10_block(self): - with pytest.raises(MindeeSourceError, match="disallowed"): - validate_url_for_source("https://10.0.0.1/file.pdf") - - def test_rejects_rfc1918_172_block(self): - with pytest.raises(MindeeSourceError, match="disallowed"): - validate_url_for_source("https://172.16.0.1/file.pdf") - - def test_rejects_rfc1918_192_block(self): - with pytest.raises(MindeeSourceError, match="disallowed"): - validate_url_for_source("https://192.168.1.1/file.pdf") - - def test_rejects_link_local(self): - with pytest.raises(MindeeSourceError, match="disallowed"): - validate_url_for_source("https://169.254.0.1/file.pdf") - - def test_rejects_unspecified(self): - with pytest.raises(MindeeSourceError, match="disallowed"): - validate_url_for_source("https://0.0.0.0/file.pdf") - - def test_rejects_multicast(self): - with pytest.raises(MindeeSourceError, match="disallowed"): - validate_url_for_source("https://224.0.0.1/file.pdf") - - -class TestValidateUrlCgnat: - def test_rejects_cgnat_start(self): - with pytest.raises(MindeeSourceError, match="disallowed"): - validate_url_for_source("https://100.64.0.1/file.pdf") - - def test_rejects_cgnat_end(self): - with pytest.raises(MindeeSourceError, match="disallowed"): - validate_url_for_source("https://100.127.255.255/file.pdf") - - def test_accepts_just_outside_cgnat(self): - # 100.128.0.1 is outside 100.64.0.0/10 - validate_url_for_source("https://100.128.0.1/file.pdf") - - -class TestValidateUrlIpv6UniqueLocal: - def test_rejects_ipv6_ula_fc(self): - with pytest.raises(MindeeSourceError, match="disallowed"): - validate_url_for_source("https://[fc00::1]/file.pdf") - - def test_rejects_ipv6_ula_fd(self): - with pytest.raises(MindeeSourceError, match="disallowed"): - validate_url_for_source("https://[fd00::1]/file.pdf")