From f37b250a368ceaa60485514c21dfbc5216f48d8f Mon Sep 17 00:00:00 2001 From: MudassarTariq Date: Fri, 6 Nov 2026 00:38:38 +0500 Subject: [PATCH 1/5] Update Python SDK request handling and prepare 3.0.1 --- README.md | 6 +- pyproject.toml | 6 +- src/ipgeolocation/_client_shared.py | 46 ++- src/ipgeolocation/_redaction.py | 95 ++++++ src/ipgeolocation/_response.py | 175 ++++++++++ src/ipgeolocation/_serde.py | 4 + src/ipgeolocation/_transport.py | 243 ++++++++------ src/ipgeolocation/async_client.py | 61 ++-- src/ipgeolocation/client.py | 61 ++-- src/ipgeolocation/config.py | 21 +- src/ipgeolocation/enums.py | 1 + src/ipgeolocation/models.py | 13 + tests/test_async_client.py | 2 +- tests/test_async_transport.py | 10 +- tests/test_client_error_mapping.py | 4 +- tests/test_client_parsing.py | 2 +- tests/test_client_request_building.py | 2 +- tests/test_http_integration.py | 2 +- tests/test_live_integration.py | 10 +- tests/test_request_protection.py | 446 ++++++++++++++++++++++++++ tests/test_transport.py | 4 + 21 files changed, 1025 insertions(+), 189 deletions(-) create mode 100644 src/ipgeolocation/_redaction.py create mode 100644 src/ipgeolocation/_response.py create mode 100644 tests/test_request_protection.py diff --git a/README.md b/README.md index a431422..f4f31b1 100644 --- a/README.md +++ b/README.md @@ -319,7 +319,7 @@ Differences from single lookup: `Content-Type: application/json` is always set b | `dma_code` | `location.dma_code` | | `*` | All optional modules available on your plan | -Supported `lang` values: `en`, `de`, `ru`, `ja`, `fr`, `cn`, `es`, `cs`, `it`, `ko`, `fa`, `pt` +Supported `lang` values: `en`, `de`, `ru`, `ja`, `fr`, `cn`, `es`, `cs`, `it`, `ko`, `fa`, `pt`, `ar` (Arabic) ## Response Metadata @@ -346,7 +346,7 @@ metadata.first_header_value("Header-Name") # first value or None ### JSON Serialization -Use `to_json()` or `to_pretty_json()` to serialize SDK objects: +Use `to_json()` or `to_pretty_json()` to print SDK objects as JSON. When you print your client settings, the SDK hides your API key. ```python from ipgeolocation import JsonOutputMode, to_pretty_json @@ -387,7 +387,7 @@ All exceptions inherit from `IpGeolocationException`. The SDK does not retry fai | `ServerErrorException` | 5xx | | `ApiException` | Any other non-2xx | -All API exceptions have `.status_code` and `.api_message` attributes. +All API exceptions have `.status_code` and `.api_message` attributes. If the error body is too large or the connection closes early, you still get the HTTP status. The SDK hides your API key in error messages, including errors for individual IPs in a typed bulk lookup. ## Links diff --git a/pyproject.toml b/pyproject.toml index be8e274..36202f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "ipgeolocationio" -version = "3.0.0" +version = "3.0.1" description = "Typed Python SDK for the IPGeolocation.io IP location API, with sync and async clients." readme = "README.md" license = "MIT" @@ -40,13 +40,15 @@ classifiers = [ dependencies = [ "httpx>=0.28.1,<1", "requests>=2.33.0,<3", - "urllib3>=2.8.0,<3", + "urllib3[zstd]>=2.8.0,<3", "anyio>=4.14.2,<5", "certifi>=2026.10.23", ] [project.optional-dependencies] dev = [ + "httpx[http2]>=0.28.1,<1", + "urllib3[brotli,zstd]>=2.8.0,<3", "pytest>=9.0.3,<10", "pytest-cov>=7.0.0,<8", "pyright>=1.1.408,<2", diff --git a/src/ipgeolocation/_client_shared.py b/src/ipgeolocation/_client_shared.py index 35bb62c..357fde1 100644 --- a/src/ipgeolocation/_client_shared.py +++ b/src/ipgeolocation/_client_shared.py @@ -6,6 +6,7 @@ from typing import Any, Dict, List, Mapping, Optional, Sequence, cast from urllib.parse import quote +from ._redaction import redact from ._serde import from_mapping from .enums import ResponseFormat from .exceptions import ( @@ -83,42 +84,47 @@ def resolve_user_agent_header( def parse_single_lookup(body: str) -> IpGeolocationResponse: try: payload = json.loads(body) - except json.JSONDecodeError as exc: + except (ValueError, RecursionError) as exc: raise SerializationException("Failed to deserialize API response") from exc if not isinstance(payload, dict): raise SerializationException("Failed to deserialize API response") try: parsed = from_mapping(IpGeolocationResponse, cast(Mapping[str, Any], payload)) - except (TypeError, ValueError) as exc: + except (TypeError, ValueError, OverflowError) as exc: raise SerializationException("Failed to deserialize API response") from exc if parsed is None: # pragma: no cover - defensive raise SerializationException("Failed to deserialize API response") return parsed -def parse_bulk_lookup(body: str) -> List[BulkLookupResult]: +def parse_bulk_lookup( + body: str, expected_count: int, api_key: Optional[str] +) -> List[BulkLookupResult]: try: payload = json.loads(body) - except json.JSONDecodeError as exc: + except (ValueError, RecursionError) as exc: raise SerializationException("Failed to deserialize bulk lookup response") from exc if not isinstance(payload, list): raise SerializationException( "Failed to deserialize bulk response: expected an array payload" ) + if len(payload) != expected_count: + raise SerializationException( + f"Expected {expected_count} bulk results, received {len(payload)}" + ) results: List[BulkLookupResult] = [] for item in payload: if isinstance(item, dict) and "message" in item and "ip" not in item: - results.append( - BulkLookupError( - error=BulkLookupErrorDetails(message=_string_or_none(item.get("message"))) - ) - ) + message = _string_or_none(item.get("message")) + if message is not None: + message = redact(message, api_key) + results.append(BulkLookupError(error=BulkLookupErrorDetails(message=message))) else: if not isinstance(item, Mapping): raise SerializationException("Failed to deserialize API response") try: parsed_item = from_mapping(IpGeolocationResponse, cast(Mapping[str, Any], item)) - except (TypeError, ValueError) as exc: + except (TypeError, ValueError, OverflowError) as exc: raise SerializationException("Failed to deserialize API response") from exc if parsed_item is None: # pragma: no cover - defensive raise SerializationException("Failed to deserialize API response") @@ -126,8 +132,8 @@ def parse_bulk_lookup(body: str) -> List[BulkLookupResult]: return results -def to_api_exception(status_code: int, body: str) -> ApiException: - api_message = extract_api_message(body) +def to_api_exception(status_code: int, body: str, api_key: Optional[str] = None) -> ApiException: + api_message = extract_api_message(body, api_key) if api_message: message = f"API request failed with HTTP status {status_code}: {api_message}" else: @@ -152,27 +158,33 @@ def to_api_exception(status_code: int, body: str) -> ApiException: return exception_type(message, status_code, api_message) -def extract_api_message(body: Optional[str]) -> Optional[str]: +def extract_api_message(body: Optional[str], api_key: Optional[str] = None) -> Optional[str]: if body is None or not str(body).strip(): return None try: payload = json.loads(body) except json.JSONDecodeError: - return body[:512] + return redact(body, api_key)[:512] + except (ValueError, RecursionError): + return None if isinstance(payload, Mapping): message = payload.get("message") if message is None: error = payload.get("error") if isinstance(error, Mapping): message = error.get("message") - return _string_or_none(message) - return body[:512] + text = _string_or_none(message) + return redact(text, api_key) if text is not None else None + return redact(body, api_key)[:512] def _string_or_none(value: Any) -> Optional[str]: if value is None: return None - return str(value) + try: + return str(value) + except (ValueError, RecursionError): + return None def parse_int_header(value: Optional[str]) -> Optional[int]: diff --git a/src/ipgeolocation/_redaction.py b/src/ipgeolocation/_redaction.py new file mode 100644 index 0000000..593b851 --- /dev/null +++ b/src/ipgeolocation/_redaction.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import json +import logging +import re +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Iterator, Optional +from urllib.parse import quote, quote_plus + +from .exceptions import ApiException, IpGeolocationException + +MASK = "[REDACTED]" +_active_key: ContextVar[Optional[str]] = ContextVar("ipgeolocation_api_key", default=None) +_percent_escape = re.compile(r"%[0-9a-fA-F]{2}") + + +def redact(text: str, key: Optional[str]) -> str: + if not key: + return text + for encoded in {quote(key, safe=""), quote_plus(key, safe="")} - {key}: + if "%" in encoded: + normalized = _percent_escape.sub(lambda m: m[0].upper(), text) + pieces: list[str] = [] + start = 0 + pos = normalized.find(encoded) + while pos >= 0: + pieces.extend((text[start:pos], MASK)) + start = pos + len(encoded) + pos = normalized.find(encoded, start) + if pieces: + text = "".join(pieces) + text[start:] + else: + text = text.replace(encoded, MASK) + for escaped in {json.dumps(key)[1:-1], json.dumps(key, ensure_ascii=False)[1:-1]} - {key}: + text = text.replace(escaped, MASK) + return text.replace(key, MASK) + + +class _RequestLogFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + key = _active_key.get() + if record.name.startswith("hpack."): + if key: + return False + if ( + record.name == "hpack.table" + and record.msg == "Evicting %s: %s from the header table" + and isinstance(record.args, tuple) + and len(record.args) == 2 + ): + record.args = (record.args[0], MASK) + if key: + message = record.getMessage() + safe = redact(message, key) + if safe != message: + record.msg, record.args = safe, () + return True + + +_log_filter = _RequestLogFilter() +for _name in ( + "hpack.hpack", + "hpack.table", + "httpx", + "httpcore.connection", + "httpcore.proxy", + "httpcore.socks", + "httpcore.http11", + "httpcore.http2", + "urllib3.connectionpool", + "urllib3.util.retry", + "urllib3.connection", +): + logging.getLogger(_name).addFilter(_log_filter) + + +@contextmanager +def protect_api_key(key: Optional[str]) -> Iterator[None]: + if key is None: + yield + return + token = _active_key.set(key) + try: + yield + except IpGeolocationException as exc: + if isinstance(exc, ApiException) and exc.api_message is not None: + exc.api_message = redact(exc.api_message, key) + exc.args = tuple(redact(arg, key) if isinstance(arg, str) else arg for arg in exc.args) + exc.__cause__ = None + exc.__context__ = None + exc.__suppress_context__ = True + raise + finally: + _active_key.reset(token) diff --git a/src/ipgeolocation/_response.py b/src/ipgeolocation/_response.py new file mode 100644 index 0000000..b95df62 --- /dev/null +++ b/src/ipgeolocation/_response.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import codecs +import zlib +from io import StringIO +from typing import Any, Iterator, Optional + +from urllib3 import response as urllib_response +from urllib3.response import HTTPResponse, MultiDecoder + +from .exceptions import TransportException + +CHUNK_SIZE = 8192 + + +class _ZlibStream: + def __init__(self, gzip: bool) -> None: + self.gzip = gzip + self.decoder: Any = zlib.decompressobj(31) if gzip else None + self.prefix = b"" + self.retry_header: bytearray | None = None + + def chunks(self, data: bytes, size: int) -> Iterator[bytes]: + if self.decoder is None: + data = self.prefix + data + if len(data) < 2: + self.prefix = data + return + self.prefix = b"" + header = int.from_bytes(data[:2], "big") + wrapped = data[0] & 15 == 8 and data[0] >> 4 <= 7 and header % 31 == 0 + self.decoder = zlib.decompressobj(15 if wrapped else -15) + if wrapped: + self.retry_header = bytearray() + if self.retry_header is not None: + if len(self.retry_header) + len(data) <= 65536: + self.retry_header.extend(data) + else: + self.retry_header = None + while True: + if self.decoder.eof: + if not self.gzip: + return + data = data.lstrip(b"\x00") + if not data: + return + self.decoder = zlib.decompressobj(31) + try: + piece = self.decoder.decompress(data, size) + except zlib.error: + if self.retry_header is None: + raise + self.decoder = zlib.decompressobj(-15) + data = bytes(self.retry_header) + self.retry_header = None + continue + if piece: + self.retry_header = None + yield piece + data = self.decoder.unused_data if self.decoder.eof else self.decoder.unconsumed_tail + if not data and len(piece) < size: + return + + def finish(self) -> bytes: + if self.decoder is None or not self.decoder.eof: + raise TransportException("Incomplete compressed response") + return b"" + + +def _bounded_brotli() -> bool: + module = getattr(urllib_response, "brotli", None) + if module is None: + return False + decoder = module.Decompressor() + process = getattr(decoder, "decompress", None) or getattr(decoder, "process", None) + if process is None: + return False + try: + process(b"", output_buffer_limit=1) + except (TypeError, AttributeError): + return False + return hasattr(decoder, "can_accept_more_data") and hasattr(decoder, "is_finished") + + +_ENCODINGS = ["gzip", "deflate"] +if "br" in HTTPResponse.CONTENT_DECODERS and _bounded_brotli(): + _ENCODINGS.append("br") +if "zstd" in HTTPResponse.CONTENT_DECODERS: + _ENCODINGS.append("zstd") +ACCEPT_ENCODING = ", ".join(_ENCODINGS) + + +class _OtherStream: + def __init__(self, coding: str) -> None: + self.brotli: Any = None + if coding == "br": + self.brotli = getattr(urllib_response, "brotli").Decompressor() + else: + self.decoder = MultiDecoder(coding) + + def chunks(self, data: bytes, size: int) -> Iterator[bytes]: + while True: + if self.brotli is None: + part = self.decoder.decompress(data, max_length=size) + done = not self.decoder.has_unconsumed_tail + else: + process = getattr(self.brotli, "decompress", None) or self.brotli.process + part = process(data, output_buffer_limit=size) + done = self.brotli.can_accept_more_data() + if part: + yield part + if done: + return + data = b"" + + def finish(self) -> bytes: + if self.brotli is not None: + if not self.brotli.is_finished(): + raise TransportException("Incomplete compressed response") + return b"" + return self.decoder.flush() + + +class ResponseBody: + def __init__(self, limit: int, charset: Optional[str], content_encoding: str) -> None: + self.limit = limit + self.size = 0 + self.text_buffer = StringIO() + self.chunk_size = min(CHUNK_SIZE, limit + 1) + try: + b" ".decode(charset or "utf-8", errors="replace") + self.text = codecs.getincrementaldecoder(charset or "utf-8")(errors="replace") + except (LookupError, ValueError): + self.text = codecs.getincrementaldecoder("utf-8")(errors="replace") + self.stages: list[_ZlibStream | _OtherStream] = [] + codings = [c.strip().lower() for c in content_encoding.split(",")] + if len(codings) > 5: + raise TransportException("Too many response encodings") + for coding in reversed(codings): + if coding in ("", "identity"): + continue + if coding in ("gzip", "x-gzip", "deflate"): + self.stages.append(_ZlibStream(coding != "deflate")) + elif coding in _ENCODINGS: + self.stages.append(_OtherStream(coding)) + else: + raise TransportException("Unsupported response encoding") + self.layer_sizes = [0] * (len(self.stages) + 1) + + def _append(self, value: str) -> None: + if value: + self.size += len(value) + if self.size > self.limit: + raise TransportException( + f"Response body exceeded max size of {self.limit} characters" + ) + self.text_buffer.write(value) + + def feed(self, data: bytes, layer: int = 0) -> None: + self.layer_sizes[layer] += len(data) + if self.layer_sizes[layer] > 4 * self.limit + 65536: + raise TransportException(f"Response body exceeded max size of {self.limit} characters") + if layer == len(self.stages): + self._append(self.text.decode(data)) + else: + for piece in self.stages[layer].chunks(data, self.chunk_size): + self.feed(piece, layer + 1) + + def finish(self) -> str: + for i, stage in enumerate(self.stages): + last = stage.finish() + if last: + self.feed(last, i + 1) + self._append(self.text.decode(b"", final=True)) + return self.text_buffer.getvalue() diff --git a/src/ipgeolocation/_serde.py b/src/ipgeolocation/_serde.py index 806db93..78cc3ac 100644 --- a/src/ipgeolocation/_serde.py +++ b/src/ipgeolocation/_serde.py @@ -26,6 +26,7 @@ get_type_hints, ) +from .config import IpGeolocationClientConfig from .enums import JsonOutputMode T = TypeVar("T") @@ -181,6 +182,9 @@ def to_plain_data(value: Any, mode: JsonOutputMode = JsonOutputMode.COMPACT) -> if isinstance(value, Enum): return value.value + if isinstance(value, IpGeolocationClientConfig): + return to_plain_data(value._for_output(), mode) + if is_dataclass(value): result = {} for field in fields(value): diff --git a/src/ipgeolocation/_transport.py b/src/ipgeolocation/_transport.py index 9d600a4..e759f34 100644 --- a/src/ipgeolocation/_transport.py +++ b/src/ipgeolocation/_transport.py @@ -2,13 +2,18 @@ from __future__ import annotations -import codecs import importlib +from contextlib import AsyncExitStack from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Protocol +from typing import Any, Dict, List, Mapping, Optional, Protocol import requests +from urllib3.exceptions import HTTPError as Urllib3Error +from urllib3.exceptions import ReadTimeoutError +from urllib3.response import HTTPResponse +from urllib3.util import parse_url +from ._response import ACCEPT_ENCODING, CHUNK_SIZE, ResponseBody from .exceptions import RequestTimeoutException, TransportException @@ -63,6 +68,49 @@ async def close(self) -> None: ... +def _same_origin(source: str, target: str) -> bool: + original, destination = parse_url(source), parse_url(target) + default = {"http": 80, "https": 443} + return ( + destination.auth is None + and original.scheme in default + and original.scheme == destination.scheme + and original.host == destination.host + and (original.port if original.port is not None else default[original.scheme]) + == (destination.port if destination.port is not None else default[original.scheme]) + ) + + +def _headers(request: HttpRequestData, defaults: Mapping[str, str]) -> Dict[str, str]: + headers = {k: ", ".join(v) for k, v in request.headers.items() if k and v} + if not any(name.lower() == "accept-encoding" for name in headers): + inherited = defaults.get("Accept-Encoding") + if inherited: + offered = [item.strip() for item in inherited.split(",")] + unsafe = {"br", "zstd"} - set(ACCEPT_ENCODING.split(", ")) + safe = [ + item for item in offered if item.partition(";")[0].strip().lower() not in unsafe + ] + if safe != offered: + headers["Accept-Encoding"] = ", ".join(safe) or "identity" + return headers + + +def _close_redirect(response: requests.Response, **kwargs: Any) -> requests.Response: + if response.is_redirect: + response.close() + return response + + +_BODY_ERRORS: tuple[type[BaseException], ...] = ( + TransportException, + ValueError, + LookupError, + Urllib3Error, + *HTTPResponse.DECODER_ERROR_CLASSES, +) + + class RequestsHttpTransport: """Requests-based HTTP transport with response-size protection.""" @@ -77,79 +125,79 @@ def __init__( raise ValueError("max_response_body_chars must be greater than zero") self._session = session or requests.Session() self._owns_session = session is None + if self._owns_session: + self._session.headers["Accept-Encoding"] = ACCEPT_ENCODING self._max_response_body_chars = max_response_body_chars def send(self, request: HttpRequestData) -> HttpResponseData: - header_map = { - key: values[0] if len(values) == 1 else ", ".join(values) - for key, values in request.headers.items() - if key is not None and values - } + timeout = (request.connect_timeout, request.read_timeout) + hooks = getattr(self._session, "hooks", {}).get("response") or () + if callable(hooks): + hooks = (hooks,) try: response = self._session.request( method=request.method, url=request.url, - headers=header_map, + headers=_headers(request, getattr(self._session, "headers", {})), data=request.body, - timeout=(request.connect_timeout, request.read_timeout), + timeout=timeout, stream=True, + allow_redirects=False, + hooks={"response": [_close_redirect, *hooks]}, ) - except requests.exceptions.Timeout as exc: + redirects = 0 + while True: + try: + next_request = response.next + if next_request is None: + return self._read_response(response) + target = next_request.url + if not target or not _same_origin(request.url, target): + return HttpResponseData( + response.status_code, "", _extract_headers(response) + ) + if redirects >= self._session.max_redirects: + raise TransportException("Too many redirects") + redirects += 1 + finally: + response.close() + options = self._session.merge_environment_settings(target, {}, True, None, None) + response = self._session.send( + next_request, timeout=timeout, allow_redirects=False, **options + ) + except (requests.exceptions.Timeout, ReadTimeoutError) as exc: raise RequestTimeoutException("HTTP request timed out") from exc - except requests.exceptions.RequestException as exc: + except (requests.exceptions.RequestException, Urllib3Error, ValueError) as exc: raise TransportException("HTTP transport error") from exc - try: - try: - body = self._read_body(response) - except requests.exceptions.Timeout as exc: - raise RequestTimeoutException("HTTP request timed out") from exc - except requests.exceptions.RequestException as exc: - raise TransportException("HTTP transport error") from exc - return HttpResponseData( - status_code=response.status_code, - body=body, - headers=_extract_headers(response), - ) - finally: - response.close() - def close(self) -> None: if self._owns_session: self._session.close() - def _read_body(self, response: requests.Response) -> str: - encoding = getattr(response, "encoding", None) or "utf-8" + def _read_response(self, response: requests.Response) -> HttpResponseData: + if response.status_code in (204, 205): + return HttpResponseData(response.status_code, "", _extract_headers(response)) try: - decoder = codecs.getincrementaldecoder(encoding)(errors="replace") - except LookupError: - decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") - body_parts: List[str] = [] - total_chars = 0 - - for chunk in response.iter_content(chunk_size=8192): - if not chunk: - continue - decoded = decoder.decode(chunk) - if decoded: - total_chars += len(decoded) - if total_chars > self._max_response_body_chars: - raise TransportException( - "Response body exceeded max size of " - f"{self._max_response_body_chars} characters" - ) - body_parts.append(decoded) - - final_chunk = decoder.decode(b"", final=True) - if final_chunk: - total_chars += len(final_chunk) - if total_chars > self._max_response_body_chars: - raise TransportException( - f"Response body exceeded max size of {self._max_response_body_chars} characters" - ) - body_parts.append(final_chunk) - - return "".join(body_parts) + raw_stream = getattr(response.raw, "stream", None) + if raw_stream is None or getattr(response, "_content_consumed", False): + chunks = response.iter_content(CHUNK_SIZE) + coding = "" + else: + chunks = raw_stream(CHUNK_SIZE, False) + coding = response.headers.get("Content-Encoding", "") + reader = ResponseBody(self._max_response_body_chars, response.encoding, coding) + for chunk in chunks: + reader.feed(chunk) + body = reader.finish() + except (requests.exceptions.RequestException,) + _BODY_ERRORS as exc: + if 200 <= response.status_code < 300: + if isinstance( + exc, (requests.exceptions.RequestException, Urllib3Error, TransportException) + ): + raise + raise TransportException("Invalid response body") from exc + body = "" + return HttpResponseData(response.status_code, body, _extract_headers(response)) def _extract_headers(response: requests.Response) -> Dict[str, List[str]]: @@ -194,55 +242,66 @@ async def send(self, request: HttpRequestData) -> HttpResponseData: httpx = _load_httpx() client = self._client if client is None: - client = httpx.AsyncClient() + client = httpx.AsyncClient(headers={"Accept-Encoding": ACCEPT_ENCODING}) self._client = client - header_map = { - key: values[0] if len(values) == 1 else ", ".join(values) - for key, values in request.headers.items() - if key is not None and values - } timeout = httpx.Timeout(request.read_timeout, connect=request.connect_timeout) - try: - async with client.stream( - method=request.method, - url=request.url, - headers=header_map, - content=request.body, - timeout=timeout, - ) as response: - body = await self._read_body(response) + async with ( + client.stream( + method=request.method, + url=request.url, + headers=_headers(request, getattr(client, "headers", {})), + content=request.body, + timeout=timeout, + follow_redirects=False, + ) as response, + AsyncExitStack() as stack, + ): + redirects = 0 + while getattr(client, "follow_redirects", False) and response.next_request: + next_request = response.next_request + if not _same_origin(request.url, str(next_request.url)): + return HttpResponseData( + response.status_code, "", _extract_httpx_headers(response.headers) + ) + if redirects >= client.max_redirects: + raise TransportException("Too many redirects") + redirects += 1 + await response.aclose() + response = await client.send(next_request, stream=True, follow_redirects=False) + stack.push_async_callback(response.aclose) + + if response.status_code in (204, 205): + return HttpResponseData( + response.status_code, "", _extract_httpx_headers(response.headers) + ) + try: + buffered = response.is_stream_consumed + coding = "" if buffered else response.headers.get("Content-Encoding", "") + reader = ResponseBody(self._max_response_body_chars, response.encoding, coding) + chunks = response.aiter_bytes(CHUNK_SIZE) if buffered else response.aiter_raw() + async for chunk in chunks: + reader.feed(chunk) + body = reader.finish() + except (httpx.HTTPError,) + _BODY_ERRORS as exc: + if 200 <= response.status_code < 300: + if isinstance(exc, (httpx.HTTPError, TransportException)): + raise + raise TransportException("Invalid response body") from exc + body = "" return HttpResponseData( - status_code=response.status_code, - body=body, - headers=_extract_httpx_headers(response.headers), + response.status_code, body, _extract_httpx_headers(response.headers) ) except httpx.TimeoutException as exc: raise RequestTimeoutException("HTTP request timed out") from exc - except httpx.HTTPError as exc: + except (httpx.HTTPError, httpx.InvalidURL, UnicodeError, Urllib3Error) as exc: raise TransportException("HTTP transport error") from exc async def close(self) -> None: if self._owns_client and self._client is not None: await self._client.aclose() - async def _read_body(self, response: Any) -> str: - body_parts: List[str] = [] - total_chars = 0 - - async for chunk in response.aiter_text(): - if not chunk: - continue - total_chars += len(chunk) - if total_chars > self._max_response_body_chars: - raise TransportException( - f"Response body exceeded max size of {self._max_response_body_chars} characters" - ) - body_parts.append(chunk) - - return "".join(body_parts) - def _extract_httpx_headers(headers: Any) -> Dict[str, List[str]]: result: Dict[str, List[str]] = {} diff --git a/src/ipgeolocation/async_client.py b/src/ipgeolocation/async_client.py index 2354952..7096751 100644 --- a/src/ipgeolocation/async_client.py +++ b/src/ipgeolocation/async_client.py @@ -16,6 +16,7 @@ to_metadata, validate_json_output, ) +from ._redaction import protect_api_key from ._transport import AsyncHttpTransport, HttpRequestData, HttpxAsyncTransport from ._version import get_version from .config import IpGeolocationClientConfig @@ -59,13 +60,14 @@ async def lookup_ip_geolocation( typed_request = cast(LookupIpGeolocationRequest, request) validate_json_output(cast(Optional[ResponseFormat], typed_request.output)) http_request = self._build_lookup_http_request(typed_request) - response, duration_ms = await self._execute_with_metrics(http_request) - if response.status_code // 100 != 2: - raise to_api_exception(response.status_code, response.body) - return ApiResponse( - data=parse_single_lookup(response.body), - metadata=to_metadata(response.status_code, duration_ms, response.headers), - ) + with protect_api_key(self._config.api_key): + response, duration_ms = await self._execute_with_metrics(http_request) + if response.status_code // 100 != 2: + raise to_api_exception(response.status_code, response.body, self._config.api_key) + return ApiResponse( + data=parse_single_lookup(response.body), + metadata=to_metadata(response.status_code, duration_ms, response.headers), + ) async def lookup_ip_geolocation_raw( self, @@ -76,13 +78,14 @@ async def lookup_ip_geolocation_raw( self._validate_lookup_request_base(request) typed_request = cast(LookupIpGeolocationRequest, request) http_request = self._build_lookup_http_request(typed_request) - response, duration_ms = await self._execute_with_metrics(http_request) - if response.status_code // 100 != 2: - raise to_api_exception(response.status_code, response.body) - return ApiResponse( - data=response.body, - metadata=to_metadata(response.status_code, duration_ms, response.headers), - ) + with protect_api_key(self._config.api_key): + response, duration_ms = await self._execute_with_metrics(http_request) + if response.status_code // 100 != 2: + raise to_api_exception(response.status_code, response.body, self._config.api_key) + return ApiResponse( + data=response.body, + metadata=to_metadata(response.status_code, duration_ms, response.headers), + ) async def bulk_lookup_ip_geolocation( self, @@ -94,13 +97,14 @@ async def bulk_lookup_ip_geolocation( typed_request = cast(BulkLookupIpGeolocationRequest, request) validate_json_output(cast(Optional[ResponseFormat], typed_request.output)) http_request = self._build_bulk_http_request(typed_request) - response, duration_ms = await self._execute_with_metrics(http_request) - if response.status_code // 100 != 2: - raise to_api_exception(response.status_code, response.body) - return ApiResponse( - data=parse_bulk_lookup(response.body), - metadata=to_metadata(response.status_code, duration_ms, response.headers), - ) + with protect_api_key(self._config.api_key): + response, duration_ms = await self._execute_with_metrics(http_request) + if response.status_code // 100 != 2: + raise to_api_exception(response.status_code, response.body, self._config.api_key) + return ApiResponse( + data=parse_bulk_lookup(response.body, len(typed_request.ips), self._config.api_key), + metadata=to_metadata(response.status_code, duration_ms, response.headers), + ) async def bulk_lookup_ip_geolocation_raw( self, @@ -111,13 +115,14 @@ async def bulk_lookup_ip_geolocation_raw( self._validate_bulk_request_base(request) typed_request = cast(BulkLookupIpGeolocationRequest, request) http_request = self._build_bulk_http_request(typed_request) - response, duration_ms = await self._execute_with_metrics(http_request) - if response.status_code // 100 != 2: - raise to_api_exception(response.status_code, response.body) - return ApiResponse( - data=response.body, - metadata=to_metadata(response.status_code, duration_ms, response.headers), - ) + with protect_api_key(self._config.api_key): + response, duration_ms = await self._execute_with_metrics(http_request) + if response.status_code // 100 != 2: + raise to_api_exception(response.status_code, response.body, self._config.api_key) + return ApiResponse( + data=response.body, + metadata=to_metadata(response.status_code, duration_ms, response.headers), + ) async def aclose(self) -> None: """Close the underlying HTTP transport when this client created it.""" diff --git a/src/ipgeolocation/client.py b/src/ipgeolocation/client.py index 1072b63..bf7a414 100644 --- a/src/ipgeolocation/client.py +++ b/src/ipgeolocation/client.py @@ -16,6 +16,7 @@ to_metadata, validate_json_output, ) +from ._redaction import protect_api_key from ._transport import HttpRequestData, HttpTransport, RequestsHttpTransport from ._version import get_version from .config import IpGeolocationClientConfig @@ -59,13 +60,14 @@ def lookup_ip_geolocation( typed_request = cast(LookupIpGeolocationRequest, request) validate_json_output(cast(Optional[ResponseFormat], typed_request.output)) http_request = self._build_lookup_http_request(typed_request) - response, duration_ms = self._execute_with_metrics(http_request) - if response.status_code // 100 != 2: - raise to_api_exception(response.status_code, response.body) - return ApiResponse( - data=parse_single_lookup(response.body), - metadata=to_metadata(response.status_code, duration_ms, response.headers), - ) + with protect_api_key(self._config.api_key): + response, duration_ms = self._execute_with_metrics(http_request) + if response.status_code // 100 != 2: + raise to_api_exception(response.status_code, response.body, self._config.api_key) + return ApiResponse( + data=parse_single_lookup(response.body), + metadata=to_metadata(response.status_code, duration_ms, response.headers), + ) def lookup_ip_geolocation_raw( self, @@ -76,13 +78,14 @@ def lookup_ip_geolocation_raw( self._validate_lookup_request_base(request) typed_request = cast(LookupIpGeolocationRequest, request) http_request = self._build_lookup_http_request(typed_request) - response, duration_ms = self._execute_with_metrics(http_request) - if response.status_code // 100 != 2: - raise to_api_exception(response.status_code, response.body) - return ApiResponse( - data=response.body, - metadata=to_metadata(response.status_code, duration_ms, response.headers), - ) + with protect_api_key(self._config.api_key): + response, duration_ms = self._execute_with_metrics(http_request) + if response.status_code // 100 != 2: + raise to_api_exception(response.status_code, response.body, self._config.api_key) + return ApiResponse( + data=response.body, + metadata=to_metadata(response.status_code, duration_ms, response.headers), + ) def bulk_lookup_ip_geolocation( self, @@ -94,13 +97,14 @@ def bulk_lookup_ip_geolocation( typed_request = cast(BulkLookupIpGeolocationRequest, request) validate_json_output(cast(Optional[ResponseFormat], typed_request.output)) http_request = self._build_bulk_http_request(typed_request) - response, duration_ms = self._execute_with_metrics(http_request) - if response.status_code // 100 != 2: - raise to_api_exception(response.status_code, response.body) - return ApiResponse( - data=parse_bulk_lookup(response.body), - metadata=to_metadata(response.status_code, duration_ms, response.headers), - ) + with protect_api_key(self._config.api_key): + response, duration_ms = self._execute_with_metrics(http_request) + if response.status_code // 100 != 2: + raise to_api_exception(response.status_code, response.body, self._config.api_key) + return ApiResponse( + data=parse_bulk_lookup(response.body, len(typed_request.ips), self._config.api_key), + metadata=to_metadata(response.status_code, duration_ms, response.headers), + ) def bulk_lookup_ip_geolocation_raw( self, @@ -111,13 +115,14 @@ def bulk_lookup_ip_geolocation_raw( self._validate_bulk_request_base(request) typed_request = cast(BulkLookupIpGeolocationRequest, request) http_request = self._build_bulk_http_request(typed_request) - response, duration_ms = self._execute_with_metrics(http_request) - if response.status_code // 100 != 2: - raise to_api_exception(response.status_code, response.body) - return ApiResponse( - data=response.body, - metadata=to_metadata(response.status_code, duration_ms, response.headers), - ) + with protect_api_key(self._config.api_key): + response, duration_ms = self._execute_with_metrics(http_request) + if response.status_code // 100 != 2: + raise to_api_exception(response.status_code, response.body, self._config.api_key) + return ApiResponse( + data=response.body, + metadata=to_metadata(response.status_code, duration_ms, response.headers), + ) def close(self) -> None: """Close the underlying HTTP transport when this client created it.""" diff --git a/src/ipgeolocation/config.py b/src/ipgeolocation/config.py index c09ea66..1ade8c0 100644 --- a/src/ipgeolocation/config.py +++ b/src/ipgeolocation/config.py @@ -3,11 +3,13 @@ from __future__ import annotations import math -from dataclasses import dataclass +from dataclasses import dataclass, fields from datetime import timedelta -from typing import Optional, Union +from typing import Dict, Optional, Union from urllib.parse import urlparse +from ._redaction import MASK, redact + TimeoutValue = Union[float, int, timedelta] @@ -53,7 +55,7 @@ def _normalize_request_origin(value: Optional[str]) -> Optional[str]: return f"{parsed_origin.scheme}://{parsed_origin.netloc}" -@dataclass(frozen=True) +@dataclass(frozen=True, repr=False) class IpGeolocationClientConfig: """Immutable configuration for :class:`IpGeolocationClient`.""" @@ -63,6 +65,19 @@ class IpGeolocationClientConfig: connect_timeout: TimeoutValue = 10.0 read_timeout: TimeoutValue = 30.0 + def _for_output(self) -> Dict[str, object]: + result = {} + for field in fields(self): + value = getattr(self, field.name) + if isinstance(value, str): + value = MASK if field.name == "api_key" else redact(value, self.api_key) + result[field.name] = value + return result + + def __repr__(self) -> str: + settings = ", ".join(f"{name}={value!r}" for name, value in self._for_output().items()) + return f"{type(self).__name__}({settings})" + def __post_init__(self) -> None: if self.api_key is not None and not isinstance(self.api_key, str): raise TypeError("api_key must be a string") diff --git a/src/ipgeolocation/enums.py b/src/ipgeolocation/enums.py index ed0c7c5..df26ed5 100644 --- a/src/ipgeolocation/enums.py +++ b/src/ipgeolocation/enums.py @@ -20,6 +20,7 @@ class Language(str, Enum): KO = "ko" FA = "fa" PT = "pt" + AR = "ar" @classmethod def from_code(cls, code: str) -> "Language": diff --git a/src/ipgeolocation/models.py b/src/ipgeolocation/models.py index 1c58138..29114f2 100644 --- a/src/ipgeolocation/models.py +++ b/src/ipgeolocation/models.py @@ -166,6 +166,8 @@ class Network: connection_type: Optional[str] = None route: Optional[str] = None is_anycast: Optional[bool] = None + is_cdn: Optional[bool] = None + cdn_provider_name: Optional[str] = None @dataclass(frozen=True) @@ -191,6 +193,17 @@ class Security: is_spam: Optional[bool] = None is_cloud_provider: Optional[bool] = None cloud_provider_name: Optional[str] = None + is_known_good_bot: Optional[bool] = None + is_corporate_gateway: Optional[bool] = None + proxy_days_seen_30d: Optional[int] = None + proxy_days_seen_7d: Optional[int] = None + proxy_provider_count_30d: Optional[int] = None + bot_confidence_score: Optional[float] = None + bot_operator_name: Optional[str] = None + bot_type: Optional[str] = None + bot_last_seen: Optional[str] = None + corporate_gateway_type: Optional[str] = None + corporate_gateway_provider_name: Optional[str] = None @dataclass(frozen=True) diff --git a/tests/test_async_client.py b/tests/test_async_client.py index 189623c..f313aba 100644 --- a/tests/test_async_client.py +++ b/tests/test_async_client.py @@ -259,7 +259,7 @@ async def scenario() -> None: transport.enqueue_response(401, '{"message":"invalid key"}') client = AsyncIpGeolocationClient( - IpGeolocationClientConfig(api_key="k"), + IpGeolocationClientConfig(api_key="test-credential"), transport=transport, ) with pytest.raises(UnauthorizedException) as exc: diff --git a/tests/test_async_transport.py b/tests/test_async_transport.py index e2996f8..a5d9596 100644 --- a/tests/test_async_transport.py +++ b/tests/test_async_transport.py @@ -26,6 +26,9 @@ def multi_items(self): class StubAsyncResponse: + is_stream_consumed = True + encoding = "utf-8" + def __init__( self, status_code: int = 200, @@ -38,11 +41,11 @@ def __init__( self.headers = StubAsyncHeaders(headers or {}) self.iter_exception = iter_exception - async def aiter_text(self): + async def aiter_bytes(self, chunk_size): if self.iter_exception is not None: raise self.iter_exception for chunk in self._chunks: - yield chunk + yield chunk.encode("utf-8") class StubAsyncStream: @@ -77,6 +80,9 @@ class FakeHttpxModule: class HTTPError(Exception): pass + class InvalidURL(Exception): + pass + class TimeoutException(HTTPError): pass diff --git a/tests/test_client_error_mapping.py b/tests/test_client_error_mapping.py index 7e53501..8ccae3f 100644 --- a/tests/test_client_error_mapping.py +++ b/tests/test_client_error_mapping.py @@ -53,7 +53,7 @@ def test_preserves_api_error_details() -> None: transport = TestTransport() transport.enqueue_response(401, '{"message":"invalid key"}') - client = IpGeolocationClient(IpGeolocationClientConfig(api_key="k"), transport=transport) + client = IpGeolocationClient(IpGeolocationClientConfig(api_key="demo"), transport=transport) with pytest.raises(ApiException) as exc: client.lookup_ip_geolocation(LookupIpGeolocationRequest()) @@ -65,7 +65,7 @@ def test_preserves_nested_api_error_details() -> None: transport = TestTransport() transport.enqueue_response(401, '{"error":{"message":"invalid key"}}') - client = IpGeolocationClient(IpGeolocationClientConfig(api_key="k"), transport=transport) + client = IpGeolocationClient(IpGeolocationClientConfig(api_key="demo"), transport=transport) with pytest.raises(ApiException) as exc: client.lookup_ip_geolocation(LookupIpGeolocationRequest()) diff --git a/tests/test_client_parsing.py b/tests/test_client_parsing.py index 3371c40..7c96b08 100644 --- a/tests/test_client_parsing.py +++ b/tests/test_client_parsing.py @@ -119,7 +119,7 @@ def test_bulk_typed_response_rejects_malformed_json() -> None: transport = TestTransport() transport.enqueue_response(200, "[{") - client = IpGeolocationClient(IpGeolocationClientConfig(api_key="k"), transport=transport) + client = IpGeolocationClient(IpGeolocationClientConfig(api_key="demo"), transport=transport) with pytest.raises(SerializationException, match="Failed to deserialize bulk lookup response"): client.bulk_lookup_ip_geolocation(BulkLookupIpGeolocationRequest(ips=["8.8.8.8"])) diff --git a/tests/test_client_request_building.py b/tests/test_client_request_building.py index 0c19695..481f285 100644 --- a/tests/test_client_request_building.py +++ b/tests/test_client_request_building.py @@ -56,7 +56,7 @@ def test_lookup_builds_expected_query_and_headers() -> None: def test_bulk_lookup_builds_payload_and_path() -> None: transport = TestTransport() - transport.enqueue_response(200, '[{"ip":"8.8.8.8"}]') + transport.enqueue_response(200, '[{"ip":"8.8.8.8"},{"ip":"1.1.1.1"}]') client = IpGeolocationClient(IpGeolocationClientConfig(api_key="bulk-key"), transport=transport) client.bulk_lookup_ip_geolocation(BulkLookupIpGeolocationRequest(ips=["8.8.8.8", "1.1.1.1"])) diff --git a/tests/test_http_integration.py b/tests/test_http_integration.py index abd6d4f..1ebaf15 100644 --- a/tests/test_http_integration.py +++ b/tests/test_http_integration.py @@ -185,7 +185,7 @@ def test_http_errors(client_type, api_url, key, exception, status): api_key=key, ) assert error.value.status_code == status - assert error.value.api_message == f"Test failure for {key}" + assert error.value.api_message == "Test failure for [REDACTED]" def test_http_invalid_json(client_type, api_url): diff --git a/tests/test_live_integration.py b/tests/test_live_integration.py index 44197e8..2198d7c 100644 --- a/tests/test_live_integration.py +++ b/tests/test_live_integration.py @@ -96,12 +96,6 @@ def test_free_plan_include_star_returns_default_response_without_additional_modu assert response.data.hostname is None -def test_free_plan_bulk_is_rejected() -> None: - with IpGeolocationClient(IpGeolocationClientConfig(api_key=FREE_KEY)) as client: - with pytest.raises(UnauthorizedException): - client.bulk_lookup_ip_geolocation(BulkLookupIpGeolocationRequest(ips=["8.8.8.8"])) - - def test_paid_plan_security_and_abuse_works() -> None: with IpGeolocationClient(IpGeolocationClientConfig(api_key=PAID_KEY)) as client: response = client.lookup_ip_geolocation( @@ -119,8 +113,8 @@ def test_paid_plan_optional_module_fields_still_require_include(field_name: str) LookupIpGeolocationRequest(ip="8.8.8.8", fields=[field_name]) ) assert getattr(response.data, field_name) is None - assert response.data.location is not None - assert response.metadata.credits_charged == 1 + assert response.data.ip == "8.8.8.8" + assert response.data.location is None def test_paid_plan_bulk_mixed_returns_success_and_error_items() -> None: diff --git a/tests/test_request_protection.py b/tests/test_request_protection.py new file mode 100644 index 0000000..2012a53 --- /dev/null +++ b/tests/test_request_protection.py @@ -0,0 +1,446 @@ +import asyncio +import gzip +import json +import logging +import secrets +import threading +import traceback +import zlib +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, quote, urlsplit + +import httpx +import pytest + +from ipgeolocation import ( + ApiException, + AsyncIpGeolocationClient, + BulkLookupIpGeolocationRequest, + IpGeolocationClient, + IpGeolocationClientConfig, + JsonOutputMode, + Language, + LookupIpGeolocationRequest, + RateLimitException, + SerializationException, + TransportException, + to_json, + to_pretty_json, +) +from ipgeolocation._transport import HttpxAsyncTransport, RequestsHttpTransport + + +@contextmanager +def server(answer): + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + data = self.rfile.read(int(self.headers.get("Content-Length", 0))) + status, headers, body = answer(self.command, urlsplit(self.path), self.headers, data) + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + for name, value in headers.items(): + self.send_header(name, value) + self.end_headers() + self.wfile.write(body) + + do_POST = do_GET + + def log_message(self, *args): + pass + + with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as listener: + thread = threading.Thread(target=listener.serve_forever, kwargs={"poll_interval": 0.01}) + thread.start() + try: + yield f"http://127.0.0.1:{listener.server_port}" + finally: + listener.shutdown() + thread.join() + + +@pytest.fixture(params=["sync", "async"]) +def call(request): + def run(url, method, lookup, *, key="example", limit=128 * 1024 * 1024, follow=False): + config = IpGeolocationClientConfig(api_key=key, base_url=url) + if request.param == "sync": + transport = RequestsHttpTransport(max_response_body_chars=limit) + try: + with IpGeolocationClient(config, transport) as client: + return getattr(client, method)(lookup) + finally: + transport.close() + + async def run_async(): + async with httpx.AsyncClient(follow_redirects=follow, trust_env=False) as http: + transport = HttpxAsyncTransport(http, max_response_body_chars=limit) + async with AsyncIpGeolocationClient(config, transport) as client: + return await getattr(client, method)(lookup) + + return asyncio.run(run_async()) + + return run + + +def test_config_output_masks_key_and_keeps_authentication_value(): + key = secrets.token_hex(12) + config = IpGeolocationClientConfig(api_key=key, base_url="https://example.test/" + key) + assert config.api_key == key + assert key not in repr(config) + for dump in (to_json, to_pretty_json): + for mode in JsonOutputMode: + text = dump([config], mode) + assert key not in text + assert json.loads(text)[0]["api_key"] == "[REDACTED]" + + +def test_logs_and_api_errors_hide_the_configured_key(call, caplog): + key = secrets.token_hex(12) + " /+" + encoded = quote(key, safe="") + caplog.set_level(logging.DEBUG) + + def respond(method, url, headers, body): + assert parse_qs(url.query)["apiKey"] == [key] + return 401, {}, json.dumps({"message": f"denied {key} {encoded}"}).encode() + + with server(respond) as url: + with pytest.raises(ApiException) as error: + call(url, "lookup_ip_geolocation", LookupIpGeolocationRequest(), key=key) + assert error.value.status_code == 401 + assert error.value.api_message == "denied [REDACTED] [REDACTED]" + assert key not in caplog.text and encoded not in caplog.text + + +def test_connection_error_drops_request_url(call): + key = secrets.token_hex(12) + with server(lambda *_: (200, {}, b"{}")) as url: + pass + with pytest.raises(TransportException) as error: + call(url, "lookup_ip_geolocation", LookupIpGeolocationRequest(), key=key) + assert key not in "".join(traceback.format_exception(error.value)) + assert error.value.__context__ is None + assert error.value.__cause__ is None + + +@pytest.mark.parametrize("status", [307, 308]) +def test_cross_origin_redirect_never_replays_bulk_data_or_headers(call, status): + delivered = [] + + def destination(*args): + delivered.append(args) + return 200, {}, b"[]" + + with server(destination) as second: + with server(lambda *_: (status, {"Location": second + "/bulk"}, b"")) as first: + req = BulkLookupIpGeolocationRequest(ips=["192.0.2.1"], headers={"X-Private": "test"}) + with pytest.raises(ApiException) as error: + call(first, "bulk_lookup_ip_geolocation", req, follow=True) + assert error.value.status_code == status + assert not delivered + + +@pytest.mark.parametrize(("status", "method"), [(302, "GET"), (307, "POST")]) +def test_same_origin_redirect_preserves_method_rules_and_skips_body(call, status, method): + def respond(actual, url, headers, body): + if url.path.endswith("ipgeo-bulk"): + return ( + status, + {"Location": "/final", "Content-Encoding": "gzip"}, + gzip.compress(b"x" * 65536), + ) + assert actual == method and headers["X-Trace"] == "retained" + assert json.loads(body)["ips"] == ["192.0.2.1"] if body else method == "GET" + return 200, {}, b'[{"ip":"192.0.2.1"}]' + + with server(respond) as url: + req = BulkLookupIpGeolocationRequest(ips=["192.0.2.1"], headers={"X-Trace": "retained"}) + result = call(url, "bulk_lookup_ip_geolocation", req, limit=32, follow=True) + assert result.data[0].data.ip == "192.0.2.1" + + +@pytest.mark.parametrize("mode", ["oversized", "truncated", "invalid-gzip"]) +def test_unreadable_error_preserves_status(call, mode): + cases = { + "oversized": ({"Content-Encoding": "gzip"}, gzip.compress(b"x" * 100_000)), + "truncated": ({"Content-Length": "200"}, b'{"message":'), + "invalid-gzip": ({"Content-Encoding": "gzip"}, b"invalid data"), + } + headers, body = cases[mode] + with server(lambda *_: (429, headers, body)) as url: + with pytest.raises(RateLimitException) as error: + call(url, "lookup_ip_geolocation", LookupIpGeolocationRequest(), limit=64) + assert error.value.status_code == 429 and error.value.api_message is None + + +@pytest.mark.parametrize("diff", [-1, 1]) +def test_typed_bulk_rejects_wrong_count_but_raw_returns_original(call, diff): + rows = json.dumps([{"ip": "192.0.2.1"}] * (2 + diff)) + req = BulkLookupIpGeolocationRequest(ips=["192.0.2.1", "192.0.2.1"]) + with server(lambda *_: (200, {}, rows.encode())) as url: + with pytest.raises(SerializationException, match="Expected 2 bulk results"): + call(url, "bulk_lookup_ip_geolocation", req) + assert call(url, "bulk_lookup_ip_geolocation_raw", req).data == rows + + +def test_bulk_hides_echoed_key_in_errors_only(call): + key = secrets.token_hex(12) + rows = json.dumps([{"message": "denied " + key}, {"ip": "192.0.2.1", "hostname": key}]) + req = BulkLookupIpGeolocationRequest(ips=["invalid", "192.0.2.1"]) + with server(lambda *_: (200, {}, rows.encode())) as url: + typed = call(url, "bulk_lookup_ip_geolocation", req, key=key).data + assert typed[0].error.message == "denied [REDACTED]" + assert typed[1].data.hostname == key + assert call(url, "bulk_lookup_ip_geolocation_raw", req, key=key).data == rows + + +def test_arabic_single_and_full_size_bulk_lookup(call): + city = "الرياض" + + def respond(method, url, headers, body): + query = parse_qs(url.query) + assert query["lang"] == ["ar"] + if method == "POST": + rows = [{"ip": ip} for ip in json.loads(body)["ips"]] + else: + rows = { + "ip": query["ip"][0], + "location": {"city": city}, + "network": {"is_cdn": True}, + "security": { + "bot_operator_name": "test", + "proxy_days_seen_7d": 2, + "is_corporate_gateway": False, + }, + } + return ( + 200, + {"Content-Encoding": "gzip"}, + gzip.compress(json.dumps(rows, ensure_ascii=False).encode()), + ) + + ips = [f"192.0.{i // 256}.{i % 256}" for i in range(50_000)] + with server(respond) as url: + single = call( + url, "lookup_ip_geolocation", LookupIpGeolocationRequest(ip=ips[0], lang=Language.AR) + ) + assert single.data.location.city == city and single.data.network.is_cdn is True + assert single.data.security.bot_operator_name == "test" + assert single.data.security.proxy_days_seen_7d == 2 + assert single.data.security.is_corporate_gateway is False + bulk = call( + url, "bulk_lookup_ip_geolocation", BulkLookupIpGeolocationRequest(ips=ips, lang="ar") + ) + assert [item.data.ip for item in bulk.data] == ips + + +@pytest.mark.parametrize("coding", ["gzip", "deflate", "raw-deflate", "gzip, deflate"]) +def test_compressed_text_limit_counts_characters(call, coding): + body = "ع" * 125 + data = body.encode() + if coding == "gzip": + wire = gzip.compress(data[:1]) + gzip.compress(data[1:]) + elif coding == "raw-deflate": + encoder = zlib.compressobj(wbits=-15) + wire = encoder.compress(data) + encoder.flush() + elif coding == "gzip, deflate": + wire = zlib.compress(gzip.compress(data)) + else: + wire = zlib.compress(data) + headers = {"Content-Encoding": coding.replace("raw-", "")} + with server(lambda *_: (200, headers, wire)) as url: + req = LookupIpGeolocationRequest() + assert call(url, "lookup_ip_geolocation_raw", req, limit=125).data == body + with pytest.raises(TransportException, match="exceeded"): + call(url, "lookup_ip_geolocation_raw", req, limit=124) + + +def test_deflate_header_and_multibyte_text_can_cross_chunks(): + from ipgeolocation._response import ResponseBody + + text = "مرحبا" + reader = ResponseBody(len(text), "utf-8", "deflate") + for byte in zlib.compress(text.encode()): + reader.feed(bytes([byte])) + assert reader.finish() == text + + +@pytest.mark.parametrize( + ("coding", "wire", "error"), + [ + ("gzip", gzip.compress(b"test")[:-3], "Incomplete"), + ("deflate", b"x", "Incomplete"), + ("bz2", b"", "Unsupported"), + ("gzip," * 5 + "gzip", b"", "Too many"), + ], +) +def test_invalid_compressed_success_is_a_transport_error(call, coding, wire, error): + with server(lambda *_: (200, {"Content-Encoding": coding}, wire)) as url: + with pytest.raises(TransportException, match=error): + call(url, "lookup_ip_geolocation_raw", LookupIpGeolocationRequest()) + + +def test_bounded_intermediate_layers(): + from ipgeolocation._response import ResponseBody + + empty_blocks = b"\x00\x00\x00\xff\xff" * 20000 + b"\x01\x00\x00\xff\xff" + reader = ResponseBody(64, "utf-8", "deflate, gzip") + with pytest.raises(TransportException, match="exceeded"): + reader.feed(gzip.compress(empty_blocks)) + + +@pytest.mark.parametrize("content", [b"plain text", "مرحبا".encode()]) +def test_nontext_charset_uses_utf8(content): + from ipgeolocation._response import ResponseBody + + reader = ResponseBody(64, "base64_codec", "identity") + reader.feed(content) + assert reader.finish() == content.decode() + + +def test_plaintext_error_redacts_before_truncation(call): + key = secrets.token_hex(16) + raw = ("x" * 507 + key + " end").encode() + with server(lambda *_: (401, {}, raw)) as url: + with pytest.raises(ApiException) as error: + call(url, "lookup_ip_geolocation", LookupIpGeolocationRequest(), key=key) + assert error.value.api_message.startswith("x" * 507 + "[") + assert len(error.value.api_message) == 512 + + +@pytest.mark.parametrize("key", ["qa+a/b", "qa example", 'qa"quoted\\id']) +def test_key_encoding_in_api_errors(call, key): + from urllib.parse import quote_plus + + encoded = quote_plus(key).replace("%2F", "%2f").replace("%2B", "%2b") + escaped = json.dumps(key)[1:-1] + wire = json.dumps({"message": f"denied {encoded} {escaped}"}).encode() + with server(lambda *_: (401, {}, wire)) as url: + with pytest.raises(ApiException) as error: + call(url, "lookup_ip_geolocation", LookupIpGeolocationRequest(), key=key) + assert error.value.api_message == "denied [REDACTED] [REDACTED]" + + +def test_redirect_loop_ends_at_client_limit(call): + with server(lambda *_: (307, {"Location": "/loop"}, b"")) as url: + with pytest.raises(TransportException, match="Too many redirects"): + call(url, "lookup_ip_geolocation", LookupIpGeolocationRequest(), follow=True) + + +@pytest.mark.parametrize("data", [b"[" * 10000 + b"]" * 10000, b"1" * 5000]) +def test_json_parser_limits_preserve_error_type(call, data): + with server(lambda *_: (429, {}, data)) as url: + with pytest.raises(RateLimitException) as error: + call(url, "lookup_ip_geolocation", LookupIpGeolocationRequest()) + assert error.value.status_code == 429 + with server(lambda *_: (200, {}, data)) as url: + with pytest.raises(SerializationException): + call(url, "lookup_ip_geolocation", LookupIpGeolocationRequest()) + with pytest.raises(SerializationException): + call(url, "bulk_lookup_ip_geolocation", BulkLookupIpGeolocationRequest(ips=["test"])) + + +def test_injected_session_headers_and_response_hook(): + import requests + + hooks = [] + encodings = [] + + def answer(method, url, headers, body): + encodings.append(headers["Accept-Encoding"]) + return (302, {"Location": "/final"}, b"") if url.path != "/final" else (200, {}, b"{}") + + with server(answer) as url, requests.Session() as session: + session.headers["Accept-Encoding"] = "identity" + session.hooks["response"] = lambda response, **kw: hooks.append(response.status_code) + config = IpGeolocationClientConfig(api_key="example", base_url=url) + with IpGeolocationClient(config, RequestsHttpTransport(session)) as client: + assert client.lookup_ip_geolocation_raw(LookupIpGeolocationRequest()).data == "{}" + assert encodings == ["identity", "identity"] and hooks == [302, 200] + + +def test_injected_async_client_keeps_encoding_header(): + async def run(): + def handle(request): + assert request.headers["Accept-Encoding"] == "identity" + return httpx.Response(200, content=b"{}") + + async with httpx.AsyncClient( + transport=httpx.MockTransport(handle), headers={"Accept-Encoding": "identity"} + ) as http: + config = IpGeolocationClientConfig(api_key="example") + async with AsyncIpGeolocationClient(config, HttpxAsyncTransport(http)) as client: + result = await client.lookup_ip_geolocation_raw(LookupIpGeolocationRequest()) + assert result.data == "{}" + + asyncio.run(run()) + + +@pytest.mark.parametrize("coding", ["br", "zstd"]) +def test_optional_response_codings(coding): + from urllib3 import response + from urllib3.exceptions import DecodeError + + from ipgeolocation._response import ACCEPT_ENCODING, ResponseBody + + if coding not in ACCEPT_ENCODING.split(", "): + pytest.skip(f"{coding} decoder is not installed") + module = response.brotli if coding == "br" else response.zstd + body = "مرحبا" * 1000 + wire = module.compress(body.encode()) + reader = ResponseBody(len(body), "utf-8", coding) + reader.feed(wire) + assert reader.finish() == body + with pytest.raises(TransportException, match="exceeded"): + ResponseBody(64, "utf-8", coding).feed(wire) + broken = ResponseBody(len(body), "utf-8", coding) + with pytest.raises((TransportException, DecodeError), match="(?i)incomplete"): + broken.feed(wire[:-1]) + broken.finish() + + +@pytest.mark.parametrize("status", [204, 205]) +def test_no_content_response_ignores_compression_headers(call, status): + with server(lambda *_: (status, {"Content-Encoding": "gzip"}, b"")) as url: + assert call(url, "lookup_ip_geolocation_raw", LookupIpGeolocationRequest()).data == "" + + +def test_raw_deflate_can_start_with_a_valid_zlib_header(): + from ipgeolocation._response import ResponseBody + + raw = bytes.fromhex("780100feff41010000ffff") + for chunks in ((raw,), (raw[:3], raw[3:])): + reader = ResponseBody(1, "utf-8", "deflate") + for chunk in chunks: + reader.feed(chunk) + assert reader.finish() == "A" + + +def test_http2_logs_hide_keys_during_request_and_later_table_eviction(caplog): + from hpack import Encoder + + from ipgeolocation._redaction import protect_api_key + + key = secrets.token_hex(16) + encoder = Encoder() + caplog.set_level(logging.DEBUG) + with protect_api_key(key): + encoder.encode( + [(b":path", ("/v3/ipgeo?apiKey=" + key).encode()), (b"x-trace", key.encode())] + ) + assert not any(record.name.startswith("hpack.") for record in caplog.records) + encoder.header_table.maxsize = 1 + encoder.encode([(b":path", b"/normal")]) + assert key not in caplog.text and "/normal" in caplog.text + + +def test_async_setup_errors_hide_secrets(): + async def run(key, base_url, headers): + config = IpGeolocationClientConfig(api_key=key, base_url=base_url) + async with AsyncIpGeolocationClient(config) as client: + with pytest.raises(TransportException) as error: + await client.lookup_ip_geolocation(LookupIpGeolocationRequest(headers=headers)) + assert key not in str(error.value) and error.value.__context__ is None + + token = secrets.token_hex(10) + asyncio.run(run(token, "http://example.test:" + token, {})) + asyncio.run(run("ä" + token, "http://example.test", {"X-Test": "ä" + token})) diff --git a/tests/test_transport.py b/tests/test_transport.py index 004fbc0..4b55ac8 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -23,6 +23,9 @@ def getlist(self, key): class StubResponse: + next = None + encoding = "utf-8" + def __init__(self, status_code=200, body="{}", headers=None, iter_exception=None) -> None: self.status_code = status_code self._body_bytes = body.encode("utf-8") @@ -49,6 +52,7 @@ def close(self) -> None: class StubSession: def __init__(self, response=None, exception=None) -> None: + self.headers = {} self.response = response self.exception = exception self.closed = False From 69071bffdec04d7b86cd24a4353449956f92e84c Mon Sep 17 00:00:00 2001 From: MudassarTariq Date: Fri, 6 Nov 2026 00:42:46 +0500 Subject: [PATCH 2/5] Hide response headers in shared HTTP/2 debug logs --- src/ipgeolocation/_redaction.py | 6 +++++- tests/test_request_protection.py | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/ipgeolocation/_redaction.py b/src/ipgeolocation/_redaction.py index 593b851..f128dad 100644 --- a/src/ipgeolocation/_redaction.py +++ b/src/ipgeolocation/_redaction.py @@ -41,7 +41,11 @@ class _RequestLogFilter(logging.Filter): def filter(self, record: logging.LogRecord) -> bool: key = _active_key.get() if record.name.startswith("hpack."): - if key: + if key or ( + record.name == "hpack.hpack" + and isinstance(record.msg, str) + and record.msg.startswith(("Decoding %s", "Decoded %s,")) + ): return False if ( record.name == "hpack.table" diff --git a/tests/test_request_protection.py b/tests/test_request_protection.py index 2012a53..c6c85bf 100644 --- a/tests/test_request_protection.py +++ b/tests/test_request_protection.py @@ -416,7 +416,7 @@ def test_raw_deflate_can_start_with_a_valid_zlib_header(): def test_http2_logs_hide_keys_during_request_and_later_table_eviction(caplog): - from hpack import Encoder + from hpack import Decoder, Encoder from ipgeolocation._redaction import protect_api_key @@ -424,10 +424,12 @@ def test_http2_logs_hide_keys_during_request_and_later_table_eviction(caplog): encoder = Encoder() caplog.set_level(logging.DEBUG) with protect_api_key(key): - encoder.encode( + block = encoder.encode( [(b":path", ("/v3/ipgeo?apiKey=" + key).encode()), (b"x-trace", key.encode())] ) assert not any(record.name.startswith("hpack.") for record in caplog.records) + assert Decoder().decode(block)[1][1] == key + assert not any(r.msg.startswith(("Decoding %s", "Decoded %s,")) for r in caplog.records) encoder.header_table.maxsize = 1 encoder.encode([(b":path", b"/normal")]) assert key not in caplog.text and "/normal" in caplog.text From f5e4336289382bb589abe636e09af47c0088d10d Mon Sep 17 00:00:00 2001 From: MudassarTariq Date: Fri, 6 Nov 2026 00:45:04 +0500 Subject: [PATCH 3/5] Set up request log filters on the first authenticated request --- src/ipgeolocation/_redaction.py | 34 +++++++++++++++++++------------- tests/test_request_protection.py | 13 ++++++++++++ 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/ipgeolocation/_redaction.py b/src/ipgeolocation/_redaction.py index f128dad..9b3ee6c 100644 --- a/src/ipgeolocation/_redaction.py +++ b/src/ipgeolocation/_redaction.py @@ -5,6 +5,7 @@ import re from contextlib import contextmanager from contextvars import ContextVar +from functools import lru_cache from typing import Iterator, Optional from urllib.parse import quote, quote_plus @@ -63,20 +64,24 @@ def filter(self, record: logging.LogRecord) -> bool: _log_filter = _RequestLogFilter() -for _name in ( - "hpack.hpack", - "hpack.table", - "httpx", - "httpcore.connection", - "httpcore.proxy", - "httpcore.socks", - "httpcore.http11", - "httpcore.http2", - "urllib3.connectionpool", - "urllib3.util.retry", - "urllib3.connection", -): - logging.getLogger(_name).addFilter(_log_filter) + + +@lru_cache(maxsize=1) +def _install_log_filter() -> None: + for name in ( + "hpack.hpack", + "hpack.table", + "httpx", + "httpcore.connection", + "httpcore.proxy", + "httpcore.socks", + "httpcore.http11", + "httpcore.http2", + "urllib3.connectionpool", + "urllib3.util.retry", + "urllib3.connection", + ): + logging.getLogger(name).addFilter(_log_filter) @contextmanager @@ -84,6 +89,7 @@ def protect_api_key(key: Optional[str]) -> Iterator[None]: if key is None: yield return + _install_log_filter() token = _active_key.set(key) try: yield diff --git a/tests/test_request_protection.py b/tests/test_request_protection.py index c6c85bf..ed09930 100644 --- a/tests/test_request_protection.py +++ b/tests/test_request_protection.py @@ -446,3 +446,16 @@ async def run(key, base_url, headers): token = secrets.token_hex(10) asyncio.run(run(token, "http://example.test:" + token, {})) asyncio.run(run("ä" + token, "http://example.test", {"X-Test": "ä" + token})) + + +def test_import_does_not_change_other_http_clients_logging(): + import subprocess + import sys + + code = ( + "import logging; from hpack import Decoder, Encoder; import ipgeolocation; " + "logging.basicConfig(level=logging.DEBUG); " + "Decoder().decode(Encoder().encode([(b'x-trace', b'ordinary')]))" + ) + result = subprocess.run([sys.executable, "-c", code], check=True, capture_output=True) + assert b"Decoded (b'x-trace', b'ordinary')" in result.stderr From 85b2fcb3615e1a8c5a34939465097cd9feaa5538 Mon Sep 17 00:00:00 2001 From: MudassarTariq Date: Fri, 6 Nov 2026 00:57:39 +0500 Subject: [PATCH 4/5] Update Python SDK documentation and 3.0.1 release notes --- CHANGELOG.md | 9 +++++++++ README.md | 16 +++------------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd55af4..ebbddf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 3.0.1 + +- Hide the configured API key in printed settings, SDK request logs and error messages, including errors in typed bulk results. +- Keep lookup data and custom headers from being sent to a different server on a redirect. +- Read compressed responses in small chunks. Keep the HTTP status if an error response is too large or cut short. +- Check that typed bulk lookups return one result per input. +- Add Arabic (`ar`) for single and bulk lookups. +- Include CDN, proxy, bot and corporate gateway details in typed results. + ## 3.0.0 - Requires Python 3.10 or newer. diff --git a/README.md b/README.md index f4f31b1..6ad5087 100644 --- a/README.md +++ b/README.md @@ -95,21 +95,12 @@ The async client exposes the same methods as coroutines. Use `async with` for cl config = IpGeolocationClientConfig(api_key="YOUR_API_KEY") ``` -**Request-origin auth**: Paid plans only, and only for single lookup. If your origin is allowlisted in the [IPGeolocation.io dashboard](https://ipgeolocation.io/), the SDK sends it in the `Origin` header. - -```python -config = IpGeolocationClientConfig(request_origin="https://app.example.com") -``` - -`request_origin` must be an absolute `http` or `https` URL with no path, query string, or fragment. Bulk lookups always require `api_key`, even if `request_origin` is set. You can set both on the same config. Single lookups require at least one of them. - ### Client Config | Field | Use | |---|---| | `api_key` | API key auth for single lookup and bulk lookup | -| `request_origin` | Paid request-origin auth for single lookup | -| `base_url` | Override the API base URL | +| `base_url` | Use the final address of your API or proxy. The standard client follows redirects only if the protocol, host and port stay the same. The async client stops at redirects by default. | | `connect_timeout` | Time to wait for the connection in seconds | | `read_timeout` | Time to wait for the response body in seconds | @@ -134,7 +125,6 @@ Responses vary by plan. | Domain lookup | No | Yes | | Bulk lookup (`/v3/ipgeo-bulk`) | No | Yes | | Non-English `lang` | No | Yes | -| Request-origin auth | No | Yes | | Optional modules via `include` | No | Yes | | `include=["*"]` | Base response only | All plan-available modules | @@ -303,7 +293,7 @@ Bulk lookup accepts the same fields as single lookup, plus: |---|---|---| | `ips` | sequence of strings | Required. Up to 50,000 IPs or domains. | -Differences from single lookup: `Content-Type: application/json` is always set by the SDK. Bulk lookup always requires `api_key` in the config, even if `request_origin` is also set. +Set `api_key` in your client settings for bulk lookups. The SDK sets `Content-Type: application/json` for you. ### `include` Values @@ -326,7 +316,7 @@ Supported `lang` values: `en`, `de`, `ru`, `ja`, `fr`, `cn`, `es`, `cs`, `it`, ` Every SDK method returns `ApiResponse(data=..., metadata=...)`. - Typed single lookup returns `IpGeolocationResponse` -- Typed bulk lookup returns a list of `BulkLookupResult` +- A typed bulk lookup returns one `BulkLookupResult` per input. If the API sends too few or too many results, the SDK raises `SerializationException`. - Raw methods return the response body as a string | Field | Type | Description | From f078740a403aba62cc447ed4a25ff8062bfed09b Mon Sep 17 00:00:00 2001 From: MudassarTariq Date: Fri, 6 Nov 2026 01:01:52 +0500 Subject: [PATCH 5/5] Set the default timeout for local HTTP tests to five seconds --- tests/test_http_integration.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/test_http_integration.py b/tests/test_http_integration.py index 1ebaf15..579959b 100644 --- a/tests/test_http_integration.py +++ b/tests/test_http_integration.py @@ -32,6 +32,7 @@ from ipgeolocation._transport import HttpRequestData, HttpxAsyncTransport, RequestsHttpTransport TEST_KEY = "TEST_KEY &+?=#%" +SIMULATION_TIMEOUT = 5 @pytest.fixture(scope="module") @@ -91,12 +92,21 @@ def client_type(request): return request.param -def _call(client_type, api_url, method, request, *, api_key=TEST_KEY, origin=None, timeout=1): +def _call( + client_type, + api_url, + method, + request, + *, + api_key=TEST_KEY, + origin=None, + timeout=SIMULATION_TIMEOUT, +): config = IpGeolocationClientConfig( api_key=api_key, request_origin=origin, base_url=api_url, - connect_timeout=min(0.1, timeout), + connect_timeout=timeout, read_timeout=timeout, ) if client_type is AsyncIpGeolocationClient: @@ -240,8 +250,8 @@ def test_http_streaming_size_limit(client_type, api_url): method="GET", headers={}, body=None, - connect_timeout=0.1, - read_timeout=1, + connect_timeout=SIMULATION_TIMEOUT, + read_timeout=SIMULATION_TIMEOUT, ) if client_type is AsyncIpGeolocationClient: