Reuse HTTPS connections, negotiate HTTP/2, add new response fields

.github/workflows/ci.yml

@@ -0,0 +1,25 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt, clippy
+
+      - run: cargo fmt --check
+
+      - run: cargo clippy --all-targets -- -D warnings
+
+      - run: cargo test

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ipdata"
-version = "0.1.1"
+version = "0.1.2"
 edition = "2021"
 description = "Official Rust client for the ipdata.co IP geolocation and threat intelligence API"
 license = "MIT"
@@ -9,10 +9,11 @@ keywords = ["ipdata", "geolocation", "ip", "api", "threat-intelligence"]
 categories = ["api-bindings", "web-programming::http-client"]
 
 [dependencies]
-reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
+reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "http2"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 thiserror = "2"
 
 [dev-dependencies]
-tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
+tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] }
+wiremock = "0.6"

README.md

@@ -87,6 +87,21 @@ let client = ipdata::IpData::eu("your-api-key");
 let client = ipdata::IpData::with_base_url("your-api-key", "https://custom-endpoint.example.com");
 ```
 
+## Connection reuse and HTTP/2
+
+An `IpData` client keeps a persistent HTTPS connection pool for as long as it lives. Create one client per process (or one per API key or endpoint), keep it, and reuse it for every lookup: each new client pays a fresh TLS handshake. Requests to api.ipdata.co are multiplexed over HTTP/2 when the platform negotiates it, and fall back to HTTP/1.1 keep-alive otherwise.
+
+`IpData` is `Send + Sync`: share one instance across tasks by reference or in an `Arc`, or `clone()` it; clones are cheap and share the same pool. There is no close method: dropping the last clone releases the pool and its connections. To bring your own `reqwest::Client` (a pool shared with the rest of your application, timeouts, a proxy or TLS settings), use `IpData::with_client`; the client is used as given, so set the `User-Agent` yourself if you want to keep the SDK's `ipdata-rust/<version>` header.
+
+```rust
+let http = reqwest::Client::builder()
+    .timeout(std::time::Duration::from_secs(10))
+    .build()?;
+let client = ipdata::IpData::with_client("your-api-key", "https://api.ipdata.co", http);
+```
+
+The default client, used by `IpData::new`, `IpData::eu` and `IpData::with_base_url`, negotiates HTTP/2 through ALPN with automatic HTTP/1.1 fallback and is tuned for reuse: pooled connections idle for 60 s are closed, HTTP/2 connections are kept alive with a PING every 30 s, and TCP keep-alive probes run every 30 s. No total request timeout is set. A bulk lookup (`POST /bulk`) whose pooled connection turns out to be dead before any response byte arrives is sent again exactly once on a fresh connection; HTTP error statuses are never retried.
+
 ## Error Handling
 
 ```rust
@@ -107,13 +122,20 @@ match client.lookup("8.8.8.8").await {
 The main response type is `IpInfo` which includes:
 
 - **Geolocation**: `city`, `region`, `country_name`, `country_code`, `continent_name`, `latitude`, `longitude`, `postal`
+- **Flags**: `is_eu` (EU member country), plus two flags the API sends only when they are true, so the typed fields read `false` when the key is absent:
+  - `is_satellite`: the IP belongs to a satellite internet provider
+  - `is_anycast`: the IP is an anycast address announced from several locations at once
 - **ASN**: `asn` (struct with `asn`, `name`, `domain`, `route`, `asn_type`)
 - **Company**: `company` (struct with `name`, `domain`, `network`, `company_type`)
 - **Carrier**: `carrier` (struct with `name`, `mcc`, `mnc`)
 - **Currency**: `currency` (struct with `name`, `code`, `symbol`, `native`, `plural`)
 - **Time Zone**: `time_zone` (struct with `name`, `abbreviation`, `offset`, `is_dst`, `current_time`)
 - **Organisation**: `organisation` (ISP / organization name)
 - **Threat**: `threat` (struct with `is_tor`, `is_vpn`, `is_proxy`, `is_anonymous`, `is_threat`, `is_bogon`, `blocklists`, `scores`, and more)
+- **Blocklist entries**: each `Blocklist` in `threat.blocklists` has `name`, `site` and `blocklist_type`, plus three fields that are `None` when a feed does not report them (ipdata's own vpn feed, for example):
+  - `first_seen`: first day the IP was seen on the list, as a `YYYY-MM-DD` string (UTC)
+  - `last_seen`: most recent day the IP was seen on the list, as a `YYYY-MM-DD` string (UTC)
+  - `days_seen`: number of days the IP has been seen on the list
 - **Languages**: `languages` (vec of `Language` with `name`, `native`, `code`)
 
 ## License

examples/lookup.rs

@@ -24,7 +24,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Bulk lookup
     let results = client.bulk(&["8.8.8.8", "1.1.1.1"]).await?;
     for result in &results {
-        println!("{} -> {}", result.ip, result.country_name.as_deref().unwrap_or("unknown"));
+        println!(
+            "{} -> {}",
+            result.ip,
+            result.country_name.as_deref().unwrap_or("unknown")
+        );
     }
 
     Ok(())

src/client.rs

@@ -1,3 +1,5 @@
+use std::time::Duration;
+
 use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT};
 
 use crate::error::{ApiErrorResponse, Error, Result};
@@ -15,8 +17,41 @@ const BULK_LIMIT: usize = 100;
 /// SDK version.
 const VERSION: &str = env!("CARGO_PKG_VERSION");
 
+/// Idle time after which the client closes a pooled connection.
+///
+/// The API drops idle connections itself after roughly two minutes; closing them
+/// earlier on the client side avoids sending a request into a socket the server has
+/// already given up on.
+const POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(60);
+
+/// Interval between HTTP/2 keep-alive PING frames.
+const HTTP2_KEEP_ALIVE_INTERVAL: Duration = Duration::from_secs(30);
+
+/// Time a keep-alive PING may stay unanswered before the connection is closed.
+const HTTP2_KEEP_ALIVE_TIMEOUT: Duration = Duration::from_secs(10);
+
+/// Interval between TCP keep-alive probes.
+const TCP_KEEPALIVE: Duration = Duration::from_secs(30);
+
 /// Client for the ipdata.co API.
 ///
+/// An `IpData` owns a [`reqwest::Client`] whose pool of HTTPS connections is kept
+/// alive and reused across requests. Build the client once (per process, or one per
+/// API key or endpoint) and reuse it for every lookup: each call to [`IpData::new`],
+/// [`IpData::eu`] or [`IpData::with_base_url`] creates a new pool, and the first
+/// request on a new pool pays a fresh TCP and TLS handshake. Requests use HTTP/2 when
+/// the server negotiates it and fall back to HTTP/1.1 keep-alive otherwise.
+///
+/// `IpData` is `Send + Sync`, so one instance can be shared across tasks and threads
+/// by reference or in an `Arc`. `Clone` is cheap: a clone shares the connection pool
+/// of the client it was cloned from. There is no close method; dropping the last clone
+/// releases the pool and its connections. Pooled connections are closed after 60
+/// seconds idle, HTTP/2 connections are kept alive with a PING every 30 seconds, and
+/// TCP keep-alive probes are sent every 30 seconds.
+///
+/// To reuse an existing [`reqwest::Client`], or to set timeouts, proxies or TLS
+/// options, use [`IpData::with_client`].
+///
 /// # Examples
 ///
 /// ```no_run
@@ -49,15 +84,35 @@ impl IpData {
 
     /// Creates a new client with a custom base URL.
     pub fn with_base_url(api_key: impl Into<String>, base_url: impl Into<String>) -> Self {
-        let mut headers = HeaderMap::new();
-        let ua = format!("ipdata-rust/{VERSION}");
-        headers.insert(USER_AGENT, HeaderValue::from_str(&ua).unwrap());
-
-        let client = reqwest::Client::builder()
-            .default_headers(headers)
-            .build()
-            .expect("failed to build HTTP client");
+        Self::with_client(api_key, base_url, build_http_client())
+    }
 
+    /// Creates a client that sends its requests through a caller-supplied
+    /// [`reqwest::Client`].
+    ///
+    /// Use this to share one connection pool with the rest of your application, or to
+    /// configure timeouts, proxies, TLS roots or other options the default client does
+    /// not set. The client is used as given: the SDK's `User-Agent` header
+    /// (`ipdata-rust/<version>`) and its keep-alive tuning are only applied by the
+    /// default client, so add them to your builder if you want them. `base_url` is the
+    /// API endpoint, normally `https://api.ipdata.co` or `https://eu-api.ipdata.co`.
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// # fn example() -> Result<(), reqwest::Error> {
+    /// let http = reqwest::Client::builder()
+    ///     .timeout(std::time::Duration::from_secs(10))
+    ///     .build()?;
+    /// let client = ipdata::IpData::with_client("your-api-key", "https://api.ipdata.co", http);
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn with_client(
+        api_key: impl Into<String>,
+        base_url: impl Into<String>,
+        client: reqwest::Client,
+    ) -> Self {
         Self {
             client,
             api_key: api_key.into(),
@@ -99,11 +154,7 @@ impl IpData {
     ///
     /// * `ip` - IPv4 or IPv6 address
     /// * `field` - Field name (e.g. `"asn"`, `"threat"`, `"currency"`)
-    pub async fn lookup_field(
-        &self,
-        ip: &str,
-        field: &str,
-    ) -> Result<serde_json::Value> {
+    pub async fn lookup_field(&self, ip: &str, field: &str) -> Result<serde_json::Value> {
         validate_ip(ip)?;
         let url = format!("{}/{}/{}", self.base_url, ip, field);
         let resp = self
@@ -159,6 +210,12 @@ impl IpData {
     /// Performs a bulk lookup for up to 100 IP addresses.
     ///
     /// Requires a paid API key. Returns results in the same order as the input.
+    ///
+    /// The lookup is a `POST`, which the HTTP stack does not retry on its own once the
+    /// request has been written. If a pooled connection turns out to be dead before any
+    /// response byte arrives (reset, EOF or broken pipe on a reused socket), the request
+    /// is sent again exactly once on a fresh connection. HTTP error statuses and
+    /// failures while reading the response body are never retried.
     pub async fn bulk(&self, ips: &[&str]) -> Result<Vec<IpInfo>> {
         if ips.is_empty() {
             return Err(Error::BulkEmpty);
@@ -172,13 +229,16 @@ impl IpData {
         }
 
         let url = format!("{}/bulk", self.base_url);
-        let resp = self
-            .client
-            .post(&url)
-            .query(&[("api-key", &self.api_key)])
-            .json(&ips)
-            .send()
-            .await?;
+        let resp = match self.post_bulk(&url, ips).await {
+            Ok(resp) => resp,
+            // The server or a middlebox can drop a pooled connection while it is idle
+            // without the client noticing until the next request fails on it. hyper
+            // transparently re-sends a request it had not yet written to such a
+            // connection; one that was already written is retried once here. The failed
+            // connection has been discarded, so the retry uses a fresh one.
+            Err(e) if is_stale_connection_error(&e) => self.post_bulk(&url, ips).await?,
+            Err(e) => return Err(e.into()),
+        };
 
         let status = resp.status();
         if !status.is_success() {
@@ -189,6 +249,16 @@ impl IpData {
         Ok(bulk.responses)
     }
 
+    /// Sends the bulk `POST` once and returns the raw response.
+    async fn post_bulk(&self, url: &str, ips: &[&str]) -> reqwest::Result<reqwest::Response> {
+        self.client
+            .post(url)
+            .query(&[("api-key", &self.api_key)])
+            .json(&ips)
+            .send()
+            .await
+    }
+
     /// Internal typed GET helper for sub-resource endpoints.
     async fn get_typed<T: serde::de::DeserializeOwned>(&self, url: &str) -> Result<T> {
         let resp = self
@@ -225,6 +295,39 @@ impl IpData {
     }
 }
 
+/// Builds the [`reqwest::Client`] used by [`IpData::new`], [`IpData::eu`] and
+/// [`IpData::with_base_url`].
+///
+/// The client identifies the SDK with a `User-Agent` default header and is tuned for
+/// connection reuse: HTTP/2 is negotiated through ALPN with automatic HTTP/1.1
+/// fallback, pooled connections are closed after 60 s idle, HTTP/2 connections are kept
+/// alive with a PING every 30 s (10 s to answer, also while idle) and TCP keep-alive
+/// probes run every 30 s. No total request timeout is set.
+fn build_http_client() -> reqwest::Client {
+    let mut headers = HeaderMap::new();
+    let ua = format!("ipdata-rust/{VERSION}");
+    headers.insert(USER_AGENT, HeaderValue::from_str(&ua).unwrap());
+
+    reqwest::Client::builder()
+        .default_headers(headers)
+        .pool_idle_timeout(POOL_IDLE_TIMEOUT)
+        .tcp_keepalive(TCP_KEEPALIVE)
+        .http2_keep_alive_interval(HTTP2_KEEP_ALIVE_INTERVAL)
+        .http2_keep_alive_timeout(HTTP2_KEEP_ALIVE_TIMEOUT)
+        .http2_keep_alive_while_idle(true)
+        .build()
+        .expect("failed to build HTTP client")
+}
+
+/// Whether a `send()` failure means the request died on an established connection
+/// before any response arrived: a reset, EOF or broken pipe on a reused socket.
+///
+/// Connection-level failures (DNS, TCP connect, TLS, proxy) and timeouts are excluded:
+/// a retry would not help, or the server may already have received the request.
+fn is_stale_connection_error(e: &reqwest::Error) -> bool {
+    e.is_request() && !e.is_connect() && !e.is_timeout()
+}
+
 /// Validates that a string is a valid IPv4 or IPv6 address.
 fn validate_ip(ip: &str) -> Result<()> {
     ip.parse::<std::net::IpAddr>()

src/types.rs

@@ -11,6 +11,20 @@ pub struct IpInfo {
     #[serde(default)]
     pub is_eu: bool,
 
+    /// Whether the IP belongs to a satellite internet provider.
+    ///
+    /// The API sends this key only when it is true; the field reads `false` when the
+    /// key is absent.
+    #[serde(default)]
+    pub is_satellite: bool,
+
+    /// Whether the IP is an anycast address, announced from several locations at once.
+    ///
+    /// The API sends this key only when it is true; the field reads `false` when the
+    /// key is absent.
+    #[serde(default)]
+    pub is_anycast: bool,
+
     /// City name.
     #[serde(default)]
     pub city: Option<String>,
@@ -217,29 +231,42 @@ pub struct Currency {
 }
 
 /// Time zone information.
+///
+/// The API sends every field as `null` when it cannot determine a time zone, for
+/// example for anycast addresses; such fields decode to their default (empty string or
+/// `false`).
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct TimeZone {
     /// IANA time zone name (e.g. "America/New_York").
-    #[serde(default)]
+    #[serde(default, deserialize_with = "null_as_default")]
     pub name: String,
 
     /// Time zone abbreviation (e.g. "EST").
-    #[serde(default, rename = "abbr")]
+    #[serde(default, rename = "abbr", deserialize_with = "null_as_default")]
     pub abbreviation: String,
 
     /// UTC offset string (e.g. "-0500").
-    #[serde(default)]
+    #[serde(default, deserialize_with = "null_as_default")]
     pub offset: String,
 
     /// Whether daylight saving time is active.
-    #[serde(default)]
+    #[serde(default, deserialize_with = "null_as_default")]
     pub is_dst: bool,
 
     /// Current time in the time zone (ISO 8601).
-    #[serde(default)]
+    #[serde(default, deserialize_with = "null_as_default")]
     pub current_time: String,
 }
 
+/// Deserializes an explicit JSON `null` as the field's default value.
+fn null_as_default<'de, D, T>(deserializer: D) -> Result<T, D::Error>
+where
+    D: serde::Deserializer<'de>,
+    T: Default + Deserialize<'de>,
+{
+    Ok(Option::<T>::deserialize(deserializer)?.unwrap_or_default())
+}
+
 /// Threat intelligence data for an IP address.
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct Threat {
@@ -328,6 +355,25 @@ pub struct Blocklist {
     /// Blocklist category / type.
     #[serde(default, rename = "type")]
     pub blocklist_type: String,
+
+    /// First day the IP was seen on this blocklist, as a `YYYY-MM-DD` date (UTC).
+    ///
+    /// Some feeds, including ipdata's own vpn feed, report no dates; the field is then
+    /// `None`.
+    #[serde(default)]
+    pub first_seen: Option<String>,
+
+    /// Most recent day the IP was seen on this blocklist, as a `YYYY-MM-DD` date (UTC).
+    ///
+    /// `None` when the feed reports no dates.
+    #[serde(default)]
+    pub last_seen: Option<String>,
+
+    /// Number of days the IP has been seen on this blocklist.
+    ///
+    /// `None` when the feed reports no dates.
+    #[serde(default)]
+    pub days_seen: Option<i32>,
 }
 
 /// Response wrapper for bulk lookups.