diff --git a/README.md b/README.md index 16996ab..3cf2825 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,51 @@ func main() { } ``` +### Connection reuse and HTTP/2 + +A `Client` keeps a persistent HTTPS connection pool for as long as it lives. +Create one client per process (or one per API key or endpoint), keep it, and +reuse it for every lookup: each new client pays a fresh TLS handshake. +Requests to api.ipdata.co are multiplexed over HTTP/2 when the platform +negotiates it, and fall back to HTTP/1.1 keep-alive otherwise. + +A `Client` is safe for concurrent use from multiple goroutines, so one +package-level or injected instance can serve all of them. Pooled connections +are closed after 60 seconds idle, idle HTTP/2 connections are health-checked +with a PING every 30 seconds, and TCP keepalive probes run every 30 seconds. +Call `client.CloseIdleConnections()` at shutdown (or before a long idle period) +to release the pooled connections eagerly; the client stays usable afterwards. + +```go +var client ipdata.Client + +func main() { + var err error + + client, err = ipdata.NewClient("YOUR_API_KEY") + if err != nil { + panic(err) + } + defer client.CloseIdleConnections() + + // Every lookup made through client reuses the same pooled connections. + data, err := client.Lookup("8.8.8.8") + // ... +} +``` + +Bulk lookups are `POST` requests, which `net/http` does not replay on its own +once they were written to a connection. The client therefore retries a bulk +request exactly once when a pooled connection turns out to be dead before any +byte of the response arrived; it never retries after a response started to +arrive, on an HTTP error status, or after a cancellation or timeout. + +If you supply your own `*http.Client` with `WithHTTPClient`, connection reuse +follows the settings of its transport. `net/http` switches its automatic HTTP/2 +setup off when an `http.Transport` sets a custom `DialContext` or +`TLSClientConfig`, so set `ForceAttemptHTTP2: true` on such a transport (the +default transport already does) to keep speaking HTTP/2 to api.ipdata.co. + ### EU Endpoint (GDPR Compliance) For GDPR compliance, use `NewEUClient` to route all requests through EU data @@ -141,6 +186,8 @@ The `IP` struct includes all fields from the ipdata API response: | `EmojiFlag` | `string` | Flag emoji character | | `EmojiUnicode` | `string` | Unicode representation | | `IsEU` | `bool` | Whether in the EU | +| `IsSatellite` | `bool` | Whether the address belongs to a satellite ISP; the API sends `is_satellite` only when true, so it reads false otherwise | +| `IsAnycast` | `bool` | Whether the address is an anycast address; the API sends `is_anycast` only when true, so it reads false otherwise | | `Organization` | `string` | Organization name | | `ASN` | `ASN` | Autonomous System Number data | | `Company` | `*Company` | Company/organization data | @@ -168,6 +215,12 @@ The `IP` struct includes all fields from the ipdata API response: **`Threat`**: `IsTOR`, `IsVPN`, `IsICloudRelay`, `IsProxy`, `IsDatacenter`, `IsAnonymous`, `IsKnownAttacker`, `IsKnownAbuser`, `IsThreat`, `IsBogon`, `Blocklists`, `Scores` +**`Blocklist`** (one entry of `Threat.Blocklists`): `Name`, `Site`, `Type`, plus three optional fields that some feeds (for example ipdata's own vpn feed) do not report: + +- `FirstSeen` (`string`): first day the address was listed, as `YYYY-MM-DD` (UTC); empty when not reported +- `LastSeen` (`string`): last day the address was listed, as `YYYY-MM-DD` (UTC); empty when not reported +- `DaysSeen` (`*int`): number of days the address was listed; `nil` when not reported + ## Contributors - [Tim Heckman](https://github.com/theckman/) - Created the first version of this library diff --git a/client.go b/client.go index 11a2970..985bcd2 100644 --- a/client.go +++ b/client.go @@ -12,19 +12,21 @@ import ( "io" "net" "net/http" + "net/http/httptrace" "net/url" "runtime" "strings" + "sync/atomic" "time" "github.com/pkg/errors" ) // Version is the package version -const Version = "0.8.0" +const Version = "0.8.2" // fqpn is the Fully Qualified Package Name for use in the client's User-Agent -const fqpn = "github.com/theckman/go-ipdata" +const fqpn = "github.com/ipdata/go" const ( apiEndpoint = "https://api.ipdata.co/" @@ -53,6 +55,14 @@ func WithHTTPClient(httpClient *http.Client) Option { // Client is the struct to represent the functionality presented by the // https://ipdata.co API. +// +// A Client owns a pool of persistent HTTPS connections (HTTP/2 where the +// server negotiates it, HTTP/1.1 keep-alive otherwise) that is reused for +// every request made through it. Build one Client per process, or one per API +// key or endpoint, and keep it for the lifetime of the program: every new +// Client pays a fresh TCP and TLS handshake. A Client is safe for concurrent +// use by multiple goroutines. Call CloseIdleConnections at shutdown to release +// the pooled connections. type Client struct { c *http.Client // http client e string // api endpoint @@ -101,6 +111,21 @@ func NewEUClient(apiKey string, opts ...Option) (Client, error) { return c, nil } +// CloseIdleConnections closes the pooled connections of the underlying +// *http.Client that are currently idle. Requests in flight are not +// interrupted, and the Client stays usable: the next request simply dials a +// new connection. Call it at shutdown, or before a long idle period, to +// release sockets eagerly instead of waiting for the 60 second idle timeout. +// It is a no-op for a Client that has no *http.Client, or whose custom +// transport does not implement CloseIdleConnections. +func (c Client) CloseIdleConnections() { + if c.c == nil { + return + } + + c.c.CloseIdleConnections() +} + type apiErr struct { Message string `json:"message"` } @@ -132,7 +157,10 @@ func (c Client) RawLookupWithContext(ctx context.Context, ip string) (*http.Resp // we can try and parse return resp, nil default: - // provide response body as error to consumer + // provide response body as error to consumer; closing the body hands + // the connection back to the pool + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) if err != nil { return nil, errors.Wrapf(err, "failed to read body from response with status code %q: %s", resp.Status, err) @@ -270,17 +298,10 @@ func (c *Client) RawBulkLookup(ips []string) (*http.Response, error) { } // RawBulkLookupWithContext is a RawBulkLookup with a provided context.Context. - func (c *Client) RawBulkLookupWithContext(ctx context.Context, ips []string) (*http.Response, error) { - // build request - req, err := newBulkPostRequestWithContext(ctx, c.e+"bulk", c.k, ips) - if err != nil { - return nil, errors.Wrap(err, "error building bulk lookup request") - } - - // action request - resp, err := c.c.Do(req) +func (c *Client) RawBulkLookupWithContext(ctx context.Context, ips []string) (*http.Response, error) { + resp, err := c.doBulkRequest(ctx, ips) if err != nil { - return nil, errors.Wrapf(err, "http request to %q failed", req.URL.Scheme+"://"+req.URL.Host+req.URL.Path) + return nil, err } switch resp.StatusCode { @@ -288,7 +309,10 @@ func (c *Client) RawBulkLookup(ips []string) (*http.Response, error) { // we can try and parse return resp, nil default: - // provide response body as error to consumer + // provide response body as error to consumer; closing the body hands + // the connection back to the pool + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) if err != nil { return nil, errors.Wrapf(err, "failed to read body from response with status code %q: %s", resp.Status, err) @@ -304,6 +328,86 @@ func (c *Client) RawBulkLookup(ips []string) (*http.Response, error) { } } +// doBulkRequest builds and sends the bulk lookup POST. +// +// The bulk endpoint is not idempotent, so net/http only replays the request on +// its own when nothing was written to the connection yet. When a pooled +// connection that the server side had already dropped is picked, the request +// bytes go out and the read then fails (EOF, reset) before any response byte +// arrived; net/http discards that connection and reports the error. In exactly +// that situation doBulkRequest sends the request one more time, over a fresh +// or another pooled connection. It never retries after a response started to +// arrive, on an HTTP status, after a cancellation or timeout, or when the +// failed connection was not a reused one. +func (c *Client) doBulkRequest(ctx context.Context, ips []string) (*http.Response, error) { + var ct connTracker + + // build request + req, err := newBulkPostRequestWithContext(ct.trace(ctx), c.e+"bulk", c.k, ips) + if err != nil { + return nil, errors.Wrap(err, "error building bulk lookup request") + } + + // action request + resp, err := c.c.Do(req) + if err != nil && ctx.Err() == nil && ct.staleConn(err) { + // the body of the first request has been consumed: build a new one + retry, rerr := newBulkPostRequestWithContext(ctx, c.e+"bulk", c.k, ips) + if rerr == nil { + resp, err = c.c.Do(retry) + } + } + + if err != nil { + return nil, errors.Wrapf(err, "http request to %q failed", req.URL.Scheme+"://"+req.URL.Host+req.URL.Path) + } + + return resp, nil +} + +// connTracker records, through net/http/httptrace, whether the connection a +// request went out on had been used before and whether any byte of the +// response arrived. Both facts are needed to tell a request that was lost on a +// stale pooled connection apart from one the server may have acted on. +type connTracker struct { + reused atomic.Bool + gotFirstByte atomic.Bool +} + +// trace returns ctx with the tracker's hooks attached; hooks already present +// in ctx keep running. +func (ct *connTracker) trace(ctx context.Context) context.Context { + return httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{ + GotConn: func(info httptrace.GotConnInfo) { + ct.reused.Store(info.Reused) + }, + GotFirstResponseByte: func() { + ct.gotFirstByte.Store(true) + }, + }) +} + +// staleConn reports whether err, returned by (*http.Client).Do, means the +// request died on a previously used connection before the server answered: +// the connection was reused, no response byte arrived, and the failure is +// neither a cancellation nor a timeout. +func (ct *connTracker) staleConn(err error) bool { + if !ct.reused.Load() || ct.gotFirstByte.Load() { + return false + } + + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + + var nerr net.Error + if errors.As(err, &nerr) && nerr.Timeout() { + return false + } + + return true +} + func newBulkPostRequestWithContext(ctx context.Context, urlStr, apiKey string, ips []string) (*http.Request, error) { if len(urlStr) == 0 { return nil, errors.New("url cannot be an empty string") @@ -401,6 +505,14 @@ func (c *Client) BulkLookupWithContext(ctx context.Context, ips []string) ([]*IP return res, nil // avoid nil interface check problem } +// newHTTPClient builds the *http.Client a Client uses unless WithHTTPClient +// supplies one. Its transport keeps a pool of persistent connections for the +// lifetime of the Client: TCP keepalive probes every 30 seconds, pooled +// connections are closed after 60 seconds idle, HTTP/2 is negotiated through +// ALPN with an HTTP/1.1 fallback (ForceAttemptHTTP2 is needed because a custom +// DialContext otherwise switches the automatic HTTP/2 setup off), and idle +// HTTP/2 connections are health checked with a PING frame every 30 seconds, +// closing the connection when no answer arrives within 15 seconds. func newHTTPClient() *http.Client { return &http.Client{ Transport: &http.Transport{ @@ -409,6 +521,11 @@ func newHTTPClient() *http.Client { Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, }).DialContext, + ForceAttemptHTTP2: true, + HTTP2: &http.HTTP2Config{ + SendPingTimeout: 30 * time.Second, + PingTimeout: 15 * time.Second, + }, MaxIdleConns: 100, IdleConnTimeout: 60 * time.Second, TLSHandshakeTimeout: 10 * time.Second, diff --git a/client_test.go b/client_test.go index 3d4c72d..92c37e3 100644 --- a/client_test.go +++ b/client_test.go @@ -5,7 +5,10 @@ package ipdata import ( + "bufio" "context" + "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "io" @@ -14,6 +17,7 @@ import ( "net/http/httptest" "net/url" "strings" + "sync/atomic" "testing" "time" @@ -111,6 +115,11 @@ func testBulkHTTPServer() *httptest.Server { fmt.Fprintf(w, `ip slice wrong inputs, want ["1.1.1.1","8.8.8.8"] got: %#v`, ips) return case 3: + if ips[0] == "98.97.34.0" && ips[1] == "8.8.8.8" && ips[2] == "187.13.250.1" { + fmt.Fprint(w, testBulkJSONNewFields) + return + } + if ips[0] == "1.1.1.1" && ips[1] == "8.8.8.8" && ips[2] == "127.0.0.1" { fmt.Fprint(w, testBulkJSONWithLocalhost) return @@ -1913,3 +1922,705 @@ const testBulkJSONWithLocalhost = `[ "message": "127.0.0.1 is a private IP address" } ]` + +// roundTripperFunc adapts a function to http.RoundTripper. It deliberately has +// no CloseIdleConnections method. +type roundTripperFunc func(*http.Request) (*http.Response, error) + +func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { + return f(r) +} + +// connCounter counts the TCP connections an *http.Server accepted. +type connCounter struct { + n atomic.Int32 +} + +func (cc *connCounter) connState(_ net.Conn, s http.ConnState) { + if s == http.StateNew { + cc.n.Add(1) + } +} + +// testReuseHandler serves canned lookup, error and bulk responses without +// checking the API key. +func testReuseHandler() http.Handler { + mux := http.NewServeMux() + + mux.HandleFunc("/76.14.47.42", func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, testJSONValid) + }) + + mux.HandleFunc("/192.168.0.1", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = io.WriteString(w, `{"message": "192.168.0.1 is a private IP address"}`) + }) + + mux.HandleFunc("/bulk", func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, testBulkJSONValid) + }) + + return mux +} + +// trustServerCert makes the transport of hc trust the certificate of the +// httptest TLS server srv. It must run before hc sends its first request. +func trustServerCert(t *testing.T, hc *http.Client, srv *httptest.Server) { + t.Helper() + + tr, ok := hc.Transport.(*http.Transport) + if !ok { + t.Fatalf("hc.Transport is %T, want *http.Transport", hc.Transport) + } + + pool := x509.NewCertPool() + pool.AddCert(srv.Certificate()) + + tr.TLSClientConfig = &tls.Config{RootCAs: pool} +} + +func Test_newHTTPClient_transport(t *testing.T) { + tr, ok := newHTTPClient().Transport.(*http.Transport) + if !ok { + t.Fatal("newHTTPClient().Transport is not an *http.Transport") + } + + if !tr.ForceAttemptHTTP2 { + t.Error("tr.ForceAttemptHTTP2 = false, want true") + } + + if tr.HTTP2 == nil { + t.Fatal("tr.HTTP2 = nil, want HTTP/2 keepalive configuration") + } + + if got, want := tr.HTTP2.SendPingTimeout, 30*time.Second; got != want { + t.Errorf("tr.HTTP2.SendPingTimeout = %s, want %s", got, want) + } + + if got, want := tr.HTTP2.PingTimeout, 15*time.Second; got != want { + t.Errorf("tr.HTTP2.PingTimeout = %s, want %s", got, want) + } + + if got, want := tr.IdleConnTimeout, 60*time.Second; got != want { + t.Errorf("tr.IdleConnTimeout = %s, want %s", got, want) + } + + if tr.Proxy == nil { + t.Error("tr.Proxy = nil, want http.ProxyFromEnvironment") + } +} + +func Test_newHTTPClient_http2(t *testing.T) { + srv := httptest.NewUnstartedServer(testReuseHandler()) + srv.EnableHTTP2 = true + srv.StartTLS() + defer srv.Close() + + hc := newHTTPClient() + trustServerCert(t, hc, srv) + + c := Client{c: hc, e: srv.URL + "/", k: "testAPIkey"} + + resp, err := c.RawLookup("76.14.47.42") + if err != nil { + t.Fatalf("RawLookup() unexpected error: %v", err) + } + + defer func() { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + }() + + if resp.Proto != "HTTP/2.0" { + t.Fatalf("resp.Proto = %q, want %q", resp.Proto, "HTTP/2.0") + } + + if resp.TLS == nil { + t.Fatal("resp.TLS = nil, want the TLS connection state") + } + + if resp.TLS.NegotiatedProtocol != "h2" { + t.Fatalf("resp.TLS.NegotiatedProtocol = %q, want %q", resp.TLS.NegotiatedProtocol, "h2") + } + + ip, err := c.Lookup("76.14.47.42") + if err != nil { + t.Fatalf("Lookup() unexpected error: %v", err) + } + + if ip.IP != "76.14.47.42" { + t.Fatalf("ip.IP = %q, want %q", ip.IP, "76.14.47.42") + } +} + +func Test_client_connectionReuse(t *testing.T) { + tests := []struct { + name string + tls bool + proto string + }{ + {name: "http1_keepalive", proto: "HTTP/1.1"}, + {name: "https_http2", tls: true, proto: "HTTP/2.0"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var conns connCounter + + srv := httptest.NewUnstartedServer(testReuseHandler()) + srv.Config.ConnState = conns.connState + + hc := newHTTPClient() + + if tt.tls { + srv.EnableHTTP2 = true + srv.StartTLS() + trustServerCert(t, hc, srv) + } else { + srv.Start() + } + + defer srv.Close() + + c := Client{c: hc, e: srv.URL + "/", k: "testAPIkey"} + + for i := 0; i < 5; i++ { + if _, err := c.Lookup("76.14.47.42"); err != nil { + t.Fatalf("Lookup() #%d unexpected error: %v", i+1, err) + } + } + + // error responses must hand their connection back as well + if _, err := c.Lookup("192.168.0.1"); err == nil { + t.Fatal("Lookup() of a private address: error expected but was nil") + } + + if _, err := c.RawLookup("192.168.0.1"); err == nil { + t.Fatal("RawLookup() of a private address: error expected but was nil") + } + + for i := 0; i < 3; i++ { + if _, err := c.BulkLookup([]string{"1.1.1.1", "8.8.8.8"}); err != nil { + t.Fatalf("BulkLookup() #%d unexpected error: %v", i+1, err) + } + } + + resp, err := c.RawLookup("76.14.47.42") + if err != nil { + t.Fatalf("RawLookup() unexpected error: %v", err) + } + + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + + if resp.Proto != tt.proto { + t.Errorf("resp.Proto = %q, want %q", resp.Proto, tt.proto) + } + + if n := conns.n.Load(); n != 1 { + t.Fatalf("server accepted %d connections for 11 sequential requests, want 1", n) + } + + // closing the idle connection forces the next request to dial again + c.CloseIdleConnections() + + if _, err := c.Lookup("76.14.47.42"); err != nil { + t.Fatalf("Lookup() after CloseIdleConnections() unexpected error: %v", err) + } + + if n := conns.n.Load(); n != 2 { + t.Fatalf("server accepted %d connections after CloseIdleConnections(), want 2", n) + } + }) + } +} + +func TestClient_CloseIdleConnections(t *testing.T) { + // a zero Client has no *http.Client and must not panic + Client{}.CloseIdleConnections() + + // no Transport: delegates to http.DefaultTransport + c, err := NewClient("testAPIkey", WithHTTPClient(&http.Client{})) + if err != nil { + t.Fatalf("NewClient() unexpected error: %v", err) + } + + c.CloseIdleConnections() + + // a transport without a CloseIdleConnections method is a no-op + rt := roundTripperFunc(func(*http.Request) (*http.Response, error) { + return nil, fmt.Errorf("not used") + }) + + c, err = NewEUClient("testAPIkey", WithHTTPClient(&http.Client{Transport: rt})) + if err != nil { + t.Fatalf("NewEUClient() unexpected error: %v", err) + } + + c.CloseIdleConnections() +} + +// actions of a scriptedServer, one per request received +const ( + srvRespond = iota // answer with 200 and the configured body + srvDrop // read the request, then close the connection without answering + srvPartial // write the start of a response, then close the connection + srvReject // answer with a 403 API error +) + +// scriptedServer is a raw HTTP/1.1 keep-alive server that follows a +// per-request script, so a test can drop a pooled connection at a precise +// point. Requests beyond the script are answered normally. +type scriptedServer struct { + ln net.Listener + body string + script []int + conns atomic.Int32 + reqs atomic.Int32 +} + +func newScriptedServer(t *testing.T, body string, script ...int) *scriptedServer { + t.Helper() + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("net.Listen() unexpected error: %v", err) + } + + s := &scriptedServer{ln: ln, body: body, script: script} + + t.Cleanup(func() { _ = ln.Close() }) + + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + + s.conns.Add(1) + + go s.serve(conn) + } + }() + + return s +} + +func (s *scriptedServer) url() string { + return "http://" + s.ln.Addr().String() + "/" +} + +func (s *scriptedServer) serve(conn net.Conn) { + defer func() { _ = conn.Close() }() + + br := bufio.NewReader(conn) + + for { + req, err := http.ReadRequest(br) + if err != nil { + return + } + + _, _ = io.Copy(io.Discard, req.Body) + _ = req.Body.Close() + + n := int(s.reqs.Add(1)) - 1 + + action := srvRespond + if n < len(s.script) { + action = s.script[n] + } + + switch action { + case srvDrop: + return + case srvPartial: + _, _ = io.WriteString(conn, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n") + return + case srvReject: + const msg = `{"message":"rejected"}` + _, _ = fmt.Fprintf(conn, "HTTP/1.1 403 Forbidden\r\nContent-Type: application/json\r\nContent-Length: %d\r\n\r\n%s", len(msg), msg) + default: + _, _ = fmt.Fprintf(conn, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: %d\r\n\r\n%s", len(s.body), s.body) + } + } +} + +func TestClient_BulkLookup_staleConnectionRetry(t *testing.T) { + ips := []string{"1.1.1.1", "8.8.8.8"} + + tests := []struct { + name string + script []int + wantErr string // substring of the second call's error, empty for success + wantReqs int32 + wantConns int32 + }{ + { + // The pooled connection dies after the request was written and + // before any response byte: the POST is replayed once, on a + // fresh connection. + name: "reused_connection_dropped_is_retried_once", + script: []int{srvRespond, srvDrop, srvRespond}, + wantReqs: 3, + wantConns: 2, + }, + { + // The replay fails as well, on a connection that was not reused: + // no further attempt is made. + name: "retry_happens_at_most_once", + script: []int{srvRespond, srvDrop, srvDrop}, + wantErr: `http request to "` + "http://", + wantReqs: 3, + wantConns: 2, + }, + { + // A response that started to arrive is never replayed. + name: "no_retry_after_response_bytes", + script: []int{srvRespond, srvPartial}, + wantErr: `http request to "` + "http://", + wantReqs: 2, + wantConns: 1, + }, + { + // An HTTP error status is never replayed. + name: "no_retry_on_http_status", + script: []int{srvRespond, srvReject}, + wantErr: "rejected", + wantReqs: 2, + wantConns: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := newScriptedServer(t, testBulkJSONValid, tt.script...) + + client := &Client{c: newHTTPClient(), e: srv.url(), k: "testAPIkey"} + defer client.CloseIdleConnections() + + // the first call leaves one kept-alive connection in the pool + if _, err := client.BulkLookup(ips); err != nil { + t.Fatalf("first BulkLookup() unexpected error: %v", err) + } + + got, err := client.BulkLookup(ips) + + if tt.wantErr == "" { + if err != nil { + t.Fatalf("second BulkLookup() unexpected error: %v", err) + } + + if len(got) != 2 || got[0] == nil || got[0].IP != "1.1.1.1" || got[1] == nil || got[1].IP != "8.8.8.8" { + t.Fatalf("second BulkLookup() = %v, want the two canned results", got) + } + } else { + if err == nil { + t.Fatal("second BulkLookup(): error expected but was nil") + } + + if !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("second BulkLookup() error = %q, should contain %q", err, tt.wantErr) + } + } + + if n := srv.reqs.Load(); n != tt.wantReqs { + t.Errorf("server received %d requests, want %d", n, tt.wantReqs) + } + + if n := srv.conns.Load(); n != tt.wantConns { + t.Errorf("server accepted %d connections, want %d", n, tt.wantConns) + } + }) + } +} + +func TestClient_BulkLookup_freshConnectionNotRetried(t *testing.T) { + ips := []string{"1.1.1.1", "8.8.8.8"} + + // the very first connection is dropped: a failure on a connection that was + // never used before is not retried + srv := newScriptedServer(t, testBulkJSONValid, srvDrop, srvRespond) + + client := &Client{c: newHTTPClient(), e: srv.url(), k: "testAPIkey"} + defer client.CloseIdleConnections() + + if _, err := client.BulkLookup(ips); err == nil { + t.Fatal("first BulkLookup(): error expected but was nil") + } + + if n := srv.reqs.Load(); n != 1 { + t.Fatalf("server received %d requests after the failed call, want 1", n) + } + + // the client is still usable afterwards + got, err := client.BulkLookup(ips) + if err != nil { + t.Fatalf("second BulkLookup() unexpected error: %v", err) + } + + if len(got) != 2 || got[0] == nil || got[0].IP != "1.1.1.1" { + t.Fatalf("second BulkLookup() = %v, want the two canned results", got) + } + + if n := srv.conns.Load(); n != 2 { + t.Fatalf("server accepted %d connections, want 2", n) + } +} + +// Fixtures for the is_satellite, is_anycast and dated blocklist additions. +// testJSONAnycast also carries a key the package does not model +// (region_type), which encoding/json has to ignore. + +// testJSONSatellite is a lookup of a satellite ISP address (excerpt). +const testJSONSatellite = `{ + "ip": "98.97.34.0", + "is_satellite": true, + "is_eu": false, + "city": "Seattle", + "country_code": "US", + "asn": { "asn": "AS14593", "name": "Space Exploration Technologies Corporation", "domain": "spacex.com", "route": "98.97.32.0/21", "type": "business" }, + "threat": { "is_tor": false, "is_vpn": false, "is_icloud_relay": false, "is_proxy": false, "is_datacenter": false, "is_anonymous": false, "is_known_attacker": false, "is_known_abuser": false, "is_threat": false, "is_bogon": false, "blocklists": [], "scores": { "vpn_score": 0, "proxy_score": 1, "threat_score": 8, "trust_score": 97 } }, + "count": "0" +}` + +// testJSONAnycast is a complete lookup of an anycast address whose blocklist +// entries carry first_seen, last_seen and days_seen. +const testJSONAnycast = `{ + "ip": "8.8.8.8", + "is_anycast": true, + "is_eu": false, + "city": null, + "region": null, + "region_code": null, + "region_type": null, + "country_name": "United States", + "country_code": "US", + "continent_name": "North America", + "continent_code": "NA", + "latitude": 39.7599983215332, + "longitude": -98.5, + "postal": null, + "calling_code": "1", + "flag": "https://ipdata.co/flags/us.png", + "emoji_flag": "🇺🇸", + "emoji_unicode": "U+1F1FA U+1F1F8", + "asn": { "asn": "AS15169", "name": "Google LLC", "domain": "about.google", "route": "8.8.8.0/24", "type": "hosting" }, + "company": { "name": "Google LLC", "domain": "about.google", "network": "8.8.8.0/24", "type": "hosting" }, + "languages": [ { "name": "English", "native": "English", "code": "en" } ], + "currency": { "name": "US Dollar", "code": "USD", "symbol": "$", "native": "$", "plural": "US dollars" }, + "time_zone": { "name": null, "abbr": null, "offset": null, "is_dst": null, "current_time": null }, + "threat": { + "is_tor": false, "is_vpn": false, "is_icloud_relay": false, "is_proxy": false, "is_datacenter": true, + "is_anonymous": false, "is_known_attacker": false, "is_known_abuser": false, "is_threat": false, "is_bogon": false, + "blocklists": [ + { "days_seen": 1, "first_seen": "2026-11-02", "last_seen": "2026-11-02", "name": "ipdata", "site": "https://ipdata.co", "type": "cloud" }, + { "days_seen": 1, "first_seen": "2026-11-02", "last_seen": "2026-11-02", "name": "ipdata", "site": "https://ipdata.co", "type": "open_resolvers" } + ], + "scores": { "vpn_score": 100, "proxy_score": 93, "threat_score": 100, "trust_score": 2 } + }, + "count": "0" +}` + +// testJSONVPNExit is a lookup of a VPN exit (excerpt): neither top-level flag +// is present, and the third blocklist entry has no dates. +const testJSONVPNExit = `{ + "ip": "187.13.250.1", + "is_eu": false, + "threat": { + "is_tor": false, "is_vpn": true, "is_icloud_relay": false, "is_proxy": false, "is_datacenter": true, + "is_anonymous": true, "is_known_attacker": true, "is_known_abuser": true, "is_threat": true, "is_bogon": false, + "blocklists": [ + { "days_seen": 2, "first_seen": "2026-10-31", "last_seen": "2026-11-02", "name": "Gluetun", "site": "https://github.com/qdm12/gluetun", "type": "vpn" }, + { "days_seen": 1, "first_seen": "2026-11-02", "last_seen": "2026-11-02", "name": "ipdata", "site": "https://ipdata.co", "type": "cloud" }, + { "name": "ipdata", "site": "https://ipdata.co", "type": "vpn" }, + { "days_seen": 2, "first_seen": "2026-10-31", "last_seen": "2026-11-02", "name": "NordVPN", "site": "https://nordvpn.com", "type": "vpn" }, + { "days_seen": 2, "first_seen": "2026-10-31", "last_seen": "2026-11-02", "name": "Wikimedia Global Blocklist", "site": "https://wikimedia.org", "type": "general" }, + { "days_seen": 2, "first_seen": "2026-10-31", "last_seen": "2026-11-02", "name": "X4BNet", "site": "https://github.com/X4BNet/lists_vpn", "type": "vpn" } + ], + "scores": { "vpn_score": 100, "proxy_score": 99, "threat_score": 100, "trust_score": 0 } + }, + "count": "0" +}` + +// testBulkJSONNewFields is a bulk response carrying the three fixtures above. +const testBulkJSONNewFields = "[" + testJSONSatellite + "," + testJSONAnycast + "," + testJSONVPNExit + "]" + +// typed values of the three fixtures; keys the API left out read false, empty +// or nil +var ( + testIPSatellite = IP{ + IP: "98.97.34.0", + ASN: ASN{ + ASN: "AS14593", + Name: "Space Exploration Technologies Corporation", + Domain: "spacex.com", + Route: "98.97.32.0/21", + Type: "business", + }, + City: "Seattle", + CountryCode: "US", + IsSatellite: true, + Threat: &Threat{ + Blocklists: []Blocklist{}, + Scores: Scores{VPNScore: 0, ProxyScore: 1, ThreatScore: 8, TrustScore: 97}, + }, + Count: "0", + } + + testIPAnycast = IP{ + IP: "8.8.8.8", + ASN: ASN{ + ASN: "AS15169", + Name: "Google LLC", + Domain: "about.google", + Route: "8.8.8.0/24", + Type: "hosting", + }, + CountryName: "United States", + CountryCode: "US", + Flag: "https://ipdata.co/flags/us.png", + EmojiFlag: "\U0001F1FA\U0001F1F8", + EmojiUnicode: "U+1F1FA U+1F1F8", + ContinentName: "North America", + ContinentCode: "NA", + Latitude: 39.7599983215332, + Longitude: -98.5, + CallingCode: "1", + IsAnycast: true, + Languages: []Language{{Name: "English", Native: "English", Code: "en"}}, + Currency: &Currency{ + Name: "US Dollar", + Code: "USD", + Symbol: "$", + Native: "$", + Plural: "US dollars", + }, + Company: &Company{ + Name: "Google LLC", + Domain: "about.google", + Network: "8.8.8.0/24", + Type: "hosting", + }, + TimeZone: &TimeZone{}, + Threat: &Threat{ + IsDatacenter: true, + Blocklists: []Blocklist{ + {Name: "ipdata", Site: "https://ipdata.co", Type: "cloud", FirstSeen: "2026-11-02", LastSeen: "2026-11-02", DaysSeen: intPtr(1)}, + {Name: "ipdata", Site: "https://ipdata.co", Type: "open_resolvers", FirstSeen: "2026-11-02", LastSeen: "2026-11-02", DaysSeen: intPtr(1)}, + }, + Scores: Scores{VPNScore: 100, ProxyScore: 93, ThreatScore: 100, TrustScore: 2}, + }, + Count: "0", + } + + testIPVPNExit = IP{ + IP: "187.13.250.1", + Threat: &Threat{ + IsVPN: true, + IsDatacenter: true, + IsAnonymous: true, + IsKnownAttacker: true, + IsKnownAbuser: true, + IsThreat: true, + Blocklists: []Blocklist{ + {Name: "Gluetun", Site: "https://github.com/qdm12/gluetun", Type: "vpn", FirstSeen: "2026-10-31", LastSeen: "2026-11-02", DaysSeen: intPtr(2)}, + {Name: "ipdata", Site: "https://ipdata.co", Type: "cloud", FirstSeen: "2026-11-02", LastSeen: "2026-11-02", DaysSeen: intPtr(1)}, + {Name: "ipdata", Site: "https://ipdata.co", Type: "vpn"}, + {Name: "NordVPN", Site: "https://nordvpn.com", Type: "vpn", FirstSeen: "2026-10-31", LastSeen: "2026-11-02", DaysSeen: intPtr(2)}, + {Name: "Wikimedia Global Blocklist", Site: "https://wikimedia.org", Type: "general", FirstSeen: "2026-10-31", LastSeen: "2026-11-02", DaysSeen: intPtr(2)}, + {Name: "X4BNet", Site: "https://github.com/X4BNet/lists_vpn", Type: "vpn", FirstSeen: "2026-10-31", LastSeen: "2026-11-02", DaysSeen: intPtr(2)}, + }, + Scores: Scores{VPNScore: 100, ProxyScore: 99, ThreatScore: 100, TrustScore: 0}, + }, + Count: "0", + } +) + +func Test_decodeIP_newFields(t *testing.T) { + tests := []struct { + name string + json string + want IP + }{ + {name: "satellite_isp", json: testJSONSatellite, want: testIPSatellite}, + {name: "anycast_with_dated_blocklists", json: testJSONAnycast, want: testIPAnycast}, + {name: "vpn_exit_entry_without_dates", json: testJSONVPNExit, want: testIPVPNExit}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := decodeIP(strings.NewReader(tt.json)) + if err != nil { + t.Fatalf("decodeIP() unexpected error: %v", err) + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatalf("IP differs: (-want +got)\n%s", diff) + } + }) + } +} + +func Test_client_Lookup_newFields(t *testing.T) { + mux := http.NewServeMux() + + mux.HandleFunc("/98.97.34.0", testAuthMiddleware(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, testJSONSatellite) + })) + + mux.HandleFunc("/8.8.8.8", testAuthMiddleware(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, testJSONAnycast) + })) + + mux.HandleFunc("/187.13.250.1", testAuthMiddleware(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, testJSONVPNExit) + })) + + server := httptest.NewServer(mux) + defer server.Close() + + c := Client{c: newHTTPClient(), e: server.URL + "/", k: "testAPIkey"} + + tests := []struct { + name string + ip string + want IP + }{ + {name: "satellite_isp", ip: "98.97.34.0", want: testIPSatellite}, + {name: "anycast_with_dated_blocklists", ip: "8.8.8.8", want: testIPAnycast}, + {name: "vpn_exit_entry_without_dates", ip: "187.13.250.1", want: testIPVPNExit}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := c.Lookup(tt.ip) + if err != nil { + t.Fatalf("Lookup(%q) unexpected error: %v", tt.ip, err) + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatalf("Lookup(%q) differs: (-want +got)\n%s", tt.ip, diff) + } + }) + } +} + +func TestClient_BulkLookup_newFields(t *testing.T) { + server := testBulkHTTPServer() + defer server.Close() + + client := &Client{c: newHTTPClient(), e: server.URL + "/", k: "testAPIkey"} + + got, err := client.BulkLookup([]string{"98.97.34.0", "8.8.8.8", "187.13.250.1"}) + if err != nil { + t.Fatalf("BulkLookup() unexpected error: %v", err) + } + + want := []*IP{&testIPSatellite, &testIPAnycast, &testIPVPNExit} + + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("IP slice differs: (-want +got)\n%s", diff) + } +} diff --git a/doc.go b/doc.go index 3a6a9e4..09b21d7 100644 --- a/doc.go +++ b/doc.go @@ -13,8 +13,18 @@ // // Example usage: // -// import "github.com/theckman/go-ipdata" +// import "github.com/ipdata/go" // -// ipd := ipdata.NewClient("") // API key is optional -// data, err := ipd.Lookup("8.8.8.8") +// ipd, err := ipdata.NewClient("YOUR_API_KEY") +// if err != nil { +// panic(err) +// } +// +// data, err := ipd.Lookup("8.8.8.8") +// +// A Client keeps a pool of persistent HTTPS connections (HTTP/2 when the +// server negotiates it) for as long as it lives, so build one Client and reuse +// it for every lookup instead of creating one per request. A Client is safe +// for concurrent use by multiple goroutines; call CloseIdleConnections at +// shutdown to release the pooled connections. package ipdata diff --git a/types.go b/types.go index dfcb241..4b52b71 100644 --- a/types.go +++ b/types.go @@ -33,6 +33,11 @@ type IP struct { IsEU bool `json:"is_eu"` + // IsSatellite and IsAnycast are present in the API response only when + // true, so they read false whenever the key is absent. + IsSatellite bool `json:"is_satellite"` + IsAnycast bool `json:"is_anycast"` + Languages []Language `json:"languages,omitempty"` Currency *Currency `json:"currency,omitempty"` @@ -109,32 +114,40 @@ type TimeZone struct { // Threat represents the threat object within the JSON response from the API. // This provides information about what type of threat this IP may be. type Threat struct { - IsTOR bool `json:"is_tor"` - IsVPN bool `json:"is_vpn"` - IsICloudRelay bool `json:"is_icloud_relay"` - IsProxy bool `json:"is_proxy"` - IsDatacenter bool `json:"is_datacenter"` - IsAnonymous bool `json:"is_anonymous"` - IsKnownAttacker bool `json:"is_known_attacker"` - IsKnownAbuser bool `json:"is_known_abuser"` - IsThreat bool `json:"is_threat"` - IsBogon bool `json:"is_bogon"` - Blocklists []Blocklist `json:"blocklists"` - Scores Scores `json:"scores"` + IsTOR bool `json:"is_tor"` + IsVPN bool `json:"is_vpn"` + IsICloudRelay bool `json:"is_icloud_relay"` + IsProxy bool `json:"is_proxy"` + IsDatacenter bool `json:"is_datacenter"` + IsAnonymous bool `json:"is_anonymous"` + IsKnownAttacker bool `json:"is_known_attacker"` + IsKnownAbuser bool `json:"is_known_abuser"` + IsThreat bool `json:"is_threat"` + IsBogon bool `json:"is_bogon"` + Blocklists []Blocklist `json:"blocklists"` + Scores Scores `json:"scores"` } +// Blocklist represents one entry of the threat.blocklists array: a feed that +// lists the IP address. FirstSeen and LastSeen are the first and last day the +// address was seen on the feed, as calendar dates in YYYY-MM-DD form (UTC) +// kept as strings, and DaysSeen is the number of days it was listed. Some +// feeds, for example ipdata's own vpn feed, report none of the three: the +// dates are then empty strings and DaysSeen is nil. type Blocklist struct { - Name string `json:"name"` - Site string `json:"site"` - Type string `json:"type"` + Name string `json:"name"` + Site string `json:"site"` + Type string `json:"type"` + FirstSeen string `json:"first_seen,omitempty"` + LastSeen string `json:"last_seen,omitempty"` + DaysSeen *int `json:"days_seen,omitempty"` } - type Scores struct { - VPNScore int `json:"vpn_score"` - ProxyScore int `json:"proxy_score"` - ThreatScore int `json:"threat_score"` - TrustScore int `json:"trust_score"` + VPNScore int `json:"vpn_score"` + ProxyScore int `json:"proxy_score"` + ThreatScore int `json:"threat_score"` + TrustScore int `json:"trust_score"` } type bulkIP struct { @@ -162,7 +175,9 @@ type bulkIP struct { CallingCode string `json:"calling_code"` - IsEU bool `json:"is_eu"` + IsEU bool `json:"is_eu"` + IsSatellite bool `json:"is_satellite"` + IsAnycast bool `json:"is_anycast"` Languages []Language `json:"languages,omitempty"` @@ -202,6 +217,8 @@ func bulkToIP(bip bulkIP) *IP { Longitude: bip.Longitude, CallingCode: bip.CallingCode, IsEU: bip.IsEU, + IsSatellite: bip.IsSatellite, + IsAnycast: bip.IsAnycast, Currency: bip.Currency, Carrier: bip.Carrier, Company: bip.Company, diff --git a/types_test.go b/types_test.go index 055c97e..a498836 100644 --- a/types_test.go +++ b/types_test.go @@ -5,6 +5,7 @@ package ipdata import ( + "encoding/json" "testing" "github.com/google/go-cmp/cmp" @@ -42,6 +43,8 @@ func Test_bulkToIP(t *testing.T) { Longitude: -122.4842, CallingCode: "1", IsEU: true, + IsSatellite: true, + IsAnycast: true, Languages: []Language{{Name: "English (US)", Native: "en-us", Code: "en"}}, Currency: &Currency{ Name: "US Dollar", @@ -76,6 +79,10 @@ func Test_bulkToIP(t *testing.T) { IsKnownAbuser: false, IsThreat: true, IsBogon: false, + Blocklists: []Blocklist{ + {Name: "ipdata", Site: "https://ipdata.co", Type: "vpn"}, + {Name: "NordVPN", Site: "https://nordvpn.com", Type: "vpn", FirstSeen: "2026-10-31", LastSeen: "2026-11-02", DaysSeen: intPtr(2)}, + }, }, Count: "1234", } @@ -104,6 +111,8 @@ func Test_bulkToIP(t *testing.T) { Longitude: -122.4842, CallingCode: "1", IsEU: true, + IsSatellite: true, + IsAnycast: true, Languages: []Language{{Name: "English (US)", Native: "en-us", Code: "en"}}, Currency: &Currency{ Name: "US Dollar", @@ -138,6 +147,10 @@ func Test_bulkToIP(t *testing.T) { IsKnownAbuser: false, IsThreat: true, IsBogon: false, + Blocklists: []Blocklist{ + {Name: "ipdata", Site: "https://ipdata.co", Type: "vpn"}, + {Name: "NordVPN", Site: "https://nordvpn.com", Type: "vpn", FirstSeen: "2026-10-31", LastSeen: "2026-11-02", DaysSeen: intPtr(2)}, + }, }, Count: "1234", } @@ -147,3 +160,49 @@ func Test_bulkToIP(t *testing.T) { t.Fatalf("IP differs: (-want +got)\n%s", diff) } } + +// intPtr returns a pointer to i, for the optional integer fields of fixtures. +func intPtr(i int) *int { + return &i +} + +func Test_Blocklist_marshalOptionalFields(t *testing.T) { + // the fields a feed did not report are left out when encoding, mirroring + // the API response + tests := []struct { + name string + in Blocklist + want string + }{ + { + name: "without_dates", + in: Blocklist{Name: "ipdata", Site: "https://ipdata.co", Type: "vpn"}, + want: `{"name":"ipdata","site":"https://ipdata.co","type":"vpn"}`, + }, + { + name: "with_dates", + in: Blocklist{ + Name: "NordVPN", + Site: "https://nordvpn.com", + Type: "vpn", + FirstSeen: "2026-10-31", + LastSeen: "2026-11-02", + DaysSeen: intPtr(2), + }, + want: `{"name":"NordVPN","site":"https://nordvpn.com","type":"vpn","first_seen":"2026-10-31","last_seen":"2026-11-02","days_seen":2}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := json.Marshal(tt.in) + if err != nil { + t.Fatalf("json.Marshal() unexpected error: %v", err) + } + + if string(got) != tt.want { + t.Fatalf("json.Marshal() = %s, want %s", got, tt.want) + } + }) + } +}