From c37e12060df503581723898ea2a1212186140ae8 Mon Sep 17 00:00:00 2001 From: MudassarTariq Date: Wed, 4 Nov 2026 20:02:26 +0500 Subject: [PATCH 1/2] Add request limits and prepare CLI 2.1.2 --- .github/release-notes/v2.1.2.md | 6 + .github/workflows/release.yml | 4 +- README.md | 28 +- cmd/commands_test.go | 70 ++++- cmd/release_test.go | 4 +- cmd/request_limits_http_test.go | 150 +++++++++++ cmd/root.go | 8 +- internal/utils/request.go | 38 ++- internal/utils/request_limits_test.go | 239 ++++++++++++++++++ postman/mocks/ipgeolocation/default.js | 35 ++- .../simulations/cli/simulation.context.yaml | 11 + scripts/test-http.cjs | 2 +- 12 files changed, 570 insertions(+), 25 deletions(-) create mode 100644 .github/release-notes/v2.1.2.md create mode 100644 cmd/request_limits_http_test.go create mode 100644 internal/utils/request_limits_test.go diff --git a/.github/release-notes/v2.1.2.md b/.github/release-notes/v2.1.2.md new file mode 100644 index 0000000..58f40f0 --- /dev/null +++ b/.github/release-notes/v2.1.2.md @@ -0,0 +1,6 @@ +- Add `--timeout` to set how long an API request can take. +- Add `--max-response-mb` to set the largest response to read. +- Stop requests that exceed these limits without printing or saving partial results. +- Add checks for large bulk lookups and Arabic requests. + +Existing commands and output formats remain available. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 860ff80..5c0b3eb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,6 +16,8 @@ jobs: with: fetch-depth: 0 persist-credentials: false + - name: Check release notes + run: test -s ".github/release-notes/$GITHUB_REF_NAME.md" - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: go-version: "1.26.10" @@ -30,4 +32,4 @@ jobs: - name: Publish archives and checksums env: GH_TOKEN: ${{ github.token }} - run: gh release create "$GITHUB_REF_NAME" dist/* --verify-tag --title "$GITHUB_REF_NAME" --generate-notes + run: gh release create "$GITHUB_REF_NAME" dist/* --verify-tag --title "$GITHUB_REF_NAME" --notes-file ".github/release-notes/$GITHUB_REF_NAME.md" diff --git a/README.md b/README.md index 6685358..780de96 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ For local HTTPS regression checks, install Node.js and Postman CLI 1.73.0, then ### Download Prebuilt Binaries #### Overview -These are prebuilt binaries for the IPGeolocation.io CLI tool, version **2.1.1**. Users can download these files directly from [GitHub Releases](https://github.com/IPGeolocation/cli/releases) or from the table below without needing to build from source. +These are prebuilt binaries for the IPGeolocation.io CLI tool, version **2.1.2**. Users can download these files directly from [GitHub Releases](https://github.com/IPGeolocation/cli/releases) or from the table below without needing to build from source. The CLI provides geolocation information, time zone, user-agent parsing, bulk IP lookups, and more. @@ -162,13 +162,13 @@ The CLI provides geolocation information, time zone, user-agent parsing, bulk IP | Platform | Architecture | File Name / Download Link | |----------|--------------|------------------------------------------------------------------------------------------------------------------------------------------------------| -| Linux | amd64 | [**ipgeolocation-2.1.1-linux-amd64.tar.gz**](https://github.com/IPGeolocation/cli/releases/download/v2.1.1/ipgeolocation-2.1.1-linux-amd64.tar.gz) | -| Linux | arm64 | [**ipgeolocation-2.1.1-linux-arm64.tar.gz**](https://github.com/IPGeolocation/cli/releases/download/v2.1.1/ipgeolocation-2.1.1-linux-arm64.tar.gz) | -| macOS | amd64 | [**ipgeolocation-2.1.1-darwin-amd64.tar.gz**](https://github.com/IPGeolocation/cli/releases/download/v2.1.1/ipgeolocation-2.1.1-darwin-amd64.tar.gz) | -| macOS | arm64 | [**ipgeolocation-2.1.1-darwin-arm64.tar.gz**](https://github.com/IPGeolocation/cli/releases/download/v2.1.1/ipgeolocation-2.1.1-darwin-arm64.tar.gz) | -| Windows | amd64 | [**ipgeolocation-2.1.1-windows-amd64.zip**](https://github.com/IPGeolocation/cli/releases/download/v2.1.1/ipgeolocation-2.1.1-windows-amd64.zip) | +| Linux | amd64 | [**ipgeolocation-2.1.2-linux-amd64.tar.gz**](https://github.com/IPGeolocation/cli/releases/download/v2.1.2/ipgeolocation-2.1.2-linux-amd64.tar.gz) | +| Linux | arm64 | [**ipgeolocation-2.1.2-linux-arm64.tar.gz**](https://github.com/IPGeolocation/cli/releases/download/v2.1.2/ipgeolocation-2.1.2-linux-arm64.tar.gz) | +| macOS | amd64 | [**ipgeolocation-2.1.2-darwin-amd64.tar.gz**](https://github.com/IPGeolocation/cli/releases/download/v2.1.2/ipgeolocation-2.1.2-darwin-amd64.tar.gz) | +| macOS | arm64 | [**ipgeolocation-2.1.2-darwin-arm64.tar.gz**](https://github.com/IPGeolocation/cli/releases/download/v2.1.2/ipgeolocation-2.1.2-darwin-arm64.tar.gz) | +| Windows | amd64 | [**ipgeolocation-2.1.2-windows-amd64.zip**](https://github.com/IPGeolocation/cli/releases/download/v2.1.2/ipgeolocation-2.1.2-windows-amd64.zip) | -Archives include [SHA256SUMS](https://github.com/IPGeolocation/cli/releases/download/v2.1.1/SHA256SUMS) for download verification. +Archives include [SHA256SUMS](https://github.com/IPGeolocation/cli/releases/download/v2.1.2/SHA256SUMS) for download verification. #### Installation Instructions @@ -176,11 +176,11 @@ Archives include [SHA256SUMS](https://github.com/IPGeolocation/cli/releases/down 1. Download the `.tar.gz` file for your architecture. 2. Extract it to a folder in your PATH, e.g., `/usr/local/bin`: ```bash - tar -xzf ipgeolocation-2.1.1-linux-amd64.tar.gz -C /usr/local/bin + tar -xzf ipgeolocation-2.1.2-linux-amd64.tar.gz -C /usr/local/bin ``` 3. Rename the binary for simplicity: ```bash - mv /usr/local/bin/ipgeolocation-2.1.1-linux-amd64 /usr/local/bin/ipgeolocation + mv /usr/local/bin/ipgeolocation-2.1.2-linux-amd64 /usr/local/bin/ipgeolocation ``` 4. Make the binary executable: ```bash @@ -195,11 +195,11 @@ Archives include [SHA256SUMS](https://github.com/IPGeolocation/cli/releases/down 1. Download the `.tar.gz` file for your architecture (amd64 or arm64). 2. Extract to a folder in your PATH, e.g., `/usr/local/bin`: ```bash - tar -xzf ipgeolocation-2.1.1-darwin-amd64.tar.gz -C /usr/local/bin + tar -xzf ipgeolocation-2.1.2-darwin-amd64.tar.gz -C /usr/local/bin ``` 3. Rename the binary: ```bash - mv /usr/local/bin/ipgeolocation-2.1.1-darwin-amd64 /usr/local/bin/ipgeolocation + mv /usr/local/bin/ipgeolocation-2.1.2-darwin-amd64 /usr/local/bin/ipgeolocation ``` 4. Make executable: ```bash @@ -212,7 +212,7 @@ Archives include [SHA256SUMS](https://github.com/IPGeolocation/cli/releases/down **Windows:** 1. Download the `.zip` file. -2. Extract the `ipgeolocation-2.1.1-windows-amd64.exe` to a folder included in your system `PATH`. +2. Extract the `ipgeolocation-2.1.2-windows-amd64.exe` to a folder included in your system `PATH`. 3. Rename the binary to `ipgeolocation.exe` for convenience. 4. Open Command Prompt and verify: ```cmd @@ -222,7 +222,7 @@ Archives include [SHA256SUMS](https://github.com/IPGeolocation/cli/releases/down > [!NOTE] > - Ensure execution permissions on Linux/macOS. > - Recommended folder for binaries: `/usr/local/bin` or any folder in your PATH. -> - Prebuilt binaries include version **2.1.1** in their filename. Rename them after extraction for easier use. +> - Prebuilt binaries include version **2.1.2** in their filename. Rename them after extraction for easier use. > - For updates, check GitHub Releases. ### Troubleshooting @@ -292,6 +292,8 @@ These flags are available for all commands: | Flag | Description | |--------------|----------------------------| | `-h, --help` | Show help for the command. | +| `--timeout` | Maximum time for one API request. Default: `5m` (5 minutes). For a longer request, use `--timeout 10m`. | +| `--max-response-mb` | Maximum response size in MB. Default: `256`. For larger bulk results, use `--max-response-mb 512` or another positive whole number. | > [!TIP] > You can also check the version for `ipgeolocation` using the `--version` flag: diff --git a/cmd/commands_test.go b/cmd/commands_test.go index 5de8e42..9cb8f70 100644 --- a/cmd/commands_test.go +++ b/cmd/commands_test.go @@ -40,6 +40,9 @@ func (fixtureTransport) RoundTrip(r *http.Request) (*http.Response, error) { "ip": "2001:db8::1", "location": map[string]string{"city": "Test City"}, "_request": map[string]interface{}{"method": r.Method, "path": r.URL.Path, "query": r.URL.Query(), "payload": payload}, } + if r.URL.Query().Get("lang") == "ar" { + result["location"] = map[string]string{"city": "مدينة"} + } var response interface{} = result if strings.HasSuffix(r.URL.Path, "-bulk") { response = []interface{}{result} @@ -85,14 +88,19 @@ func TestCLIProcess(t *testing.T) { t.Fatal("missing CLI arguments") } -func runCLI(t *testing.T, home string, args ...string) string { - t.Helper() +func newCLIProcess(home string, args ...string) *exec.Cmd { process := exec.Command(os.Args[0], append([]string{"-test.run=^TestCLIProcess$", "--"}, args...)...) if binary := os.Getenv("CLI_TEST_BINARY"); binary != "" { process = exec.Command(binary, args...) } process.Dir = home process.Env = append(os.Environ(), "CLI_TEST_PROCESS=1", "HOME="+home, "USERPROFILE="+home, "GORACE=atexit_sleep_ms=0") + return process +} + +func runCLI(t *testing.T, home string, args ...string) string { + t.Helper() + process := newCLIProcess(home, args...) output, err := process.CombinedOutput() if err != nil { t.Fatalf("CLI %v failed: %v\n%s", args, err, output) @@ -100,6 +108,62 @@ func runCLI(t *testing.T, home string, args ...string) string { return strings.TrimSpace(string(output)) } +func TestCLIArabic(t *testing.T) { + home := configuredHome(t, testKey) + for _, args := range [][]string{ + {"ipgeo", "--ip", "8.8.8.8"}, + {"bulk-ip-geo", "--ips", "8.8.8.8,1.1.1.1"}, + {"timezone", "--ip", "8.8.8.8"}, + {"astronomy", "--ip", "8.8.8.8"}, + {"astronomy-timeseries", "--ip", "8.8.8.8", "--start-date", "2026-11-01", "--end-date", "2026-11-02"}, + } { + t.Run(args[0], func(t *testing.T) { + output := runCLI(t, home, append(args, "--lang", "ar", "--output", "raw")...) + var response interface{} + if err := json.Unmarshal([]byte(output), &response); err != nil { + t.Fatalf("Arabic response invalid: %v", err) + } + if args[0] == "bulk-ip-geo" { + response = response.([]interface{})[0] + } + result := response.(map[string]interface{}) + query := result["_request"].(map[string]interface{})["query"].(map[string]interface{}) + if !reflect.DeepEqual(query["lang"], []interface{}{"ar"}) || result["location"].(map[string]interface{})["city"] != "مدينة" { + t.Fatal("Arabic request or response changed") + } + }) + } +} + +func TestCLIRequestLimitFlags(t *testing.T) { + home := configuredHome(t, testKey) + if output := runCLI(t, home, "my-ip", "--timeout", "10m", "--max-response-mb", "512", "--output", "raw"); !strings.Contains(output, "2001:db8::1") { + t.Fatalf("limit overrides failed: %s", output) + } + for _, args := range [][]string{ + {"ipgeo", "--timeout", "0s"}, + {"ipgeo", "--timeout=-1s"}, + {"ipgeo", "--timeout", "not-a-duration"}, + {"ipgeo", "--timeout", "999999999999999h"}, + {"ipgeo", "--max-response-mb", "0"}, + {"ipgeo", "--max-response-mb=-1"}, + {"ipgeo", "--max-response-mb", "1.5"}, + {"ipgeo", "--max-response-mb", "9223372036855"}, + } { + process := newCLIProcess(home, args...) + output, err := process.CombinedOutput() + if err == nil || !strings.Contains(string(output), "Error") || strings.Contains(string(output), `"_request"`) || strings.Contains(string(output), testKey) { + t.Fatalf("invalid limits not rejected: args=%v err=%v output=%s", args, err, output) + } + } + for _, command := range []string{"ipgeo", "bulk-ip-geo", "my-ip", "config"} { + output := runCLI(t, home, command, "--help") + if !strings.Contains(output, "--timeout") || !strings.Contains(output, "--max-response-mb") || !strings.Contains(output, "default 5m0s") || !strings.Contains(output, "default 256") { + t.Fatalf("request controls absent from help for %s", command) + } + } +} + func configuredHome(t *testing.T, key string) string { t.Helper() home := t.TempDir() @@ -203,7 +267,7 @@ func TestCLIConfigAndBulkFiles(t *testing.T) { if output := runCLI(t, home, "config"); strings.Contains(output, "short") || !strings.Contains(output, "********") { t.Fatalf("short key exposed: %s", output) } - if output := runCLI(t, home, "--version"); output != "ipgeolocation version 2.1.1" { + if output := runCLI(t, home, "--version"); output != "ipgeolocation version 2.1.2" { t.Fatalf("incorrect version: %s", output) } if output := runCLI(t, home, "--help"); !strings.Contains(output, "time-conversion") { diff --git a/cmd/release_test.go b/cmd/release_test.go index 99f9eb6..e3c313c 100644 --- a/cmd/release_test.go +++ b/cmd/release_test.go @@ -20,7 +20,7 @@ func TestReleaseGuardsKeepExistingArtifacts(t *testing.T) { for _, tc := range []struct{ compiler, tag, want string }{ {"go1.24.5", "", "Use Go"}, {"go1.26.9", "", "Use Go"}, {"go1.27.2", "", "Use Go"}, {"go1.28rc1", "", "Use Go"}, {"go1.26.10", "v2.1.0", "Release tag must match"}, - {"go1.26.10", "v2.1.1", "checks-started"}, {"go1.27.3", "", "checks-started"}, {"go1.28.0", "", "checks-started"}, + {"go1.26.10", "v2.1.2", "checks-started"}, {"go1.27.3", "", "checks-started"}, {"go1.28.0", "", "checks-started"}, } { t.Run(tc.compiler+tc.tag, func(t *testing.T) { dir := t.TempDir() @@ -29,7 +29,7 @@ func TestReleaseGuardsKeepExistingArtifacts(t *testing.T) { } for name, content := range map[string]string{ "build-release.sh": string(source), "dist/keep": "existing archive", - "go": "#!/bin/sh\ncase \"$1\" in\nenv) echo \"$TEST_COMPILER\";;\nrun) echo 'ipgeolocation version 2.1.1';;\n*) echo checks-started >&2; exit 7;;\nesac\n", + "go": "#!/bin/sh\ncase \"$1\" in\nenv) echo \"$TEST_COMPILER\";;\nrun) echo 'ipgeolocation version 2.1.2';;\n*) echo checks-started >&2; exit 7;;\nesac\n", "govulncheck": "#!/bin/sh\nexit 0\n", } { if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0700); err != nil { diff --git a/cmd/request_limits_http_test.go b/cmd/request_limits_http_test.go new file mode 100644 index 0000000..65990f3 --- /dev/null +++ b/cmd/request_limits_http_test.go @@ -0,0 +1,150 @@ +package cmd + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestCLIRequestLimitsHTTP(t *testing.T) { + if os.Getenv("CLI_HTTP_TEST") != "1" { + t.Skip("requires the local HTTP simulation") + } + for _, key := range []string{"SLOW_HEADERS", "SLOW_BODY"} { + home := configuredHome(t, key) + for _, args := range [][]string{{"ipgeo"}, {"parse-user-agent", "--user-agent", "test"}} { + output := runCLI(t, home, append(args, "--timeout", "100ms")...) + if !strings.Contains(output, "request timed out") || strings.Contains(output, key) || strings.Contains(output, `"_request"`) { + t.Fatalf("timeout did not stop safely: %s", output) + } + } + output := runCLI(t, home, "ipgeo", "--timeout", "2s", "--output", "raw") + if !json.Valid([]byte(output)) || !strings.Contains(output, "2001:db8::1") { + t.Fatalf("longer timeout did not complete: %s", output) + } + } + for _, key := range []string{"LIMIT_RESPONSE", "SLOW_BODY"} { + home := configuredHome(t, key) + for _, command := range []string{"bulk-ip-geo", "bulk-ip-security"} { + for _, exists := range []bool{false, true} { + name := fmt.Sprintf("%s-%v", command, exists) + file := filepath.Join(home, name+".json") + if exists { + if err := os.WriteFile(file, []byte("keep existing results"), 0600); err != nil { + t.Fatal(err) + } + } + args := []string{command, "--ips", "8.8.8.8", "--output-file", name} + want := "response exceeds 1 MB" + if key == "LIMIT_RESPONSE" { + args = append(args, "--max-response-mb", "1") + } else { + args = append(args, "--timeout", "100ms") + want = "request timed out" + } + output := runCLI(t, home, args...) + if !strings.Contains(output, want) || strings.Contains(output, key) || strings.Contains(output, `"_request"`) || strings.Contains(output, "Output saved") { + t.Fatalf("failed bulk request produced output: %s", output) + } + data, err := os.ReadFile(file) + if exists { + if err != nil || string(data) != "keep existing results" { + t.Fatalf("failed request changed output file: %v", err) + } + } else if !os.IsNotExist(err) { + t.Fatal("failed request created an output file") + } + } + } + if key == "LIMIT_RESPONSE" { + output := runCLI(t, home, "bulk-ip-geo", "--ips", "8.8.8.8", "--max-response-mb", "2", "--output", "raw", "--output-file", "complete") + if !strings.Contains(output, "Output saved to file:") { + t.Fatal("larger response limit did not complete") + } + data, err := os.ReadFile(filepath.Join(home, "complete.json")) + var result []struct{ Padding string } + if err != nil || json.Unmarshal(data, &result) != nil || len(result) != 1 || len(result[0].Padding) != 1_100_000 { + t.Fatal("larger response limit lost data") + } + } + } +} + +func TestCLILargeBulkHTTP(t *testing.T) { + if os.Getenv("CLI_HTTP_TEST") != "1" { + t.Skip("requires the local HTTP simulation") + } + home := configuredHome(t, "BULK_50K") + ips, err := os.Create(filepath.Join(home, "ips.txt")) + if err != nil { + t.Fatal(err) + } + writer := bufio.NewWriter(ips) + for i := 0; i < 50_000; i++ { + fmt.Fprintf(writer, "2001:db8::%x\n", i) + } + if err := writer.Flush(); err != nil { + t.Fatal(err) + } + ips.Close() + for _, command := range []string{"bulk-ip-geo", "bulk-ip-security"} { + t.Run(command, func(t *testing.T) { + output, err := os.Create(filepath.Join(home, command+".stdout")) + if err != nil { + t.Fatal(err) + } + defer output.Close() + args := []string{command, "--file", "ips.txt", "--output", "raw"} + city := "Test City" + if command == "bulk-ip-geo" { + args = append(args, "--lang", "ar") + city = "مدينة" + } + process := newCLIProcess(home, args...) + var stderr bytes.Buffer + process.Stdout, process.Stderr = output, &stderr + if err := process.Run(); err != nil { + t.Fatalf("large bulk request failed: %v %s", err, stderr.String()) + } + info, err := output.Stat() + if err != nil || info.Size() < 200_000_000 || info.Size() > 256_000_000 { + t.Fatalf("unexpected large response size: %v %v", info, err) + } + if _, err := output.Seek(0, io.SeekStart); err != nil { + t.Fatal(err) + } + decoder := json.NewDecoder(output) + if token, err := decoder.Token(); err != nil || token != json.Delim('[') { + t.Fatalf("bulk output is not a JSON array: %v %v", token, err) + } + count := 0 + for decoder.More() { + var result struct { + IP string + Location struct{ City string } + Company struct{ Name string } + } + if err := decoder.Decode(&result); err != nil { + t.Fatal(err) + } + if result.IP != fmt.Sprintf("2001:db8::%x", count) || result.Location.City != city || len(result.Company.Name) != 4000 { + t.Fatalf("bulk result %d changed", count) + } + count++ + } + if token, err := decoder.Token(); err != nil || token != json.Delim(']') || count != 50_000 { + t.Fatalf("bulk output incomplete: count=%d token=%v err=%v", count, token, err) + } + if _, err := decoder.Token(); err != io.EOF { + t.Fatalf("unexpected data after bulk results: %v", err) + } + t.Logf("Verified %d ordered results and %d output bytes with default limits", count, info.Size()) + }) + } +} diff --git a/cmd/root.go b/cmd/root.go index 169a65c..7ca4294 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -9,6 +9,7 @@ import ( "os" "github.com/IPGeolocation/cli/v2/ascii" + "github.com/IPGeolocation/cli/v2/internal/utils" "github.com/spf13/cobra" ) @@ -17,7 +18,10 @@ import ( var rootCmd = &cobra.Command{ Use: "ipgeolocation", Short: "CLI for accessing IPGeolocation.io API endpoints", - Version: "2.1.1", + Version: "2.1.2", + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + return utils.ValidateRequestLimits() + }, Long: `ipgeolocation is a command-line interface (CLI) tool for accessing all major endpoints of the IPGeolocation.io API. It allows you to retrieve IP geolocation data, timezone information, currency details, user-agent parsing, and more, directly from your terminal. @@ -53,4 +57,6 @@ func Execute() { // init sets up the root command with flags. func init() { rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") + rootCmd.PersistentFlags().DurationVar(&utils.RequestTimeout, "timeout", utils.RequestTimeout, "Maximum time for one API request (e.g. 30s or 5m)") + rootCmd.PersistentFlags().Int64Var(&utils.MaxResponseMB, "max-response-mb", utils.MaxResponseMB, "Maximum response size in MB") } diff --git a/internal/utils/request.go b/internal/utils/request.go index 27dbe4e..a13669a 100644 --- a/internal/utils/request.go +++ b/internal/utils/request.go @@ -2,14 +2,33 @@ package utils import ( "bytes" + "context" "encoding/json" "fmt" "io" + "math" "net/http" "net/url" "strings" + "time" ) +var RequestTimeout = 5 * time.Minute +var MaxResponseMB int64 = 256 + +func ValidateRequestLimits() error { + if RequestTimeout <= 0 { + return fmt.Errorf("request timeout must be greater than zero") + } + if MaxResponseMB <= 0 { + return fmt.Errorf("response size limit must be greater than zero") + } + if MaxResponseMB > math.MaxInt64/1_000_000 { + return fmt.Errorf("response size limit is too large") + } + return nil +} + func Get(apiURL string) ([]byte, error) { return request(http.MethodGet, apiURL, nil, nil) } @@ -32,13 +51,22 @@ func (e *requestError) Error() string { return e.message } func (e *requestError) Unwrap() error { return e.cause } func request(method, apiURL string, data io.Reader, headers map[string]string) (body []byte, err error) { - req, err := http.NewRequest(method, apiURL, data) + if err := ValidateRequestLimits(); err != nil { + return nil, err + } + maxBytes := MaxResponseMB * 1_000_000 + ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, method, apiURL, data) if err != nil { return nil, fmt.Errorf("failed to create request: invalid URL") } // API keys can appear in transport errors and in error responses from the server. defer func() { + if err != nil && ctx.Err() == context.DeadlineExceeded { + err = fmt.Errorf("request timed out; increase --timeout for longer requests: %w", err) + } if key := req.URL.Query().Get("apiKey"); err != nil && key != "" { encoded, _ := json.Marshal(key) message := strings.NewReplacer( @@ -62,10 +90,16 @@ func request(method, apiURL string, data io.Reader, headers map[string]string) ( } defer resp.Body.Close() - body, err = io.ReadAll(resp.Body) + if resp.ContentLength > maxBytes { + return nil, fmt.Errorf("response exceeds %d MB; increase --max-response-mb for larger results", MaxResponseMB) + } + body, err = io.ReadAll(io.LimitReader(resp.Body, maxBytes+1)) if err != nil { return nil, fmt.Errorf("failed to read response: %w", err) } + if int64(len(body)) > maxBytes { + return nil, fmt.Errorf("response exceeds %d MB; increase --max-response-mb for larger results", MaxResponseMB) + } if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("server error: %s", string(body)) } diff --git a/internal/utils/request_limits_test.go b/internal/utils/request_limits_test.go new file mode 100644 index 0000000..6f8425a --- /dev/null +++ b/internal/utils/request_limits_test.go @@ -0,0 +1,239 @@ +package utils + +import ( + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "math" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func setRequestLimits(t *testing.T, timeout time.Duration, megabytes int64) { + t.Helper() + oldTimeout, oldSize := RequestTimeout, MaxResponseMB + t.Cleanup(func() { RequestTimeout, MaxResponseMB = oldTimeout, oldSize }) + RequestTimeout, MaxResponseMB = timeout, megabytes +} + +type countedBody struct { + io.Reader + read int + closed bool +} + +func (b *countedBody) Read(p []byte) (int, error) { + n, err := b.Reader.Read(p) + b.read += n + return n, err +} + +func (b *countedBody) Close() error { b.closed = true; return nil } + +func TestRequestsHaveDeadline(t *testing.T) { + original := http.DefaultClient + t.Cleanup(func() { http.DefaultClient = original }) + http.DefaultClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if _, ok := r.Context().Deadline(); !ok { + t.Error("request has no deadline") + } + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{}`))}, nil + })} + if _, err := Get("https://api.example.test/"); err != nil { + t.Fatal(err) + } + if _, err := PostJSON("https://api.example.test/", nil, nil); err != nil { + t.Fatal(err) + } +} + +func TestRequestsRejectAdvertisedOversizedBody(t *testing.T) { + original := http.DefaultClient + t.Cleanup(func() { http.DefaultClient = original }) + http.DefaultClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusOK, ContentLength: 1 << 62, Body: io.NopCloser(strings.NewReader(`{}`))}, nil + })} + for _, call := range []func(string) ([]byte, error){Get, func(u string) ([]byte, error) { return PostJSON(u, nil, nil) }} { + if body, err := call("https://api.example.test/"); err == nil || body != nil { + t.Errorf("oversized response was accepted: body=%q err=%v", body, err) + } + } +} + +func TestRequestSizeBoundaries(t *testing.T) { + setRequestLimits(t, time.Minute, 1) + original := http.DefaultClient + t.Cleanup(func() { http.DefaultClient = original }) + for _, method := range []string{http.MethodGet, http.MethodPost} { + for _, size := range []int{999_999, 1_000_000, 1_000_001} { + for _, knownLength := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%d/known=%v", method, size, knownLength), func(t *testing.T) { + body := &countedBody{Reader: strings.NewReader(strings.Repeat("x", size))} + var requestContext context.Context + http.DefaultClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + requestContext = r.Context() + length := int64(-1) + if knownLength { + length = int64(size) + } + return &http.Response{StatusCode: http.StatusOK, ContentLength: length, Body: body}, nil + })} + result, err := request(method, "https://api.example.test/?apiKey=fixture-secret", nil, nil) + if size <= 1_000_000 { + if err != nil || len(result) != size { + t.Fatalf("valid body changed: length=%d err=%v", len(result), err) + } + } else if err == nil || result != nil || !strings.Contains(err.Error(), "response exceeds 1 MB") || strings.Contains(err.Error(), "fixture-secret") { + t.Fatalf("oversized response not rejected cleanly: length=%d err=%v", len(result), err) + } + if !body.closed || body.read > 1_000_001 || requestContext.Err() != context.Canceled { + t.Fatalf("response not stopped: closed=%v read=%d context=%v", body.closed, body.read, requestContext.Err()) + } + if knownLength && size > 1_000_000 && body.read != 0 { + t.Fatal("read a body already known to be too large") + } + }) + } + } + } +} + +func TestRequestLimitAppliesAfterGzipDecoding(t *testing.T) { + setRequestLimits(t, time.Minute, 1) + for _, size := range []int{1_000_000, 1_000_001} { + t.Run(fmt.Sprint(size), func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Encoding", "gzip") + writer := gzip.NewWriter(w) + io.WriteString(writer, strings.Repeat("x", size)) + writer.Close() + })) + defer server.Close() + body, err := Get(server.URL) + if size == 1_000_000 { + if err != nil || len(body) != size { + t.Fatalf("valid decoded body changed: length=%d err=%v", len(body), err) + } + } else if err == nil || body != nil || !strings.Contains(err.Error(), "response exceeds") { + t.Fatalf("oversized decoded body accepted: length=%d err=%v", len(body), err) + } + }) + } +} + +func TestRequestTimeoutStopsHeadersAndBody(t *testing.T) { + setRequestLimits(t, 100*time.Millisecond, 1) + for _, phase := range []string{"headers", "body", "continuous-body"} { + for _, method := range []string{http.MethodGet, http.MethodPost} { + t.Run(method+"/"+phase, func(t *testing.T) { + closed := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer close(closed) + if phase != "headers" { + io.WriteString(w, "prefix") + w.(http.Flusher).Flush() + } + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-r.Context().Done(): + return + case <-ticker.C: + if phase == "continuous-body" { + io.WriteString(w, "x") + w.(http.Flusher).Flush() + } + } + } + })) + defer server.Close() + done := make(chan struct{}) + var body []byte + var err error + go func() { + body, err = request(method, server.URL+"?apiKey=fixture-secret", nil, nil) + close(done) + }() + select { + case <-done: + case <-time.After(3 * time.Second): + server.CloseClientConnections() + <-done + t.Fatal("request did not stop at its deadline") + } + if !errors.Is(err, context.DeadlineExceeded) || body != nil || !strings.Contains(err.Error(), "request timed out") || strings.Contains(err.Error(), "fixture-secret") { + t.Fatalf("unsafe timeout result: body=%q err=%v", body, err) + } + select { + case <-closed: + case <-time.After(3 * time.Second): + t.Fatal("server request was not canceled") + } + }) + } + } +} + +func TestRedirectsKeepOneDeadline(t *testing.T) { + setRequestLimits(t, 10*time.Minute, 512) + original := http.DefaultClient + t.Cleanup(func() { http.DefaultClient = original }) + var deadline time.Time + requests := 0 + http.DefaultClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + current, ok := r.Context().Deadline() + if !ok { + t.Fatal("missing deadline") + } + if requests == 0 { + deadline = current + if remaining := time.Until(current); remaining < 9*time.Minute || remaining > 10*time.Minute { + t.Fatalf("timeout override not used: %s", remaining) + } + } else if current != deadline { + t.Fatal("redirect changed the deadline") + } + requests++ + response := &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{}`))} + if requests < 3 { + response.StatusCode = http.StatusFound + response.Header.Set("Location", fmt.Sprintf("/step%d", requests)) + } + return response, nil + })} + if _, err := Get("https://api.example.test/"); err != nil || requests != 3 { + t.Fatalf("redirect handling changed: requests=%d err=%v", requests, err) + } +} + +func TestInvalidLimitsDoNotSendRequest(t *testing.T) { + original := http.DefaultClient + t.Cleanup(func() { http.DefaultClient = original }) + http.DefaultClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + t.Error("sent request with invalid limits") + return nil, errors.New("unexpected request") + })} + for _, tc := range []struct { + timeout time.Duration + size int64 + }{{0, 1}, {-1, 1}, {time.Second, 0}, {time.Second, -1}, {time.Second, math.MaxInt64/1_000_000 + 1}} { + t.Run(fmt.Sprintf("%s/%d", tc.timeout, tc.size), func(t *testing.T) { + setRequestLimits(t, tc.timeout, tc.size) + for _, call := range []func(string) ([]byte, error){Get, func(u string) ([]byte, error) { return PostJSON(u, nil, nil) }} { + if body, err := call("https://api.example.test/"); err == nil || body != nil { + t.Fatalf("invalid limits accepted: %q %v", body, err) + } + } + }) + } + setRequestLimits(t, time.Duration(math.MaxInt64), math.MaxInt64/1_000_000) + if err := ValidateRequestLimits(); err != nil { + t.Fatalf("valid large limits rejected: %v", err) + } +} diff --git a/postman/mocks/ipgeolocation/default.js b/postman/mocks/ipgeolocation/default.js index 48502f3..2889a0b 100644 --- a/postman/mocks/ipgeolocation/default.js +++ b/postman/mocks/ipgeolocation/default.js @@ -1,4 +1,5 @@ const http = require('node:http'); +const { Readable } = require('node:stream'); const endpoints = new Set([ '/getip', '/v3/ipgeo', '/v3/security', '/v3/abuse', '/v3/asn', '/v3/timezone', @@ -16,7 +17,7 @@ http.createServer((req, res) => { const statuses = { BAD_KEY: 401, FORBIDDEN: 403, RATE_LIMIT: 429, SERVER_ERROR: 503 }; if (statuses[key]) return reply(statuses[key], { message: `Test failure for ${key}` }); if (key === 'BAD_JSON') { res.writeHead(200); return res.end('{'); } - if (url.pathname !== '/getip' && !['TEST_KEY', 'TEST_KEY &+?=#%'].includes(key)) return reply(401, { message: 'Invalid test key' }); + if (url.pathname !== '/getip' && !['TEST_KEY', 'TEST_KEY &+?=#%', 'SLOW_HEADERS', 'SLOW_BODY', 'LIMIT_RESPONSE', 'BULK_50K'].includes(key)) return reply(401, { message: 'Invalid test key' }); let payload = null; if (isPost) { if (req.headers['content-type'] !== 'application/json') return reply(400, { message: 'Expected JSON' }); @@ -25,8 +26,38 @@ http.createServer((req, res) => { } const query = Object.fromEntries([...url.searchParams.keys()].map((name) => [name, url.searchParams.getAll(name)])); const result = { - ip: '2001:db8::1', location: { city: 'Test City' }, + ip: '2001:db8::1', location: { city: url.searchParams.get('lang') === 'ar' ? 'مدينة' : 'Test City' }, _request: { method: req.method, path: url.pathname, query, payload }, }; + if (key === 'BULK_50K') { + if (!Array.isArray(payload?.ips) || payload.ips.length !== 50000) return reply(400, { message: 'Expected 50000 fixture addresses' }); + const companyName = 'x'.repeat(4000); + const records = Readable.from((function* () { + yield '['; + for (let i = 0; i < payload.ips.length; i++) { + if (i) yield ','; + yield JSON.stringify({ ip: payload.ips[i], location: result.location, company: { name: companyName } }); + } + yield ']'; + })()); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.once('close', () => records.destroy()); + records.pipe(res); + return; + } + if (key === 'LIMIT_RESPONSE') result.padding = 'x'.repeat(1100000); + if (key === 'SLOW_HEADERS' || key === 'SLOW_BODY') { + const encoded = JSON.stringify(url.pathname.endsWith('-bulk') ? [result] : result); + if (key === 'SLOW_BODY') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.write(encoded.slice(0, 1)); + } + const timer = setTimeout(() => { + if (key === 'SLOW_HEADERS') res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(key === 'SLOW_BODY' ? encoded.slice(1) : encoded); + }, 750); + res.once('close', () => clearTimeout(timer)); + return; + } reply(200, url.pathname.endsWith('-bulk') ? [result] : result); }).listen(process.env.PORT || 4500); diff --git a/postman/simulations/cli/simulation.context.yaml b/postman/simulations/cli/simulation.context.yaml index 16fb5aa..9bc2104 100644 --- a/postman/simulations/cli/simulation.context.yaml +++ b/postman/simulations/cli/simulation.context.yaml @@ -23,8 +23,19 @@ assumptions: verification: Reproduced during the preceding Go integration checks with this installed CLI version. source: scripts/test-http.cjs; internal/utils/request_test.go updated_at: '2026-10-29' + - id: request-limits-and-large-bulk + status: supported + claim: The CLI stops requests that exceed the time or size limit. Bulk results within those limits remain complete. + reason: Local HTTPS tests used delayed responses, oversized responses and large bulk responses. + verification: On November 4, 2026, both bulk commands returned all 50,000 records in order, with responses over 203 MB under the default limits. Arabic text was preserved. Failed requests did not print partial results or create or overwrite output files. Longer time and larger size settings also passed. These tests used local fixtures, not the live API. + source: cmd/request_limits_http_test.go; internal/utils/request_limits_test.go + updated_at: '2026-11-04' fingerprints: - id: cli-v2.1.1 date: '2026-10-29' commit: df51101 plus the CLI request and release changes in this branch. cli: Postman CLI 1.73.0; Go 1.26.10 and 1.27.3. + - id: cli-v2.1.2 + date: '2026-11-04' + commit: The 2.1.2 changes based on 7e33066. + cli: Go 1.27.3 and Postman CLI 1.73.0. diff --git a/scripts/test-http.cjs b/scripts/test-http.cjs index 3a4f1a9..77bf438 100644 --- a/scripts/test-http.cjs +++ b/scripts/test-http.cjs @@ -43,7 +43,7 @@ async function main() { const [code] = await once(build, 'exit'); if (code !== 0) throw new Error('CLI build failed'); } - const checks = spawn('go', ['test', './cmd', '-run', '^TestCLI(Commands|HTTPErrors)$', '-count=1', '-v'], { + const checks = spawn('go', ['test', './cmd', '-run', '^TestCLI(Commands|HTTPErrors|Arabic|RequestLimitsHTTP|LargeBulkHTTP)$', '-count=1', '-v'], { cwd: path.resolve(__dirname, '..'), stdio: 'inherit', env: { ...process.env, From c471c3a323bb9b63db0150185d4a8b7136dac5f5 Mon Sep 17 00:00:00 2001 From: MudassarTariq Date: Wed, 4 Nov 2026 20:21:07 +0500 Subject: [PATCH 2/2] Use Intel macOS runners for CLI checks --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 49df101..bb20d78 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -14,7 +14,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - os: [ubuntu-24.04, macos-15, windows-2025] + os: [ubuntu-24.04, macos-15-intel, windows-2025] go: ["1.26.10"] include: - os: ubuntu-24.04