From e94538a59f086db922f190627727d3760a87cdbe Mon Sep 17 00:00:00 2001 From: Daud Ahmad Date: Fri, 6 Nov 2026 00:52:54 +0500 Subject: [PATCH] feat: add CSV output format Add `--output csv` to every command, alongside pretty, raw, table and yaml. A top-level JSON array becomes one row per record, which is what the bulk endpoints return; any other response becomes a single row. Nested objects flatten into dotted column names such as `location.city`. An array of scalars joins into one cell with `|`, while an array of objects keeps a column per element so its fields stay addressable. Columns are the sorted union of the keys of every row, so the same response always produces the same file. Numbers are read with json.Decoder.UseNumber and written exactly as the API sent them. Decoding into float64 would render `current_time_unix` as 1.772535675329e+09 and round long identifiers. Values are guarded against spreadsheet formula injection, since organization names and whois text come from third parties. A leading `=`, `@`, tab or carriage return is quoted off, as is a `+` or `-` followed by a non-digit. Signed data is left intact, so the `-122.08421` longitude, the `+1` calling code, the `+1.00H` and `-1.00H` DST durations and the lone `-` the astronomy endpoint uses for an absent sun or moon status all survive unchanged. On the bulk commands `--output-file` now follows `--output`: csv saves a .csv file, every other format keeps saving the .json it always has. The extension is appended only when the given name lacks it, so `--output-file results` and `--output-file results.csv` both write results.csv. This also fixes `--output-file=output.json`, which previously wrote output.json.json. Co-Authored-By: Claude Opus 5 --- README.md | 79 ++++++++++--- cmd/abuse.go | 9 +- cmd/asn.go | 9 +- cmd/astronomy.go | 9 +- cmd/astronomyTimeseries.go | 9 +- cmd/bulkIpSecurity.go | 34 ++++-- cmd/commands_test.go | 214 +++++++++++++++++++++++++++++++++- cmd/ipSecurity.go | 9 +- cmd/ipgeo.go | 14 ++- cmd/ipgeoBulk.go | 34 ++++-- cmd/myIp.go | 9 +- cmd/parseBulkUserAgents.go | 9 +- cmd/timeConversion.go | 9 +- cmd/timezone.go | 9 +- cmd/userAgent.go | 9 +- internal/utils/csv.go | 187 +++++++++++++++++++++++++++++ internal/utils/csv_test.go | 203 ++++++++++++++++++++++++++++++++ internal/utils/output.go | 16 +++ internal/utils/output_test.go | 41 +++++++ 19 files changed, 869 insertions(+), 43 deletions(-) create mode 100644 internal/utils/csv.go create mode 100644 internal/utils/csv_test.go create mode 100644 internal/utils/output_test.go diff --git a/README.md b/README.md index a9683e7..e35f81e 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ Based on: - [`bulk-ip-geo` Usage](#bulk-ip-geo-usage) - [Flags for `bulk-ip-geo`](#flags-for-bulk-ip-geo) - [Output Formats](#output-formats) + - [CSV Output](#csv-output) - [`ip-security` Command](#ip-security-command) - [`ip-security` Usage](#ip-security-usage) - [Flags for `ip-security`](#flags-for-ip-security) @@ -332,7 +333,7 @@ ipgeolocation my-ip [flags] | Flag | Type | Default | Description | |------------|--------|----------|----------------------------------------------------------| -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `text`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`, `text`. | > [!NOTE] > - The `my-ip` command does not require an API key. @@ -382,7 +383,7 @@ ipgeolocation ipgeo [flags] | `--fields` | string[] | `[]` | Return only specific fields (e.g. `location,asn.organization`). | | `--exclude` | string[] | `[]` | Exclude fields from output. `--excludes` is also accepted. | | `--lang` | string | `""` | Response language. | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | > [!NOTE] > Available language options are listed in [Response in multiple languages](https://ipgeolocation.io/documentation/ip-location-api.html#response-in-multiple-languages) @@ -979,8 +980,8 @@ ipgeolocation bulk-ip-geo [flags] | `--exclude` | string[] | `[]` | Exclude fields (e.g. `currency`). | | `--fields` | string[] | `[]` | Return only specific fields (e.g. `location,time_zone`). | | `--lang` | string | `""` | Response language (if supported). | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | -| `--output-file` | string | `""` | Save output to JSON file. Example: `--output-file results` | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | +| `--output-file` | string | `""` | Save output to a file. Writes `.csv` with `--output csv`, otherwise `.json`; the extension is added only if the name lacks it. Example: `--output-file results` or `--output-file results.csv` | For further information, please visit [IP Geolocation API Documentation](https://ipgeolocation.io/documentation/ip-location-api.html). @@ -1010,12 +1011,56 @@ Save results to JSON file: ipgeolocation bulk-ip-geo --ips=8.8.8.8,1.1.1.1 --output-file=output.json ``` +Results will be written to the `output.json` file. The `.json` is not added twice, because the name already ends in it. + #### Output Formats - **pretty** (default): Human-readable formatted JSON. - **raw**: Raw API response. - **table**: Tabular display of common fields. - **yaml**: YAML-formatted output. -- **json file**: If `--output-file` is provided, results are saved to a `.json` file. +- **csv**: Spreadsheet-friendly CSV. See [CSV Output](#csv-output). +- **file**: If `--output-file` is provided, results are saved to a file. With `--output csv` the file is `.csv`; otherwise it is `.json`. The extension is appended only when the name does not already end in it, so `results` and `results.csv` both save to `results.csv`. + +### CSV Output +Every command accepts `--output csv`, which renders the response as CSV for spreadsheets, `awk`, `cut`, or a database import. + +```bash +ipgeolocation ipgeo --ip 8.8.8.8 --output csv +``` +Sample output: +```csv +asn.as_number,asn.country,asn.organization,country_metadata.calling_code,country_metadata.languages,currency.code,ip,location.city,location.is_eu,location.latitude,time_zone.name,time_zone.offset +AS15169,US,Google LLC,+1,en-US|es-US|haw|fr,USD,8.8.8.8,Mountain View,false,37.42240,America/Los_Angeles,-8 +``` + +How a response becomes CSV: + +| Response shape | CSV result | +|---|---| +| A top-level array, as the bulk commands return | One row per record | +| Anything else | A single row | +| A nested object | Dotted column names, e.g. `location.city` | +| An array of scalars | Joined into one cell with `\|`, e.g. `en-US\|es-US\|haw` | +| An array of objects | A column per element, e.g. `routes.0.prefix` | +| `null`, `[]` or `{}` | An empty cell | + +Notes: +- Columns are the union of the fields of every row, sorted by name, so the same response always produces the same file. Where one record lacks a field another has, its cell is empty. +- Numbers are written exactly as the API returned them, so Unix timestamps and long identifiers keep every digit. +- Values containing a comma, quote, or newline are quoted per RFC 4180. A value that itself contains `|` is indistinguishable from the array separator. +- A value beginning with `=`, `@`, a tab, or a carriage return is prefixed with `'` so a spreadsheet does not run it as a formula, as is a `+` or `-` followed by a non-digit, such as `+SUM(A1)`. A sign followed by a digit, and a sign on its own, are data and are left alone, so a `-122.08421` coordinate, a `-8` offset, a `+1` calling code, the `+1.00H` and `-1.00H` DST durations, and the lone `-` the astronomy endpoint reports for an absent `sun_status` or `moon_status` all stay exactly as the API sent them. + +Save CSV straight to a file with `--output-file` on the bulk commands: +```bash +ipgeolocation bulk-ip-geo --file=ips.txt --output csv --output-file results +``` +This writes `results.csv`. Passing `--output-file results.csv` writes the same file — the extension is added only when it is missing. Without `--output csv` the same flag writes `results.json`. + +For the other commands, redirect the output: +```bash +ipgeolocation ipgeo --ip 8.8.8.8 --output csv > ipgeo.csv +``` + ### `ip-security` Command Look up IP security information using the `ipgeolocation.io` API. @@ -1031,7 +1076,7 @@ ipgeolocation ip-security [flags] | `--ip` | string | `""` | IPv4 or IPv6 address. | | `--exclude` | string[] | `[]` | Exclude fields from output. | | `--fields` | string[] | `[]` | Return only specific fields (e.g. `security.threat_score`). | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | > [!NOTE] > IP Security API is only available in the Paid Plan @@ -1121,8 +1166,8 @@ ipgeolocation bulk-ip-security [flags] | `--file` | string | `""` | Path to a text file containing IPs (one per line). | | `--exclude` | string[] | `[]` | Exclude fields (e.g. `security.is_tor`). | | `--fields` | string[] | `[]` | Return only specific fields (e.g. `security.threat_score`). | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | -| `--output-file` | string | `""` | Save output to JSON file. Example: `--output-file results` | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | +| `--output-file` | string | `""` | Save output to a file. Writes `.csv` with `--output csv`, otherwise `.json`; the extension is added only if the name lacks it. Example: `--output-file results` or `--output-file results.csv` | #### `bulk-ip-security` Examples Look up 3 IP addresses: @@ -1135,6 +1180,8 @@ Look up from a file: ipgeolocation bulk-ip-security --file=ips.txt --output-file=output ``` +Results will be written to the `output.json` file. + Output as YAML: ```bash ipgeolocation bulk-ip-security --ips=8.8.8.8 --output=yaml @@ -1162,7 +1209,7 @@ ipgeolocation asn [flags] | `--include` | string[] | `[]` | Include extra fields in output.(e.g., `peers,downstreams,upstreams,routes,whois_response`) | | `--exclude` | string[] | `[]` | Exclude fields from output. | | `--fields` | string[] | `[]` | Return only specific fields (e.g. `ip,organization`). | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | > [!NOTE] > ASN API is only available in the Paid Plan @@ -1313,7 +1360,7 @@ ipgeolocation abuse [flags] | `--ip` | string | `""` | IPv4 or IPv6 address. | | `--exclude` | string[] | `[]` | Exclude fields from output. | | `--fields` | string[] | `[]` | Return only specific fields (e.g. `ip,organization`). | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | > [!NOTE] > Abuse Contact API is only available in the Paid Plan @@ -1395,7 +1442,7 @@ ipgeolocation timezone [flags] | `--iata` | string | `""` | IATA code (e.g. DXB). | | `--icao` | string | `""` | ICAO code (e.g. KATL). | | `--lo` | string | `""` | LO code (e.g. DEBER). | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | #### Get time zone info about your current IP ```bash @@ -1792,7 +1839,7 @@ ipgeolocation time-conversion [flags] | `--lo_from` | string | `""` | LO code to convert from. | | `--lo_to` | string | `""` | LO code to convert to. | | `--time` | string | `""` | Time to convert. | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | #### Convert Current Time from One Timezone to Another ```bash @@ -1868,7 +1915,7 @@ ipgeolocation astronomy [flags] | `--lang` | string | `""` | Response language (if supported). | | `--tz` | string | `""` | IANA time zone for output timestamps. | | `--elevation` | float64 | `0` | Elevation. | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | #### Lookup Astronomy API by Coordinates Get astronomy info about a specific latitude and longitude: @@ -2119,7 +2166,7 @@ ipgeolocation astronomy-timeseries [flags] | `--lang` | string | `""` | Response language (if supported). | | `--start-date` | string | `""` | Start date (e.g. 2023-01-01) Only YYYY-MM-DD. | | `--end-date` | string | `""` | End date (e.g. 2023-12-31) Only YYYY-MM-DD | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | > [!NOTE] > - The `start-date` and `end-date` flags are required. @@ -2256,7 +2303,7 @@ ipgeolocation parse-user-agent [flags] | Flag | Type | Default | Description | |----------------|--------|----------|--------------------------------------------------| | `--user-agent` | string | `""` | User agent string. | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | For further information, please visit [User Agent API Documentation](https://ipgeolocation.io/documentation/user-agent-api.html). @@ -2313,7 +2360,7 @@ ipgeolocation parse-bulk-user-agents [flags] | Flag | Type | Default | Description | |-----------------|----------|----------|--------------------------------------------------| | `--user-agents` | string[] | `[]` | User agent strings. | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | For further information, please visit [User Agent API bulk parsing documentation](https://ipgeolocation.io/documentation/user-agent-api.html#parse-bulk-user-agent-strings). diff --git a/cmd/abuse.go b/cmd/abuse.go index 6f8aef7..2b841d0 100644 --- a/cmd/abuse.go +++ b/cmd/abuse.go @@ -66,6 +66,13 @@ Example: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -84,7 +91,7 @@ func init() { abuseCmd.Flags().StringVar(&abuseFlags.IP, "ip", "", "IPv4 or IPv6 address (e.g. 8.8.8.8)") abuseCmd.Flags().StringSliceVar(&abuseFlags.Excludes, "exclude", []string{}, "Fields to exclude from the output") abuseCmd.Flags().StringSliceVar(&abuseFlags.Fields, "fields", []string{}, "Get Specific Fields to include in the output") - abuseCmd.Flags().StringVar(&abuseFlags.Output, "output", "", "Output format: yaml, raw, table, yaml") + abuseCmd.Flags().StringVar(&abuseFlags.Output, "output", "", "Output format: pretty, raw, table, yaml, csv") rootCmd.AddCommand(abuseCmd) } diff --git a/cmd/asn.go b/cmd/asn.go index aab6e32..1123b44 100644 --- a/cmd/asn.go +++ b/cmd/asn.go @@ -76,6 +76,13 @@ Getting peering relationships for an ASN number: ipgeolocation asn --asn 12345 - fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -96,7 +103,7 @@ func init() { asnCmd.Flags().StringSliceVar(&asnFlags.Include, "include", []string{}, "To include additional values in the output") asnCmd.Flags().StringSliceVar(&asnFlags.Excludes, "exclude", []string{}, "Fields to exclude from the output") asnCmd.Flags().StringSliceVar(&asnFlags.Fields, "fields", []string{}, "Get Specific Fields to include in the output") - asnCmd.Flags().StringVar(&asnFlags.Output, "output", "pretty", "Output format: pretty, raw, table") + asnCmd.Flags().StringVar(&asnFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv") rootCmd.AddCommand(asnCmd) } diff --git a/cmd/astronomy.go b/cmd/astronomy.go index 50b01fc..fd9f56e 100644 --- a/cmd/astronomy.go +++ b/cmd/astronomy.go @@ -99,6 +99,13 @@ API Reference: https://ipgeolocation.io/astronomy-api.html fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -122,7 +129,7 @@ func init() { astronomyCmd.Flags().Float64Var(&astronomyFlags.Longitude, "longitude", 0, "Longitude (e.g. -122.4194)") astronomyCmd.Flags().StringVar(&astronomyFlags.Language, "lang", "", "Language code (e.g. en)") astronomyCmd.Flags().Float64Var(&astronomyFlags.Elevation, "elevation", 0, "Elevation (e.g. 1000)") - astronomyCmd.Flags().StringVar(&astronomyFlags.Output, "output", "pretty", "Output format: pretty, raw, table") + astronomyCmd.Flags().StringVar(&astronomyFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv") rootCmd.AddCommand(astronomyCmd) } diff --git a/cmd/astronomyTimeseries.go b/cmd/astronomyTimeseries.go index 6b3a6e8..d29312a 100644 --- a/cmd/astronomyTimeseries.go +++ b/cmd/astronomyTimeseries.go @@ -94,6 +94,13 @@ Examples: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -117,7 +124,7 @@ func init() { astronomyTimeseriesCmd.Flags().Float64Var(&astronomyTimeseriesFlags.Latitude, "latitude", 0, "Latitude (e.g. 37.7749)") astronomyTimeseriesCmd.Flags().Float64Var(&astronomyTimeseriesFlags.Longitude, "longitude", 0, "Longitude (e.g. -122.4194)") astronomyTimeseriesCmd.Flags().StringVar(&astronomyTimeseriesFlags.Language, "lang", "", "Language code (e.g. en)") - astronomyTimeseriesCmd.Flags().StringVar(&astronomyTimeseriesFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml") + astronomyTimeseriesCmd.Flags().StringVar(&astronomyTimeseriesFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv") rootCmd.AddCommand(astronomyTimeseriesCmd) } diff --git a/cmd/bulkIpSecurity.go b/cmd/bulkIpSecurity.go index 5baa1e4..355ec80 100644 --- a/cmd/bulkIpSecurity.go +++ b/cmd/bulkIpSecurity.go @@ -96,6 +96,13 @@ Retrieving security information for IP addresses from a file: ipgeolocation bulk fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -108,13 +115,26 @@ Retrieving security information for IP addresses from a file: ipgeolocation bulk fmt.Println(string(pretty)) } if bulkSecurityFlags.OutputFile != "" { - pretty, _ := json.MarshalIndent(result, "", " ") - err := os.WriteFile(bulkSecurityFlags.OutputFile+".json", pretty, 0644) - if err != nil { - fmt.Println("Error writing JSON to file:", err) + // The saved file matches --output, so --output csv saves .csv and + // every other format keeps saving the .json it always has. The + // extension is only appended when the given name lacks it. + extension := ".json" + data, _ := json.MarshalIndent(result, "", " ") + if bulkSecurityFlags.Output == "csv" { + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + extension = ".csv" + data = []byte(csvData) + } + path := utils.OutputFilePath(bulkSecurityFlags.OutputFile, extension) + if err := os.WriteFile(path, data, 0644); err != nil { + fmt.Println("Error writing output to file:", err) return } - fmt.Println("Output saved to file:", bulkSecurityFlags.OutputFile+".json") + fmt.Println("Output saved to file:", path) } }, } @@ -123,9 +143,9 @@ func init() { bulkIpSecurityCmd.Flags().StringSliceVar(&bulkSecurityFlags.IPs, "ips", []string{}, "IPs") bulkIpSecurityCmd.Flags().StringSliceVar(&bulkSecurityFlags.Excludes, "exclude", []string{}, "Fields to exclude from the output") bulkIpSecurityCmd.Flags().StringSliceVar(&bulkSecurityFlags.Fields, "fields", []string{}, "Get Specific Fields to include in the output") - bulkIpSecurityCmd.Flags().StringVar(&bulkSecurityFlags.Output, "output", "pretty", "Output format: pretty, raw, table") + bulkIpSecurityCmd.Flags().StringVar(&bulkSecurityFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv") bulkIpSecurityCmd.Flags().StringVar(&bulkSecurityFlags.File, "file", "", "Path to a text file containing IPs (one per line)") - bulkIpSecurityCmd.Flags().StringVar(&bulkSecurityFlags.OutputFile, "output-file", "", "Save output to a file (JSON only)") + bulkIpSecurityCmd.Flags().StringVar(&bulkSecurityFlags.OutputFile, "output-file", "", "Save output to a file; the .csv or .json extension is added only if missing") rootCmd.AddCommand(bulkIpSecurityCmd) } diff --git a/cmd/commands_test.go b/cmd/commands_test.go index 9cb8f70..81d4bd5 100644 --- a/cmd/commands_test.go +++ b/cmd/commands_test.go @@ -3,6 +3,7 @@ package cmd import ( "crypto/tls" "crypto/x509" + "encoding/csv" "encoding/json" "fmt" "io" @@ -201,7 +202,7 @@ func TestCLICommands(t *testing.T) { if tc.path != "/getip" { tc.query.Set("apiKey", testKey) } - for _, format := range []string{"raw", "pretty", "yaml", "table"} { + for _, format := range []string{"raw", "pretty", "yaml", "table", "csv"} { t.Run(tc.name+"/"+format, func(t *testing.T) { args := append([]string{tc.name}, tc.flags...) output := runCLI(t, home, append(args, "--output", format)...) @@ -211,6 +212,20 @@ func TestCLICommands(t *testing.T) { } return } + if format == "csv" { + rows := parseCSV(t, output) + if len(rows) != 1 { + t.Fatalf("expected one CSV row, got %d:\n%s", len(rows), output) + } + // Nested fields keep their path, so location.city survives flattening. + if rows[0]["ip"] != "2001:db8::1" || rows[0]["location.city"] != "Test City" { + t.Fatalf("CSV lost result fields: %s", output) + } + if _, charted := rows[0]["_request.path"]; !charted { + t.Fatalf("CSV lost the echoed request: %s", output) + } + return + } var result interface{} var err error if format == "yaml" { @@ -301,3 +316,200 @@ func TestCLIConfigAndBulkFiles(t *testing.T) { } } } + +func parseCSV(t *testing.T, output string) []map[string]string { + t.Helper() + records, err := csv.NewReader(strings.NewReader(output)).ReadAll() + if err != nil { + t.Fatalf("invalid CSV: %v\n%s", err, output) + } + if len(records) == 0 { + return nil + } + rows := make([]map[string]string, 0, len(records)-1) + for _, record := range records[1:] { + row := map[string]string{} + for i, header := range records[0] { + row[header] = record[i] + } + rows = append(rows, row) + } + return rows +} + +// --output-file follows --output: csv saves .csv, everything else still saves +// the .json it always did. +func TestCLICSVOutputFile(t *testing.T) { + home := configuredHome(t, testKey) + for _, command := range []string{"bulk-ip-geo", "bulk-ip-security"} { + t.Run(command, func(t *testing.T) { + name := command + "-csv" + output := runCLI(t, home, command, "--ips", "8.8.8.8,1.1.1.1", "--output", "csv", "--output-file", name) + if !strings.Contains(output, "Output saved to file: "+name+".csv") { + t.Fatalf("csv file not reported: %s", output) + } + if _, err := os.Stat(filepath.Join(home, name+".json")); !os.IsNotExist(err) { + t.Fatalf("csv run also wrote a .json file: %v", err) + } + + saved, err := os.ReadFile(filepath.Join(home, name+".csv")) + if err != nil { + t.Fatal(err) + } + rows := parseCSV(t, string(saved)) + if len(rows) != 1 || rows[0]["ip"] != "2001:db8::1" || rows[0]["location.city"] != "Test City" { + t.Fatalf("saved CSV lost fields: %s", saved) + } + // Both requested addresses ride in one cell, joined by the separator. + if rows[0]["_request.payload.ips"] != "8.8.8.8|1.1.1.1" { + t.Fatalf("array cell changed: %q", rows[0]["_request.payload.ips"]) + } + // The file and stdout must agree. + if strings.TrimSpace(strings.TrimSuffix(output, "Output saved to file: "+name+".csv")) != strings.TrimSpace(string(saved)) { + t.Fatalf("file and stdout disagree:\n%s\n---\n%s", output, saved) + } + + // Without --output csv the JSON file is unchanged. + plain := command + "-plain" + if output := runCLI(t, home, command, "--ips", "8.8.8.8", "--output-file", plain); !strings.Contains(output, "Output saved to file: "+plain+".json") { + t.Fatalf("json file regressed: %s", output) + } + data, err := os.ReadFile(filepath.Join(home, plain+".json")) + if err != nil { + t.Fatal(err) + } + var results []map[string]interface{} + if err := json.Unmarshal(data, &results); err != nil { + t.Fatalf("json file is no longer JSON: %v", err) + } + if _, err := os.Stat(filepath.Join(home, plain+".csv")); !os.IsNotExist(err) { + t.Fatalf("plain run wrote a .csv file: %v", err) + } + }) + } +} + +// A request that fails must leave no file behind and must not overwrite one, +// for CSV exactly as for JSON. +func TestCLICSVOutputFileNotWrittenOnFailure(t *testing.T) { + for _, key := range []string{"BAD_KEY", "SERVER_ERROR", "BAD_JSON"} { + t.Run(key, func(t *testing.T) { + home := configuredHome(t, key) + for _, command := range []string{"bulk-ip-geo", "bulk-ip-security"} { + for _, exists := range []bool{false, true} { + name := fmt.Sprintf("%s-%v", command, exists) + file := filepath.Join(home, name+".csv") + if exists { + if err := os.WriteFile(file, []byte("keep existing results"), 0600); err != nil { + t.Fatal(err) + } + } + output := runCLI(t, home, command, "--ips", "8.8.8.8", "--output", "csv", "--output-file", name) + if strings.Contains(output, "Output saved") || strings.Contains(output, key) { + t.Fatalf("failed request reported a saved file: %s", output) + } + data, err := os.ReadFile(file) + if exists { + if err != nil || string(data) != "keep existing results" { + t.Fatalf("failed request changed the output file: %v", err) + } + } else if !os.IsNotExist(err) { + t.Fatalf("failed request created an output file: %v", err) + } + } + } + }) + } +} + +// --output-file adds .csv or .json only when the given name lacks it, so a +// name the user already suffixed is used as it stands and never doubles up. +func TestCLIOutputFileExtension(t *testing.T) { + for _, tc := range []struct { + given, format, want string + }{ + // A bare name gains the extension that matches --output. + {"plain", "", "plain.json"}, + {"plain-csv", "csv", "plain-csv.csv"}, + + // A name that already carries it is used as given. + {"suffixed.json", "", "suffixed.json"}, + {"suffixed.csv", "csv", "suffixed.csv"}, + + // Matching ignores case. + {"upper.JSON", "", "upper.JSON"}, + {"upper.CSV", "csv", "upper.CSV"}, + + // A non-matching extension is kept, not swapped. + {"mixed.json", "csv", "mixed.json.csv"}, + {"mixed.csv", "", "mixed.csv.json"}, + + // A dot that is not the extension is left alone. + {"2026.report", "", "2026.report.json"}, + } { + for _, command := range []string{"bulk-ip-geo", "bulk-ip-security"} { + name := tc.given + label := command + "/" + name + "/" + tc.format + t.Run(label, func(t *testing.T) { + directory := t.TempDir() + // A fresh HOME per case so leftover files cannot mask a mistake. + if output := runCLI(t, directory, "config", "--apikey", testKey); !strings.Contains(output, "saved") { + t.Fatalf("config failed: %s", output) + } + args := []string{command, "--ips", "8.8.8.8", "--output-file", name} + if tc.format != "" { + args = append(args, "--output", tc.format) + } + output := runCLI(t, directory, args...) + if !strings.Contains(output, "Output saved to file: "+tc.want) { + t.Fatalf("expected %q, got: %s", tc.want, output) + } + + // Exactly one result file exists, and it is the expected one. + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatal(err) + } + var produced []string + for _, entry := range entries { + if !entry.IsDir() && !strings.HasPrefix(entry.Name(), ".") { + produced = append(produced, entry.Name()) + } + } + if len(produced) != 1 || produced[0] != tc.want { + t.Fatalf("expected only %q, found %v", tc.want, produced) + } + + // And it holds the format that was asked for. + data, err := os.ReadFile(filepath.Join(directory, tc.want)) + if err != nil { + t.Fatal(err) + } + if tc.format == "csv" { + if rows := parseCSV(t, string(data)); len(rows) != 1 || rows[0]["ip"] != "2001:db8::1" { + t.Fatalf("saved file is not the expected CSV: %s", data) + } + } else { + var results []map[string]interface{} + if err := json.Unmarshal(data, &results); err != nil { + t.Fatalf("saved file is not JSON: %v\n%s", err, data) + } + } + }) + } + } +} + +func TestCLICSVHelpAndFallback(t *testing.T) { + home := configuredHome(t, testKey) + for _, command := range []string{"ipgeo", "bulk-ip-geo", "my-ip", "timezone", "asn", "abuse", "parse-user-agent"} { + if output := runCLI(t, home, command, "--help"); !strings.Contains(output, "csv") { + t.Fatalf("csv missing from %s help: %s", command, output) + } + } + // An unrecognised format still falls back to pretty JSON, as it always has. + output := runCLI(t, home, "ipgeo", "--output", "csvv") + if !json.Valid([]byte(output)) { + t.Fatalf("unknown format no longer falls back to pretty: %s", output) + } +} diff --git a/cmd/ipSecurity.go b/cmd/ipSecurity.go index 32fe026..1047131 100644 --- a/cmd/ipSecurity.go +++ b/cmd/ipSecurity.go @@ -76,6 +76,13 @@ Notes: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -94,7 +101,7 @@ func init() { ipSecurityCmd.Flags().StringVar(&securityFlags.IP, "ip", "", "IPv4 or IPv6 address (e.g. 8.8.8.8)") ipSecurityCmd.Flags().StringSliceVar(&securityFlags.Excludes, "exclude", []string{}, "Fields to exclude from the output") ipSecurityCmd.Flags().StringSliceVar(&securityFlags.Fields, "fields", []string{}, "Get Specific Fields to include in the output") - ipSecurityCmd.Flags().StringVar(&securityFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml") + ipSecurityCmd.Flags().StringVar(&securityFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv") rootCmd.AddCommand(ipSecurityCmd) } diff --git a/cmd/ipgeo.go b/cmd/ipgeo.go index de0f56b..2e4c3ea 100644 --- a/cmd/ipgeo.go +++ b/cmd/ipgeo.go @@ -27,7 +27,7 @@ and coordinates using the ipgeolocation.io IPGeo API (v2). You can query any public IP address or domain name, and customize the output using include/exclude/fields parameters as supported by the API. -Available output formats include: pretty (default), raw (JSON), table, and YAML. +Available output formats include: pretty (default), raw (JSON), table, YAML, and CSV. Examples: @@ -46,6 +46,9 @@ Examples: # Exclude unnecessary fields ipgeolocation ipgeo --exclude currency,time_zone + # Save a spreadsheet-friendly CSV + ipgeolocation ipgeo --ip 8.8.8.8 --output csv > ipgeo.csv + Notes: - You must have a valid API key configured using: ipgeolocation config --apikey= - If no --ip flag is provided, it defaults to your current IP address.`, @@ -95,6 +98,13 @@ Notes: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -115,7 +125,7 @@ func init() { ipgeoCmd.Flags().StringSliceVar(&ipgeoFlags.Excludes, "exclude", []string{}, "Fields to exclude from the output") ipgeoCmd.Flags().StringSliceVar(&ipgeoFlags.Fields, "fields", []string{}, "Get Specific Fields to include in the output") ipgeoCmd.Flags().StringVar(&ipgeoFlags.Language, "lang", "", "Language for the output") - ipgeoCmd.Flags().StringVar(&ipgeoFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml") + ipgeoCmd.Flags().StringVar(&ipgeoFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv") // --excludes was the original name of --exclude; keep accepting it so existing scripts don't break. ipgeoCmd.Flags().SetNormalizeFunc(func(f *pflag.FlagSet, name string) pflag.NormalizedName { diff --git a/cmd/ipgeoBulk.go b/cmd/ipgeoBulk.go index 495703a..23d4f69 100644 --- a/cmd/ipgeoBulk.go +++ b/cmd/ipgeoBulk.go @@ -109,6 +109,13 @@ Examples: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -122,13 +129,26 @@ Examples: fmt.Println(string(pretty)) } if bulkIpgeoFlags.OutputFile != "" { - pretty, _ := json.MarshalIndent(result, "", " ") - err := os.WriteFile(bulkIpgeoFlags.OutputFile+".json", pretty, 0644) - if err != nil { - fmt.Println("Error writing JSON to file:", err) + // The saved file matches --output, so --output csv saves .csv and + // every other format keeps saving the .json it always has. The + // extension is only appended when the given name lacks it. + extension := ".json" + data, _ := json.MarshalIndent(result, "", " ") + if bulkIpgeoFlags.Output == "csv" { + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + extension = ".csv" + data = []byte(csvData) + } + path := utils.OutputFilePath(bulkIpgeoFlags.OutputFile, extension) + if err := os.WriteFile(path, data, 0644); err != nil { + fmt.Println("Error writing output to file:", err) return } - fmt.Println("Output saved to file:", bulkIpgeoFlags.OutputFile+".json") + fmt.Println("Output saved to file:", path) } }, } @@ -139,9 +159,9 @@ func init() { bulkIpgeoCmd.Flags().StringSliceVar(&bulkIpgeoFlags.Excludes, "exclude", []string{}, "Fields to exclude from the output") bulkIpgeoCmd.Flags().StringSliceVar(&bulkIpgeoFlags.Fields, "fields", []string{}, "Get Specific Fields to include in the output") bulkIpgeoCmd.Flags().StringVar(&bulkIpgeoFlags.Language, "lang", "", "Language for the output") - bulkIpgeoCmd.Flags().StringVar(&bulkIpgeoFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml") + bulkIpgeoCmd.Flags().StringVar(&bulkIpgeoFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv") bulkIpgeoCmd.Flags().StringVar(&bulkIpgeoFlags.File, "file", "", "Path to a text file containing IPs (one per line)") - bulkIpgeoCmd.Flags().StringVar(&bulkIpgeoFlags.OutputFile, "output-file", "", "Save output to a file (JSON only)") + bulkIpgeoCmd.Flags().StringVar(&bulkIpgeoFlags.OutputFile, "output-file", "", "Save output to a file; the .csv or .json extension is added only if missing") rootCmd.AddCommand(bulkIpgeoCmd) } diff --git a/cmd/myIp.go b/cmd/myIp.go index 04051d6..041c358 100644 --- a/cmd/myIp.go +++ b/cmd/myIp.go @@ -54,6 +54,13 @@ Notes: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -69,7 +76,7 @@ Notes: } func init() { - myIPCmd.Flags().StringVar(&myIPFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, text") + myIPCmd.Flags().StringVar(&myIPFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv, text") rootCmd.AddCommand(myIPCmd) } diff --git a/cmd/parseBulkUserAgents.go b/cmd/parseBulkUserAgents.go index 51f28a1..992ac05 100644 --- a/cmd/parseBulkUserAgents.go +++ b/cmd/parseBulkUserAgents.go @@ -80,6 +80,13 @@ Note: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -97,7 +104,7 @@ Note: func init() { parseBulkUserAgentsCmd.Flags().StringSliceVar(&bulkUserAgentsFlags.UserAgents, "user-agents", []string{}, "User Agents") - parseBulkUserAgentsCmd.Flags().StringVar(&bulkUserAgentsFlags.Output, "output", "", "Output format: raw, table, yaml") + parseBulkUserAgentsCmd.Flags().StringVar(&bulkUserAgentsFlags.Output, "output", "", "Output format: pretty, raw, table, yaml, csv") rootCmd.AddCommand(parseBulkUserAgentsCmd) } diff --git a/cmd/timeConversion.go b/cmd/timeConversion.go index 8ec5cf6..cb8152d 100644 --- a/cmd/timeConversion.go +++ b/cmd/timeConversion.go @@ -136,6 +136,13 @@ Notes: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -166,7 +173,7 @@ func init() { timeConversionCmd.Flags().StringVar(&timeConversionFlags.LoCodeFrom, "lo_from", "", "LO code from") timeConversionCmd.Flags().StringVar(&timeConversionFlags.LoCodeTo, "lo_to", "", "LO code to") timeConversionCmd.Flags().StringVar(&timeConversionFlags.Time, "time", "", "Time") - timeConversionCmd.Flags().StringVar(&timeConversionFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml") + timeConversionCmd.Flags().StringVar(&timeConversionFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv") rootCmd.AddCommand(timeConversionCmd) } diff --git a/cmd/timezone.go b/cmd/timezone.go index b44716f..ded15e7 100644 --- a/cmd/timezone.go +++ b/cmd/timezone.go @@ -113,6 +113,13 @@ Note: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -137,7 +144,7 @@ func init() { timezoneCmd.Flags().StringVar(&timezoneFlags.IcaoCode, "icao", "", "ICAO code (e.g. KATL)") timezoneCmd.Flags().StringVar(&timezoneFlags.LoCode, "lo", "", "LO code (e.g. DEBER)") timezoneCmd.Flags().StringVar(&timezoneFlags.Language, "lang", "", "Language code (e.g. en)") - timezoneCmd.Flags().StringVar(&timezoneFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml") + timezoneCmd.Flags().StringVar(&timezoneFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv") rootCmd.AddCommand(timezoneCmd) } diff --git a/cmd/userAgent.go b/cmd/userAgent.go index 794bf59..147b943 100644 --- a/cmd/userAgent.go +++ b/cmd/userAgent.go @@ -66,6 +66,13 @@ Examples: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -83,7 +90,7 @@ Examples: func init() { userAgentCmd.Flags().StringVar(&userAgentFlags.UserAgent, "user-agent", "", "User Agent") - userAgentCmd.Flags().StringVar(&userAgentFlags.Output, "output", "", "Output format: raw, table, yaml") + userAgentCmd.Flags().StringVar(&userAgentFlags.Output, "output", "", "Output format: pretty, raw, table, yaml, csv") rootCmd.AddCommand(userAgentCmd) } diff --git a/internal/utils/csv.go b/internal/utils/csv.go new file mode 100644 index 0000000..8a832ac --- /dev/null +++ b/internal/utils/csv.go @@ -0,0 +1,187 @@ +package utils + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "sort" + "strconv" + "strings" +) + +// ArraySeparator joins the elements of an array of scalars inside a single CSV cell. +const ArraySeparator = "|" + +// ConvertToCSV renders a JSON API response as CSV text. +// +// A top-level JSON array becomes one row per element, which is what the bulk +// endpoints return; any other response becomes a single row. Nested objects are +// flattened into dotted column names ("location.city"). An array whose elements +// are all scalars is joined into one cell with ArraySeparator; an array holding +// objects keeps a column per element instead ("routes.0.prefix"), so its fields +// stay addressable. Columns are the union of the keys of every row, sorted, so +// the same response always produces the same file. +func ConvertToCSV(body []byte) (string, error) { + decoder := json.NewDecoder(bytes.NewReader(body)) + // Numbers stay exactly as the API sent them; float64 would turn a Unix + // timestamp into 1.772535675329e+09 and lose digits off long IDs. + decoder.UseNumber() + + var data interface{} + if err := decoder.Decode(&data); err != nil { + return "", fmt.Errorf("invalid JSON: %w", err) + } + + records, isList := data.([]interface{}) + if !isList { + records = []interface{}{data} + } + + rows := make([]map[string]string, 0, len(records)) + columns := map[string]bool{} + for _, record := range records { + row := flattenRecord(record) + for column := range row { + columns[column] = true + } + rows = append(rows, row) + } + + if len(columns) == 0 { + return "", nil + } + + headers := make([]string, 0, len(columns)) + for column := range columns { + headers = append(headers, column) + } + sort.Strings(headers) + + var buffer bytes.Buffer + writer := csv.NewWriter(&buffer) + if err := writer.Write(headers); err != nil { + return "", err + } + for _, row := range rows { + line := make([]string, len(headers)) + for i, header := range headers { + line[i] = escapeFormula(row[header]) + } + if err := writer.Write(line); err != nil { + return "", err + } + } + writer.Flush() + if err := writer.Error(); err != nil { + return "", err + } + + return buffer.String(), nil +} + +// flattenRecord turns one record into its CSV cells, keyed by column name. +func flattenRecord(record interface{}) map[string]string { + row := map[string]string{} + flattenValue("", record, row) + // A record that is a bare scalar, an empty object or an empty array leaves + // its cell unnamed; give it a column rather than writing a blank header. + if value, unnamed := row[""]; unnamed { + delete(row, "") + row["value"] = value + } + return row +} + +func flattenValue(prefix string, value interface{}, row map[string]string) { + switch typed := value.(type) { + case map[string]interface{}: + if len(typed) == 0 { + row[prefix] = "" + return + } + for key, nested := range typed { + flattenValue(joinColumn(prefix, key), nested, row) + } + case []interface{}: + if len(typed) == 0 { + row[prefix] = "" + return + } + if scalarsOnly(typed) { + parts := make([]string, len(typed)) + for i, item := range typed { + parts[i] = scalarToString(item) + } + row[prefix] = strings.Join(parts, ArraySeparator) + return + } + for i, item := range typed { + flattenValue(joinColumn(prefix, strconv.Itoa(i)), item, row) + } + default: + row[prefix] = scalarToString(typed) + } +} + +func joinColumn(prefix, key string) string { + if prefix == "" { + return key + } + return prefix + "." + key +} + +func scalarsOnly(values []interface{}) bool { + for _, value := range values { + switch value.(type) { + case map[string]interface{}, []interface{}: + return false + } + } + return true +} + +func scalarToString(value interface{}) string { + switch typed := value.(type) { + case nil: + return "" + case string: + return typed + case bool: + return strconv.FormatBool(typed) + case json.Number: + return typed.String() + case float64: + return strconv.FormatFloat(typed, 'f', -1, 64) + default: + return fmt.Sprintf("%v", typed) + } +} + +// escapeFormula stops a spreadsheet from running a cell as a formula. Response +// values such as organization names and whois text come from third parties, so +// a leading =, @ or control character is quoted off. +// +// A leading + or - is only quoted when a non-digit follows it. Signed numbers +// and signed data are ordinary values here, such as a "-122.08421" longitude, +// a "+1" calling code, the "+1.00H" and "-1.00H" DST durations and the lone +// "-" the astronomy endpoint uses for an absent sun or moon status; quoting +// those would corrupt real columns. A sign followed by anything else is the +// shape a formula takes, as in "+SUM(A1)" or "-cmd|' /c calc'!A0". +func escapeFormula(value string) string { + if value == "" { + return value + } + switch value[0] { + case '=', '@', '\t', '\r': + return "'" + value + case '+', '-': + // A lone sign is a value in its own right: the astronomy endpoint + // reports sun_status and moon_status as "-". It cannot be a formula. + if len(value) == 1 || value[1] == '.' || (value[1] >= '0' && value[1] <= '9') { + return value + } + return "'" + value + } + return value +} diff --git a/internal/utils/csv_test.go b/internal/utils/csv_test.go new file mode 100644 index 0000000..2f26475 --- /dev/null +++ b/internal/utils/csv_test.go @@ -0,0 +1,203 @@ +package utils + +import ( + "encoding/csv" + "strings" + "testing" +) + +func csvOf(t *testing.T, body string) string { + t.Helper() + output, err := ConvertToCSV([]byte(body)) + if err != nil { + t.Fatalf("ConvertToCSV(%s) failed: %v", body, err) + } + return output +} + +func TestConvertToCSVShapes(t *testing.T) { + for _, tc := range []struct { + name, body, want string + }{ + { + name: "nested objects become dotted columns", + body: `{"ip":"8.8.8.8","location":{"city":"Mountain View","is_eu":false}}`, + want: "ip,location.city,location.is_eu\n8.8.8.8,Mountain View,false\n", + }, + { + // The bulk endpoints answer with an array; each element is a record. + name: "top-level array becomes one row per element", + body: `[{"ip":"8.8.8.8"},{"ip":"1.1.1.1"}]`, + want: "ip\n8.8.8.8\n1.1.1.1\n", + }, + { + // Records need not agree on their keys; every column still appears. + name: "columns are the union across rows", + body: `[{"ip":"8.8.8.8","city":"MV"},{"ip":"1.1.1.1","asn":"AS13335"}]`, + want: "asn,city,ip\n,MV,8.8.8.8\nAS13335,,1.1.1.1\n", + }, + { + name: "array of scalars joins into one cell", + body: `{"languages":["en-US","es-US","haw"]}`, + want: "languages\nen-US|es-US|haw\n", + }, + { + // Keeping a column per field beats hiding objects in a joined cell. + name: "array of objects keeps a column per element", + body: `{"routes":[{"prefix":"8.8.8.0/24"},{"prefix":"8.8.4.0/24"}]}`, + want: "routes.0.prefix,routes.1.prefix\n8.8.8.0/24,8.8.4.0/24\n", + }, + { + name: "array mixing scalars and objects is indexed", + body: `{"items":["a",{"b":1}]}`, + want: "items.0,items.1.b\na,1\n", + }, + { + name: "empty containers leave an empty cell", + body: `{"ip":"8.8.8.8","tags":[],"meta":{}}`, + want: "ip,meta,tags\n8.8.8.8,,\n", + }, + { + name: "null becomes an empty cell", + body: `{"ip":"8.8.8.8","city":null}`, + want: "city,ip\n,8.8.8.8\n", + }, + { + name: "booleans keep their JSON spelling", + body: `{"is_eu":true,"is_dst":false}`, + want: "is_dst,is_eu\nfalse,true\n", + }, + { + name: "deep nesting keeps the whole path", + body: `{"time_zone":{"dst_end":{"gap":false,"utc_time":"2026-11-01 09:00"}}}`, + want: "time_zone.dst_end.gap,time_zone.dst_end.utc_time\nfalse,2026-11-01 09:00\n", + }, + { + name: "a bare scalar response gets a value column", + body: `"8.8.8.8"`, + want: "value\n8.8.8.8\n", + }, + { + name: "an empty object yields a single empty row", + body: `{}`, + want: "value\n\n", + }, + { + name: "unicode survives unchanged", + body: `{"city":"مدينة","flag":"🇺🇸"}`, + want: "city,flag\nمدينة,🇺🇸\n", + }, + } { + t.Run(tc.name, func(t *testing.T) { + if got := csvOf(t, tc.body); got != tc.want { + t.Fatalf("CSV changed:\ngot %q\nwant %q", got, tc.want) + } + }) + } +} + +// An empty bulk result has no columns to describe, so it writes nothing rather +// than a stray header line. +func TestConvertToCSVEmptyArray(t *testing.T) { + if got := csvOf(t, `[]`); got != "" { + t.Fatalf("empty array produced %q", got) + } +} + +// float64 would render a Unix timestamp as 1.772535675329e+09 and round long +// identifiers, so numbers must pass through exactly as the API sent them. +func TestConvertToCSVKeepsNumbersExact(t *testing.T) { + body := `{"unix":1772535675.329,"big":12345678901234567890,"neg":-122.08421,"zero":0,"exp":1e3,"offset":-8}` + want := "big,exp,neg,offset,unix,zero\n12345678901234567890,1e3,-122.08421,-8,1772535675.329,0\n" + if got := csvOf(t, body); got != want { + t.Fatalf("number formatting changed:\ngot %q\nwant %q", got, want) + } +} + +// Separators, quotes and newlines inside values must survive a round trip. +func TestConvertToCSVQuotesSpecialCharacters(t *testing.T) { + body := `{"org":"Google, LLC","quote":"say \"hi\"","line":"a\nb","tab":"a\tb"}` + output := csvOf(t, body) + + rows, err := csv.NewReader(strings.NewReader(output)).ReadAll() + if err != nil { + t.Fatalf("output is not valid CSV: %v\n%s", err, output) + } + if len(rows) != 2 { + t.Fatalf("expected a header and one row, got %d rows", len(rows)) + } + values := map[string]string{} + for i, header := range rows[0] { + values[header] = rows[1][i] + } + for header, want := range map[string]string{ + "org": "Google, LLC", "quote": `say "hi"`, "line": "a\nb", "tab": "a\tb", + } { + if values[header] != want { + t.Fatalf("%s round-tripped as %q, want %q", header, values[header], want) + } + } +} + +// Response values come from third parties, so a cell must not open as a formula +// in a spreadsheet. Numbers stay untouched so coordinates and offsets survive. +func TestConvertToCSVEscapesFormulas(t *testing.T) { + for value, want := range map[string]string{ + // Formulas are quoted off. + `=1+1`: `'=1+1`, + `=cmd|' /c calc'!A0`: `'=cmd|' /c calc'!A0`, + `+SUM(A1)`: `'+SUM(A1)`, + `-cmd|' /c calc'!A0`: `'-cmd|' /c calc'!A0`, + `@import`: `'@import`, + "\tlead": "'\tlead", + "\rlead": "'\rlead", + // A lone sign is real data: the astronomy endpoint reports an absent + // sun_status or moon_status as "-". It cannot be a formula. + "+": "+", + "-": "-", + // Signed data is left exactly as the API sent it. The DST duration + // field carries both signs, so the two must be treated alike. + "-1.00H": "-1.00H", + "+1.00H": "+1.00H", + "-122.08421": "-122.08421", + "+.5": "+.5", + "-8": "-8", + "+1": "+1", + // Ordinary text is untouched. + "Google LLC": "Google LLC", + "": "", + "not=a=formula": "not=a=formula", + "مدينة": "مدينة", + } { + if got := escapeFormula(value); got != want { + t.Fatalf("escapeFormula(%q) = %q, want %q", value, got, want) + } + } + + // And it reaches the rendered output. + if got := csvOf(t, `{"org":"=1+1","lat":"-37.4"}`); got != "lat,org\n-37.4,'=1+1\n" { + t.Fatalf("escaping missing from output: %q", got) + } +} + +func TestConvertToCSVRejectsInvalidJSON(t *testing.T) { + for _, body := range []string{``, `{`, `not json`, `{"a":}`} { + if _, err := ConvertToCSV([]byte(body)); err == nil { + t.Fatalf("invalid JSON %q was accepted", body) + } + } +} + +// Go randomises map iteration, so column order has to be imposed, not observed. +func TestConvertToCSVIsDeterministic(t *testing.T) { + body := `{"z":1,"a":2,"m":{"y":3,"b":4},"list":["q","p"]}` + first := csvOf(t, body) + for i := 0; i < 50; i++ { + if got := csvOf(t, body); got != first { + t.Fatalf("output varies between runs:\n%q\n%q", first, got) + } + } + if !strings.HasPrefix(first, "a,list,m.b,m.y,z\n") { + t.Fatalf("columns are not sorted: %q", first) + } +} diff --git a/internal/utils/output.go b/internal/utils/output.go index f3968a7..7242043 100644 --- a/internal/utils/output.go +++ b/internal/utils/output.go @@ -2,6 +2,7 @@ package utils import ( "fmt" + "path/filepath" "strings" ) @@ -42,3 +43,18 @@ func ToTitle(s string) string { func ConvertToXML(data interface{}) string { return "" } + +// OutputFilePath works out the file an --output-file value should write to. +// The extension is appended only when the name does not already end in it, so +// "results" and "results.csv" both save to results.csv and neither doubles up. +// +// An extension that does not match is left in place rather than swapped, so +// --output-file results.json with --output csv writes results.json.csv. That +// keeps the name the caller gave and never silently retargets a different file. +// The comparison ignores case, so results.CSV is accepted as it stands. +func OutputFilePath(name, extension string) string { + if strings.EqualFold(filepath.Ext(name), extension) { + return name + } + return name + extension +} diff --git a/internal/utils/output_test.go b/internal/utils/output_test.go new file mode 100644 index 0000000..0282ce0 --- /dev/null +++ b/internal/utils/output_test.go @@ -0,0 +1,41 @@ +package utils + +import "testing" + +func TestOutputFilePath(t *testing.T) { + for _, tc := range []struct { + name, extension, want string + }{ + // A bare name gains the extension. + {"results", ".csv", "results.csv"}, + {"results", ".json", "results.json"}, + {"out/results", ".csv", "out/results.csv"}, + + // A name that already carries it is left alone, so it never doubles up. + {"results.csv", ".csv", "results.csv"}, + {"results.json", ".json", "results.json"}, + {"out/results.json", ".json", "out/results.json"}, + + // Matching ignores case, which matters on Windows. + {"results.CSV", ".csv", "results.CSV"}, + {"results.Json", ".json", "results.Json"}, + + // A different extension is kept rather than swapped: the caller's name + // survives and no unrelated file is targeted. + {"results.json", ".csv", "results.json.csv"}, + {"results.csv", ".json", "results.csv.json"}, + {"results.tar.gz", ".csv", "results.tar.gz.csv"}, + + // Dots inside a name are not an extension. + {"my.results", ".csv", "my.results.csv"}, + {"2026-01-01.report", ".json", "2026-01-01.report.json"}, + + // Degenerate shapes still produce a usable path. + {".csv", ".csv", ".csv"}, + {"results.", ".csv", "results..csv"}, + } { + if got := OutputFilePath(tc.name, tc.extension); got != tc.want { + t.Fatalf("OutputFilePath(%q, %q) = %q, want %q", tc.name, tc.extension, got, tc.want) + } + } +}