diff --git a/README.md b/README.md index a9683e7..e35f81e 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ Based on: - [`bulk-ip-geo` Usage](#bulk-ip-geo-usage) - [Flags for `bulk-ip-geo`](#flags-for-bulk-ip-geo) - [Output Formats](#output-formats) + - [CSV Output](#csv-output) - [`ip-security` Command](#ip-security-command) - [`ip-security` Usage](#ip-security-usage) - [Flags for `ip-security`](#flags-for-ip-security) @@ -332,7 +333,7 @@ ipgeolocation my-ip [flags] | Flag | Type | Default | Description | |------------|--------|----------|----------------------------------------------------------| -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `text`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`, `text`. | > [!NOTE] > - The `my-ip` command does not require an API key. @@ -382,7 +383,7 @@ ipgeolocation ipgeo [flags] | `--fields` | string[] | `[]` | Return only specific fields (e.g. `location,asn.organization`). | | `--exclude` | string[] | `[]` | Exclude fields from output. `--excludes` is also accepted. | | `--lang` | string | `""` | Response language. | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | > [!NOTE] > Available language options are listed in [Response in multiple languages](https://ipgeolocation.io/documentation/ip-location-api.html#response-in-multiple-languages) @@ -979,8 +980,8 @@ ipgeolocation bulk-ip-geo [flags] | `--exclude` | string[] | `[]` | Exclude fields (e.g. `currency`). | | `--fields` | string[] | `[]` | Return only specific fields (e.g. `location,time_zone`). | | `--lang` | string | `""` | Response language (if supported). | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | -| `--output-file` | string | `""` | Save output to JSON file. Example: `--output-file results` | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | +| `--output-file` | string | `""` | Save output to a file. Writes `.csv` with `--output csv`, otherwise `.json`; the extension is added only if the name lacks it. Example: `--output-file results` or `--output-file results.csv` | For further information, please visit [IP Geolocation API Documentation](https://ipgeolocation.io/documentation/ip-location-api.html). @@ -1010,12 +1011,56 @@ Save results to JSON file: ipgeolocation bulk-ip-geo --ips=8.8.8.8,1.1.1.1 --output-file=output.json ``` +Results will be written to the `output.json` file. The `.json` is not added twice, because the name already ends in it. + #### Output Formats - **pretty** (default): Human-readable formatted JSON. - **raw**: Raw API response. - **table**: Tabular display of common fields. - **yaml**: YAML-formatted output. -- **json file**: If `--output-file` is provided, results are saved to a `.json` file. +- **csv**: Spreadsheet-friendly CSV. See [CSV Output](#csv-output). +- **file**: If `--output-file` is provided, results are saved to a file. With `--output csv` the file is `.csv`; otherwise it is `.json`. The extension is appended only when the name does not already end in it, so `results` and `results.csv` both save to `results.csv`. + +### CSV Output +Every command accepts `--output csv`, which renders the response as CSV for spreadsheets, `awk`, `cut`, or a database import. + +```bash +ipgeolocation ipgeo --ip 8.8.8.8 --output csv +``` +Sample output: +```csv +asn.as_number,asn.country,asn.organization,country_metadata.calling_code,country_metadata.languages,currency.code,ip,location.city,location.is_eu,location.latitude,time_zone.name,time_zone.offset +AS15169,US,Google LLC,+1,en-US|es-US|haw|fr,USD,8.8.8.8,Mountain View,false,37.42240,America/Los_Angeles,-8 +``` + +How a response becomes CSV: + +| Response shape | CSV result | +|---|---| +| A top-level array, as the bulk commands return | One row per record | +| Anything else | A single row | +| A nested object | Dotted column names, e.g. `location.city` | +| An array of scalars | Joined into one cell with `\|`, e.g. `en-US\|es-US\|haw` | +| An array of objects | A column per element, e.g. `routes.0.prefix` | +| `null`, `[]` or `{}` | An empty cell | + +Notes: +- Columns are the union of the fields of every row, sorted by name, so the same response always produces the same file. Where one record lacks a field another has, its cell is empty. +- Numbers are written exactly as the API returned them, so Unix timestamps and long identifiers keep every digit. +- Values containing a comma, quote, or newline are quoted per RFC 4180. A value that itself contains `|` is indistinguishable from the array separator. +- A value beginning with `=`, `@`, a tab, or a carriage return is prefixed with `'` so a spreadsheet does not run it as a formula, as is a `+` or `-` followed by a non-digit, such as `+SUM(A1)`. A sign followed by a digit, and a sign on its own, are data and are left alone, so a `-122.08421` coordinate, a `-8` offset, a `+1` calling code, the `+1.00H` and `-1.00H` DST durations, and the lone `-` the astronomy endpoint reports for an absent `sun_status` or `moon_status` all stay exactly as the API sent them. + +Save CSV straight to a file with `--output-file` on the bulk commands: +```bash +ipgeolocation bulk-ip-geo --file=ips.txt --output csv --output-file results +``` +This writes `results.csv`. Passing `--output-file results.csv` writes the same file — the extension is added only when it is missing. Without `--output csv` the same flag writes `results.json`. + +For the other commands, redirect the output: +```bash +ipgeolocation ipgeo --ip 8.8.8.8 --output csv > ipgeo.csv +``` + ### `ip-security` Command Look up IP security information using the `ipgeolocation.io` API. @@ -1031,7 +1076,7 @@ ipgeolocation ip-security [flags] | `--ip` | string | `""` | IPv4 or IPv6 address. | | `--exclude` | string[] | `[]` | Exclude fields from output. | | `--fields` | string[] | `[]` | Return only specific fields (e.g. `security.threat_score`). | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | > [!NOTE] > IP Security API is only available in the Paid Plan @@ -1121,8 +1166,8 @@ ipgeolocation bulk-ip-security [flags] | `--file` | string | `""` | Path to a text file containing IPs (one per line). | | `--exclude` | string[] | `[]` | Exclude fields (e.g. `security.is_tor`). | | `--fields` | string[] | `[]` | Return only specific fields (e.g. `security.threat_score`). | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | -| `--output-file` | string | `""` | Save output to JSON file. Example: `--output-file results` | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | +| `--output-file` | string | `""` | Save output to a file. Writes `.csv` with `--output csv`, otherwise `.json`; the extension is added only if the name lacks it. Example: `--output-file results` or `--output-file results.csv` | #### `bulk-ip-security` Examples Look up 3 IP addresses: @@ -1135,6 +1180,8 @@ Look up from a file: ipgeolocation bulk-ip-security --file=ips.txt --output-file=output ``` +Results will be written to the `output.json` file. + Output as YAML: ```bash ipgeolocation bulk-ip-security --ips=8.8.8.8 --output=yaml @@ -1162,7 +1209,7 @@ ipgeolocation asn [flags] | `--include` | string[] | `[]` | Include extra fields in output.(e.g., `peers,downstreams,upstreams,routes,whois_response`) | | `--exclude` | string[] | `[]` | Exclude fields from output. | | `--fields` | string[] | `[]` | Return only specific fields (e.g. `ip,organization`). | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | > [!NOTE] > ASN API is only available in the Paid Plan @@ -1313,7 +1360,7 @@ ipgeolocation abuse [flags] | `--ip` | string | `""` | IPv4 or IPv6 address. | | `--exclude` | string[] | `[]` | Exclude fields from output. | | `--fields` | string[] | `[]` | Return only specific fields (e.g. `ip,organization`). | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | > [!NOTE] > Abuse Contact API is only available in the Paid Plan @@ -1395,7 +1442,7 @@ ipgeolocation timezone [flags] | `--iata` | string | `""` | IATA code (e.g. DXB). | | `--icao` | string | `""` | ICAO code (e.g. KATL). | | `--lo` | string | `""` | LO code (e.g. DEBER). | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | #### Get time zone info about your current IP ```bash @@ -1792,7 +1839,7 @@ ipgeolocation time-conversion [flags] | `--lo_from` | string | `""` | LO code to convert from. | | `--lo_to` | string | `""` | LO code to convert to. | | `--time` | string | `""` | Time to convert. | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | #### Convert Current Time from One Timezone to Another ```bash @@ -1868,7 +1915,7 @@ ipgeolocation astronomy [flags] | `--lang` | string | `""` | Response language (if supported). | | `--tz` | string | `""` | IANA time zone for output timestamps. | | `--elevation` | float64 | `0` | Elevation. | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | #### Lookup Astronomy API by Coordinates Get astronomy info about a specific latitude and longitude: @@ -2119,7 +2166,7 @@ ipgeolocation astronomy-timeseries [flags] | `--lang` | string | `""` | Response language (if supported). | | `--start-date` | string | `""` | Start date (e.g. 2023-01-01) Only YYYY-MM-DD. | | `--end-date` | string | `""` | End date (e.g. 2023-12-31) Only YYYY-MM-DD | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | > [!NOTE] > - The `start-date` and `end-date` flags are required. @@ -2256,7 +2303,7 @@ ipgeolocation parse-user-agent [flags] | Flag | Type | Default | Description | |----------------|--------|----------|--------------------------------------------------| | `--user-agent` | string | `""` | User agent string. | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | For further information, please visit [User Agent API Documentation](https://ipgeolocation.io/documentation/user-agent-api.html). @@ -2313,7 +2360,7 @@ ipgeolocation parse-bulk-user-agents [flags] | Flag | Type | Default | Description | |-----------------|----------|----------|--------------------------------------------------| | `--user-agents` | string[] | `[]` | User agent strings. | -| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`. | +| `--output` | string | `pretty` | Output format: `pretty`, `raw`, `table`, `yaml`, `csv`. | For further information, please visit [User Agent API bulk parsing documentation](https://ipgeolocation.io/documentation/user-agent-api.html#parse-bulk-user-agent-strings). diff --git a/cmd/abuse.go b/cmd/abuse.go index 6f8aef7..2b841d0 100644 --- a/cmd/abuse.go +++ b/cmd/abuse.go @@ -66,6 +66,13 @@ Example: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -84,7 +91,7 @@ func init() { abuseCmd.Flags().StringVar(&abuseFlags.IP, "ip", "", "IPv4 or IPv6 address (e.g. 8.8.8.8)") abuseCmd.Flags().StringSliceVar(&abuseFlags.Excludes, "exclude", []string{}, "Fields to exclude from the output") abuseCmd.Flags().StringSliceVar(&abuseFlags.Fields, "fields", []string{}, "Get Specific Fields to include in the output") - abuseCmd.Flags().StringVar(&abuseFlags.Output, "output", "", "Output format: yaml, raw, table, yaml") + abuseCmd.Flags().StringVar(&abuseFlags.Output, "output", "", "Output format: pretty, raw, table, yaml, csv") rootCmd.AddCommand(abuseCmd) } diff --git a/cmd/asn.go b/cmd/asn.go index aab6e32..1123b44 100644 --- a/cmd/asn.go +++ b/cmd/asn.go @@ -76,6 +76,13 @@ Getting peering relationships for an ASN number: ipgeolocation asn --asn 12345 - fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -96,7 +103,7 @@ func init() { asnCmd.Flags().StringSliceVar(&asnFlags.Include, "include", []string{}, "To include additional values in the output") asnCmd.Flags().StringSliceVar(&asnFlags.Excludes, "exclude", []string{}, "Fields to exclude from the output") asnCmd.Flags().StringSliceVar(&asnFlags.Fields, "fields", []string{}, "Get Specific Fields to include in the output") - asnCmd.Flags().StringVar(&asnFlags.Output, "output", "pretty", "Output format: pretty, raw, table") + asnCmd.Flags().StringVar(&asnFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv") rootCmd.AddCommand(asnCmd) } diff --git a/cmd/astronomy.go b/cmd/astronomy.go index 50b01fc..fd9f56e 100644 --- a/cmd/astronomy.go +++ b/cmd/astronomy.go @@ -99,6 +99,13 @@ API Reference: https://ipgeolocation.io/astronomy-api.html fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -122,7 +129,7 @@ func init() { astronomyCmd.Flags().Float64Var(&astronomyFlags.Longitude, "longitude", 0, "Longitude (e.g. -122.4194)") astronomyCmd.Flags().StringVar(&astronomyFlags.Language, "lang", "", "Language code (e.g. en)") astronomyCmd.Flags().Float64Var(&astronomyFlags.Elevation, "elevation", 0, "Elevation (e.g. 1000)") - astronomyCmd.Flags().StringVar(&astronomyFlags.Output, "output", "pretty", "Output format: pretty, raw, table") + astronomyCmd.Flags().StringVar(&astronomyFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv") rootCmd.AddCommand(astronomyCmd) } diff --git a/cmd/astronomyTimeseries.go b/cmd/astronomyTimeseries.go index 6b3a6e8..d29312a 100644 --- a/cmd/astronomyTimeseries.go +++ b/cmd/astronomyTimeseries.go @@ -94,6 +94,13 @@ Examples: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -117,7 +124,7 @@ func init() { astronomyTimeseriesCmd.Flags().Float64Var(&astronomyTimeseriesFlags.Latitude, "latitude", 0, "Latitude (e.g. 37.7749)") astronomyTimeseriesCmd.Flags().Float64Var(&astronomyTimeseriesFlags.Longitude, "longitude", 0, "Longitude (e.g. -122.4194)") astronomyTimeseriesCmd.Flags().StringVar(&astronomyTimeseriesFlags.Language, "lang", "", "Language code (e.g. en)") - astronomyTimeseriesCmd.Flags().StringVar(&astronomyTimeseriesFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml") + astronomyTimeseriesCmd.Flags().StringVar(&astronomyTimeseriesFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv") rootCmd.AddCommand(astronomyTimeseriesCmd) } diff --git a/cmd/bulkIpSecurity.go b/cmd/bulkIpSecurity.go index 5baa1e4..355ec80 100644 --- a/cmd/bulkIpSecurity.go +++ b/cmd/bulkIpSecurity.go @@ -96,6 +96,13 @@ Retrieving security information for IP addresses from a file: ipgeolocation bulk fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -108,13 +115,26 @@ Retrieving security information for IP addresses from a file: ipgeolocation bulk fmt.Println(string(pretty)) } if bulkSecurityFlags.OutputFile != "" { - pretty, _ := json.MarshalIndent(result, "", " ") - err := os.WriteFile(bulkSecurityFlags.OutputFile+".json", pretty, 0644) - if err != nil { - fmt.Println("Error writing JSON to file:", err) + // The saved file matches --output, so --output csv saves .csv and + // every other format keeps saving the .json it always has. The + // extension is only appended when the given name lacks it. + extension := ".json" + data, _ := json.MarshalIndent(result, "", " ") + if bulkSecurityFlags.Output == "csv" { + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + extension = ".csv" + data = []byte(csvData) + } + path := utils.OutputFilePath(bulkSecurityFlags.OutputFile, extension) + if err := os.WriteFile(path, data, 0644); err != nil { + fmt.Println("Error writing output to file:", err) return } - fmt.Println("Output saved to file:", bulkSecurityFlags.OutputFile+".json") + fmt.Println("Output saved to file:", path) } }, } @@ -123,9 +143,9 @@ func init() { bulkIpSecurityCmd.Flags().StringSliceVar(&bulkSecurityFlags.IPs, "ips", []string{}, "IPs") bulkIpSecurityCmd.Flags().StringSliceVar(&bulkSecurityFlags.Excludes, "exclude", []string{}, "Fields to exclude from the output") bulkIpSecurityCmd.Flags().StringSliceVar(&bulkSecurityFlags.Fields, "fields", []string{}, "Get Specific Fields to include in the output") - bulkIpSecurityCmd.Flags().StringVar(&bulkSecurityFlags.Output, "output", "pretty", "Output format: pretty, raw, table") + bulkIpSecurityCmd.Flags().StringVar(&bulkSecurityFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv") bulkIpSecurityCmd.Flags().StringVar(&bulkSecurityFlags.File, "file", "", "Path to a text file containing IPs (one per line)") - bulkIpSecurityCmd.Flags().StringVar(&bulkSecurityFlags.OutputFile, "output-file", "", "Save output to a file (JSON only)") + bulkIpSecurityCmd.Flags().StringVar(&bulkSecurityFlags.OutputFile, "output-file", "", "Save output to a file; the .csv or .json extension is added only if missing") rootCmd.AddCommand(bulkIpSecurityCmd) } diff --git a/cmd/commands_test.go b/cmd/commands_test.go index 9cb8f70..81d4bd5 100644 --- a/cmd/commands_test.go +++ b/cmd/commands_test.go @@ -3,6 +3,7 @@ package cmd import ( "crypto/tls" "crypto/x509" + "encoding/csv" "encoding/json" "fmt" "io" @@ -201,7 +202,7 @@ func TestCLICommands(t *testing.T) { if tc.path != "/getip" { tc.query.Set("apiKey", testKey) } - for _, format := range []string{"raw", "pretty", "yaml", "table"} { + for _, format := range []string{"raw", "pretty", "yaml", "table", "csv"} { t.Run(tc.name+"/"+format, func(t *testing.T) { args := append([]string{tc.name}, tc.flags...) output := runCLI(t, home, append(args, "--output", format)...) @@ -211,6 +212,20 @@ func TestCLICommands(t *testing.T) { } return } + if format == "csv" { + rows := parseCSV(t, output) + if len(rows) != 1 { + t.Fatalf("expected one CSV row, got %d:\n%s", len(rows), output) + } + // Nested fields keep their path, so location.city survives flattening. + if rows[0]["ip"] != "2001:db8::1" || rows[0]["location.city"] != "Test City" { + t.Fatalf("CSV lost result fields: %s", output) + } + if _, charted := rows[0]["_request.path"]; !charted { + t.Fatalf("CSV lost the echoed request: %s", output) + } + return + } var result interface{} var err error if format == "yaml" { @@ -301,3 +316,200 @@ func TestCLIConfigAndBulkFiles(t *testing.T) { } } } + +func parseCSV(t *testing.T, output string) []map[string]string { + t.Helper() + records, err := csv.NewReader(strings.NewReader(output)).ReadAll() + if err != nil { + t.Fatalf("invalid CSV: %v\n%s", err, output) + } + if len(records) == 0 { + return nil + } + rows := make([]map[string]string, 0, len(records)-1) + for _, record := range records[1:] { + row := map[string]string{} + for i, header := range records[0] { + row[header] = record[i] + } + rows = append(rows, row) + } + return rows +} + +// --output-file follows --output: csv saves .csv, everything else still saves +// the .json it always did. +func TestCLICSVOutputFile(t *testing.T) { + home := configuredHome(t, testKey) + for _, command := range []string{"bulk-ip-geo", "bulk-ip-security"} { + t.Run(command, func(t *testing.T) { + name := command + "-csv" + output := runCLI(t, home, command, "--ips", "8.8.8.8,1.1.1.1", "--output", "csv", "--output-file", name) + if !strings.Contains(output, "Output saved to file: "+name+".csv") { + t.Fatalf("csv file not reported: %s", output) + } + if _, err := os.Stat(filepath.Join(home, name+".json")); !os.IsNotExist(err) { + t.Fatalf("csv run also wrote a .json file: %v", err) + } + + saved, err := os.ReadFile(filepath.Join(home, name+".csv")) + if err != nil { + t.Fatal(err) + } + rows := parseCSV(t, string(saved)) + if len(rows) != 1 || rows[0]["ip"] != "2001:db8::1" || rows[0]["location.city"] != "Test City" { + t.Fatalf("saved CSV lost fields: %s", saved) + } + // Both requested addresses ride in one cell, joined by the separator. + if rows[0]["_request.payload.ips"] != "8.8.8.8|1.1.1.1" { + t.Fatalf("array cell changed: %q", rows[0]["_request.payload.ips"]) + } + // The file and stdout must agree. + if strings.TrimSpace(strings.TrimSuffix(output, "Output saved to file: "+name+".csv")) != strings.TrimSpace(string(saved)) { + t.Fatalf("file and stdout disagree:\n%s\n---\n%s", output, saved) + } + + // Without --output csv the JSON file is unchanged. + plain := command + "-plain" + if output := runCLI(t, home, command, "--ips", "8.8.8.8", "--output-file", plain); !strings.Contains(output, "Output saved to file: "+plain+".json") { + t.Fatalf("json file regressed: %s", output) + } + data, err := os.ReadFile(filepath.Join(home, plain+".json")) + if err != nil { + t.Fatal(err) + } + var results []map[string]interface{} + if err := json.Unmarshal(data, &results); err != nil { + t.Fatalf("json file is no longer JSON: %v", err) + } + if _, err := os.Stat(filepath.Join(home, plain+".csv")); !os.IsNotExist(err) { + t.Fatalf("plain run wrote a .csv file: %v", err) + } + }) + } +} + +// A request that fails must leave no file behind and must not overwrite one, +// for CSV exactly as for JSON. +func TestCLICSVOutputFileNotWrittenOnFailure(t *testing.T) { + for _, key := range []string{"BAD_KEY", "SERVER_ERROR", "BAD_JSON"} { + t.Run(key, func(t *testing.T) { + home := configuredHome(t, key) + for _, command := range []string{"bulk-ip-geo", "bulk-ip-security"} { + for _, exists := range []bool{false, true} { + name := fmt.Sprintf("%s-%v", command, exists) + file := filepath.Join(home, name+".csv") + if exists { + if err := os.WriteFile(file, []byte("keep existing results"), 0600); err != nil { + t.Fatal(err) + } + } + output := runCLI(t, home, command, "--ips", "8.8.8.8", "--output", "csv", "--output-file", name) + if strings.Contains(output, "Output saved") || strings.Contains(output, key) { + t.Fatalf("failed request reported a saved file: %s", output) + } + data, err := os.ReadFile(file) + if exists { + if err != nil || string(data) != "keep existing results" { + t.Fatalf("failed request changed the output file: %v", err) + } + } else if !os.IsNotExist(err) { + t.Fatalf("failed request created an output file: %v", err) + } + } + } + }) + } +} + +// --output-file adds .csv or .json only when the given name lacks it, so a +// name the user already suffixed is used as it stands and never doubles up. +func TestCLIOutputFileExtension(t *testing.T) { + for _, tc := range []struct { + given, format, want string + }{ + // A bare name gains the extension that matches --output. + {"plain", "", "plain.json"}, + {"plain-csv", "csv", "plain-csv.csv"}, + + // A name that already carries it is used as given. + {"suffixed.json", "", "suffixed.json"}, + {"suffixed.csv", "csv", "suffixed.csv"}, + + // Matching ignores case. + {"upper.JSON", "", "upper.JSON"}, + {"upper.CSV", "csv", "upper.CSV"}, + + // A non-matching extension is kept, not swapped. + {"mixed.json", "csv", "mixed.json.csv"}, + {"mixed.csv", "", "mixed.csv.json"}, + + // A dot that is not the extension is left alone. + {"2026.report", "", "2026.report.json"}, + } { + for _, command := range []string{"bulk-ip-geo", "bulk-ip-security"} { + name := tc.given + label := command + "/" + name + "/" + tc.format + t.Run(label, func(t *testing.T) { + directory := t.TempDir() + // A fresh HOME per case so leftover files cannot mask a mistake. + if output := runCLI(t, directory, "config", "--apikey", testKey); !strings.Contains(output, "saved") { + t.Fatalf("config failed: %s", output) + } + args := []string{command, "--ips", "8.8.8.8", "--output-file", name} + if tc.format != "" { + args = append(args, "--output", tc.format) + } + output := runCLI(t, directory, args...) + if !strings.Contains(output, "Output saved to file: "+tc.want) { + t.Fatalf("expected %q, got: %s", tc.want, output) + } + + // Exactly one result file exists, and it is the expected one. + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatal(err) + } + var produced []string + for _, entry := range entries { + if !entry.IsDir() && !strings.HasPrefix(entry.Name(), ".") { + produced = append(produced, entry.Name()) + } + } + if len(produced) != 1 || produced[0] != tc.want { + t.Fatalf("expected only %q, found %v", tc.want, produced) + } + + // And it holds the format that was asked for. + data, err := os.ReadFile(filepath.Join(directory, tc.want)) + if err != nil { + t.Fatal(err) + } + if tc.format == "csv" { + if rows := parseCSV(t, string(data)); len(rows) != 1 || rows[0]["ip"] != "2001:db8::1" { + t.Fatalf("saved file is not the expected CSV: %s", data) + } + } else { + var results []map[string]interface{} + if err := json.Unmarshal(data, &results); err != nil { + t.Fatalf("saved file is not JSON: %v\n%s", err, data) + } + } + }) + } + } +} + +func TestCLICSVHelpAndFallback(t *testing.T) { + home := configuredHome(t, testKey) + for _, command := range []string{"ipgeo", "bulk-ip-geo", "my-ip", "timezone", "asn", "abuse", "parse-user-agent"} { + if output := runCLI(t, home, command, "--help"); !strings.Contains(output, "csv") { + t.Fatalf("csv missing from %s help: %s", command, output) + } + } + // An unrecognised format still falls back to pretty JSON, as it always has. + output := runCLI(t, home, "ipgeo", "--output", "csvv") + if !json.Valid([]byte(output)) { + t.Fatalf("unknown format no longer falls back to pretty: %s", output) + } +} diff --git a/cmd/ipSecurity.go b/cmd/ipSecurity.go index 32fe026..1047131 100644 --- a/cmd/ipSecurity.go +++ b/cmd/ipSecurity.go @@ -76,6 +76,13 @@ Notes: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -94,7 +101,7 @@ func init() { ipSecurityCmd.Flags().StringVar(&securityFlags.IP, "ip", "", "IPv4 or IPv6 address (e.g. 8.8.8.8)") ipSecurityCmd.Flags().StringSliceVar(&securityFlags.Excludes, "exclude", []string{}, "Fields to exclude from the output") ipSecurityCmd.Flags().StringSliceVar(&securityFlags.Fields, "fields", []string{}, "Get Specific Fields to include in the output") - ipSecurityCmd.Flags().StringVar(&securityFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml") + ipSecurityCmd.Flags().StringVar(&securityFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv") rootCmd.AddCommand(ipSecurityCmd) } diff --git a/cmd/ipgeo.go b/cmd/ipgeo.go index de0f56b..2e4c3ea 100644 --- a/cmd/ipgeo.go +++ b/cmd/ipgeo.go @@ -27,7 +27,7 @@ and coordinates using the ipgeolocation.io IPGeo API (v2). You can query any public IP address or domain name, and customize the output using include/exclude/fields parameters as supported by the API. -Available output formats include: pretty (default), raw (JSON), table, and YAML. +Available output formats include: pretty (default), raw (JSON), table, YAML, and CSV. Examples: @@ -46,6 +46,9 @@ Examples: # Exclude unnecessary fields ipgeolocation ipgeo --exclude currency,time_zone + # Save a spreadsheet-friendly CSV + ipgeolocation ipgeo --ip 8.8.8.8 --output csv > ipgeo.csv + Notes: - You must have a valid API key configured using: ipgeolocation config --apikey= - If no --ip flag is provided, it defaults to your current IP address.`, @@ -95,6 +98,13 @@ Notes: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -115,7 +125,7 @@ func init() { ipgeoCmd.Flags().StringSliceVar(&ipgeoFlags.Excludes, "exclude", []string{}, "Fields to exclude from the output") ipgeoCmd.Flags().StringSliceVar(&ipgeoFlags.Fields, "fields", []string{}, "Get Specific Fields to include in the output") ipgeoCmd.Flags().StringVar(&ipgeoFlags.Language, "lang", "", "Language for the output") - ipgeoCmd.Flags().StringVar(&ipgeoFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml") + ipgeoCmd.Flags().StringVar(&ipgeoFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv") // --excludes was the original name of --exclude; keep accepting it so existing scripts don't break. ipgeoCmd.Flags().SetNormalizeFunc(func(f *pflag.FlagSet, name string) pflag.NormalizedName { diff --git a/cmd/ipgeoBulk.go b/cmd/ipgeoBulk.go index 495703a..23d4f69 100644 --- a/cmd/ipgeoBulk.go +++ b/cmd/ipgeoBulk.go @@ -109,6 +109,13 @@ Examples: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -122,13 +129,26 @@ Examples: fmt.Println(string(pretty)) } if bulkIpgeoFlags.OutputFile != "" { - pretty, _ := json.MarshalIndent(result, "", " ") - err := os.WriteFile(bulkIpgeoFlags.OutputFile+".json", pretty, 0644) - if err != nil { - fmt.Println("Error writing JSON to file:", err) + // The saved file matches --output, so --output csv saves .csv and + // every other format keeps saving the .json it always has. The + // extension is only appended when the given name lacks it. + extension := ".json" + data, _ := json.MarshalIndent(result, "", " ") + if bulkIpgeoFlags.Output == "csv" { + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + extension = ".csv" + data = []byte(csvData) + } + path := utils.OutputFilePath(bulkIpgeoFlags.OutputFile, extension) + if err := os.WriteFile(path, data, 0644); err != nil { + fmt.Println("Error writing output to file:", err) return } - fmt.Println("Output saved to file:", bulkIpgeoFlags.OutputFile+".json") + fmt.Println("Output saved to file:", path) } }, } @@ -139,9 +159,9 @@ func init() { bulkIpgeoCmd.Flags().StringSliceVar(&bulkIpgeoFlags.Excludes, "exclude", []string{}, "Fields to exclude from the output") bulkIpgeoCmd.Flags().StringSliceVar(&bulkIpgeoFlags.Fields, "fields", []string{}, "Get Specific Fields to include in the output") bulkIpgeoCmd.Flags().StringVar(&bulkIpgeoFlags.Language, "lang", "", "Language for the output") - bulkIpgeoCmd.Flags().StringVar(&bulkIpgeoFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml") + bulkIpgeoCmd.Flags().StringVar(&bulkIpgeoFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv") bulkIpgeoCmd.Flags().StringVar(&bulkIpgeoFlags.File, "file", "", "Path to a text file containing IPs (one per line)") - bulkIpgeoCmd.Flags().StringVar(&bulkIpgeoFlags.OutputFile, "output-file", "", "Save output to a file (JSON only)") + bulkIpgeoCmd.Flags().StringVar(&bulkIpgeoFlags.OutputFile, "output-file", "", "Save output to a file; the .csv or .json extension is added only if missing") rootCmd.AddCommand(bulkIpgeoCmd) } diff --git a/cmd/myIp.go b/cmd/myIp.go index 04051d6..041c358 100644 --- a/cmd/myIp.go +++ b/cmd/myIp.go @@ -54,6 +54,13 @@ Notes: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -69,7 +76,7 @@ Notes: } func init() { - myIPCmd.Flags().StringVar(&myIPFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, text") + myIPCmd.Flags().StringVar(&myIPFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv, text") rootCmd.AddCommand(myIPCmd) } diff --git a/cmd/parseBulkUserAgents.go b/cmd/parseBulkUserAgents.go index 51f28a1..992ac05 100644 --- a/cmd/parseBulkUserAgents.go +++ b/cmd/parseBulkUserAgents.go @@ -80,6 +80,13 @@ Note: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -97,7 +104,7 @@ Note: func init() { parseBulkUserAgentsCmd.Flags().StringSliceVar(&bulkUserAgentsFlags.UserAgents, "user-agents", []string{}, "User Agents") - parseBulkUserAgentsCmd.Flags().StringVar(&bulkUserAgentsFlags.Output, "output", "", "Output format: raw, table, yaml") + parseBulkUserAgentsCmd.Flags().StringVar(&bulkUserAgentsFlags.Output, "output", "", "Output format: pretty, raw, table, yaml, csv") rootCmd.AddCommand(parseBulkUserAgentsCmd) } diff --git a/cmd/timeConversion.go b/cmd/timeConversion.go index 8ec5cf6..cb8152d 100644 --- a/cmd/timeConversion.go +++ b/cmd/timeConversion.go @@ -136,6 +136,13 @@ Notes: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -166,7 +173,7 @@ func init() { timeConversionCmd.Flags().StringVar(&timeConversionFlags.LoCodeFrom, "lo_from", "", "LO code from") timeConversionCmd.Flags().StringVar(&timeConversionFlags.LoCodeTo, "lo_to", "", "LO code to") timeConversionCmd.Flags().StringVar(&timeConversionFlags.Time, "time", "", "Time") - timeConversionCmd.Flags().StringVar(&timeConversionFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml") + timeConversionCmd.Flags().StringVar(&timeConversionFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv") rootCmd.AddCommand(timeConversionCmd) } diff --git a/cmd/timezone.go b/cmd/timezone.go index b44716f..ded15e7 100644 --- a/cmd/timezone.go +++ b/cmd/timezone.go @@ -113,6 +113,13 @@ Note: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -137,7 +144,7 @@ func init() { timezoneCmd.Flags().StringVar(&timezoneFlags.IcaoCode, "icao", "", "ICAO code (e.g. KATL)") timezoneCmd.Flags().StringVar(&timezoneFlags.LoCode, "lo", "", "LO code (e.g. DEBER)") timezoneCmd.Flags().StringVar(&timezoneFlags.Language, "lang", "", "Language code (e.g. en)") - timezoneCmd.Flags().StringVar(&timezoneFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml") + timezoneCmd.Flags().StringVar(&timezoneFlags.Output, "output", "pretty", "Output format: pretty, raw, table, yaml, csv") rootCmd.AddCommand(timezoneCmd) } diff --git a/cmd/userAgent.go b/cmd/userAgent.go index 794bf59..147b943 100644 --- a/cmd/userAgent.go +++ b/cmd/userAgent.go @@ -66,6 +66,13 @@ Examples: fmt.Println(string(body)) case "table": utils.PrintAsTable(result, 0) + case "csv": + csvData, err := utils.ConvertToCSV(body) + if err != nil { + fmt.Println("Error converting to CSV:", err) + return + } + fmt.Print(csvData) case "yaml": yamlData, err := yaml.Marshal(result) if err != nil { @@ -83,7 +90,7 @@ Examples: func init() { userAgentCmd.Flags().StringVar(&userAgentFlags.UserAgent, "user-agent", "", "User Agent") - userAgentCmd.Flags().StringVar(&userAgentFlags.Output, "output", "", "Output format: raw, table, yaml") + userAgentCmd.Flags().StringVar(&userAgentFlags.Output, "output", "", "Output format: pretty, raw, table, yaml, csv") rootCmd.AddCommand(userAgentCmd) } diff --git a/internal/utils/csv.go b/internal/utils/csv.go new file mode 100644 index 0000000..8a832ac --- /dev/null +++ b/internal/utils/csv.go @@ -0,0 +1,187 @@ +package utils + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "sort" + "strconv" + "strings" +) + +// ArraySeparator joins the elements of an array of scalars inside a single CSV cell. +const ArraySeparator = "|" + +// ConvertToCSV renders a JSON API response as CSV text. +// +// A top-level JSON array becomes one row per element, which is what the bulk +// endpoints return; any other response becomes a single row. Nested objects are +// flattened into dotted column names ("location.city"). An array whose elements +// are all scalars is joined into one cell with ArraySeparator; an array holding +// objects keeps a column per element instead ("routes.0.prefix"), so its fields +// stay addressable. Columns are the union of the keys of every row, sorted, so +// the same response always produces the same file. +func ConvertToCSV(body []byte) (string, error) { + decoder := json.NewDecoder(bytes.NewReader(body)) + // Numbers stay exactly as the API sent them; float64 would turn a Unix + // timestamp into 1.772535675329e+09 and lose digits off long IDs. + decoder.UseNumber() + + var data interface{} + if err := decoder.Decode(&data); err != nil { + return "", fmt.Errorf("invalid JSON: %w", err) + } + + records, isList := data.([]interface{}) + if !isList { + records = []interface{}{data} + } + + rows := make([]map[string]string, 0, len(records)) + columns := map[string]bool{} + for _, record := range records { + row := flattenRecord(record) + for column := range row { + columns[column] = true + } + rows = append(rows, row) + } + + if len(columns) == 0 { + return "", nil + } + + headers := make([]string, 0, len(columns)) + for column := range columns { + headers = append(headers, column) + } + sort.Strings(headers) + + var buffer bytes.Buffer + writer := csv.NewWriter(&buffer) + if err := writer.Write(headers); err != nil { + return "", err + } + for _, row := range rows { + line := make([]string, len(headers)) + for i, header := range headers { + line[i] = escapeFormula(row[header]) + } + if err := writer.Write(line); err != nil { + return "", err + } + } + writer.Flush() + if err := writer.Error(); err != nil { + return "", err + } + + return buffer.String(), nil +} + +// flattenRecord turns one record into its CSV cells, keyed by column name. +func flattenRecord(record interface{}) map[string]string { + row := map[string]string{} + flattenValue("", record, row) + // A record that is a bare scalar, an empty object or an empty array leaves + // its cell unnamed; give it a column rather than writing a blank header. + if value, unnamed := row[""]; unnamed { + delete(row, "") + row["value"] = value + } + return row +} + +func flattenValue(prefix string, value interface{}, row map[string]string) { + switch typed := value.(type) { + case map[string]interface{}: + if len(typed) == 0 { + row[prefix] = "" + return + } + for key, nested := range typed { + flattenValue(joinColumn(prefix, key), nested, row) + } + case []interface{}: + if len(typed) == 0 { + row[prefix] = "" + return + } + if scalarsOnly(typed) { + parts := make([]string, len(typed)) + for i, item := range typed { + parts[i] = scalarToString(item) + } + row[prefix] = strings.Join(parts, ArraySeparator) + return + } + for i, item := range typed { + flattenValue(joinColumn(prefix, strconv.Itoa(i)), item, row) + } + default: + row[prefix] = scalarToString(typed) + } +} + +func joinColumn(prefix, key string) string { + if prefix == "" { + return key + } + return prefix + "." + key +} + +func scalarsOnly(values []interface{}) bool { + for _, value := range values { + switch value.(type) { + case map[string]interface{}, []interface{}: + return false + } + } + return true +} + +func scalarToString(value interface{}) string { + switch typed := value.(type) { + case nil: + return "" + case string: + return typed + case bool: + return strconv.FormatBool(typed) + case json.Number: + return typed.String() + case float64: + return strconv.FormatFloat(typed, 'f', -1, 64) + default: + return fmt.Sprintf("%v", typed) + } +} + +// escapeFormula stops a spreadsheet from running a cell as a formula. Response +// values such as organization names and whois text come from third parties, so +// a leading =, @ or control character is quoted off. +// +// A leading + or - is only quoted when a non-digit follows it. Signed numbers +// and signed data are ordinary values here, such as a "-122.08421" longitude, +// a "+1" calling code, the "+1.00H" and "-1.00H" DST durations and the lone +// "-" the astronomy endpoint uses for an absent sun or moon status; quoting +// those would corrupt real columns. A sign followed by anything else is the +// shape a formula takes, as in "+SUM(A1)" or "-cmd|' /c calc'!A0". +func escapeFormula(value string) string { + if value == "" { + return value + } + switch value[0] { + case '=', '@', '\t', '\r': + return "'" + value + case '+', '-': + // A lone sign is a value in its own right: the astronomy endpoint + // reports sun_status and moon_status as "-". It cannot be a formula. + if len(value) == 1 || value[1] == '.' || (value[1] >= '0' && value[1] <= '9') { + return value + } + return "'" + value + } + return value +} diff --git a/internal/utils/csv_test.go b/internal/utils/csv_test.go new file mode 100644 index 0000000..2f26475 --- /dev/null +++ b/internal/utils/csv_test.go @@ -0,0 +1,203 @@ +package utils + +import ( + "encoding/csv" + "strings" + "testing" +) + +func csvOf(t *testing.T, body string) string { + t.Helper() + output, err := ConvertToCSV([]byte(body)) + if err != nil { + t.Fatalf("ConvertToCSV(%s) failed: %v", body, err) + } + return output +} + +func TestConvertToCSVShapes(t *testing.T) { + for _, tc := range []struct { + name, body, want string + }{ + { + name: "nested objects become dotted columns", + body: `{"ip":"8.8.8.8","location":{"city":"Mountain View","is_eu":false}}`, + want: "ip,location.city,location.is_eu\n8.8.8.8,Mountain View,false\n", + }, + { + // The bulk endpoints answer with an array; each element is a record. + name: "top-level array becomes one row per element", + body: `[{"ip":"8.8.8.8"},{"ip":"1.1.1.1"}]`, + want: "ip\n8.8.8.8\n1.1.1.1\n", + }, + { + // Records need not agree on their keys; every column still appears. + name: "columns are the union across rows", + body: `[{"ip":"8.8.8.8","city":"MV"},{"ip":"1.1.1.1","asn":"AS13335"}]`, + want: "asn,city,ip\n,MV,8.8.8.8\nAS13335,,1.1.1.1\n", + }, + { + name: "array of scalars joins into one cell", + body: `{"languages":["en-US","es-US","haw"]}`, + want: "languages\nen-US|es-US|haw\n", + }, + { + // Keeping a column per field beats hiding objects in a joined cell. + name: "array of objects keeps a column per element", + body: `{"routes":[{"prefix":"8.8.8.0/24"},{"prefix":"8.8.4.0/24"}]}`, + want: "routes.0.prefix,routes.1.prefix\n8.8.8.0/24,8.8.4.0/24\n", + }, + { + name: "array mixing scalars and objects is indexed", + body: `{"items":["a",{"b":1}]}`, + want: "items.0,items.1.b\na,1\n", + }, + { + name: "empty containers leave an empty cell", + body: `{"ip":"8.8.8.8","tags":[],"meta":{}}`, + want: "ip,meta,tags\n8.8.8.8,,\n", + }, + { + name: "null becomes an empty cell", + body: `{"ip":"8.8.8.8","city":null}`, + want: "city,ip\n,8.8.8.8\n", + }, + { + name: "booleans keep their JSON spelling", + body: `{"is_eu":true,"is_dst":false}`, + want: "is_dst,is_eu\nfalse,true\n", + }, + { + name: "deep nesting keeps the whole path", + body: `{"time_zone":{"dst_end":{"gap":false,"utc_time":"2026-11-01 09:00"}}}`, + want: "time_zone.dst_end.gap,time_zone.dst_end.utc_time\nfalse,2026-11-01 09:00\n", + }, + { + name: "a bare scalar response gets a value column", + body: `"8.8.8.8"`, + want: "value\n8.8.8.8\n", + }, + { + name: "an empty object yields a single empty row", + body: `{}`, + want: "value\n\n", + }, + { + name: "unicode survives unchanged", + body: `{"city":"مدينة","flag":"🇺🇸"}`, + want: "city,flag\nمدينة,🇺🇸\n", + }, + } { + t.Run(tc.name, func(t *testing.T) { + if got := csvOf(t, tc.body); got != tc.want { + t.Fatalf("CSV changed:\ngot %q\nwant %q", got, tc.want) + } + }) + } +} + +// An empty bulk result has no columns to describe, so it writes nothing rather +// than a stray header line. +func TestConvertToCSVEmptyArray(t *testing.T) { + if got := csvOf(t, `[]`); got != "" { + t.Fatalf("empty array produced %q", got) + } +} + +// float64 would render a Unix timestamp as 1.772535675329e+09 and round long +// identifiers, so numbers must pass through exactly as the API sent them. +func TestConvertToCSVKeepsNumbersExact(t *testing.T) { + body := `{"unix":1772535675.329,"big":12345678901234567890,"neg":-122.08421,"zero":0,"exp":1e3,"offset":-8}` + want := "big,exp,neg,offset,unix,zero\n12345678901234567890,1e3,-122.08421,-8,1772535675.329,0\n" + if got := csvOf(t, body); got != want { + t.Fatalf("number formatting changed:\ngot %q\nwant %q", got, want) + } +} + +// Separators, quotes and newlines inside values must survive a round trip. +func TestConvertToCSVQuotesSpecialCharacters(t *testing.T) { + body := `{"org":"Google, LLC","quote":"say \"hi\"","line":"a\nb","tab":"a\tb"}` + output := csvOf(t, body) + + rows, err := csv.NewReader(strings.NewReader(output)).ReadAll() + if err != nil { + t.Fatalf("output is not valid CSV: %v\n%s", err, output) + } + if len(rows) != 2 { + t.Fatalf("expected a header and one row, got %d rows", len(rows)) + } + values := map[string]string{} + for i, header := range rows[0] { + values[header] = rows[1][i] + } + for header, want := range map[string]string{ + "org": "Google, LLC", "quote": `say "hi"`, "line": "a\nb", "tab": "a\tb", + } { + if values[header] != want { + t.Fatalf("%s round-tripped as %q, want %q", header, values[header], want) + } + } +} + +// Response values come from third parties, so a cell must not open as a formula +// in a spreadsheet. Numbers stay untouched so coordinates and offsets survive. +func TestConvertToCSVEscapesFormulas(t *testing.T) { + for value, want := range map[string]string{ + // Formulas are quoted off. + `=1+1`: `'=1+1`, + `=cmd|' /c calc'!A0`: `'=cmd|' /c calc'!A0`, + `+SUM(A1)`: `'+SUM(A1)`, + `-cmd|' /c calc'!A0`: `'-cmd|' /c calc'!A0`, + `@import`: `'@import`, + "\tlead": "'\tlead", + "\rlead": "'\rlead", + // A lone sign is real data: the astronomy endpoint reports an absent + // sun_status or moon_status as "-". It cannot be a formula. + "+": "+", + "-": "-", + // Signed data is left exactly as the API sent it. The DST duration + // field carries both signs, so the two must be treated alike. + "-1.00H": "-1.00H", + "+1.00H": "+1.00H", + "-122.08421": "-122.08421", + "+.5": "+.5", + "-8": "-8", + "+1": "+1", + // Ordinary text is untouched. + "Google LLC": "Google LLC", + "": "", + "not=a=formula": "not=a=formula", + "مدينة": "مدينة", + } { + if got := escapeFormula(value); got != want { + t.Fatalf("escapeFormula(%q) = %q, want %q", value, got, want) + } + } + + // And it reaches the rendered output. + if got := csvOf(t, `{"org":"=1+1","lat":"-37.4"}`); got != "lat,org\n-37.4,'=1+1\n" { + t.Fatalf("escaping missing from output: %q", got) + } +} + +func TestConvertToCSVRejectsInvalidJSON(t *testing.T) { + for _, body := range []string{``, `{`, `not json`, `{"a":}`} { + if _, err := ConvertToCSV([]byte(body)); err == nil { + t.Fatalf("invalid JSON %q was accepted", body) + } + } +} + +// Go randomises map iteration, so column order has to be imposed, not observed. +func TestConvertToCSVIsDeterministic(t *testing.T) { + body := `{"z":1,"a":2,"m":{"y":3,"b":4},"list":["q","p"]}` + first := csvOf(t, body) + for i := 0; i < 50; i++ { + if got := csvOf(t, body); got != first { + t.Fatalf("output varies between runs:\n%q\n%q", first, got) + } + } + if !strings.HasPrefix(first, "a,list,m.b,m.y,z\n") { + t.Fatalf("columns are not sorted: %q", first) + } +} diff --git a/internal/utils/output.go b/internal/utils/output.go index f3968a7..7242043 100644 --- a/internal/utils/output.go +++ b/internal/utils/output.go @@ -2,6 +2,7 @@ package utils import ( "fmt" + "path/filepath" "strings" ) @@ -42,3 +43,18 @@ func ToTitle(s string) string { func ConvertToXML(data interface{}) string { return "" } + +// OutputFilePath works out the file an --output-file value should write to. +// The extension is appended only when the name does not already end in it, so +// "results" and "results.csv" both save to results.csv and neither doubles up. +// +// An extension that does not match is left in place rather than swapped, so +// --output-file results.json with --output csv writes results.json.csv. That +// keeps the name the caller gave and never silently retargets a different file. +// The comparison ignores case, so results.CSV is accepted as it stands. +func OutputFilePath(name, extension string) string { + if strings.EqualFold(filepath.Ext(name), extension) { + return name + } + return name + extension +} diff --git a/internal/utils/output_test.go b/internal/utils/output_test.go new file mode 100644 index 0000000..0282ce0 --- /dev/null +++ b/internal/utils/output_test.go @@ -0,0 +1,41 @@ +package utils + +import "testing" + +func TestOutputFilePath(t *testing.T) { + for _, tc := range []struct { + name, extension, want string + }{ + // A bare name gains the extension. + {"results", ".csv", "results.csv"}, + {"results", ".json", "results.json"}, + {"out/results", ".csv", "out/results.csv"}, + + // A name that already carries it is left alone, so it never doubles up. + {"results.csv", ".csv", "results.csv"}, + {"results.json", ".json", "results.json"}, + {"out/results.json", ".json", "out/results.json"}, + + // Matching ignores case, which matters on Windows. + {"results.CSV", ".csv", "results.CSV"}, + {"results.Json", ".json", "results.Json"}, + + // A different extension is kept rather than swapped: the caller's name + // survives and no unrelated file is targeted. + {"results.json", ".csv", "results.json.csv"}, + {"results.csv", ".json", "results.csv.json"}, + {"results.tar.gz", ".csv", "results.tar.gz.csv"}, + + // Dots inside a name are not an extension. + {"my.results", ".csv", "my.results.csv"}, + {"2026-01-01.report", ".json", "2026-01-01.report.json"}, + + // Degenerate shapes still produce a usable path. + {".csv", ".csv", ".csv"}, + {"results.", ".csv", "results..csv"}, + } { + if got := OutputFilePath(tc.name, tc.extension); got != tc.want { + t.Fatalf("OutputFilePath(%q, %q) = %q, want %q", tc.name, tc.extension, got, tc.want) + } + } +}