diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 0d38d54..51d7486 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -16,11 +16,11 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -34,14 +34,14 @@ jobs: build-n-publish: name: Build and publish to PyPI if: github.event_name == 'push' - # the pinned Python 3.10.4 below has no build for Ubuntu 26.04 either + # kept on the same image as the test job runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.10.4 - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - name: Set up Python 3.13 + uses: actions/setup-python@v7 with: - python-version: 3.10.4 + python-version: "3.13" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -53,11 +53,7 @@ jobs: env: IPDATA_API_KEY: ${{ secrets.IPDATA_API_KEY }} - name: Install build - run: >- - python -m - pip install - build - --user + run: python -m pip install build - name: Build a binary wheel and a source tarball run: >- python -m diff --git a/README.md b/README.md index 7ab6dab..77976cc 100644 --- a/README.md +++ b/README.md @@ -544,10 +544,10 @@ ipdata batch my_ip_backlog.csv --output geolocation_results.json --format JSON ### Batch lookup with output to CSV file ``` -ipdata batch my_ip_backlog.csv --output results.csv --output-format CSV --fields ip --fields country_code +ipdata batch my_ip_backlog.csv --output results.csv --format CSV --fields ip --fields country_code ``` -The `--fields` option is required to use CSV output. +Either `--fields` or `--exclude` is required to use CSV output. #### Example Results diff --git a/setup.cfg b/setup.cfg index 2b3cca7..4d14fb8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,6 +15,7 @@ classifiers = Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 Programming Language :: Python :: 3.13 + Programming Language :: Python :: 3.14 License :: OSI Approved :: MIT License Operating System :: OS Independent diff --git a/src/ipdata/cli.py b/src/ipdata/cli.py index d71b67e..01bbbb5 100644 --- a/src/ipdata/cli.py +++ b/src/ipdata/cli.py @@ -380,26 +380,28 @@ def batch(ctx, input, fields, output, format, exclude): :param input: A list of resources to lookup either IP addresses or ASNs. You could mix them however this could break output processing when writing to CSV :param fields: A list of valid fields to extract from the individual responses + :param exclude: A list of fields to leave out, every other field is extracted. Mutually exclusive with fields :param output: The path to write the results to - :param format: The format to write the results to. When using the CSV format it is required to provide fields. + :param format: The format to write the results to. When using the CSV format it is required to provide fields or exclude. """ - if format == "CSV" and not fields: - print( - f'You need to specify a "--fields" argument with a list of fields to extract to get results in CSV. To get entire responses use JSON.', - file=stderr, - ) - return - # enforce mutual exclusivity of fields and exclude if exclude and fields: raise click.ClickException( "'--fields / -f' and '--exclude / -e' are mutually exclusive." ) - # if the user wants to exclude some fields, get all the fields in fields that are not in exclude + # if the user wants to exclude some fields, request all the other ones. They are sorted so that the + # CSV columns always come out in the same order. if exclude: - fields = set(ipdata.valid_fields).difference(set(exclude)) + fields = sorted(IPData.valid_fields.difference(exclude)) + + if format == "CSV" and not fields: + print( + 'You need to specify a "--fields" argument with a list of fields to extract (or "--exclude" with the fields to leave out) to get results in CSV. To get entire responses use JSON.', + file=stderr, + ) + return # Prepare requests ipdata = IPData(ctx.obj["api-key"]) @@ -427,6 +429,7 @@ def batch(ctx, input, fields, output, format, exclude): f"threat_{sub_field}" for sub_field in [ "is_tor", + "is_vpn", "is_icloud_relay", "is_proxy", "is_datacenter", @@ -436,6 +439,7 @@ def batch(ctx, input, fields, output, format, exclude): "is_threat", "is_bogon", "blocklists", + "scores", ] ] continue diff --git a/src/ipdata/ipdata.py b/src/ipdata/ipdata.py index f21ddfe..068c317 100644 --- a/src/ipdata/ipdata.py +++ b/src/ipdata/ipdata.py @@ -60,7 +60,8 @@ def __getattr__(*args): if type(val) is dict: return DotDict(val) elif type(val) is list: - return [DotDict(item) for item in val] + # wrap the dicts in a list of objects; a list of scalars is returned as it is + return [DotDict(item) if type(item) is dict else item for item in val] return val __setattr__ = dict.__setitem__ @@ -349,7 +350,7 @@ def bulk(self, resources, fields=[]): """ Lookup up to 100 resources in one request. Makes a POST request wth the resources as a JSON array and the specified fields. - :param resources: A list of resources. This can be a list of IP addresses or ASNs or a mix of both which is not recommended unless you handle it during parsing. Mixing resources might lead to weird behavior when writing results to CSV. + :param resources: A list or set of resources. This can be a list of IP addresses or ASNs or a mix of both which is not recommended unless you handle it during parsing. Mixing resources might lead to weird behavior when writing results to CSV. :param fields: A collection of API fields to be returned. :returns: A DotDict object with the data contained under the 'responses' key. @@ -368,9 +369,9 @@ def bulk(self, resources, fields=[]): self._validate_fields(fields) query_params = {"fields": ",".join(fields)} - # Make the requests + # Make the requests (as a list: a set passes the validation above but is not JSON serialisable) try: - response = self._request("POST", "/bulk", params=query_params, json=resources) + response = self._request("POST", "/bulk", params=query_params, json=list(resources)) except Exception as e: raise IPDataException(f"Error when looking up {resources} - {e}") diff --git a/src/ipdata/test_cli.py b/src/ipdata/test_cli.py new file mode 100644 index 0000000..9e0a2e5 --- /dev/null +++ b/src/ipdata/test_cli.py @@ -0,0 +1,143 @@ +""" +Offline tests for the command line interface. The IPData client that the commands create is replaced by one +wired to an httpx.MockTransport, so nothing here talks to the live API. +""" +import csv +import json + +import httpx +import pytest +from click.testing import CliRunner + +from ipdata import cli as cli_module +from ipdata.ipdata import IPData + +ASN = {"asn": "AS15169", "name": "Google LLC", "domain": "about.google", "route": "8.8.8.0/24", "type": "hosting"} + +# the members of the threat object, in the order the API returns them +THREAT = { + "is_tor": False, + "is_vpn": True, + "is_icloud_relay": False, + "is_proxy": False, + "is_datacenter": False, + "is_anonymous": True, + "is_known_attacker": False, + "is_known_abuser": False, + "is_threat": False, + "is_bogon": False, + "blocklists": [ + { + "name": "ipdata vpn", + "site": "https://ipdata.co", + "type": "vpn", + "first_seen": "2026-10-31", + "last_seen": "2026-11-02", + "days_seen": 2, + } + ], + "scores": {"vpn_score": 100, "proxy_score": 0, "threat_score": 0, "trust_score": 0}, +} + + +def bulk_response(request): + """Answers POST /bulk with one response per requested resource, in request order.""" + responses = [ + {"ip": ip, "country_code": "US", "asn": ASN, "threat": THREAT} for ip in json.loads(request.content) + ] + return httpx.Response(200, json=responses) + + +@pytest.fixture +def cli_requests(monkeypatch, tmp_path): + """ + Points the CLI at a stored API key and makes it create clients that record their requests and answer them + with bulk_response instead of sending them. Returns the list of recorded requests. + """ + key_file = tmp_path / "api-key" + key_file.write_text("test") + monkeypatch.setattr(cli_module, "API_KEY_FILE", str(key_file)) + + requests = [] + + def handler(request): + requests.append(request) + return bulk_response(request) + + class RecordingIPData(IPData): + def __init__(self, api_key=None, **kwargs): + super().__init__(api_key, transport=httpx.MockTransport(handler), **kwargs) + + monkeypatch.setattr(cli_module, "IPData", RecordingIPData) + return requests + + +def run_batch(tmp_path, *options, resources=("8.8.8.8", "1.1.1.1")): + """Runs 'ipdata batch' on a file listing 'resources' and returns the click result and the output path.""" + source = tmp_path / "ips.txt" + source.write_text("".join(f"{resource}\n" for resource in resources)) + output = tmp_path / "results" + result = CliRunner().invoke(cli_module.cli, ["batch", str(source), "--output", str(output), *options]) + return result, output + + +def test_batch_exclude_requests_every_other_field(cli_requests, tmp_path): + result, output = run_batch(tmp_path, "--exclude", "threat", "-e", "asn") + + assert result.exit_code == 0, result.output + assert result.exception is None + assert len(cli_requests) == 1 + requested = cli_requests[0].url.params["fields"].split(",") + assert set(requested) == IPData.valid_fields - {"threat", "asn"} + assert requested == sorted(requested) + assert [json.loads(line)["ip"] for line in output.read_text().splitlines()] == ["8.8.8.8", "1.1.1.1"] + + +def test_batch_exclude_with_csv_output(cli_requests, tmp_path): + result, output = run_batch(tmp_path, "--format", "CSV", "--exclude", "asn", "-e", "company") + + assert result.exit_code == 0, result.output + reader = csv.DictReader(output.read_text().splitlines()) + rows = list(reader) + assert len(rows) == 2 + assert not any(column.startswith(("asn_", "company_")) for column in reader.fieldnames) + # the remaining fields are requested, and therefore written, in sorted order + plain_columns = [column for column in reader.fieldnames if not column.startswith("threat_")] + assert plain_columns == sorted(plain_columns) + assert "ip" in plain_columns and "country_code" in plain_columns + assert rows[0]["ip"] == "8.8.8.8" + assert rows[0]["country_code"] == "US" + assert rows[0]["threat_is_tor"] == "False" + assert rows[1]["ip"] == "1.1.1.1" + + +def test_batch_rejects_fields_and_exclude_together(cli_requests, tmp_path): + result, output = run_batch(tmp_path, "-f", "ip", "-e", "asn") + + assert result.exit_code != 0 + assert "mutually exclusive" in result.output + assert cli_requests == [] + + +def test_batch_csv_needs_fields_or_exclude(cli_requests, tmp_path): + result, output = run_batch(tmp_path, "--format", "CSV") + + assert result.exit_code == 0 + assert cli_requests == [] + assert not output.exists() + + +def test_batch_csv_threat_columns_include_is_vpn_and_scores(cli_requests, tmp_path): + result, output = run_batch(tmp_path, "--format", "CSV", "-f", "ip", "-f", "threat", resources=("8.8.8.8",)) + + assert result.exit_code == 0, result.output + reader = csv.DictReader(output.read_text().splitlines()) + rows = list(reader) + assert reader.fieldnames == ["ip"] + [f"threat_{member}" for member in THREAT] + assert rows[0]["ip"] == "8.8.8.8" + assert rows[0]["threat_is_vpn"] == "True" + assert rows[0]["threat_is_tor"] == "False" + # nested values are written as their Python representation, the blocklist dates included + assert rows[0]["threat_blocklists"] == str(THREAT["blocklists"]) + assert "'first_seen': '2026-10-31'" in rows[0]["threat_blocklists"] + assert rows[0]["threat_scores"] == str(THREAT["scores"]) diff --git a/src/ipdata/test_client.py b/src/ipdata/test_client.py index fb8d93e..94c81f3 100644 --- a/src/ipdata/test_client.py +++ b/src/ipdata/test_client.py @@ -12,7 +12,7 @@ import ipdata from ipdata import ipdata as ipdata_module -from ipdata.ipdata import IPData, IPDataException +from ipdata.ipdata import DotDict, IPData, IPDataException LOOKUP_RESPONSE = { "ip": "8.8.8.8", @@ -139,6 +139,23 @@ def test_lookup_returns_error_payload_with_status(): assert data.message.startswith("You have not provided") +def test_dotdict_wraps_dict_list_items_and_leaves_the_others_alone(): + data = DotDict( + { + "numbers": [1, 2, 3], + "mixed": [{"name": "a"}, "b", None], + "nested": {"entries": [{"name": "c"}]}, + } + ) + + # dot access to a list of scalars used to raise TypeError + assert data.numbers == [1, 2, 3] + assert data.mixed[0].name == "a" + assert data.mixed[1:] == ["b", None] + assert data.nested.entries[0].name == "c" + assert data.missing is None + + def test_bulk_posts_json_with_the_client_timeout(): responses = [dict(LOOKUP_RESPONSE), dict(LOOKUP_RESPONSE, ip="1.1.1.1")] client, recorder = make_client(httpx.Response(200, json=responses), timeout=7) @@ -166,6 +183,18 @@ def test_lookup_uses_the_default_timeout(): assert recorder.requests[0].extensions["timeout"] == {"connect": 60, "read": 60, "write": 60, "pool": 60} +def test_bulk_accepts_a_set_of_resources(): + responses = [dict(LOOKUP_RESPONSE), dict(LOOKUP_RESPONSE, ip="1.1.1.1")] + client, recorder = make_client(httpx.Response(200, json=responses)) + + data = client.bulk({"8.8.8.8", "1.1.1.1"}, fields=["ip"]) + + # a set passes the validation but is not JSON serialisable, so it is sent as a list + assert sorted(json.loads(recorder.requests[0].content)) == ["1.1.1.1", "8.8.8.8"] + assert data.status == 200 + assert [r.ip for r in data.responses] == ["8.8.8.8", "1.1.1.1"] + + def test_bulk_non_200_returns_payload_with_status(): client, _ = make_client(httpx.Response(403, json={"message": "Forbidden"})) diff --git a/src/ipdata/test_ipdata.py b/src/ipdata/test_ipdata.py index a6dd63e..4d6385a 100644 --- a/src/ipdata/test_ipdata.py +++ b/src/ipdata/test_ipdata.py @@ -50,9 +50,8 @@ def test_bulk_v6(ips): # Check we got all results assert len(data.responses) == len(ips) - # Capitalize to handle 2620:11A:A000::' == '2620:11a:a000::' - for pair in zip(ips, [response.ip for response in data.responses]): - assert pair[0].upper() == pair[0].upper() + # Check we got a response for every requested IP. Capitalize to handle '2620:11A:A000::' == '2620:11a:a000::' + assert sorted(response.ip.upper() for response in data.responses) == sorted(ip.upper() for ip in ips) @given(st.lists(st.ip_addresses(network="8.8.8.0/24"), min_size=1, max_size=100)) @@ -65,5 +64,5 @@ def test_bulk_v4(ips): # Check we got all results assert len(data.responses) == len(ips) - for pair in zip(ips, [response.ip for response in data.responses]): - assert pair[0] == pair[0] + # Check we got a response for every requested IP + assert sorted(response.ip for response in data.responses) == sorted(ips)