From 99e726966300a47e9e36b8a79ffbded3e62e2357 Mon Sep 17 00:00:00 2001 From: bobbyxng Date: Fri, 25 Sep 2026 00:05:22 +0200 Subject: [PATCH 01/11] Rename OSM cleaning/building to clean.py and build_network.py Replace clean_osm_data.py/build_osm_network.py and the build.smk/clean.smk rules with clean.py/build_network.py/network.smk, moving output paths from osm/retrieve|clean|build to retrieve|clean|build directly. Also fixes a crash in relation-to-line merging for route relations whose members are all nodes (no way geometry at all) - previously a raw KeyError, now correctly dropped as having nothing to merge. --- tests/integration_test.py | 4 +- workflow/rules/build.smk | 33 ------- workflow/rules/clean.smk | 55 ------------ workflow/rules/network.smk | 85 +++++++++++++++++++ workflow/rules/retrieve.smk | 12 +-- ...{build_osm_network.py => build_network.py} | 8 +- .../scripts/{clean_osm_data.py => clean.py} | 44 +++++++++- workflow/scripts/retrieve_osm_overpass.py | 4 +- workflow/scripts/retrieve_osm_pbf.py | 4 +- 9 files changed, 141 insertions(+), 108 deletions(-) delete mode 100644 workflow/rules/build.smk delete mode 100644 workflow/rules/clean.smk create mode 100644 workflow/rules/network.smk rename workflow/scripts/{build_osm_network.py => build_network.py} (99%) rename workflow/scripts/{clean_osm_data.py => clean.py} (95%) diff --git a/tests/integration_test.py b/tests/integration_test.py index d26e957..ad9089c 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -81,7 +81,7 @@ def test_snakemake_integration_testing(module_path, tmp_path): shutil.copytree(tmp_path / "logs", log_dir, dirs_exist_ok=True) assert result.returncode == 0, run_log.read_text(encoding="utf-8") - output_dir = tmp_path / "resources/grid-builder/osm/retrieve" + output_dir = tmp_path / "resources/grid-builder/retrieve" for feature in ("substations_way", "lines_way"): payload = json.loads( (output_dir / f"benin_{feature}.json").read_text(encoding="utf-8") @@ -90,7 +90,7 @@ def test_snakemake_integration_testing(module_path, tmp_path): assert elements, f"No {feature} records retrieved" assert all(item["geometry"] for item in elements) - build_dir = tmp_path / "resources/grid-builder/osm/build" + build_dir = tmp_path / "resources/grid-builder/build" for component in ("buses", "lines", "transformers"): with (build_dir / "csv" / f"{component}.csv").open(encoding="utf-8") as file: rows = list(csv.DictReader(file)) diff --git a/workflow/rules/build.smk b/workflow/rules/build.smk deleted file mode 100644 index 97e926c..0000000 --- a/workflow/rules/build.smk +++ /dev/null @@ -1,33 +0,0 @@ -# SPDX-FileCopyrightText: Contributors to grid-builder /build_osm_network.log", - conda: - "../envs/network.yaml" - threads: 1 - params: - station_merge_radius_m=config["network"]["station_merge_radius_m"], - remove_under_construction=config["network"]["remove_under_construction"], - remove_after=config["network"]["remove_after"], - crs=config["crs"].model_dump(mode="json"), - message: - "Building a connected generic OSM network." - script: - "../scripts/build_osm_network.py" diff --git a/workflow/rules/clean.smk b/workflow/rules/clean.smk deleted file mode 100644 index d3d24e3..0000000 --- a/workflow/rules/clean.smk +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-FileCopyrightText: Contributors to grid-builder /osm/retrieve/{country}_lines_way.json", - country=config["countries"], - ), - cables_way=expand( - "/osm/retrieve/{country}_cables_way.json", - country=config["countries"], - ), - substations_way=expand( - "/osm/retrieve/{country}_substations_way.json", - country=config["countries"], - ), - substations_node=expand( - "/osm/retrieve/{country}_substations_node.json", - country=config["countries"], - ), - substations_relation=expand( - "/osm/retrieve/{country}_substations_relation.json", - country=config["countries"], - ), - routes_relation=( - expand( - "/osm/retrieve/{country}_routes_relation.json", - country=config["countries"], - ) - if config["retrieve"]["include_relations"] - else [] - ), - output: - substations="/osm/clean/substations.geojson", - substations_polygon="/osm/clean/substations_polygon.geojson", - lines="/osm/clean/lines.geojson", - log: - "/clean_osm_data.log", - conda: - "../envs/network.yaml" - threads: 1 - params: - network=config["network"].model_dump(mode="json"), - regions={ - code: value.model_dump(mode="json") - for code, value in config["regions"].items() - }, - crs=config["crs"].model_dump(mode="json"), - message: - "Cleaning retrieved OSM power features." - script: - "../scripts/clean_osm_data.py" diff --git a/workflow/rules/network.smk b/workflow/rules/network.smk new file mode 100644 index 0000000..3d5476b --- /dev/null +++ b/workflow/rules/network.smk @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: Contributors to grid-builder /retrieve/{country}_lines_way.json", + country=config["countries"], + ), + cables_way=expand( + "/retrieve/{country}_cables_way.json", + country=config["countries"], + ), + substations_way=expand( + "/retrieve/{country}_substations_way.json", + country=config["countries"], + ), + substations_node=expand( + "/retrieve/{country}_substations_node.json", + country=config["countries"], + ), + substations_relation=expand( + "/retrieve/{country}_substations_relation.json", + country=config["countries"], + ), + routes_relation=( + expand( + "/retrieve/{country}_routes_relation.json", + country=config["countries"], + ) + if config["network"]["include_relations"] + else [] + ), + output: + substations="/clean/substations.geojson", + substations_polygon="/clean/substations_polygon.geojson", + lines="/clean/lines.geojson", + log: + "/clean.log", + conda: + "../envs/network.yaml" + threads: 1 + params: + network=config["network"].model_dump(mode="json"), + regions={ + code: value.model_dump(mode="json") + for code, value in config["regions"].items() + }, + crs=config["crs"].model_dump(mode="json"), + message: + "Cleaning retrieved OSM power features." + script: + "../scripts/clean.py" + + +rule build_network: + input: + substations=rules.clean.output.substations, + substations_polygon=rules.clean.output.substations_polygon, + lines=rules.clean.output.lines, + output: + buses="/build/csv/buses.csv", + lines="/build/csv/lines.csv", + transformers="/build/csv/transformers.csv", + buses_geojson="/build/geojson/buses.geojson", + lines_geojson="/build/geojson/lines.geojson", + transformers_geojson="/build/geojson/transformers.geojson", + stations_polygon="/build/geojson/stations_polygon.geojson", + buses_polygon="/build/geojson/buses_polygon.geojson", + log: + "/build_network.log", + conda: + "../envs/network.yaml" + threads: 1 + params: + station_merge_radius_m=config["network"]["station_merge_radius_m"], + remove_under_construction=config["network"]["remove_under_construction"], + remove_after=config["network"]["remove_after"], + crs=config["crs"].model_dump(mode="json"), + message: + "Building a connected generic OSM network." + script: + "../scripts/build_network.py" diff --git a/workflow/rules/retrieve.smk b/workflow/rules/retrieve.smk index 05e731f..cc6144f 100644 --- a/workflow/rules/retrieve.smk +++ b/workflow/rules/retrieve.smk @@ -6,9 +6,9 @@ from pathlib import Path # Both retrieve_osm_pbf and retrieve_osm_overpass produce this same fixed # set of six files per country (routes_relation included even when -# retrieve.include_relations is off, just empty) — see either script's +# network.include_relations is off, just empty) — see either script's # module docstring for why relations aren't optional at the retrieval layer -# even though clean_osm_data only ever reads routes_relation.json when the +# even though clean only ever reads routes_relation.json when the # config flag is on. Only one of the two rules below is ever defined, since # retrieve.source picks exactly one implementation for the same output # paths — defining both unconditionally would make Snakemake's DAG @@ -22,7 +22,7 @@ _OSM_FEATURES = [ "routes_relation", ] _OSM_OUTPUTS = { - feature: f"/osm/retrieve/{{country}}_{feature}.json" + feature: f"/retrieve/{{country}}_{feature}.json" for feature in _OSM_FEATURES } @@ -38,7 +38,7 @@ if config["retrieve"]["source"] == "geofabrik": "../envs/retrieve.yaml" threads: 1 params: - include_relations=config["retrieve"]["include_relations"], + include_relations=config["network"]["include_relations"], force_redownload=config["retrieve"]["force_redownload"], data_dir=str(Path(workflow.basedir).parent / "data" / "earth-osm"), message: @@ -57,7 +57,7 @@ elif config["retrieve"]["source"] == "overpass": "../envs/retrieve.yaml" threads: 1 params: - include_relations=config["retrieve"]["include_relations"], + include_relations=config["network"]["include_relations"], overpass_api=config["retrieve"]["overpass_api"].model_dump(mode="json"), message: "Retrieve OSM power features for one country from the Overpass API." @@ -68,7 +68,7 @@ elif config["retrieve"]["source"] == "overpass": rule retrieve_osm_all: input: expand( - "/osm/retrieve/{country}_{feature}.json", + "/retrieve/{country}_{feature}.json", country=config["countries"], feature=_OSM_FEATURES, ), diff --git a/workflow/scripts/build_osm_network.py b/workflow/scripts/build_network.py similarity index 99% rename from workflow/scripts/build_osm_network.py rename to workflow/scripts/build_network.py index adb8704..e4803af 100644 --- a/workflow/scripts/build_osm_network.py +++ b/workflow/scripts/build_network.py @@ -708,7 +708,7 @@ def _add_transformers(buses: gpd.GeoDataFrame, geo_crs: str) -> gpd.GeoDataFrame return all_transformers[["transformer_id", *columns]] -def build_osm_network( +def build_network( substations: gpd.GeoDataFrame, substations_polygon: gpd.GeoDataFrame, lines: gpd.GeoDataFrame, @@ -724,7 +724,7 @@ def build_osm_network( gpd.GeoDataFrame, gpd.GeoDataFrame, ]: - """Create buses, AC lines, and transformers from clean_osm_data's output. + """Create buses, AC lines, and transformers from clean's output. Also returns two polygon views for visualisation: ``stations_polygon`` (the clustered station shapes from station-seed buffering, keyed by @@ -942,10 +942,10 @@ def _write_components( if "snakemake" not in globals(): from scripts._helpers import mock_snakemake - snakemake = mock_snakemake("build_osm_network") + snakemake = mock_snakemake("build_network") configure_logging(snakemake.log[0]) - buses, lines, transformers, stations_polygon, buses_polygon = build_osm_network( + buses, lines, transformers, stations_polygon, buses_polygon = build_network( gpd.read_file(snakemake.input.substations), gpd.read_file(snakemake.input.substations_polygon), gpd.read_file(snakemake.input.lines), diff --git a/workflow/scripts/clean_osm_data.py b/workflow/scripts/clean.py similarity index 95% rename from workflow/scripts/clean_osm_data.py rename to workflow/scripts/clean.py index 52b261b..32f5fcd 100644 --- a/workflow/scripts/clean_osm_data.py +++ b/workflow/scripts/clean.py @@ -66,6 +66,17 @@ def _apply_corrections(column: pd.Series, steps: list[dict[str, Any]]) -> pd.Ser for step in steps: if "lower" in step: column = column.str.lower() + elif "exact" in step: + # Whole-value match, unlike "replace" below: some OSM contributors + # tag voltage with a crude word like "medium" instead of a number, + # but "replace"'s substring match would also corrupt that same + # word appearing inside unrelated freeform text (seen for real: + # a garbage voltage tag containing "amperes" and "equipment" had + # every "m" in it expanded into "33000" by a substring rule for + # exactly this case, ballooning into a 30+ digit value that + # overflowed int64 in _filter_by_voltage). + pattern, replacement = step["exact"] + column = column.where(column != pattern, replacement) else: pattern, replacement = step["replace"] column = column.str.replace(pattern, replacement, regex=False) @@ -189,6 +200,13 @@ def _filter_by_voltage( list_voltages = df["voltage"].str.split(";").explode().unique().astype(str) list_voltages = list_voltages[np.vectorize(str.isnumeric)(list_voltages)] + # A purely-numeric string still isn't necessarily a real voltage: crowd- + # sourced OSM tags occasionally clean up into a huge digit string (e.g. a + # freeform note misusing the voltage key) that overflows astype(int)'s + # fixed-width C long. The highest real-world transmission voltage is + # ~1,100 kV (7 digits), so anything past 9 digits is unambiguously noise, + # not a value some future config's min_voltage might legitimately want. + list_voltages = list_voltages[np.vectorize(len)(list_voltages) <= 9] list_voltages = list_voltages.astype(int) list_voltages = list_voltages[list_voltages >= int(min_voltage)] list_voltages = list_voltages.astype(str) @@ -667,11 +685,29 @@ def _check_if_ways_in_multi(members: list[str], longer_list: Any) -> bool: def _create_line(row: pd.Series) -> tuple[Any, list[str]]: - """Merge a relation's member ways into one line, dropping closed rings (substations).""" + """Merge a relation's member ways into one line, dropping closed rings (substations). + + A relation whose members are all nodes (route relations occasionally + tag only their endpoint substations, no way) has no "geometry" key on + any member at all, so ``pd.json_normalize`` never creates that column; + without the guard below, ``dropna(subset=["geometry"])`` raises + ``KeyError`` instead of just finding nothing to merge. The empty-``df`` + short-circuit after it sidesteps a second, separate pandas quirk: on an + empty-but-multi-column frame, ``.apply(..., axis=1)`` can't infer a + per-row return shape and hands back an empty *DataFrame* rather than a + Series, which then fails to assign into a single column. The caller + already drops any relation whose resulting geometry isn't a + LineString, which ``linemerge([])`` (an empty GeometryCollection) + satisfies for free. + """ df = pd.json_normalize(row["members"]) + if "geometry" not in df.columns: + df["geometry"] = pd.NA df["ref"] = df["ref"].astype(str) df["ways"] = "way/" + df["ref"] df = df.dropna(subset=["geometry"]) + if df.empty: + return linemerge([]), [] df["geometry"] = df.apply(_create_linestring, axis=1) closed_geom = df["geometry"].apply(lambda x: x.is_closed) @@ -916,7 +952,7 @@ def _region_ac_hz( return _format_hz(hz) -def clean_osm_data( +def clean( inputs: dict[str, list[str]], network: dict[str, Any], regions: dict[str, Any], @@ -1163,11 +1199,11 @@ def clean_osm_data( if "snakemake" not in globals(): from scripts._helpers import mock_snakemake - snakemake = mock_snakemake("clean_osm_data") + snakemake = mock_snakemake("clean") configure_logging(snakemake.log[0]) inputs = {name: list(paths) for name, paths in snakemake.input.items()} - buses, polygons, lines = clean_osm_data( + buses, polygons, lines = clean( inputs, snakemake.params.network, snakemake.params.regions, diff --git a/workflow/scripts/retrieve_osm_overpass.py b/workflow/scripts/retrieve_osm_overpass.py index 5234363..6495dde 100644 --- a/workflow/scripts/retrieve_osm_overpass.py +++ b/workflow/scripts/retrieve_osm_overpass.py @@ -15,7 +15,7 @@ plain-node substations. Output is raw Overpass JSON (``{"elements": [...]}``) — the same shape -retrieve_osm_pbf.py produces from a local PBF file, so clean_osm_data.py's +retrieve_osm_pbf.py produces from a local PBF file, so clean.py's importers don't need to know which source produced their input. """ @@ -98,7 +98,7 @@ def _normalise_node_geometry(payload: dict[str, Any]) -> dict[str, Any]: Overpass's own JSON puts a node's location directly on ``lat``/``lon`` (there's no member/way to have a "geometry" list of), but retrieve_osm_pbf.py normalises nodes to the same list-of-points shape as - everything else, so clean_osm_data.py can treat all three element types + everything else, so clean.py can treat all three element types uniformly regardless of source. """ elements = [] diff --git a/workflow/scripts/retrieve_osm_pbf.py b/workflow/scripts/retrieve_osm_pbf.py index e76daa6..ee1cadb 100644 --- a/workflow/scripts/retrieve_osm_pbf.py +++ b/workflow/scripts/retrieve_osm_pbf.py @@ -8,7 +8,7 @@ resolution and PBF download/caching (``get_region_tuple``, ``download_region_pbf``) — reading the file itself is done directly with osmium, since earth-osm's own PBF parser (and its Overpass client) parses -relations internally but never exports them, and this way clean_osm_data.py +relations internally but never exports them, and this way clean.py gets the exact same file shape regardless of whether the data came from a local PBF (this script) or a live Overpass query (retrieve_osm_overpass.py). @@ -321,7 +321,7 @@ def _extract_from_filtered_pbf( # Always written, even empty when include_relations is off: this keeps # every retrieval rule producing the same fixed six files per country, # so whether a routes_relation.json is actually read is decided in one - # place (clean_osm_data's rule input), not duplicated into every writer. + # place (clean.py's rule input), not duplicated into every writer. return { "lines_way": {"elements": resolver.ways_by_feature["line"]}, "cables_way": {"elements": resolver.ways_by_feature["cable"]}, From 7108f88d4b2643378447b245e03a3dcf94c4df86 Mon Sep 17 00:00:00 2001 From: bobbyxng Date: Fri, 25 Sep 2026 00:06:34 +0200 Subject: [PATCH 02/11] Fix voltage corrections corrupting garbage tags into huge numbers The "m"/"medium"/"low"/etc. voltage corrections matched as substrings, so any unrelated text containing those letters got mangled - one Philippines line's freeform garbage tag ballooned into a 30+ digit number and crashed the pipeline. Match those rules on the whole value instead, and drop any voltage that's still implausibly long as a defensive backstop. --- tests/test_network_processing.py | 63 ++++++++++++++++++-------- workflow/internal/tag_corrections.yaml | 25 +++++++--- 2 files changed, 62 insertions(+), 26 deletions(-) diff --git a/tests/test_network_processing.py b/tests/test_network_processing.py index 3a4e2e7..e896207 100644 --- a/tests/test_network_processing.py +++ b/tests/test_network_processing.py @@ -6,8 +6,13 @@ import pandas as pd from shapely.geometry import LineString -from workflow.scripts.build_osm_network import build_osm_network -from workflow.scripts.clean_osm_data import _region_ac_hz, clean_osm_data +from workflow.scripts.build_network import build_network +from workflow.scripts.clean import ( + _apply_corrections, + _filter_by_voltage, + _region_ac_hz, + clean, +) def _write(path, elements): @@ -46,6 +51,30 @@ def test_region_ac_hz_handles_null_frequency_override(): assert _region_ac_hz("US", _NETWORK, regions_us) == "60" +def test_apply_corrections_exact_matches_whole_value_only(): + """An ``exact`` step only fires on a value equal to the pattern, unlike ``replace``. + + A freeform voltage tag containing an unrelated "m" (e.g. "amperes") + must survive untouched, where a substring "replace" would corrupt it. + """ + column = pd.Series(["m", "amperes", "medium"]) + steps = [{"exact": ["m", "33000"]}, {"exact": ["medium", "99000"]}] + result = _apply_corrections(column, steps) + assert list(result) == ["33000", "amperes", "99000"] + + +def test_filter_by_voltage_drops_oversized_garbage_without_crashing(): + """A voltage tag that cleans up into an implausibly long digit string. + + (e.g. freeform text misusing the voltage key) is dropped as noise + instead of overflowing ``astype(int)``'s fixed-width C long. + """ + df = pd.DataFrame({"voltage": ["230000", "9" * 15]}) + filtered, list_voltages = _filter_by_voltage(df, min_voltage=220000) + assert list(list_voltages) == ["230000"] + assert list(filtered["voltage"]) == ["230000"] + + def test_cleaner_removes_line_in_overlapping_substation_polygons(tmp_path): """Containment filtering remains index-safe when polygons overlap.""" square = _ring(4.0, 50.0, 4.1, 50.1) @@ -85,7 +114,7 @@ def test_cleaner_removes_line_in_overlapping_substation_polygons(tmp_path): "substations_way": [str(substations_path)], "lines_way": [str(lines_path)], } - buses, polygons, lines = clean_osm_data(inputs, _NETWORK, {}, _GEO_CRS) + buses, polygons, lines = clean(inputs, _NETWORK, {}, _GEO_CRS) assert set(buses["bus_id"]) == {"way/1", "way/2"} assert len(polygons) == 2 @@ -136,7 +165,7 @@ def _member(ref: int, lon0: float, lon1: float) -> dict: ) inputs = {"lines_way": [str(lines_path)], "routes_relation": [str(relations_path)]} - _, _, lines = clean_osm_data(inputs, _NETWORK, {}, _GEO_CRS) + _, _, lines = clean(inputs, _NETWORK, {}, _GEO_CRS) assert len(lines) == 1 assert lines.iloc[0]["line_id"] == "relation/99" @@ -188,7 +217,7 @@ def test_builder_merges_compatible_segments_through_virtual_bus(monkeypatch): crs="EPSG:4326", ) - station_seeds = build_osm_network.__globals__["_create_station_seeds"] + station_seeds = build_network.__globals__["_create_station_seeds"] captured = {} def capture_station_merge_radius(*args, **kwargs): @@ -196,22 +225,18 @@ def capture_station_merge_radius(*args, **kwargs): return station_seeds(*args, **kwargs) monkeypatch.setitem( - build_osm_network.__globals__, - "_create_station_seeds", - capture_station_merge_radius, + build_network.__globals__, "_create_station_seeds", capture_station_merge_radius ) - buses, built_lines, transformers, stations_polygon, buses_polygon = ( - build_osm_network( - substations, - substations_polygon, - lines, - False, - None, - _GEO_CRS, - _DISTANCE_CRS, - station_merge_radius_m=1, - ) + buses, built_lines, transformers, stations_polygon, buses_polygon = build_network( + substations, + substations_polygon, + lines, + False, + None, + _GEO_CRS, + _DISTANCE_CRS, + station_merge_radius_m=1, ) assert len(buses) == 2 diff --git a/workflow/internal/tag_corrections.yaml b/workflow/internal/tag_corrections.yaml index bdf8ecb..ec83725 100644 --- a/workflow/internal/tag_corrections.yaml +++ b/workflow/internal/tag_corrections.yaml @@ -1,5 +1,5 @@ # Module data that users cannot modify: ordered, literal string corrections -# for known-bad OSM tag values, applied by clean_osm_data.py's `_clean_*` +# for known-bad OSM tag values, applied by clean.py's `_clean_*` # functions before their general syntax cleanup. Order matters — a later # entry can depend on an earlier one having already run — and each list is # applied verbatim, so add new entries at the point in the sequence where @@ -13,6 +13,17 @@ # uppercase V, but by the time it runs the column is already lowercased, so # it can never match. Preserved as-is rather than "fixed" to keep behaviour # identical to the reference. +# +# One deliberate deviation: PyPSA-Eur applies "low"/"minor"/"medium"/"med"/ +# "m"/"high" as substring replacements (like everything else in this list), +# but they're meant to catch a tag whose *entire* value is that word (some +# OSM contributors classify voltage qualitatively instead of numerically). +# As a substring match, "m" in particular corrupts any unrelated freeform +# text containing the letter m - seen for real on a Philippines line tagged +# voltage="...Max Amperes 100 Amps...", where every stray "m" expanded into +# "33000" and produced a 30+ digit value that overflowed int64 downstream. +# `exact` (whole-value match) below fixes that without changing behaviour +# for any tag that's genuinely just "medium"/"m"/etc. voltage: - lower: true @@ -23,12 +34,12 @@ voltage: - replace: ["kvv/", ""] - replace: ["11000l400", "11000"] - replace: ["(temp 150000)", ""] - - replace: ["low", "1000"] - - replace: ["minor", "1000"] - - replace: ["medium", "33000"] - - replace: ["med", "33000"] - - replace: ["m", "33000"] - - replace: ["high", "150000"] + - exact: ["low", "1000"] + - exact: ["minor", "1000"] + - exact: ["medium", "33000"] + - exact: ["med", "33000"] + - exact: ["m", "33000"] + - exact: ["high", "150000"] - replace: ["23000-109000", "109000"] - replace: ["380000>220000", "380000;220000"] - replace: [":", ";"] From f1d42cdcd7827c7b012443f7755577d57bfe1177 Mon Sep 17 00:00:00 2001 From: bobbyxng Date: Fri, 25 Sep 2026 00:06:45 +0200 Subject: [PATCH 03/11] Add a self-contained interactive map of the built network New build_interactive_map rule renders buses/lines/transformers/stations as a standalone HTML map (pydeck), with layer toggles, voltage and text search, multi-circuit line rendering, light/dark theming, and click-through OSM links - now the workflow's default target. Also builds it in the integration test. --- pixi.lock | 19 + pixi.toml | 1 + tests/integration/Snakefile | 3 +- workflow/Snakefile | 8 +- workflow/envs/network.yaml | 1 + workflow/internal/colors.yaml | 12 + workflow/rules/plot.smk | 26 + workflow/scripts/build_interactive_map.py | 1103 +++++++++++++++++++++ 8 files changed, 1168 insertions(+), 5 deletions(-) create mode 100644 workflow/internal/colors.yaml create mode 100644 workflow/rules/plot.smk create mode 100644 workflow/scripts/build_interactive_map.py diff --git a/pixi.lock b/pixi.lock index 2d630c7..44715b2 100644 --- a/pixi.lock +++ b/pixi.lock @@ -318,6 +318,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pybind11-abi-11-hc364b38_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pycparser-2.22-pyh29332c3_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pydantic-2.13.3-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pydeck-0.9.3-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pydot-4.0.1-pyhcf101f3_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pygments-2.20.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.3.2-pyhcf101f3_0.conda @@ -464,6 +465,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pybind11-abi-11-hc364b38_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pycparser-2.22-pyh29332c3_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pydantic-2.13.3-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pydeck-0.9.3-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pydot-4.0.1-pyhcf101f3_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pygments-2.20.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.3.2-pyhcf101f3_0.conda @@ -782,6 +784,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pybind11-abi-11-hc364b38_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pycparser-2.22-pyh29332c3_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pydantic-2.13.3-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pydeck-0.9.3-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pydot-4.0.1-pyhcf101f3_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pygments-2.20.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.3.2-pyhcf101f3_0.conda @@ -5160,6 +5163,22 @@ packages: license_family: MIT size: 346352 timestamp: 1776728341165 +- conda: https://conda.anaconda.org/conda-forge/noarch/pydeck-0.9.3-pyhd8ed1ab_0.conda + sha256: de766dda257d65795a8011d6e03bfc02f6e0ca0ce7bb3d07fa5a1cd02b23ac2b + md5: 7c94c948b32d681345f134ae11892fee + depends: + - jinja2 >=2.10.1 + - numpy >=1.16.4 + - python >=3.10 + constrains: + - ipywidgets >=7,<8 + - ipykernel >=5.1.2 + - traitlets >=4.3.2 + license: Apache-2.0 + license_family: Apache + run_exports: {} + size: 8357588 + timestamp: 1783048446690 - conda: https://conda.anaconda.org/conda-forge/noarch/pydot-4.0.1-pyhcf101f3_2.conda sha256: af7213a8ca077895e7e10c8f33d5de3436b8a26828422e8a113cc59c9277a3e2 md5: 15f6d0866b0997c5302fc230a566bc72 diff --git a/pixi.toml b/pixi.toml index 5c27184..8c1aafb 100644 --- a/pixi.toml +++ b/pixi.toml @@ -32,6 +32,7 @@ osmium-tool = ">=1.19,<2" pandas = ">=2,<3" pyogrio = ">=0.10,<1" pyosmium = ">=4,<5" +pydeck = ">=0.9,<1" requests = ">=2,<3" shapely = ">=2,<3" diff --git a/tests/integration/Snakefile b/tests/integration/Snakefile index 2925d03..e1f7e10 100644 --- a/tests/integration/Snakefile +++ b/tests/integration/Snakefile @@ -18,4 +18,5 @@ use rule * from grid_builder as grid_builder_* rule all: default_target: True input: - rules.grid_builder_build_osm_network.output, + rules.grid_builder_build_network.output, + rules.grid_builder_build_interactive_map.output, diff --git a/workflow/Snakefile b/workflow/Snakefile index 97a0527..b8ed000 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -13,7 +13,7 @@ min_version("9.19") # Define pathvars to expose OSM retrieval outputs for downstream use. pathvars: # OSM retrieval outputs by country and feature - osm_retrieve="/osm/retrieve", + osm_retrieve="/retrieve", # Default configuration file generated from pydantic schema. @@ -30,11 +30,11 @@ with open(workflow.source_path("internal/settings.yaml"), "r") as f: include: "rules/retrieve.smk" -include: "rules/clean.smk" -include: "rules/build.smk" +include: "rules/network.smk" +include: "rules/plot.smk" rule all: default_target: True input: - rules.build_osm_network.output, + rules.build_interactive_map.output, diff --git a/workflow/envs/network.yaml b/workflow/envs/network.yaml index 6789e12..451eca8 100644 --- a/workflow/envs/network.yaml +++ b/workflow/envs/network.yaml @@ -10,5 +10,6 @@ dependencies: - pandas >=2,<3 - pyogrio >=0.10,<1 - pyproj >=3.7,<4 + - pydeck >=0.9,<1 - pyyaml >=6,<7 - shapely >=2,<3 diff --git a/workflow/internal/colors.yaml b/workflow/internal/colors.yaml new file mode 100644 index 0000000..c2dbd01 --- /dev/null +++ b/workflow/internal/colors.yaml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Contributors to grid-builder /build_interactive_map.log", + conda: + "../envs/network.yaml" + threads: 1 + params: + crs=config["crs"].model_dump(mode="json"), + interactive_map=config["interactive_map"].model_dump(mode="json"), + message: + "Building an interactive OSM network map." + script: + "../scripts/build_interactive_map.py" diff --git a/workflow/scripts/build_interactive_map.py b/workflow/scripts/build_interactive_map.py new file mode 100644 index 0000000..02525ba --- /dev/null +++ b/workflow/scripts/build_interactive_map.py @@ -0,0 +1,1103 @@ +# SPDX-FileCopyrightText: Contributors to grid-builder gpd.GeoDataFrame: + """Create an HTML table from the fields actually present in a layer.""" + frame = frame.copy() + columns = [column for column in frame if column != "geometry"] + + def value_text(column: str, value: Any) -> str: + if isinstance(value, (list, tuple, dict, set)): + value = ", ".join(map(str, value)) + if pd.isna(value): + return "" + value = str(value) + if column == "osm_ids": + return "
".join( + f'{html.escape(item)}' + for item in value.split(";") + ) + return html.escape(value) + + frame["tooltip_html"] = frame.apply( + lambda row: ( + "" + + "".join( + f"" + for column in columns + ) + + "
{html.escape(column)}{value_text(column, row[column])}
" + ), + axis=1, + ) + return frame + + +def _coord(point: Any, decimals: int) -> list[float]: + """Round a 2D point to ``decimals`` places, as a plain ``[lon, lat]`` pair.""" + return [round(point[0], decimals), round(point[1], decimals)] + + +def line_colors(voltages: pd.Series) -> list[list[int]]: + """Return the PyPSA-Eur voltage palette from internal configuration.""" + bands = load_internal_yaml("colors.yaml")["lines"]["voltage"] + + def rgba(color: str) -> list[int]: + color = color.removeprefix("#") + return [int(color[index : index + 2], 16) for index in range(0, 6, 2)] + [150] + + def color(voltage: float) -> str: + for voltage_range, value in bands.items(): + minimum, maximum = voltage_range.split("-", maxsplit=1) + if ( + float(minimum) + <= voltage + <= (float(maximum) if maximum != "inf" else float("inf")) + ): + return value + raise ValueError(f"No line colour configured for {voltage} kV.") + + return [rgba(color(voltage)) for voltage in voltages] + + +def path_layer( + frame: gpd.GeoDataFrame, + name: str, + color: list[int] | str, + *, + geo_crs: str, + distance_crs: str, + coord_decimals: int, + simplify_m: float | None = None, + auto_highlight: bool = True, +) -> pdk.Layer | None: + """Render lines/transformers as a PathLayer, including MultiLineString geometries.""" + if frame.empty: + return None + if simplify_m is not None: + frame = frame.copy() + frame["geometry"] = ( + frame.geometry.to_crs(distance_crs).simplify(simplify_m).to_crs(geo_crs) + ) + data = tooltip(frame) + data["path"] = data.geometry.map( + lambda line: ( + [ + [_coord(point, coord_decimals) for point in item.coords] + for item in line.geoms + ] + if line.geom_type == "MultiLineString" + else [_coord(point, coord_decimals) for point in line.coords] + ) + ) + return pdk.Layer( + "PathLayer", + data=data.drop(columns="geometry"), + get_path="path", + get_color=color, + width_min_pixels=2, + pickable=True, + auto_highlight=auto_highlight, + parameters={"depthTest": False}, + id=name, + ) + + +def polygon_layer( + frame: gpd.GeoDataFrame, + name: str, + color: list[int], + *, + geo_crs: str, + distance_crs: str, + coord_decimals: int, + simplify_m: float | None = None, + extruded: bool = False, +) -> pdk.Layer | None: + """Render station and bus polygons, including MultiPolygon geometries.""" + if frame.empty: + return None + if simplify_m is not None: + frame = frame.copy() + frame["geometry"] = ( + frame.geometry.to_crs(distance_crs).simplify(simplify_m).to_crs(geo_crs) + ) + data = tooltip(frame) + data["polygon"] = data.geometry.map( + lambda polygon: ( + [ + [_coord(point, coord_decimals) for point in item.exterior.coords] + for item in polygon.geoms + ] + if polygon.geom_type == "MultiPolygon" + else [_coord(point, coord_decimals) for point in polygon.exterior.coords] + ) + ) + extrusion_kwargs: dict[str, Any] = {} + if extruded: + extrusion_kwargs.update( + extruded=True, + wireframe=True, + get_elevation=200, + get_line_color=[255, 255, 255], + ) + return pdk.Layer( + "PolygonLayer", + data=data.drop(columns="geometry"), + get_polygon="polygon", + get_fill_color=color, + pickable=True, + auto_highlight=True, + parameters={"depthTest": False}, + id=name, + **extrusion_kwargs, + ) + + +def build_map( + buses: gpd.GeoDataFrame, + lines: gpd.GeoDataFrame, + transformers: gpd.GeoDataFrame, + stations: gpd.GeoDataFrame, + bus_polygons: gpd.GeoDataFrame, + *, + geo_crs: str, + distance_crs: str, + stations_simplify_m: float | None, + buses_polygon_simplify_m: float | None, + lines_simplify_m: float | None, + coord_decimals: int, +) -> pdk.Deck: + """Create the generic AC map without requiring DC links or converters.""" + lines = lines.copy() + if not lines.empty: + lines["color"] = line_colors(lines["voltage_kv"]) + layers = [ + layer + for layer in ( + polygon_layer( + stations, + "Stations", + [0, 80, 255, 60], + geo_crs=geo_crs, + distance_crs=distance_crs, + coord_decimals=coord_decimals, + simplify_m=stations_simplify_m, + ), + polygon_layer( + bus_polygons, + "Bus polygons", + [255, 0, 155, 50], + geo_crs=geo_crs, + distance_crs=distance_crs, + coord_decimals=coord_decimals, + simplify_m=buses_polygon_simplify_m, + extruded=True, + ), + path_layer( + lines, + "Lines", + "color", + geo_crs=geo_crs, + distance_crs=distance_crs, + coord_decimals=coord_decimals, + simplify_m=lines_simplify_m, + auto_highlight=False, + ), + path_layer( + transformers, + "Transformers", + [255, 255, 0, 180], + geo_crs=geo_crs, + distance_crs=distance_crs, + coord_decimals=coord_decimals, + ), + ) + if layer + ] + if not buses.empty: + data = tooltip(buses) + data["position"] = data.geometry.map( + lambda point: _coord((point.x, point.y), coord_decimals) + ) + layers.append( + pdk.Layer( + "ColumnLayer", + data=data.drop(columns="geometry"), + get_position="position", + get_fill_color=[255, 0, 155, 180], + radius=20, + get_elevation=10, + pickable=True, + auto_highlight=True, + parameters={"depthTest": False}, + id="Buses", + ) + ) + geometry = pd.concat([buses.geometry, lines.geometry], ignore_index=True) + center = geometry.union_all().centroid if not geometry.empty else None + return pdk.Deck( + layers=layers, + map_style="https://basemaps.cartocdn.com/gl/dark-matter-gl-style/style.json", + initial_view_state=pdk.ViewState( + longitude=center.x if center else 0, + latitude=center.y if center else 0, + zoom=5, + pitch=25, + ), + tooltip={"html": "{tooltip_html}"}, + ) + + +def inject_controls(deck: pdk.Deck) -> str: + """Inject the release-map UI, adapted to grid-builder's available layers.""" + page = deck.to_html(as_string=True) + # PyDeck 0.9 varies the indentation before createDeck's closing delimiter, + # so inject at the final script closing tag instead of matching whitespace. + script, closing_tag = page.rsplit("\n ", maxsplit=1) + page = script + "\nwindow.deck = deckInstance;\n " + closing_tag + # This is deliberately self-contained: maps are distributed as standalone HTML. + controls = r""" + + + + + +
+
+Search all fields + +
Use & for AND, | for OR, ( ) to group
+ +
+
+Filter by voltage (kV) +
+ +
+
+Show components +
+ +
+
+""" + return page.replace( + '
', controls + '
' + ).replace("pydeck", "grid-builder OSM network") + + +def compress_html(page: str) -> str: + """Strip whitespace and compact pydeck's embedded JSON to shrink the output.""" + page = re.sub(r"", "", page, flags=re.DOTALL) + + def minify_css(match: re.Match[str]) -> str: + css = re.sub(r"\s+", " ", match.group(1)) + css = re.sub(r"\s*([{};:,])\s*", r"\1", css) + return f"" + + page = re.sub(r"", minify_css, page, flags=re.DOTALL) + + def compress_json_in_script(match: re.Match[str]) -> str: + script = match.group(0) + json_match = re.search(r"const jsonInput = (\{.*?\});", script, re.DOTALL) + if json_match: + try: + compact = json.dumps( + json.loads(json_match.group(1)), separators=(",", ":") + ) + script = script.replace(json_match.group(1), compact) + except json.JSONDecodeError: + pass + return script + + # This targets only pydeck's own auto-generated script (the one that + # starts with its jsonInput blob), never our hand-written controls + # script below it, which is left readable. + page = re.sub( + r"", + compress_json_in_script, + page, + flags=re.DOTALL, + ) + + parts = re.split(r"()", page, flags=re.DOTALL) + for index, part in enumerate(parts): + if not part.startswith(")", page, flags=re.DOTALL) + parts = re.split( + r"()", page, flags=re.DOTALL | re.IGNORECASE + ) for index, part in enumerate(parts): - if not part.startswith(")", page, flags=re.DOTALL | re.IGNORECASE - ) + parts = re.split(r"()", page, flags=re.DOTALL | re.IGNORECASE) for index, part in enumerate(parts): if not part.lower().startswith("