From 19bd6a5114f138f218961be65fdaf08f9754cb6d Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 26 Sep 2026 13:20:11 +0200 Subject: [PATCH] fix(vag_vehicle): unzip without overflowing gopher-lua's value stack A dataset of more than a few kB failed on a box with "registry overflow". FTW runs gopher-lua, whose table.concat puts every item of the range on a value stack of about 5,000 slots, and the unzip joined its whole output in one call. It now joins at most 256 items per call. The tests run C Lua, which has no such limit, so they passed. The VAG harness now refuses table.concat ranges of 2,000 items or more, and a 360 kB fixture makes the unzip flush. Put back the old join and the test fails on a 98,346-item concat. Run in FTW's own host (NewLuaDriver, real json_decode and telemetry store, HTTP stubbed), 100 kB of JSON took 55 ms per poll and 1 MB took 0.6 s on an Apple M-series core. A poll has 10 seconds and a Raspberry Pi is several times slower, so the dataset cap drops from 4 MiB to 2 MiB. Version 0.1.1, since 0.1.0 is published. Signed-off-by: Fredrik Ahlgren Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +++ SUPPORT_STATUS.md | 4 +-- devices.yaml | 2 +- drivers/lua/vag_vehicle.lua | 33 +++++++++++++---- .../tests/lua_harness/test_vag_vehicle.lua | 35 ++++++++++++++++--- drivers/tests/test_vag_vehicle.py | 16 ++++++++- index.yaml | 6 ++-- manifests/vag_vehicle.yaml | 6 ++-- support-status.json | 2 +- 9 files changed, 86 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2001ee7..dca8b25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## vag_vehicle 0.1.1 + +A dataset of more than a few kB failed on a box with "registry overflow". FTW runs gopher-lua, whose `table.concat` puts every item of the range on a value stack of about 5,000 slots, and the unzip joined its whole output in one call. It now joins at most 256 items per call. The test harness refuses longer `table.concat` ranges too, since the C Lua the tests run has no such limit. The dataset cap drops from 4 MiB to 2 MiB: in FTW's host, 1 MB of JSON took 0.6 s to unzip and read on an Apple M-series core, a poll has 10 seconds, and a Raspberry Pi is several times slower. + ## vag_vehicle 0.1.0 Read-only VW, Audi, Škoda, SEAT and Cupra telemetry from the VW Group EU Data Act portal, the one owner door left now that We Connect's third-party APIs are closed. The owner orders a continuous 15-minute All Data delivery on the portal and pastes the Cookie header of a logged-in portal session: the Lua host has no cookie jar and cannot complete the portal's OIDC login, so the driver stops when that session ends. Every 5 minutes it lists the delivered files, downloads the newest one with content, unzips it in Lua (streamed zip entries included, 4 MiB cap) and emits `DerVehicle`: SoC, charge limit, charging state and time to full. Data keys follow VW's data dictionary V5.0. The newest file at start has an unknown age, so it is reported stale until a newer file arrives. Portal data are about 15 minutes old, so Core will mostly show this SoC rather than steer by it. Cloud access is not a charging prerequisite. Porsche, wake and charge start are out of scope. Not yet run against the portal or a car. diff --git a/SUPPORT_STATUS.md b/SUPPORT_STATUS.md index 64af16e..9cb249c 100644 --- a/SUPPORT_STATUS.md +++ b/SUPPORT_STATUS.md @@ -166,8 +166,8 @@ Catalog source is not proof that a target can install or run a driver. | teslamate_vehicle | 0.1.0 | blixt-l1 | not_assessed | — | not_recorded | not_assessed | | tibber | 1.1.2 | ftw-core | not_assessed | — | not_recorded | not_assessed | | tibber | 1.1.2 | blixt-l1 | not_assessed | — | not_recorded | not_assessed | -| vag_vehicle | 0.1.0 | ftw-core | not_assessed | — | not_recorded | not_assessed | -| vag_vehicle | 0.1.0 | blixt-l1 | not_assessed | — | not_recorded | not_assessed | +| vag_vehicle | 0.1.1 | ftw-core | not_assessed | — | not_recorded | not_assessed | +| vag_vehicle | 0.1.1 | blixt-l1 | not_assessed | — | not_recorded | not_assessed | | varta | 1.1.1 | ftw-core | not_assessed | — | not_recorded | not_assessed | | varta | 1.1.1 | blixt-l1 | not_assessed | — | not_recorded | not_assessed | | victron | 2.1.2 | ftw-core | not_assessed | — | not_recorded | not_assessed | diff --git a/devices.yaml b/devices.yaml index 6a61407..c7a171f 100644 --- a/devices.yaml +++ b/devices.yaml @@ -2051,7 +2051,7 @@ manufacturers: protocols: - protocol: http driver: "vag_vehicle" - version: "0.1.0" + version: "0.1.1" ders: [vehicle] control: false firmware_versions: "" diff --git a/drivers/lua/vag_vehicle.lua b/drivers/lua/vag_vehicle.lua index 9615ae5..9b92ed3 100644 --- a/drivers/lua/vag_vehicle.lua +++ b/drivers/lua/vag_vehicle.lua @@ -52,7 +52,7 @@ DRIVER = { id = "vag_vehicle", name = "VAG Vehicle (EU Data Act)", manufacturer = "Volkswagen Group", - version = "0.1.0", + version = "0.1.1", protocols = { "http" }, capabilities = { "vehicle" }, read_only = true, @@ -165,10 +165,31 @@ local function u32le(s, i) end -- A dataset larger than this is refused rather than inflated: the charging --- fields need far less, and every driver shares the host's memory. -local MAX_JSON_BYTES = 4194304 +-- fields need far less, every driver shares the host's memory, and a poll +-- has 10 seconds. In FTW's gopher-lua, 1 MB of JSON took 0.6 s on an Apple +-- M-series core; a Raspberry Pi is several times slower. +local MAX_JSON_BYTES = 2097152 -- DEFLATE back-references reach at most this far back. local WINDOW = 32768 +-- FTW runs gopher-lua, whose table.concat puts every item of the range on +-- a value stack of about 5,000 slots, so a long range fails with "registry +-- overflow". Join at most JOIN_STEP items per call. +local JOIN_STEP = 256 + +local function join(parts, first, last) + local level = {} + for i = first, last, JOIN_STEP do + level[#level + 1] = table.concat(parts, "", i, math.min(i + JOIN_STEP - 1, last)) + end + while #level > 1 do + local up = {} + for i = 1, #level, JOIN_STEP do + up[#up + 1] = table.concat(level, "", i, math.min(i + JOIN_STEP - 1, #level)) + end + level = up + end + return level[1] or "" +end local function inflate_raw(src) local pos = 1 @@ -180,7 +201,7 @@ local function inflate_raw(src) local function flush() local keep = n - WINDOW - chunks[#chunks + 1] = table.concat(out, "", 1, keep) + chunks[#chunks + 1] = join(out, 1, keep) flushed = flushed + keep for i = 1, WINDOW do out[i] = out[keep + i] end for i = WINDOW + 1, n do out[i] = nil end @@ -421,8 +442,8 @@ local function inflate_raw(src) return nil end if bfinal == 1 then - chunks[#chunks + 1] = table.concat(out, "", 1, n) - return table.concat(chunks) + chunks[#chunks + 1] = join(out, 1, n) + return join(chunks, 1, #chunks) end end end diff --git a/drivers/tests/lua_harness/test_vag_vehicle.lua b/drivers/tests/lua_harness/test_vag_vehicle.lua index e7387ae..85b75b5 100644 --- a/drivers/tests/lua_harness/test_vag_vehicle.lua +++ b/drivers/tests/lua_harness/test_vag_vehicle.lua @@ -1,12 +1,24 @@ -- Telemetry-only VAG / EU Data Act vehicle driver. --- Args: stored.zip deflated.zip streamed.zip oversized.zip nosoc.zip +-- Args: stored.zip deflated.zip streamed.zip oversized.zip nosoc.zip large.zip dofile("drivers/tests/lua_harness/host_mock.lua") -local stored_path, deflated_path, streamed_path, oversized_path, nosoc_path = - arg[1], arg[2], arg[3], arg[4], arg[5] -assert(stored_path and deflated_path and streamed_path and oversized_path and nosoc_path, - "usage: test_vag_vehicle.lua stored.zip deflated.zip streamed.zip oversized.zip nosoc.zip") +-- FTW runs gopher-lua, whose table.concat puts every item of the range on a +-- value stack of about 5,000 slots. C Lua has no such limit, so refuse long +-- ranges here too, or a driver that needs them passes this test and fails +-- on a box with "registry overflow". +local c_concat = table.concat +table.concat = function(t, sep, i, j) + i = i or 1 + j = j or #t + assert(j - i < 2000, "table.concat over " .. (j - i + 1) .. " items overflows gopher-lua") + return c_concat(t, sep, i, j) +end + +local stored_path, deflated_path, streamed_path, oversized_path, nosoc_path, large_path = + arg[1], arg[2], arg[3], arg[4], arg[5], arg[6] +assert(stored_path and deflated_path and streamed_path and oversized_path and nosoc_path and large_path, + "usage: test_vag_vehicle.lua stored.zip deflated.zip streamed.zip oversized.zip nosoc.zip large.zip") local function read_bin(path) local f = assert(io.open(path, "rb")) @@ -20,6 +32,7 @@ local deflated_zip = read_bin(deflated_path) local streamed_zip = read_bin(streamed_path) local oversized_zip = read_bin(oversized_path) local nosoc_zip = read_bin(nosoc_path) +local large_zip = read_bin(large_path) assert(#stored_zip > 0 and #deflated_zip > 0 and #streamed_zip > 0, "empty zip fixture") local VIN = "WVWZZZTESTVIN0001" @@ -170,6 +183,18 @@ assert(logged("dataset too large"), "oversized dataset is logged") driver_poll() assert(#host._download_log == 1, "a refused file is not downloaded again") +-- A few hundred kB of data points: the unzip flushes chunks, and every join +-- stays short enough for gopher-lua. +boot() +listing({ file(OLD, "2026-09-26T06:45:00Z") }) +host._downloads[OLD] = stored_zip +driver_poll() +listing({ file(OLD, "2026-09-26T06:45:00Z"), file(FILE, "2026-09-26T07:00:00Z") }) +host._downloads[FILE] = large_zip +driver_poll() +assert(last_row().soc == 71, "large dataset decodes, got " .. tostring(last_row().soc)) +assert(last_row().soc_fresh == true, "large dataset is a fresh reading") + -- A new file without SoC is read once and replays the last reading. boot() listing({ file(OLD, "2026-09-26T06:45:00Z") }) diff --git a/drivers/tests/test_vag_vehicle.py b/drivers/tests/test_vag_vehicle.py index f6ac5a6..9f22573 100644 --- a/drivers/tests/test_vag_vehicle.py +++ b/drivers/tests/test_vag_vehicle.py @@ -56,11 +56,24 @@ def _zip_streamed() -> bytes: def _zip_oversized() -> bytes: - # About 5 MB of JSON that deflates to a few kB: over the driver's 4 MiB cap. + # About 5 MB of JSON that deflates to a few kB: over the driver's 2 MiB cap. text = json.dumps({"vin": "WVWZZZTESTVIN0001", "Data": [], "pad": " " * 5_000_000}) return _zip(zipfile.ZIP_DEFLATED, text) +def _zip_large() -> bytes: + # About 360 kB of data points: enough that unzipping flushes chunks. + rows = [ + {"key": "00000000-0000-3000-8000-%012d" % i, "dataFieldName": "mileage", + "value": str(i), "timestampUtc": "2026-09-26T07:00:00Z"} + for i in range(3000) + ] + rows.append({"key": "162c2a75-edf4-3990-b8ed-7c600b3dbc40", + "dataFieldName": "battery_level_HV.value", "value": "71", + "timestampUtc": "2026-09-26T07:00:00Z"}) + return _zip(zipfile.ZIP_DEFLATED, json.dumps({"vin": "WVWZZZTESTVIN0001", "Data": rows})) + + def test_vag_vehicle_dataset_and_freshness(tmp_path: Path) -> None: fixtures = { "stored": _zip(zipfile.ZIP_STORED), @@ -68,6 +81,7 @@ def test_vag_vehicle_dataset_and_freshness(tmp_path: Path) -> None: "streamed": _zip_streamed(), "oversized": _zip_oversized(), "nosoc": _zip(zipfile.ZIP_DEFLATED, NO_SOC), + "large": _zip_large(), } paths = [] for name, blob in fixtures.items(): diff --git a/index.yaml b/index.yaml index 3cd21a4..16d629e 100644 --- a/index.yaml +++ b/index.yaml @@ -748,15 +748,15 @@ drivers: size_bytes: 12472 sha256: "3cf10b93755fa8b840375f53343e91b5efa98ddffe119a5679b43ee4bfb61c88" - name: "vag_vehicle" - version: "0.1.0" + version: "0.1.1" tier: community protocol: http connectivity: cloud setup: [vendor_portal] ders: [vehicle] control: false - size_bytes: 27869 - sha256: "eaa1b985db5deb897a1aa95596eeb8bedc6c0ac50cabcf14e273a9070edfe227" + size_bytes: 28621 + sha256: "907bb6f845765771a026d3b63f2080f941a6225a7fb51789898ee37e890f0856" - name: "varta" version: "1.1.1" tier: community diff --git a/manifests/vag_vehicle.yaml b/manifests/vag_vehicle.yaml index ab5d14d..7be15fa 100644 --- a/manifests/vag_vehicle.yaml +++ b/manifests/vag_vehicle.yaml @@ -1,5 +1,5 @@ name: "vag_vehicle" -version: "0.1.0" +version: "0.1.1" tier: community author: "Sourceful Labs AB" protocol: http @@ -21,9 +21,9 @@ upstream_docs: kind: other url_stability: stable min_host_version: "2.0.0" -size_bytes: 27869 +size_bytes: 28621 dkb_id: "vag_vehicle" -sha256: "eaa1b985db5deb897a1aa95596eeb8bedc6c0ac50cabcf14e273a9070edfe227" +sha256: "907bb6f845765771a026d3b63f2080f941a6225a7fb51789898ee37e890f0856" signature: "" bytecode_sha256: "" diff --git a/support-status.json b/support-status.json index a102449..0a04364 100644 --- a/support-status.json +++ b/support-status.json @@ -1703,7 +1703,7 @@ }, { "catalog_source": true, - "catalog_version": "0.1.0", + "catalog_version": "0.1.1", "driver_id": "vag_vehicle", "targets": { "blixt-l1": {