From 1883f51a7330336258208186bd3e85fa2726130c Mon Sep 17 00:00:00 2001 From: TuxedoFish <“harryliversedge@gmail.com”> Date: Thu, 10 Sep 2026 11:46:12 +0100 Subject: [PATCH] Remove crackL2BookFast, keep the general simdjson path as the sole l2Book parser crackL2BookFast (a hand-rolled scanTo/memchr byte-scanner) showed no measurable speedup over crackL2Book in the hyperliquid-md-benchmark replay harness - fast path vs general path percentiles overlapped across repeated runs, unlike the clean, consistent effect measured for the validateJson toggle. It also duplicated parsing logic that had to be kept in sync with crackL2Book, and entirely bypassed simdjson's structural parsing (and the validateJson safety net) whenever it succeeded, on the theory that the byte pattern looked right - a separate correctness surface for no measured benefit. Removes it, WebsocketParsingUtils::scanTo/parseUint64Fast (now unused), and the fast-path-specific test comments/names. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_018Le73N2EZjGLsCqmCScAse --- src/websocket/WebsocketMessageParser.cpp | 100 ----------------------- src/websocket/WebsocketParsingUtils.cpp | 18 ---- src/websocket/WebsocketParsingUtils.h | 9 +- tests/websocket_parser_test.cpp | 12 +-- 4 files changed, 6 insertions(+), 133 deletions(-) diff --git a/src/websocket/WebsocketMessageParser.cpp b/src/websocket/WebsocketMessageParser.cpp index f23cb0b..0f8681a 100644 --- a/src/websocket/WebsocketMessageParser.cpp +++ b/src/websocket/WebsocketMessageParser.cpp @@ -1,7 +1,5 @@ #include "hyperliquid/websocket/WebsocketMessageParser.h" -#include #include -#include #include #include "../config/Logger.h" #include "WebsocketParsingUtils.h" @@ -10,94 +8,6 @@ namespace hyperliquid { - static void parseLevels(const char*& p, const char* end, - std::array& levels, - uint8_t& count, Side side) - { - while (p < end && count < L2_BOOK_MAX_LEVELS) - { - // Find next "px":" - const char* px = WebsocketParsingUtils::scanTo(p, end, "\"px\":\"", 6); - if (!px) return; - - // Check if we crossed the ],[ boundary (bid/ask split) - // by seeing if there's a ],[ between our current position and px - for (const char* scan = p; scan < px - 1; ++scan) - { - if (scan[0] == ']' && scan[1] == ',' && scan[2] == '[') - { - p = scan + 3; - return; // done with this side - } - } - - const char* pxEnd = static_cast(memchr(px, '"', end - px)); - if (!pxEnd) return; - - const char* sz = WebsocketParsingUtils::scanTo(pxEnd, end, "\"sz\":\"", 6); - if (!sz) return; - const char* szEnd = static_cast(memchr(sz, '"', end - sz)); - if (!szEnd) return; - - const char* nPos = WebsocketParsingUtils::scanTo(szEnd, end, "\"n\":", 4); - if (!nPos) return; - int n = 0; - while (nPos < end && *nPos >= '0' && *nPos <= '9') - { - n = n * 10 + (*nPos - '0'); - nPos++; - } - - levels[count].side = side; - levels[count].px = std::string_view(px, pxEnd - px); - levels[count].sz = std::string_view(sz, szEnd - sz); - levels[count].n = n; - count++; - - p = nPos; - } - } - - static bool crackL2BookFast(std::string_view msg, WebsocketMessageHandler& listener) - { - L2BookSnapshot snapshot; - snapshot.numBids = 0; - snapshot.numAsks = 0; - - const char* p = msg.data(); - const char* end = p + msg.size(); - - // Find "coin":" — extract coin name up to closing quote - p = WebsocketParsingUtils::scanTo(p, end, "\"coin\":\"", 8); - if (!p) return false; - const char* coinEnd = static_cast(memchr(p, '"', end - p)); - if (!coinEnd) return false; - snapshot.coin = std::string(p, coinEnd - p); - - // Find "time": — parse uint64 - p = WebsocketParsingUtils::scanTo(coinEnd, end, "\"time\":", 7); - if (!p) return false; - snapshot.time = WebsocketParsingUtils::parseUint64Fast(p, end); - - // Find start of levels array: "levels":[[ - p = WebsocketParsingUtils::scanTo(p, end, "[[", 2); - if (!p) return false; - - // Parse bid levels until we hit ],[ boundary - parseLevels(p, end, snapshot.bids, snapshot.numBids, Side::Bid); - - // If parseLevels exited due to count limit (not finding ],[), - // we need to skip past the ],[ boundary before parsing asks. - const char* boundary = WebsocketParsingUtils::scanTo(p, end, "],[", 3); - if (boundary) p = boundary; - - // Parse ask levels - parseLevels(p, end, snapshot.asks, snapshot.numAsks, Side::Ask); - - listener.onL2Book(snapshot); - return true; - } - struct WebsocketMessageParser::Impl { simdjson::ondemand::parser parser; @@ -127,15 +37,6 @@ namespace hyperliquid void crack(std::string_view message, WebsocketMessageHandler& listener) { - // Fast path: l2Book is the most common message type. - // Detect via cheap string scan and parse without simdjson. - if (message.size() > 30 && message.find("\"l2Book\"") != std::string_view::npos) - { - if (crackL2BookFast(message, listener)) - return; - // Fall through to simdjson if fast parse failed - } - padded = simdjson::padded_string(message.data(), message.size()); auto doc = parser.iterate(padded); @@ -164,7 +65,6 @@ namespace hyperliquid if (channel == "l2Book") { - // Fast path already handled this above — only here as fallback auto data = doc["data"].get_object().value(); crackL2Book(data, listener); } diff --git a/src/websocket/WebsocketParsingUtils.cpp b/src/websocket/WebsocketParsingUtils.cpp index e6e6fa2..98cc04e 100644 --- a/src/websocket/WebsocketParsingUtils.cpp +++ b/src/websocket/WebsocketParsingUtils.cpp @@ -2,7 +2,6 @@ #include #include -#include #include namespace hyperliquid @@ -63,21 +62,4 @@ namespace hyperliquid inflateEnd(&stream); return ret == Z_STREAM_END; } - - const char* WebsocketParsingUtils::scanTo(const char* p, const char* end, const char* pattern, size_t len) - { - const char* found = static_cast(memmem(p, end - p, pattern, len)); - return found ? found + len : nullptr; - } - - uint64_t WebsocketParsingUtils::parseUint64Fast(const char*& p, const char* end) - { - uint64_t val = 0; - while (p < end && *p >= '0' && *p <= '9') - { - val = val * 10 + (*p - '0'); - p++; - } - return val; - } } diff --git a/src/websocket/WebsocketParsingUtils.h b/src/websocket/WebsocketParsingUtils.h index cc4b7a2..515eeba 100644 --- a/src/websocket/WebsocketParsingUtils.h +++ b/src/websocket/WebsocketParsingUtils.h @@ -7,8 +7,8 @@ namespace hyperliquid { - // Low-level, stateless helpers shared by the fast-path (non-simdjson) l2Book parser and the - // fastAssetCtxs decompression path. No dependency on simdjson or any response type. + // Low-level, stateless helper for the fastAssetCtxs decompression path. No dependency on + // simdjson or any response type. class WebsocketParsingUtils { public: @@ -16,10 +16,5 @@ namespace hyperliquid // windowBits=-15 selects raw DEFLATE (RFC 1951): no zlib/gzip header or checksum. static bool inflateRawDeflate(const std::vector& compressed, std::string& out); - - // Find pattern in [p, end), return pointer past the pattern, or nullptr. - static const char* scanTo(const char* p, const char* end, const char* pattern, size_t len); - - static uint64_t parseUint64Fast(const char*& p, const char* end); }; } diff --git a/tests/websocket_parser_test.cpp b/tests/websocket_parser_test.cpp index c4ed0ea..8bd322a 100644 --- a/tests/websocket_parser_test.cpp +++ b/tests/websocket_parser_test.cpp @@ -501,10 +501,8 @@ TEST(WebsocketParser, TwapStates) EXPECT_EQ(state.timestamp, 1700000000000u); } -TEST(WebsocketParser, L2BookFastPath) +TEST(WebsocketParser, L2BookCompact) { - // Byte-for-byte matches what crackL2BookFast's raw string scan expects - // ("coin":", "time":, [[ ... ], [ ... ) - exercises the fast path. static const std::string kMsg = R"({"channel":"l2Book","data":{"coin":"BTC","time":1700000000000,"levels":[[)" R"({"px":"29800.0","sz":"1.5","n":2},{"px":"29799.0","sz":"0.5","n":1}],[)" @@ -526,12 +524,10 @@ TEST(WebsocketParser, L2BookFastPath) EXPECT_EQ(snapshot.asks[0].px, "29801.0"); } -TEST(WebsocketParser, L2BookFallbackPath) +TEST(WebsocketParser, L2BookWithWhitespace) { - // A space after "coin": breaks crackL2BookFast's exact literal scan ("coin":\"), so this - // falls through to the simdjson-based crackL2Book - the path with the dangling-reference - // bug (fixed: the inner per-side array must be bound to a named variable before the - // range-for, same as the outer per-book array already was). + // The dangling-reference bug (fixed: the inner per-side array must be bound to a named + // variable before the range-for, same as the outer per-book array already was). static const std::string kMsg = R"({ "channel": "l2Book", "data": {