From b576359aacad6d5352f5cb470f583c580cded9d9 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sun, 18 Oct 2026 12:45:25 -0400 Subject: [PATCH] Avoid copying the type AST at each LowCardinality level CreateColumnFromAst() binds the LowCardinality child as "const auto nested = GetASTChildElement(ast, 0)", which copies the whole subtree. With nested LowCardinality(Nullable(...)) levels every level makes another copy while the recursion still holds the earlier ones, so peak memory grows with depth times subtree width: 200 levels over an 8000-element Tuple took about 220 MiB before the type was rejected. Bind the child by reference instead. --- clickhouse/columns/factory.cpp | 2 +- ut/CreateColumnByType_ut.cpp | 62 ++++++++++++++++++++++++++++++++-- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/clickhouse/columns/factory.cpp b/clickhouse/columns/factory.cpp index 2fb0d39f..0dae4ca6 100644 --- a/clickhouse/columns/factory.cpp +++ b/clickhouse/columns/factory.cpp @@ -218,7 +218,7 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti break; } case TypeAst::LowCardinality: { - const auto nested = GetASTChildElement(ast, 0); + const auto& nested = GetASTChildElement(ast, 0); if (settings.low_cardinality_as_wrapped_column) { switch (nested.code) { // TODO (nemkov): update this to maximize code reuse. diff --git a/ut/CreateColumnByType_ut.cpp b/ut/CreateColumnByType_ut.cpp index 0f1fbc4f..62c6ce28 100644 --- a/ut/CreateColumnByType_ut.cpp +++ b/ut/CreateColumnByType_ut.cpp @@ -1,14 +1,15 @@ #include -#include #include +#include +#include #include #include #include #include -#include - #include +#include +#include #include namespace { @@ -97,6 +98,61 @@ TEST(CreateColumnByType, LowCardinalityGeneralInnerTypes) { } } +#if defined(__linux__) +namespace { +long PeakRssKiB() { + std::ifstream status("/proc/self/status"); + std::string line; + while (std::getline(status, line)) { + if (line.rfind("VmHWM:", 0) == 0) { + return std::stol(line.substr(6)); + } + } + return -1; +} + +bool ResetPeakRss() { + std::ofstream clear_refs("/proc/self/clear_refs"); + clear_refs << "5"; + clear_refs.close(); + return !clear_refs.fail(); +} +} // namespace +#endif + +TEST(CreateColumnByType, NestedLowCardinalityDoesNotCopyAst) { +#if !defined(__linux__) + GTEST_SKIP() << "peak RSS is read from /proc/self/status"; +#else + // Each LowCardinality level used to copy its whole subtree, so peak memory + // grew with depth * width (about 220 MiB here) instead of width. + const size_t depth = 200; + const size_t width = 8000; + std::string type_name; + for (size_t i = 0; i < depth; ++i) { + type_name += "LowCardinality(Nullable("; + } + type_name += "Tuple(UInt8"; + for (size_t i = 1; i < width; ++i) { + type_name += ", UInt8"; + } + type_name += ")"; + type_name.append(2 * depth, ')'); + + if (!ResetPeakRss() || PeakRssKiB() < 0) { + GTEST_SKIP() << "cannot reset or read peak RSS"; + } + const long before = PeakRssKiB(); + // Whether this type is supported is not the subject; only the memory used + // while building or rejecting it is. + try { + CreateColumnByType(type_name); + } catch (const std::exception&) { + } + EXPECT_LT(PeakRssKiB() - before, 16 * 1024); +#endif +} + TEST(CreateColumnByType, LowCardinality) { // In the default (non-wrapped) mode, LowCardinality(String)/LowCardinality(FixedString) map to // the base ColumnLowCardinality (like Array/Nullable/Tuple/Map do), and the strongly-typed