From bb9073e19f5630a9069d6348d188680c67b27e57 Mon Sep 17 00:00:00 2001 From: zwy991114 <643044969@qq.com> Date: Tue, 25 Aug 2026 14:41:34 +0800 Subject: [PATCH] [feature](dictionary) Add FLAT layout for dictionary --- be/src/exec/operator/dict_sink_operator.cpp | 8 + be/src/exprs/function/flat_dictionary.cpp | 217 ++++++++++++++++++ be/src/exprs/function/flat_dictionary.h | 97 ++++++++ be/src/exprs/function/function_format.cpp | 1 + .../function/function_flat_dict_test.cpp | 196 ++++++++++++++++ .../apache/doris/dictionary/LayoutType.java | 2 +- .../nereids/parser/LogicalPlanBuilder.java | 2 +- .../expressions/functions/scalar/DictGet.java | 2 +- .../functions/scalar/DictGetMany.java | 3 +- .../commands/info/CreateDictionaryInfo.java | 8 + gensrc/thrift/DataSinks.thrift | 1 + .../test_dict_load_and_get_flat.out | 15 ++ .../test_dict_load_and_get_flat.groovy | 157 +++++++++++++ .../suites/dictionary_p0/test_ddl.groovy | 49 +++- 14 files changed, 753 insertions(+), 5 deletions(-) create mode 100644 be/src/exprs/function/flat_dictionary.cpp create mode 100644 be/src/exprs/function/flat_dictionary.h create mode 100644 be/test/exprs/function/function_flat_dict_test.cpp create mode 100644 regression-test/data/dictionary_p0/dictionary_load_and_get/test_dict_load_and_get_flat.out create mode 100644 regression-test/suites/dictionary_p0/dictionary_load_and_get/test_dict_load_and_get_flat.groovy diff --git a/be/src/exec/operator/dict_sink_operator.cpp b/be/src/exec/operator/dict_sink_operator.cpp index 8f8a5a13685bb4..a5ee45c155b0ae 100644 --- a/be/src/exec/operator/dict_sink_operator.cpp +++ b/be/src/exec/operator/dict_sink_operator.cpp @@ -23,6 +23,7 @@ #include "exprs/function/complex_hash_map_dictionary.h" #include "exprs/function/dictionary_factory.h" #include "exprs/function/dictionary_util.h" +#include "exprs/function/flat_dictionary.h" #include "exprs/function/ip_address_dictionary.h" namespace doris { @@ -87,6 +88,13 @@ Status DictSinkLocalState::load_dict(RuntimeState* state) { dict = create_complex_hash_map_dict_from_column(dict_name, key_data, value_data); break; } + case TDictLayoutType::type::FLAT: { + if (key_data.size() != 1) { + return Status::InvalidArgument("FLAT dict key size must be 1"); + } + dict = create_flat_dict_from_column(dict_name, key_data[0], value_data); + break; + } default: return Status::InvalidArgument("Unknown layout type"); } diff --git a/be/src/exprs/function/flat_dictionary.cpp b/be/src/exprs/function/flat_dictionary.cpp new file mode 100644 index 00000000000000..4e25de6b16d537 --- /dev/null +++ b/be/src/exprs/function/flat_dictionary.cpp @@ -0,0 +1,217 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exprs/function/flat_dictionary.h" + +#include + +#include "core/assert_cast.h" +#include "core/column/column.h" +#include "core/column/column_nullable.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_decimal.h" // IWYU pragma: keep +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" // IWYU pragma: keep +#include "core/data_type/primitive_type.h" +#include "core/types.h" +#include "exec/common/template_helpers.hpp" +#include "exprs/function/dictionary.h" +#include "runtime/thread_context.h" + +namespace doris { + +FlatDictionary::~FlatDictionary() { + if (_mem_tracker) { + // These buffers were allocated under the dictionary factory's tracker; switch + // back to it before freeing so the memory is credited to the tracker that owns + // it, instead of whatever query/RPC tracker happens to be active on this thread + // when the dictionary is replaced/deleted/releases its last reference. + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker); + std::vector {}.swap(_value_row_index); + std::vector {}.swap(_loaded_keys); + } +} + +size_t FlatDictionary::allocated_bytes() const { + auto vec_mem = [](const auto& vec) { + return vec.capacity() * sizeof(typename std::decay_t::value_type); + }; + return IDictionary::allocated_bytes() + vec_mem(_value_row_index) + vec_mem(_loaded_keys); +} + +// Invoke func with the concrete integer key column (ColumnInt8/16/32/64/128). +// Returns false if the primitive type is not an integer type. +template +static bool visit_int_key_column(PrimitiveType type, const IColumn* key_column, Func&& func) { + switch (type) { + case TYPE_TINYINT: + func(assert_cast(key_column)); + return true; + case TYPE_SMALLINT: + func(assert_cast(key_column)); + return true; + case TYPE_INT: + func(assert_cast(key_column)); + return true; + case TYPE_BIGINT: + func(assert_cast(key_column)); + return true; + case TYPE_LARGEINT: + func(assert_cast(key_column)); + return true; + default: + return false; + } +} + +void FlatDictionary::load_data(const ColumnPtr& key_column, const DataTypePtr& key_type, + const std::vector& values_column) { + // load value columns first (column-wise storage in the base class) + load_values(values_column); + + const auto* real_key_column = remove_nullable(key_column).get(); + const auto rows = real_key_column->size(); + + auto load_keys = [&](const auto* typed_key_column) { + for (size_t i = 0; i < rows; i++) { + auto raw_key = typed_key_column->get_element(i); + // A flat dictionary key is used directly as an array index. Negative + // keys have no valid slot; reject them as unqualified dictionary data. + if (raw_key < 0) { + throw doris::Exception( + ErrorCode::INVALID_ARGUMENT, + DICT_DATA_ERROR_TAG + + "FlatDictionary key must be non-negative, got a negative key"); + } + // Reject keys that would force an oversized array BEFORE any resize, + // so a single sparse-huge key cannot explode memory. The per-dict + // memory_limit check only happens after allocation, so it is not enough. + // NOTE: compare in 128-bit arithmetic by promoting raw_key; narrowing + // raw_key to 64 bits first would drop high bits (e.g. 2^64 -> 0) and let an + // out-of-range key slip through. + if (static_cast<__int128>(raw_key) >= static_cast<__int128>(MAX_ARRAY_SIZE)) { + throw doris::Exception( + ErrorCode::INVALID_ARGUMENT, + DICT_DATA_ERROR_TAG + "FlatDictionary key exceeds max array size {}", + MAX_ARRAY_SIZE); + } + // raw_key is now guaranteed in [0, MAX_ARRAY_SIZE), so narrowing is safe. + auto key = static_cast(raw_key); + if (key >= _loaded_keys.size()) { + _loaded_keys.resize(key + 1, false); + _value_row_index.resize(key + 1, 0); + } + // Duplicate keys map ambiguously to a single slot; reject them, mirroring + // HashMapDictionary's duplicate-key rejection. + if (_loaded_keys[key]) { + throw doris::Exception( + ErrorCode::INVALID_ARGUMENT, + DICT_DATA_ERROR_TAG + "The key has duplicate data in FlatDictionary"); + } + _loaded_keys[key] = true; + _value_row_index[key] = i; + } + }; + + if (!visit_int_key_column(key_type->get_primitive_type(), real_key_column, load_keys)) { + throw doris::Exception(ErrorCode::INVALID_ARGUMENT, + DICT_DATA_ERROR_TAG + + "FlatDictionary only support integer key , input key type " + "is {} ", + key_type->get_name()); + } +} + +ColumnPtr FlatDictionary::get_column(const std::string& attribute_name, + const DataTypePtr& attribute_type, const ColumnPtr& key_column, + const DataTypePtr& key_type) const { + if (have_nullable({attribute_type}) || have_nullable({key_type})) { + throw doris::Exception( + ErrorCode::INTERNAL_ERROR, + "FlatDictionary get_column attribute_type or key_type must not be nullable type"); + } + if (!is_int(key_type->get_primitive_type())) { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "FlatDictionary only support integer type key , input key type is " + "{} ", + key_type->get_name()); + } + + const auto rows = key_column->size(); + MutableColumnPtr res_column = attribute_type->create_column(); + ColumnUInt8::MutablePtr res_null = ColumnUInt8::create(rows, false); + auto& res_null_map = res_null->get_data(); + const auto& value_data = _values_data[attribute_index(attribute_name)]; + + // resolve each query key to a value row index, or mark it as not found + IColumn::Selector value_index = IColumn::Selector(rows); + NullMap key_not_found = NullMap(rows, false); + + const auto* real_key_column = remove_nullable(key_column).get(); + const auto* null_key = check_and_get_column(key_column.get()); + + auto resolve_keys = [&](const auto* typed_key_column) { + for (size_t i = 0; i < rows; i++) { + if (null_key != nullptr && null_key->is_null_at(i)) { + key_not_found[i] = true; + continue; + } + auto raw_key = typed_key_column->get_element(i); + // Compare in 128-bit arithmetic by promoting raw_key before narrowing; + // narrowing first would let a large key (e.g. 2^64, low bits 0) alias to a + // present slot instead of resolving to not-found. + if (raw_key < 0 || + static_cast<__int128>(raw_key) >= static_cast<__int128>(_loaded_keys.size()) || + !_loaded_keys[static_cast(raw_key)]) { + key_not_found[i] = true; + } else { + value_index[i] = + static_cast(_value_row_index[static_cast(raw_key)]); + } + } + }; + + if (!visit_int_key_column(key_type->get_primitive_type(), real_key_column, resolve_keys)) { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, "FlatDictionary unexpected key type {} ", + key_type->get_name()); + } + + std::visit( + [&](auto&& arg, auto value_is_nullable) { + using ValueDataType = std::decay_t; + using OutputColumnType = ValueDataType::OutputColumnType; + auto* res_real_column = assert_cast(res_column.get()); + const auto* value_column = arg.get(); + const auto* value_null_map = arg.get_null_map(); + for (size_t i = 0; i < rows; i++) { + if (key_not_found[i]) { + // if input key is not found, set the result column to null + res_real_column->insert_default(); + res_null_map[i] = true; + } else { + set_value_data(res_real_column, res_null_map[i], + value_column, value_null_map, + value_index[i]); + } + } + }, + value_data, attribute_nullable_variant(attribute_index(attribute_name))); + + return ColumnNullable::create(std::move(res_column), std::move(res_null)); +} + +} // namespace doris diff --git a/be/src/exprs/function/flat_dictionary.h b/be/src/exprs/function/flat_dictionary.h new file mode 100644 index 00000000000000..b861a0357f9d9a --- /dev/null +++ b/be/src/exprs/function/flat_dictionary.h @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/exception.h" +#include "common/status.h" +#include "core/block/columns_with_type_and_name.h" +#include "core/column/column.h" +#include "core/data_type/data_type.h" +#include "exprs/function/dictionary.h" + +namespace doris { + +// FlatDictionary stores a single UInt64 key used directly as an array index. +// It mirrors ClickHouse's flat dictionary layout: value attributes are stored +// column-wise (via IDictionary::_values_data), a presence bitmap distinguishes +// absent keys from present-with-default, and the key-indexed structure grows to +// key + 1. A key that is greater than or equal to max_array_size is rejected at +// load time, before any allocation, to prevent a sparse huge key from exploding +// memory (the per-dictionary memory_limit check happens only after allocation). +class FlatDictionary : public IDictionary { +public: + // ClickHouse default flat dictionary maximum array size. + static constexpr size_t MAX_ARRAY_SIZE = 500000; + + FlatDictionary(std::string name, std::vector attributes) + : IDictionary(std::move(name), std::move(attributes)) {} + + ~FlatDictionary() override; + + ColumnPtr get_column(const std::string& attribute_name, const DataTypePtr& attribute_type, + const ColumnPtr& key_column, const DataTypePtr& key_type) const override; + + static DictionaryPtr create_flat_dict(const std::string& name, const ColumnPtr& key_column, + const DataTypePtr& key_type, + const ColumnsWithTypeAndName& values_data) { + std::vector attributes; + std::vector values_column; + for (const auto& att : values_data) { + attributes.push_back({att.name, att.type}); + values_column.push_back(att.column); + } + auto dict = std::make_shared(name, attributes); + dict->load_data(key_column, key_type, values_column); + return dict; + } + + size_t allocated_bytes() const override; + +private: + void load_data(const ColumnPtr& key_column, const DataTypePtr& key_type, + const std::vector& values_column); + + // _value_row_index[key] gives the source row index of the value for that key. + // Only meaningful when _loaded_keys[key] is true. + std::vector _value_row_index; + + // _loaded_keys[key] marks whether the key was present in the source data. + std::vector _loaded_keys; +}; + +inline DictionaryPtr create_flat_dict_from_column(const std::string& name, + const ColumnWithTypeAndName& key_data, + const ColumnsWithTypeAndName& values_data) { + auto key_column = key_data.column; + auto key_type = key_data.type; + if (!is_int(key_type->get_primitive_type())) { + throw doris::Exception( + ErrorCode::INVALID_ARGUMENT, + DICT_DATA_ERROR_TAG + + "FlatDictionary only support integer key , input key type is {} ", + key_type->get_name()); + } + + DictionaryPtr dict = FlatDictionary::create_flat_dict(name, key_column, key_type, values_data); + return dict; +} +} // namespace doris diff --git a/be/src/exprs/function/function_format.cpp b/be/src/exprs/function/function_format.cpp index cc88de71c7d14e..31a45229357d74 100644 --- a/be/src/exprs/function/function_format.cpp +++ b/be/src/exprs/function/function_format.cpp @@ -26,6 +26,7 @@ #include "core/column/column.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" #include "core/data_type/define_primitive_type.h" #include "core/types.h" #include "exprs/function/cast_type_to_either.h" diff --git a/be/test/exprs/function/function_flat_dict_test.cpp b/be/test/exprs/function/function_flat_dict_test.cpp new file mode 100644 index 00000000000000..55757dde423d3a --- /dev/null +++ b/be/test/exprs/function/function_flat_dict_test.cpp @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include + +#include "common/exception.h" +#include "core/column/column_nullable.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "exprs/function/dictionary.h" +#include "exprs/function/flat_dictionary.h" + +namespace doris { + +template +static ColumnPtr flat_create_column_with_data(std::vector datas) { + auto column = DataType::ColumnType::create(); + if constexpr (std::is_same_v) { + for (auto data : datas) { + column->insert_data(data.data(), data.size()); + } + } else { + for (auto data : datas) { + column->insert_value(data); + } + } + return std::move(column); +} + +template +static ColumnWithTypeAndName flat_create_column(std::vector datas, + std::string name) { + return ColumnWithTypeAndName(flat_create_column_with_data(datas), + std::make_shared(), name); +} + +// Build a FLAT dictionary from an integer key column and value columns. +static DictionaryPtr build_flat_dict(const ColumnWithTypeAndName& key_data, + const ColumnsWithTypeAndName& values_data) { + return create_flat_dict_from_column("flat_dict", key_data, values_data); +} + +// Happy path: keys 0,1 plus a sparse key; look up present and missing keys. +TEST(FlatDictTest, HappyPathAndMissingKey) { + auto key = flat_create_column({0, 1, 100}, "key"); + auto dict = build_flat_dict( + key, ColumnsWithTypeAndName {flat_create_column({10, 11, 12}, "v")}); + + // query keys: 0 (hit), 1 (hit), 100 (hit sparse), 5 (miss) + auto query = flat_create_column({0, 1, 100, 5}, "q"); + auto result = + dict->get_column("v", std::make_shared(), query.column, query.type); + + ASSERT_EQ(result->size(), 4); + const auto* nullable = assert_cast(result.get()); + const auto* data = assert_cast(nullable->get_nested_column_ptr().get()); + + EXPECT_FALSE(nullable->is_null_at(0)); + EXPECT_EQ(data->get_element(0), 10); + EXPECT_FALSE(nullable->is_null_at(1)); + EXPECT_EQ(data->get_element(1), 11); + EXPECT_FALSE(nullable->is_null_at(2)); + EXPECT_EQ(data->get_element(2), 12); + // missing key -> null + EXPECT_TRUE(nullable->is_null_at(3)); +} + +// Key exactly at the max boundary (MAX_ARRAY_SIZE - 1) is accepted. +TEST(FlatDictTest, KeyAtMaxBoundaryAccepted) { + int64_t boundary = FlatDictionary::MAX_ARRAY_SIZE - 1; + auto key = flat_create_column({boundary}, "key"); + auto dict = build_flat_dict( + key, ColumnsWithTypeAndName {flat_create_column({777}, "v")}); + + auto query = flat_create_column({boundary}, "q"); + auto result = + dict->get_column("v", std::make_shared(), query.column, query.type); + const auto* nullable = assert_cast(result.get()); + const auto* data = assert_cast(nullable->get_nested_column_ptr().get()); + EXPECT_FALSE(nullable->is_null_at(0)); + EXPECT_EQ(data->get_element(0), 777); +} + +// Key >= MAX_ARRAY_SIZE is rejected at load time (before allocation). +TEST(FlatDictTest, KeyAboveMaxRejected) { + int64_t over = FlatDictionary::MAX_ARRAY_SIZE; + auto key = flat_create_column({over}, "key"); + EXPECT_THROW(build_flat_dict( + key, ColumnsWithTypeAndName {flat_create_column({1}, "v")}), + doris::Exception); +} + +// Negative key is rejected at load time. +TEST(FlatDictTest, NegativeKeyRejected) { + auto key = flat_create_column({-1}, "key"); + EXPECT_THROW(build_flat_dict( + key, ColumnsWithTypeAndName {flat_create_column({1}, "v")}), + doris::Exception); +} + +// Duplicate key is rejected at load time. +TEST(FlatDictTest, DuplicateKeyRejected) { + auto key = flat_create_column({3, 3}, "key"); + EXPECT_THROW(build_flat_dict(key, ColumnsWithTypeAndName {flat_create_column( + {1, 2}, "v")}), + doris::Exception); +} + +// Nullable value column: a present key whose value is null returns null. +TEST(FlatDictTest, NullableValue) { + auto key = flat_create_column({0, 1}, "key"); + // value column is nullable; row 1 is null + auto nested = DataTypeInt64::ColumnType::create(); + nested->insert_value(100); + nested->insert_value(0); + auto null_map = ColumnUInt8::create(); + null_map->insert_value(0); + null_map->insert_value(1); + auto nullable_value = ColumnNullable::create(std::move(nested), std::move(null_map)); + ColumnWithTypeAndName value_data( + std::move(nullable_value), + std::make_shared(std::make_shared()), "v"); + + auto dict = build_flat_dict(key, ColumnsWithTypeAndName {value_data}); + auto query = flat_create_column({0, 1}, "q"); + auto result = + dict->get_column("v", std::make_shared(), query.column, query.type); + const auto* nullable = assert_cast(result.get()); + const auto* data = assert_cast(nullable->get_nested_column_ptr().get()); + EXPECT_FALSE(nullable->is_null_at(0)); + EXPECT_EQ(data->get_element(0), 100); + // present key but null value -> null + EXPECT_TRUE(nullable->is_null_at(1)); +} + +// allocated_bytes should be non-zero and include the key structures. +TEST(FlatDictTest, AllocatedBytes) { + auto key = flat_create_column({0, 1, 2}, "key"); + auto dict = build_flat_dict( + key, ColumnsWithTypeAndName {flat_create_column({1, 2, 3}, "v")}); + EXPECT_GT(dict->allocated_bytes(), 0); +} + +// A LARGEINT key that exceeds MAX_ARRAY_SIZE but whose low 64 bits are small +// (e.g. 2^64, whose low bits are 0) must still be rejected at load time. The +// range check must be done in 128-bit arithmetic, not after narrowing. +TEST(FlatDictTest, LargeIntKeyOverMaxLowBitsZeroRejected) { + Int128 over = (Int128 {1} << 64); // 18446744073709551616; low 64 bits = 0 + auto key = flat_create_column({over}, "key"); + EXPECT_THROW(build_flat_dict( + key, ColumnsWithTypeAndName {flat_create_column({1}, "v")}), + doris::Exception); +} + +// Looking up a LARGEINT value whose low 64 bits collide with a present key +// (2^64 low bits = 0) must return not-found, NOT the entry for key 0. +TEST(FlatDictTest, LargeIntLookupOverMaxNotFound) { + // dictionary has key 0 present + auto key = flat_create_column({Int128 {0}}, "key"); + auto dict = build_flat_dict( + key, ColumnsWithTypeAndName {flat_create_column({42}, "v")}); + + // query key 2^64 (low 64 bits = 0) must NOT alias to key 0 + Int128 alias = (Int128 {1} << 64); + auto query = flat_create_column({Int128 {0}, alias}, "q"); + auto result = + dict->get_column("v", std::make_shared(), query.column, query.type); + const auto* nullable = assert_cast(result.get()); + const auto* data = assert_cast(nullable->get_nested_column_ptr().get()); + // key 0 -> hit + EXPECT_FALSE(nullable->is_null_at(0)); + EXPECT_EQ(data->get_element(0), 42); + // key 2^64 -> must be not found (null), not key 0's value + EXPECT_TRUE(nullable->is_null_at(1)); +} + +} // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/dictionary/LayoutType.java b/fe/fe-core/src/main/java/org/apache/doris/dictionary/LayoutType.java index aa7b560191ef1c..353e6e35e3ad1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/dictionary/LayoutType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/dictionary/LayoutType.java @@ -18,7 +18,7 @@ package org.apache.doris.dictionary; public enum LayoutType { - IP_TRIE, HASH_MAP; + IP_TRIE, HASH_MAP, FLAT; public static LayoutType of(String name) { return LayoutType.valueOf(name.toUpperCase()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 667724bb6bf326..a0a8bacdfe106a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -9723,7 +9723,7 @@ public LogicalPlan visitCreateDictionary(CreateDictionaryContext ctx) { layoutType = LayoutType.of(ctx.layoutType.getText()); } catch (IllegalArgumentException e) { throw new AnalysisException( - "Unknown layout type: " + ctx.layoutType.getText() + ". must be IP_TRIE or HASH_MAP"); + "Unknown layout type: " + ctx.layoutType.getText() + ". must be IP_TRIE, HASH_MAP or FLAT"); } return new CreateDictionaryCommand(ctx.EXISTS() != null, // if not exists diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGet.java index f374213c0483dc..dacff0b5a71f9f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGet.java @@ -106,7 +106,7 @@ public Pair customSignatureDict() { // Do type coercion manually because the function signature accept any initially. DataType queryType = getArgumentType(2); - if (dictionary.getLayout() == LayoutType.HASH_MAP) { + if (dictionary.getLayout() == LayoutType.HASH_MAP || dictionary.getLayout() == LayoutType.FLAT) { List colTypes = dictionary.getKeyColumnTypes(); if (colTypes.size() != 1) { // multi-key dict should only use dict_get_many throw new AnalysisException("dict_get() only support one key column"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetMany.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetMany.java index 51b31dd7104ac2..e5aa35338de7b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetMany.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetMany.java @@ -132,7 +132,8 @@ public Pair customSignatureDict() { DataType queryType = field.getDataType(); DataType targetType = targetTypes.get(i); - if (dictionary.getLayout() == LayoutType.HASH_MAP) { + if (dictionary.getLayout() == LayoutType.HASH_MAP + || dictionary.getLayout() == LayoutType.FLAT) { Optional castType = TypeCoercionUtils.implicitCast(queryType, targetType); if (castType.isPresent() && !castType.get().equals(queryType)) { queryType = castType.get(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateDictionaryInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateDictionaryInfo.java index 9e18e41c2a0834..43da0e7f2771b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateDictionaryInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateDictionaryInfo.java @@ -189,6 +189,9 @@ private void validateAndSetColumns(Table table) throws DdlException { if (getLayout() == LayoutType.IP_TRIE && columns.stream().filter(c -> c.isKey()).count() != 1) { throw new DdlException("IP_TRIE layout requires exactly one key column"); } + if (getLayout() == LayoutType.FLAT && columns.stream().filter(c -> c.isKey()).count() != 1) { + throw new DdlException("FLAT layout requires exactly one key column"); + } // Validate each dictionary column exists in source table and set its type for (DictionaryColumnDefinition columnDef : columns) { @@ -219,6 +222,11 @@ private void validateKeyColumn(Column source) throws DdlException { throw new DdlException("Key column " + source.getName() + " must be String type for IP_TRIE layout"); } } + if (getLayout() == LayoutType.FLAT) { + if (!source.getType().isFixedPointType()) { + throw new DdlException("Key column " + source.getName() + " must be integer type for FLAT layout"); + } + } } private void validateAndSetProperties() throws DdlException { diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift index 1c776b8306644d..9d474c3abbd1f1 100644 --- a/gensrc/thrift/DataSinks.thrift +++ b/gensrc/thrift/DataSinks.thrift @@ -560,6 +560,7 @@ struct TIcebergMergeSink { enum TDictLayoutType { HASH_MAP = 0, IP_TRIE = 1, + FLAT = 2, } struct TDictionarySink { diff --git a/regression-test/data/dictionary_p0/dictionary_load_and_get/test_dict_load_and_get_flat.out b/regression-test/data/dictionary_p0/dictionary_load_and_get/test_dict_load_and_get_flat.out new file mode 100644 index 00000000000000..07a6b7ca0df7c5 --- /dev/null +++ b/regression-test/data/dictionary_p0/dictionary_load_and_get/test_dict_load_and_get_flat.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_hit -- +ABC +abc +sparse + +-- !sql_constant -- +abc ABC sparse \N + +-- !sql_nullable_value -- +def \N + +-- !sql_get_many -- +{"str_not_null":"abc", "int_not_null":100} + diff --git a/regression-test/suites/dictionary_p0/dictionary_load_and_get/test_dict_load_and_get_flat.groovy b/regression-test/suites/dictionary_p0/dictionary_load_and_get/test_dict_load_and_get_flat.groovy new file mode 100644 index 00000000000000..12651eb8a99eec --- /dev/null +++ b/regression-test/suites/dictionary_p0/dictionary_load_and_get/test_dict_load_and_get_flat.groovy @@ -0,0 +1,157 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_dict_load_and_get_flat") { + sql "drop database if exists test_dict_load_and_get_flat_db" + sql "create database test_dict_load_and_get_flat_db" + sql "use test_dict_load_and_get_flat_db" + + // ---- duplicate key rejection ---- + sql """ + create table flat_single_key_with_duplicate( + k0 int not null, + v0 varchar not null + ) + DISTRIBUTED BY HASH(`k0`) BUCKETS auto + properties("replication_num" = "1"); + """ + + sql """insert into flat_single_key_with_duplicate values(1, 'abc');""" + sql """insert into flat_single_key_with_duplicate values(1, 'def');""" + + sql """ + create dictionary dc_flat_with_duplicate using flat_single_key_with_duplicate + ( + k0 KEY, + v0 VALUE + ) + LAYOUT(FLAT) + properties('data_lifetime'='600'); + """ + + boolean sawDuplicateError = false + for (int i = 0; i < 30; i++) { + try { + sql "refresh dictionary dc_flat_with_duplicate" + assertTrue(false, "refresh should fail on duplicate key") + } catch (Exception e) { + if (e.getMessage().contains("The key has duplicate data in FlatDictionary")) { + sawDuplicateError = true + break + } else { + logger.info("refresh dc_flat_with_duplicate failed: " + e.getMessage()) + } + } + sleep(1000) + } + assertTrue(sawDuplicateError, "refresh dc_flat_with_duplicate did not report duplicate error") + + // ---- happy path: dense keys 0/1, sparse key 100, plus a missing key ---- + sql """ + create table flat_single_key_without_duplicate( + k0 int not null, + str_not_null string not null, + str_null string null, + int_not_null int not null, + int_null int null + ) + DISTRIBUTED BY HASH(`k0`) BUCKETS auto + properties("replication_num" = "1"); + """ + + sql """insert into flat_single_key_without_duplicate values(0, 'abc', 'def', 100, 10000);""" + sql """insert into flat_single_key_without_duplicate values(1, 'ABC', null, 200, null);""" + sql """insert into flat_single_key_without_duplicate values(100, 'sparse', 'S', 300, 30000);""" + + sql """ + create dictionary dc_flat_without_duplicate using flat_single_key_without_duplicate + ( + k0 KEY, + str_not_null VALUE, + str_null VALUE, + int_not_null VALUE, + int_null VALUE + ) + LAYOUT(FLAT) + properties('data_lifetime'='600'); + """ + waitDictionaryReady("dc_flat_without_duplicate") + + // present keys (0, 1, 100) return their values; missing key (5) returns null + order_qt_sql_hit """ + select dict_get("test_dict_load_and_get_flat_db.dc_flat_without_duplicate", "str_not_null", k0) as v + from flat_single_key_without_duplicate order by k0; + """ + + // constant lookups: 0 hit, 1 hit, 100 hit sparse, 5 miss -> null + order_qt_sql_constant """ + select dict_get("test_dict_load_and_get_flat_db.dc_flat_without_duplicate", "str_not_null", 0), + dict_get("test_dict_load_and_get_flat_db.dc_flat_without_duplicate", "str_not_null", 1), + dict_get("test_dict_load_and_get_flat_db.dc_flat_without_duplicate", "str_not_null", 100), + dict_get("test_dict_load_and_get_flat_db.dc_flat_without_duplicate", "str_not_null", 5); + """ + + // nullable value column: key 1 has null str_null -> null + order_qt_sql_nullable_value """ + select dict_get("test_dict_load_and_get_flat_db.dc_flat_without_duplicate", "str_null", 0), + dict_get("test_dict_load_and_get_flat_db.dc_flat_without_duplicate", "str_null", 1); + """ + + // dict_get_many with a single-field key on a FLAT dictionary + order_qt_sql_get_many """ + select dict_get_many("test_dict_load_and_get_flat_db.dc_flat_without_duplicate", + ["str_not_null", "int_not_null"], struct(0)); + """ + + // ---- key over MAX_ARRAY_SIZE (500000) is rejected at load ---- + sql """ + create table flat_over_max( + k0 bigint not null, + v0 int not null + ) + DISTRIBUTED BY HASH(`k0`) BUCKETS auto + properties("replication_num" = "1"); + """ + sql """insert into flat_over_max values(500000, 1);""" + + sql """ + create dictionary dc_flat_over_max using flat_over_max + ( + k0 KEY, + v0 VALUE + ) + LAYOUT(FLAT) + properties('data_lifetime'='600'); + """ + + boolean sawOverMaxError = false + for (int i = 0; i < 30; i++) { + try { + sql "refresh dictionary dc_flat_over_max" + assertTrue(false, "refresh should fail on over-max key") + } catch (Exception e) { + if (e.getMessage().contains("exceeds max array size")) { + sawOverMaxError = true + break + } else { + logger.info("refresh dc_flat_over_max failed: " + e.getMessage()) + } + } + sleep(1000) + } + assertTrue(sawOverMaxError, "refresh dc_flat_over_max did not report over-max error") +} diff --git a/regression-test/suites/dictionary_p0/test_ddl.groovy b/regression-test/suites/dictionary_p0/test_ddl.groovy index af21b3b5c95e1b..e5fea0b1246bb2 100644 --- a/regression-test/suites/dictionary_p0/test_ddl.groovy +++ b/regression-test/suites/dictionary_p0/test_ddl.groovy @@ -175,7 +175,7 @@ suite("test_ddl") { )LAYOUT(xxx) properties('data_lifetime'='600'); """ - exception "Unknown layout type: xxx. must be IP_TRIE or HASH_MAP" + exception "Unknown layout type: xxx. must be IP_TRIE, HASH_MAP or FLAT" } test { // wrong type for ip_trie @@ -190,6 +190,53 @@ suite("test_ddl") { exception "Key column k0 must be String type for IP_TRIE layout" } + // FLAT layout DDL validation + sql """ + create table flat_int_table( + ik int not null, + iv varchar not null, + ik2 int not null + ) + DISTRIBUTED BY HASH(`ik`) BUCKETS auto + properties("replication_num" = "1"); + """ + + test { // FLAT requires exactly one key column + sql """ + create dictionary dic_flat_multi using flat_int_table + ( + ik KEY, + ik2 KEY, + iv VALUE + )LAYOUT(FLAT) + properties('data_lifetime'='600'); + """ + exception "FLAT layout requires exactly one key column" + } + + test { // FLAT requires integer key type (dc.k1 is varchar) + sql """ + create dictionary dic_flat_str using dc + ( + k1 KEY, + k0 VALUE + )LAYOUT(FLAT) + properties('data_lifetime'='600'); + """ + exception "must be integer type for FLAT layout" + } + + // FLAT normal creation with integer key + sql """ + create dictionary dic_flat_ok using flat_int_table + ( + ik KEY, + iv VALUE + )LAYOUT(FLAT) + properties('data_lifetime'='600'); + """ + sql "drop dictionary dic_flat_ok" + test { // no data_lifetime sql """ create dictionary dic_trie using dc