diff --git a/docs/source/user_guide/data_types.rst b/docs/source/user_guide/data_types.rst index 19de6ac2a..f9a9e3baa 100644 --- a/docs/source/user_guide/data_types.rst +++ b/docs/source/user_guide/data_types.rst @@ -160,14 +160,129 @@ and `Arrow DataTypes .default-value``). Paimon C++ + does not validate or apply default values for any type yet. + - File or global indexes on TIME columns. * - ``TIMESTAMP`` diff --git a/include/paimon/defs.h b/include/paimon/defs.h index dbb12038a..4576a8a4d 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -52,6 +52,9 @@ enum class FieldType { VARIANT = 17, /// Fixed-length dense vector represented by Arrow FixedSizeList. VECTOR = 18, + /// TIME(p) without time zone, stored as milliseconds of the day in an int32 (Arrow + /// `time32[ms]`) whatever p is, matching Paimon Java. + TIME = 19, UNKNOWN = 128, }; diff --git a/include/paimon/predicate/literal.h b/include/paimon/predicate/literal.h index 168d6a451..e35963795 100644 --- a/include/paimon/predicate/literal.h +++ b/include/paimon/predicate/literal.h @@ -31,7 +31,7 @@ namespace paimon { /// Literal represents a constant value used in predicate expressions. /// /// Literal support BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, STRING, BINARY, -/// TIMESTAMP, DECIMAL, DATE. +/// TIMESTAMP, DECIMAL, DATE, TIME. class PAIMON_EXPORT Literal { public: /// Creates a null literal of the specified type. @@ -62,10 +62,11 @@ class PAIMON_EXPORT Literal { /// if false, the caller must ensure the data remains valid. Literal(FieldType binary_type, const char* str, size_t size, bool own_data); - /// Creates a date literal from an integer value. - /// @param date_type Must be `DATE` field type. - /// @param date_value Date value as days since epoch (1970-01-01). - Literal(FieldType date_type, int32_t date_value); + /// Creates a date or time literal from an integer value. + /// @param field_type Must be `DATE` or `TIME` field type. + /// @param value For `DATE`, days since epoch (1970-01-01); for `TIME`, milliseconds of the + /// day (0 to 86399999). + Literal(FieldType field_type, int32_t value); Literal(const Literal& other); ~Literal(); diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index c89d7f452..5583e78e1 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -259,10 +259,14 @@ set(PAIMON_CORE_SRCS core/casting/string_to_date_cast_executor.cpp core/casting/string_to_decimal_cast_executor.cpp core/casting/string_to_numeric_primitive_cast_executor.cpp + core/casting/string_to_time_cast_executor.cpp core/casting/string_to_timestamp_cast_executor.cpp + core/casting/time_to_string_cast_executor.cpp + core/casting/time_to_timestamp_cast_executor.cpp core/casting/timestamp_to_date_cast_executor.cpp core/casting/timestamp_to_numeric_primitive_cast_executor.cpp core/casting/timestamp_to_string_cast_executor.cpp + core/casting/timestamp_to_time_cast_executor.cpp core/casting/timestamp_to_timestamp_cast_executor.cpp core/casting/casting_utils.cpp core/catalog/catalog.cpp diff --git a/src/paimon/common/data/binary_array_writer.cpp b/src/paimon/common/data/binary_array_writer.cpp index 2a0961f41..f445c283a 100644 --- a/src/paimon/common/data/binary_array_writer.cpp +++ b/src/paimon/common/data/binary_array_writer.cpp @@ -71,6 +71,7 @@ void BinaryArrayWriter::SetNullAt(int32_t ordinal, const arrow::Type::type& arro SetNullValue(ordinal); return; case arrow::Type::type::DATE32: + case arrow::Type::type::TIME32: case arrow::Type::type::INT32: SetNullValue(ordinal); return; @@ -124,6 +125,7 @@ int32_t BinaryArrayWriter::GetElementSize(const arrow::Type::type& arrow_type) { case arrow::Type::type::INT16: return sizeof(int16_t); case arrow::Type::type::DATE32: + case arrow::Type::type::TIME32: case arrow::Type::type::INT32: return sizeof(int32_t); case arrow::Type::type::INT64: diff --git a/src/paimon/common/data/binary_row_writer.cpp b/src/paimon/common/data/binary_row_writer.cpp index 6f0020832..103fc77a1 100644 --- a/src/paimon/common/data/binary_row_writer.cpp +++ b/src/paimon/common/data/binary_row_writer.cpp @@ -96,7 +96,8 @@ Result BinaryRowWriter::CreateFieldSetter( }; break; } - case arrow::Type::type::DATE32: { + case arrow::Type::type::DATE32: + case arrow::Type::type::TIME32: { field_setter = [field_idx](const VariantType& field, BinaryRowWriter* writer) -> void { return writer->WriteInt(field_idx, DataDefine::GetVariantValue(field)); }; diff --git a/src/paimon/common/data/binary_row_writer_test.cpp b/src/paimon/common/data/binary_row_writer_test.cpp index fc5ca42a2..5b84e7965 100644 --- a/src/paimon/common/data/binary_row_writer_test.cpp +++ b/src/paimon/common/data/binary_row_writer_test.cpp @@ -39,7 +39,7 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { TEST(BinaryRowWriterTest, TestFieldSetter) { - int32_t arity = 24; + int32_t arity = 25; BinaryRow row(arity); BinaryRowWriter writer(&row, /*initial_size=*/0, GetDefaultPool().get()); writer.Reset(); @@ -106,6 +106,9 @@ TEST(BinaryRowWriterTest, TestFieldSetter) { ASSERT_OK_AND_ASSIGN( BinaryRowWriter::FieldSetterFunc setter23, BinaryRowWriter::CreateFieldSetter(23, arrow::timestamp(arrow::TimeUnit::NANO, timezone))); + ASSERT_OK_AND_ASSIGN( + BinaryRowWriter::FieldSetterFunc setter24, + BinaryRowWriter::CreateFieldSetter(24, arrow::time32(arrow::TimeUnit::MILLI))); setter0(true, &writer); setter1(static_cast(1), &writer); @@ -135,6 +138,7 @@ TEST(BinaryRowWriterTest, TestFieldSetter) { setter21(Timestamp(1758030902002l, 0), &writer); setter22(Timestamp(1758030902002l, 2000), &writer); setter23(Timestamp(1758030902002l, 2002), &writer); + setter24(45296123, &writer); writer.Complete(); @@ -208,6 +212,9 @@ TEST(BinaryRowWriterTest, TestFieldSetter) { ASSERT_OK_AND_ASSIGN(InternalRow::FieldGetterFunc getter23, InternalRow::CreateFieldGetter(23, arrow::timestamp(arrow::TimeUnit::NANO), /*use_view=*/true)); + ASSERT_OK_AND_ASSIGN(InternalRow::FieldGetterFunc getter24, + InternalRow::CreateFieldGetter(24, arrow::time32(arrow::TimeUnit::MILLI), + /*use_view=*/true)); ASSERT_EQ(DataDefine::GetVariantValue(getter0(row)), true); ASSERT_EQ(DataDefine::GetVariantValue(getter1(row)), static_cast(1)); @@ -240,6 +247,7 @@ TEST(BinaryRowWriterTest, TestFieldSetter) { Timestamp(1758030902002l, 2000)); ASSERT_EQ(DataDefine::GetVariantValue(getter23(row)), Timestamp(1758030902002l, 2002)); + ASSERT_EQ(DataDefine::GetVariantValue(getter24(row)), 45296123); } TEST(BinaryRowWriterTest, TestFieldSetterWithNull) { diff --git a/src/paimon/common/data/columnar/columnar_array.h b/src/paimon/common/data/columnar/columnar_array.h index 773e46f53..e4a928d8b 100644 --- a/src/paimon/common/data/columnar/columnar_array.h +++ b/src/paimon/common/data/columnar/columnar_array.h @@ -86,7 +86,7 @@ class ColumnarArray : public InternalArray { } int32_t GetInt(int32_t pos) const override { - return ColumnarUtils::GetGenericValue(array_, offset_ + pos); + return ColumnarUtils::GetInt32Value(array_, offset_ + pos); } int32_t GetDate(int32_t pos) const override { diff --git a/src/paimon/common/data/columnar/columnar_batch_context.h b/src/paimon/common/data/columnar/columnar_batch_context.h index adbe54160..e0b052d55 100644 --- a/src/paimon/common/data/columnar/columnar_batch_context.h +++ b/src/paimon/common/data/columnar/columnar_batch_context.h @@ -50,7 +50,8 @@ struct CachedColumnMeta { !arrow::bit_util::GetBit(null_bitmap, array_offset + row_id); } - /// Fast fixed-width value access (INT8/INT16/INT32/INT64/FLOAT/DOUBLE/DATE32/TIMESTAMP). + /// Fast fixed-width value access + /// (INT8/INT16/INT32/INT64/FLOAT/DOUBLE/DATE32/TIME32/TIMESTAMP). template inline T GetFixed(int64_t row_id) const { return reinterpret_cast(values_data)[array_offset + row_id]; @@ -103,6 +104,7 @@ struct ColumnarBatchContext { case arrow::Type::INT16: case arrow::Type::INT32: case arrow::Type::DATE32: + case arrow::Type::TIME32: case arrow::Type::INT64: case arrow::Type::FLOAT: case arrow::Type::DOUBLE: { diff --git a/src/paimon/common/data/columnar/columnar_row.h b/src/paimon/common/data/columnar/columnar_row.h index f3d20dba0..5a0537737 100644 --- a/src/paimon/common/data/columnar/columnar_row.h +++ b/src/paimon/common/data/columnar/columnar_row.h @@ -104,7 +104,7 @@ class ColumnarRow : public InternalRow { } int32_t GetInt(int32_t pos) const override { - return ColumnarUtils::GetGenericValue(array_vec_[pos], row_id_); + return ColumnarUtils::GetInt32Value(array_vec_[pos], row_id_); } int32_t GetDate(int32_t pos) const override { diff --git a/src/paimon/common/data/columnar/columnar_row_test.cpp b/src/paimon/common/data/columnar/columnar_row_test.cpp index d1a5c6003..228b5d8b9 100644 --- a/src/paimon/common/data/columnar/columnar_row_test.cpp +++ b/src/paimon/common/data/columnar/columnar_row_test.cpp @@ -85,19 +85,24 @@ TEST(ColumnarRowTest, TestSimple) { TEST(ColumnarRowRefTest, TestSimple) { auto pool = GetDefaultPool(); std::shared_ptr target_type = - arrow::struct_({arrow::field("f1", arrow::int32()), arrow::field("f2", arrow::utf8())}); + arrow::struct_({arrow::field("f1", arrow::int32()), arrow::field("f2", arrow::utf8()), + arrow::field("f3", arrow::time32(arrow::TimeUnit::MILLI))}); auto f1 = arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([1, 2, 3])").ValueOrDie(); auto f2 = arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"(["alpha", "beta", "gamma"])") .ValueOrDie(); - auto data = arrow::StructArray::Make({f1, f2}, target_type->fields()).ValueOrDie(); + auto f3 = arrow::ipc::internal::json::ArrayFromJSON(arrow::time32(arrow::TimeUnit::MILLI), + R"([0, 45296123, 86399999])") + .ValueOrDie(); + auto data = arrow::StructArray::Make({f1, f2, f3}, target_type->fields()).ValueOrDie(); auto ctx = std::make_shared(data->fields(), pool); ColumnarRowRef row(ctx, 1); - ASSERT_EQ(row.GetFieldCount(), 2); + ASSERT_EQ(row.GetFieldCount(), 3); ASSERT_EQ(row.GetInt(0), 2); ASSERT_EQ(std::string(row.GetStringView(1)), "beta"); + ASSERT_EQ(row.GetInt(2), 45296123); auto row_kind = row.GetRowKind(); ASSERT_TRUE(row_kind.ok()); diff --git a/src/paimon/common/data/columnar/columnar_utils.h b/src/paimon/common/data/columnar/columnar_utils.h index 2ab413e32..9acb2c8f2 100644 --- a/src/paimon/common/data/columnar/columnar_utils.h +++ b/src/paimon/common/data/columnar/columnar_utils.h @@ -47,6 +47,14 @@ class ColumnarUtils { return typed_array->Value(pos); } + /// Reads an int32 value from an INT32 or a TIME32 array. + static int32_t GetInt32Value(const arrow::Array* array, int32_t pos) { + if (array->type_id() == arrow::Type::TIME32) { + return GetGenericValue(array, pos); + } + return GetGenericValue(array, pos); + } + static std::string_view GetView(const arrow::Array* array, int32_t pos) { auto type_id = array->type_id(); bool is_dict = (type_id == arrow::Type::type::DICTIONARY); diff --git a/src/paimon/common/data/data_define.h b/src/paimon/common/data/data_define.h index 376c0de5d..8328f4e07 100644 --- a/src/paimon/common/data/data_define.h +++ b/src/paimon/common/data/data_define.h @@ -150,6 +150,8 @@ class DataDefine { return Literal(GetVariantValue(value)); case arrow::Type::type::DATE32: return Literal(FieldType::DATE, GetVariantValue(value)); + case arrow::Type::type::TIME32: + return Literal(FieldType::TIME, GetVariantValue(value)); default: return Status::Invalid( fmt::format("Not support arrow type {} in VariantValueToLiteral", diff --git a/src/paimon/common/data/data_define_test.cpp b/src/paimon/common/data/data_define_test.cpp index 04d1e6f92..7f17dc950 100644 --- a/src/paimon/common/data/data_define_test.cpp +++ b/src/paimon/common/data/data_define_test.cpp @@ -236,6 +236,13 @@ TEST(DataDefineTest, VariantValueToLiteral) { DataDefine::VariantValueToLiteral(value, arrow::Type::type::INT32)); ASSERT_EQ(literal.GetValue(), 100000); } + { + // TIME32 + VariantType value = static_cast(45296123); + ASSERT_OK_AND_ASSIGN(auto literal, + DataDefine::VariantValueToLiteral(value, arrow::Type::type::TIME32)); + ASSERT_EQ(literal, Literal(FieldType::TIME, 45296123)); + } { // INT64 VariantType value = static_cast(123456789L); diff --git a/src/paimon/common/data/internal_row.cpp b/src/paimon/common/data/internal_row.cpp index b7c011b2e..1d1b1a9f7 100644 --- a/src/paimon/common/data/internal_row.cpp +++ b/src/paimon/common/data/internal_row.cpp @@ -53,7 +53,8 @@ Result InternalRow::CreateFieldGetter( }; break; } - case arrow::Type::type::INT32: { + case arrow::Type::type::INT32: + case arrow::Type::type::TIME32: { field_getter = [field_idx](const InternalRow& row) -> VariantType { return row.GetInt(field_idx); }; diff --git a/src/paimon/common/data/serializer/binary_serializer_utils.cpp b/src/paimon/common/data/serializer/binary_serializer_utils.cpp index d72a2b0b1..753389dae 100644 --- a/src/paimon/common/data/serializer/binary_serializer_utils.cpp +++ b/src/paimon/common/data/serializer/binary_serializer_utils.cpp @@ -127,7 +127,8 @@ Status BinarySerializerUtils::WriteBinaryData(const std::shared_ptrWriteInt(pos, getter->GetDate(pos)); break; } - case arrow::Type::type::INT32: { + case arrow::Type::type::INT32: + case arrow::Type::type::TIME32: { writer->WriteInt(pos, getter->GetInt(pos)); break; } diff --git a/src/paimon/common/data/serializer/binary_serializer_utils_test.cpp b/src/paimon/common/data/serializer/binary_serializer_utils_test.cpp index f557d4698..6429ed8ba 100644 --- a/src/paimon/common/data/serializer/binary_serializer_utils_test.cpp +++ b/src/paimon/common/data/serializer/binary_serializer_utils_test.cpp @@ -38,11 +38,13 @@ TEST(BinarySerializerUtilsTest, TestSimple) { arrow::field("f7", arrow::float64()), arrow::field("f8", arrow::utf8()), arrow::field("f9", arrow::binary()), arrow::field("f10", arrow::date32()), arrow::field("f11", arrow::timestamp(arrow::TimeUnit::NANO)), - arrow::field("f12", arrow::decimal128(5, 2))}); + arrow::field("f12", arrow::decimal128(5, 2)), + arrow::field("f13", arrow::time32(arrow::TimeUnit::MILLI)), + arrow::field("f14", arrow::list(arrow::time32(arrow::TimeUnit::MILLI)))}); std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(arrow_type, R"([ - [true, 0, 32767, 2147483647, null, 4294967295, 0.5, 1.141, "20250327", "banana", 2026, 5000001, "5.12"] + [true, 0, 32767, 2147483647, null, 4294967295, 0.5, 1.141, "20250327", "banana", 2026, 5000001, "5.12", 45296123, [0, null, 86399999]] ])") .ValueOrDie(); auto struct_array = std::dynamic_pointer_cast(array); @@ -54,7 +56,7 @@ TEST(BinarySerializerUtilsTest, TestSimple) { columnar_row, arrow_type, pool.get())); // check result - ASSERT_EQ(binary_row->GetFieldCount(), 13); + ASSERT_EQ(binary_row->GetFieldCount(), 15); ASSERT_FALSE(binary_row->IsNullAt(0)); ASSERT_EQ(binary_row->GetBoolean(0), true); @@ -73,6 +75,12 @@ TEST(BinarySerializerUtilsTest, TestSimple) { Timestamp(/*millisecond=*/5, /*nano_of_millisecond*/ 1)); ASSERT_EQ(binary_row->GetDecimal(12, 5, 2), Decimal(5, 2, DecimalUtils::StrToInt128("512").value())); + ASSERT_EQ(binary_row->GetInt(13), 45296123); + auto times = binary_row->GetArray(14); + ASSERT_EQ(times->Size(), 3); + ASSERT_EQ(times->GetInt(0), 0); + ASSERT_TRUE(times->IsNullAt(1)); + ASSERT_EQ(times->GetInt(2), 86399999); } TEST(BinarySerializerUtilsTest, TestNestedType) { diff --git a/src/paimon/common/data/serializer/row_compacted_serializer.cpp b/src/paimon/common/data/serializer/row_compacted_serializer.cpp index 1fd618a39..65977a06e 100644 --- a/src/paimon/common/data/serializer/row_compacted_serializer.cpp +++ b/src/paimon/common/data/serializer/row_compacted_serializer.cpp @@ -66,7 +66,8 @@ Result RowCompactedSerializer::CompareField(const FieldInfo& field_info return val1 == val2 ? 0 : (val1 < val2 ? -1 : 1); } case arrow::Type::type::INT32: - case arrow::Type::type::DATE32: { + case arrow::Type::type::DATE32: + case arrow::Type::type::TIME32: { auto val1 = reader1->ReadValue(); auto val2 = reader2->ReadValue(); return val1 == val2 ? 0 : (val1 < val2 ? -1 : 1); @@ -227,7 +228,8 @@ Result RowCompactedSerializer::CreateFieldR break; } case arrow::Type::type::INT32: - case arrow::Type::type::DATE32: { + case arrow::Type::type::DATE32: + case arrow::Type::type::TIME32: { field_reader = [](int32_t pos, RowReader* reader) -> Result { return VariantType(reader->ReadValue()); }; @@ -351,7 +353,8 @@ Result RowCompactedSerializer::CreateFieldW break; } case arrow::Type::type::INT32: - case arrow::Type::type::DATE32: { + case arrow::Type::type::DATE32: + case arrow::Type::type::TIME32: { field_writer = [](int32_t pos, const VariantType& field, RowWriter* writer) -> Status { return writer->WriteValue(DataDefine::GetVariantValue(field)); }; diff --git a/src/paimon/common/data/serializer/row_compacted_serializer_test.cpp b/src/paimon/common/data/serializer/row_compacted_serializer_test.cpp index cf11108df..089d10eab 100644 --- a/src/paimon/common/data/serializer/row_compacted_serializer_test.cpp +++ b/src/paimon/common/data/serializer/row_compacted_serializer_test.cpp @@ -513,13 +513,14 @@ TEST(RowCompactedSerializerTest, TestListType) { auto pool = GetDefaultPool(); // prepare data auto inner_child1 = arrow::field("inner1", arrow::list(arrow::int32())); - auto arrow_type = arrow::struct_({inner_child1}); + auto inner_child2 = arrow::field("inner2", arrow::list(arrow::time32(arrow::TimeUnit::MILLI))); + auto arrow_type = arrow::struct_({inner_child1, inner_child2}); // each inner child per row std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(arrow_type, R"([ -[[5, 6, 7]], -[[1, 2, 3]], -[[4]] +[[5, 6, 7], [0, null, 86399999]], +[[1, 2, 3], [123]], +[[4], []] ])") .ValueOrDie(); auto struct_array = std::dynamic_pointer_cast(array); @@ -535,10 +536,17 @@ TEST(RowCompactedSerializerTest, TestListType) { ASSERT_OK_AND_ASSIGN(auto row, serializer->Deserialize(bytes)); // check result - ASSERT_EQ(row->GetFieldCount(), 1); + ASSERT_EQ(row->GetFieldCount(), 2); // for inner_child1 ASSERT_EQ(row->GetArray(0)->ToIntArray().value(), std::vector({5, 6, 7})); + + // for inner_child2 + auto times = row->GetArray(1); + ASSERT_EQ(times->Size(), 3); + ASSERT_EQ(times->GetInt(0), 0); + ASSERT_TRUE(times->IsNullAt(1)); + ASSERT_EQ(times->GetInt(2), 86399999); } TEST(RowCompactedSerializerTest, TestSliceComparator) { @@ -816,6 +824,21 @@ TEST(RowCompactedSerializerTest, TestSliceComparator) { .ValueOrDie(); check_result(type, array); } + { + auto type = arrow::struct_({arrow::field("f15", arrow::time32(arrow::TimeUnit::MILLI))}); + std::shared_ptr array = + arrow::ipc::internal::json::ArrayFromJSON(type, R"([[123], [86399999]])").ValueOrDie(); + check_result(type, array); + + auto struct_array = std::dynamic_pointer_cast(array); + ASSERT_OK_AND_ASSIGN(auto serializer, + RowCompactedSerializer::Create(arrow::schema(type->fields()), pool)); + ColumnarRow columnar_row(/*struct_array=*/nullptr, struct_array->fields(), pool, + /*row_id=*/1); + ASSERT_OK_AND_ASSIGN(auto bytes, serializer->SerializeToBytes(columnar_row)); + ASSERT_OK_AND_ASSIGN(auto row, serializer->Deserialize(bytes)); + ASSERT_EQ(row->GetInt(0), 86399999); + } } } // namespace paimon::test diff --git a/src/paimon/common/data/variant/variant_get.cpp b/src/paimon/common/data/variant/variant_get.cpp index e7b0a564f..76549fd95 100644 --- a/src/paimon/common/data/variant/variant_get.cpp +++ b/src/paimon/common/data/variant/variant_get.cpp @@ -29,6 +29,7 @@ #include "paimon/common/data/variant/variant_json_utils.h" #include "paimon/common/data/variant/variant_path_segment.h" #include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" @@ -41,6 +42,22 @@ namespace paimon { namespace { +/// Rejects a TIME target, also nested, whose unit is not milliseconds: the cast results are +/// milliseconds of the day, which a builder of another unit would misread. +Status ValidateTargetType(const arrow::DataType& target_type) { + if (target_type.id() == arrow::Type::type::TIME32) { + if (checked_cast(target_type).unit() != arrow::TimeUnit::MILLI) { + return Status::Invalid( + fmt::format("Only millisecond TIME is supported: {}", target_type.ToString())); + } + return Status::OK(); + } + for (const std::shared_ptr& child : target_type.fields()) { + PAIMON_RETURN_NOT_OK(ValidateTargetType(*child->type())); + } + return Status::OK(); +} + Result> InvalidCast(const std::shared_ptr& variant, const std::shared_ptr& target_type, const VariantCastArgs& cast_args) { @@ -184,13 +201,14 @@ Result> CastVariant(const std::shared_ptr return input; } - PAIMON_ASSIGN_OR_RAISE(FieldType input_field_type, - FieldTypeUtils::ConvertToFieldType(input_type->id())); PAIMON_ASSIGN_OR_RAISE(FieldType target_field_type, FieldTypeUtils::ConvertToFieldType(target_type->id())); - std::shared_ptr executor = - CastExecutorFactory::GetCastExecutorFactory()->GetCastExecutor(input_field_type, - target_field_type); + // The executor depends on the whole input type, e.g. a TIMESTAMP_LTZ casts to TIME in the + // local time zone. + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr executor, + CastExecutorFactory::GetCastExecutorFactory()->GetCastExecutor( + DataField(/*id=*/0, arrow::field("input", input_type)), target_field_type)); if (executor == nullptr) { return InvalidCast(variant, target_type, cast_args); } @@ -235,6 +253,9 @@ Status AppendLiteralToBuilder(const Literal& literal, case arrow::Type::type::DATE32: return ToPaimonStatus( checked_cast(builder)->Append(literal.GetValue())); + case arrow::Type::type::TIME32: + return ToPaimonStatus( + checked_cast(builder)->Append(literal.GetValue())); case arrow::Type::type::TIMESTAMP: { auto timestamp = literal.GetValue(); const auto& timestamp_type = static_cast(*target_type); @@ -268,13 +289,10 @@ Status AppendLiteralToBuilder(const Literal& literal, } } -} // namespace - -Status VariantGetExecutor::CastToBuilder(const std::shared_ptr& variant, - const std::shared_ptr& target_field, - const VariantCastArgs& cast_args, - const std::shared_ptr& pool, - arrow::ArrayBuilder* builder) { +Status CastToBuilderImpl(const std::shared_ptr& variant, + const std::shared_ptr& target_field, + const VariantCastArgs& cast_args, const std::shared_ptr& pool, + arrow::ArrayBuilder* builder) { if (variant == nullptr) { return ToPaimonStatus(builder->AppendNull()); } @@ -318,8 +336,8 @@ Status VariantGetExecutor::CastToBuilder(const std::shared_ptr& const std::shared_ptr& child_field = struct_type.field(i); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr child, variant->GetFieldByKey(child_field->name())); - PAIMON_RETURN_NOT_OK(CastToBuilder(child, child_field, cast_args, pool, - struct_builder->field_builder(i))); + PAIMON_RETURN_NOT_OK(CastToBuilderImpl(child, child_field, cast_args, pool, + struct_builder->field_builder(i))); } return Status::OK(); } @@ -341,8 +359,9 @@ Status VariantGetExecutor::CastToBuilder(const std::shared_ptr& PAIMON_RETURN_NOT_OK_FROM_ARROW( checked_cast(map_builder->key_builder()) ->Append(field->key)); - PAIMON_RETURN_NOT_OK(CastToBuilder(field->value, map_type.item_field(), cast_args, - pool, map_builder->item_builder())); + PAIMON_RETURN_NOT_OK(CastToBuilderImpl(field->value, map_type.item_field(), + cast_args, pool, + map_builder->item_builder())); } return Status::OK(); } @@ -357,8 +376,8 @@ Status VariantGetExecutor::CastToBuilder(const std::shared_ptr& for (int32_t i = 0; i < array_size; ++i) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr element, variant->GetElementAtIndex(i)); - PAIMON_RETURN_NOT_OK(CastToBuilder(element, list_type.value_field(), cast_args, - pool, list_builder->value_builder())); + PAIMON_RETURN_NOT_OK(CastToBuilderImpl(element, list_type.value_field(), cast_args, + pool, list_builder->value_builder())); } return Status::OK(); } @@ -373,6 +392,17 @@ Status VariantGetExecutor::CastToBuilder(const std::shared_ptr& } } +} // namespace + +Status VariantGetExecutor::CastToBuilder(const std::shared_ptr& variant, + const std::shared_ptr& target_field, + const VariantCastArgs& cast_args, + const std::shared_ptr& pool, + arrow::ArrayBuilder* builder) { + PAIMON_RETURN_NOT_OK(ValidateTargetType(*target_field->type())); + return CastToBuilderImpl(variant, target_field, cast_args, pool, builder); +} + Result> VariantGetExecutor::GetAsArrow( const std::shared_ptr& variant, const std::string& path, const std::shared_ptr& target_field, const VariantCastArgs& cast_args, @@ -413,6 +443,7 @@ Result> VariantGetExecutor::ExtractByPath( Result> VariantGetExecutor::Get( const std::shared_ptr& variant, const std::string& path, const std::shared_ptr& target_type, const VariantCastArgs& cast_args) { + PAIMON_RETURN_NOT_OK(ValidateTargetType(*target_type)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr extracted, ExtractByPath(variant, path)); if (extracted == nullptr) { return std::optional(std::nullopt); diff --git a/src/paimon/common/data/variant/variant_get_test.cpp b/src/paimon/common/data/variant/variant_get_test.cpp index 7d34d68ea..1e1e218d1 100644 --- a/src/paimon/common/data/variant/variant_get_test.cpp +++ b/src/paimon/common/data/variant/variant_get_test.cpp @@ -21,6 +21,8 @@ #include #include +#include +#include #include "arrow/api.h" #include "gtest/gtest.h" @@ -31,6 +33,7 @@ #include "paimon/data/decimal.h" #include "paimon/data/timestamp.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/testing/utils/timezone_guard.h" namespace paimon::test { @@ -456,6 +459,12 @@ TEST_F(VariantGetTest, ScalarArrowBuilders) { std::shared_ptr date = build_array(d, "$", arrow::date32()); ASSERT_EQ(static_cast(*date).Value(0), 19000); + std::shared_ptr time_string = + BuildScalar([](VariantBuilder& b) { return b.AppendString("12:34:56.123"); }); + std::shared_ptr time = + build_array(time_string, "$", arrow::time32(arrow::TimeUnit::MILLI)); + ASSERT_EQ(static_cast(*time).Value(0), 45296123); + // Only the microsecond unit matches the variant's internal timestamp representation; other // units require a timestamp->timestamp literal cast that is intentionally unsupported. std::shared_ptr ts = @@ -465,6 +474,53 @@ TEST_F(VariantGetTest, ScalarArrowBuilders) { ASSERT_EQ(static_cast(*timestamp).Value(0), 1700000000123456); } +TEST_F(VariantGetTest, NonMillisecondTimeTargetIsRejected) { + std::shared_ptr time_string = + BuildScalar([](VariantBuilder& b) { return b.AppendString("00:00:01"); }); + auto time_s = arrow::time32(arrow::TimeUnit::SECOND); + ASSERT_NOK_WITH_MSG(VariantGetExecutor::Get(time_string, "$", time_s, cast_args_), + "Only millisecond TIME is supported: time32[s]"); + ASSERT_NOK_WITH_MSG(VariantGetExecutor::GetAsArrow(time_string, "$", arrow::field("t", time_s), + cast_args_, pool_, arrow_pool_), + "Only millisecond TIME is supported: time32[s]"); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr object, + GenericVariant::FromJson(R"({"t": "00:00:01"})", pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + GenericVariant::FromJson(R"(["00:00:01"])", pool_)); + std::vector, std::shared_ptr>> + nested_cases = {{object, arrow::struct_({arrow::field("t", time_s)})}, + {array, arrow::list(time_s)}, + {object, arrow::map(arrow::utf8(), time_s)}}; + for (const auto& [variant, type] : nested_cases) { + ASSERT_NOK_WITH_MSG(VariantGetExecutor::GetAsArrow(variant, "$", arrow::field("x", type), + cast_args_, pool_, arrow_pool_), + "Only millisecond TIME is supported: time32[s]"); + } +} + +TEST_F(VariantGetTest, TimestampToTime) { + auto time_type = arrow::time32(arrow::TimeUnit::MILLI); + std::shared_ptr ntz = + BuildScalar([](VariantBuilder& b) { return b.AppendTimestampNtz(1686227696123456); }); + ASSERT_OK_AND_ASSIGN(std::optional ntz_time, + VariantGetExecutor::Get(ntz, "$", time_type, cast_args_)); + ASSERT_TRUE(ntz_time.has_value()); + ASSERT_EQ(*ntz_time, Literal(FieldType::TIME, 45296123)); + + TimezoneGuard tz_guard("Asia/Shanghai"); + std::shared_ptr ltz = + BuildScalar([](VariantBuilder& b) { return b.AppendTimestamp(0); }); + ASSERT_OK_AND_ASSIGN(std::optional ltz_time, + VariantGetExecutor::Get(ltz, "$", time_type, cast_args_)); + ASSERT_TRUE(ltz_time.has_value()); + ASSERT_EQ(*ltz_time, Literal(FieldType::TIME, 28800000)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr ltz_array, + VariantGetExecutor::GetAsArrow(ltz, "$", arrow::field("t", time_type), + cast_args_, pool_, arrow_pool_)); + ASSERT_EQ(static_cast(*ltz_array).Value(0), 28800000); +} + TEST_F(VariantGetTest, ListFromNonArrayIsNull) { auto target = arrow::field("l", arrow::list(arrow::int64())); // An object cannot cast to a list: SQL NULL when fail_on_error is false ... diff --git a/src/paimon/common/predicate/literal.cpp b/src/paimon/common/predicate/literal.cpp index d679c2ccf..a6d0b38e8 100644 --- a/src/paimon/common/predicate/literal.cpp +++ b/src/paimon/common/predicate/literal.cpp @@ -77,6 +77,7 @@ class Literal::Impl { std::hash{}(value_.DecimalVal.LowBits()) * 17 + std::hash{}(value_.DecimalVal.Scale()); case FieldType::DATE: + case FieldType::TIME: return std::hash{}(value_.IntVal); default: return 0; @@ -168,10 +169,10 @@ Literal::Literal(FieldType binary_type, const char* str, size_t size, bool own_d } } -Literal::Literal(FieldType date_type, int32_t date_value) : impl_(std::make_unique()) { - impl_->type_ = date_type; +Literal::Literal(FieldType field_type, int32_t value) : impl_(std::make_unique()) { + impl_->type_ = field_type; impl_->is_null_ = false; - impl_->value_.IntVal = date_value; + impl_->value_.IntVal = value; impl_->hash_code_ = impl_->CalculateHashCode(); } @@ -289,6 +290,7 @@ std::string Literal::ToString() const { sstream << impl_->value_.DecimalVal.ToString(); break; case FieldType::DATE: + case FieldType::TIME: sstream << impl_->value_.IntVal; break; default: @@ -355,6 +357,7 @@ Result Literal::CompareTo(const Literal& other) const { case FieldType::DECIMAL: return impl_->value_.DecimalVal.CompareTo(other.impl_->value_.DecimalVal); case FieldType::DATE: + case FieldType::TIME: return impl_->value_.IntVal == other.impl_->value_.IntVal ? 0 : ((impl_->value_.IntVal < other.impl_->value_.IntVal) ? -1 : 1); diff --git a/src/paimon/common/predicate/literal_converter.cpp b/src/paimon/common/predicate/literal_converter.cpp index 4604bddca..0b5ddcb4e 100644 --- a/src/paimon/common/predicate/literal_converter.cpp +++ b/src/paimon/common/predicate/literal_converter.cpp @@ -19,6 +19,7 @@ #include "paimon/common/predicate/literal_converter.h" #include +#include #include "arrow/array/array_base.h" #include "arrow/array/array_binary.h" @@ -85,6 +86,10 @@ Result LiteralConverter::ConvertLiteralsFromString(const FieldType& typ PAIMON_ASSIGN_OR_RAISE(int32_t value, StringUtils::StringToDate(value_str)); return Literal(FieldType::DATE, value); } + case FieldType::TIME: { + PAIMON_ASSIGN_OR_RAISE(int32_t value, StringUtils::StringToTime(value_str)); + return Literal(FieldType::TIME, value); + } case FieldType::BIGINT: { auto value = StringUtils::StringToValue(value_str); if (value == std::nullopt) { @@ -172,6 +177,8 @@ Result LiteralConverter::ConvertLiteralsFromRow( } case FieldType::DATE: return Literal(FieldType::DATE, row.GetInt(field_idx)); + case FieldType::TIME: + return Literal(FieldType::TIME, row.GetInt(field_idx)); case FieldType::ARRAY: case FieldType::VECTOR: case FieldType::MAP: @@ -210,6 +217,8 @@ Result> LiteralConverter::ConvertLiteralsFromArray(const ar return GetLiteralFromDecimalArray(array); case arrow::Type::type::DATE32: return GetLiteralFromDateArray(array); + case arrow::Type::type::TIME32: + return GetLiteralFromTimeArray(array); case arrow::Type::type::DICTIONARY: { const auto& dict_array = checked_cast(array); auto* dict_type = checked_cast(dict_array.type().get()); @@ -280,6 +289,29 @@ std::vector LiteralConverter::GetLiteralFromDateArray(const arrow::Arra return literals; } +Result> LiteralConverter::GetLiteralFromTimeArray(const arrow::Array& array) { + using ArrayType = typename arrow::TypeTraits::ArrayType; + const auto& array_(checked_cast(array)); + // A TIME literal holds milliseconds of the day, so a value of another unit would be silently + // reinterpreted, e.g. 1 of time32[s] would become 1 millisecond. + const auto& time_type = checked_cast(*array_.type()); + if (time_type.unit() != arrow::TimeUnit::MILLI) { + return Status::Invalid( + fmt::format("Not support literal on arrow {} type, TIME literals require time32[ms]", + array_.type()->ToString())); + } + std::vector literals; + literals.reserve(array_.length()); + for (int64_t i = 0; i < array_.length(); i++) { + if (array_.IsNull(i)) { + literals.emplace_back(FieldType::TIME); + } else { + literals.emplace_back(FieldType::TIME, array_.Value(i)); + } + } + return literals; +} + std::vector LiteralConverter::GetLiteralFromTimestampArray(const arrow::Array& array) { using ArrayType = typename arrow::TypeTraits::ArrayType; const auto& array_(checked_cast(array)); @@ -317,11 +349,12 @@ arrow::TimeUnit::type LiteralConverter::MinRequiredTimeUnit(const std::vector
  • +// Appends every literal, read by `extract`, to a `BuilderType` made from `builder_args`, keeping a +// null literal a null. +template Result> BuildArray(const std::vector& literals, - Extract extract, arrow::MemoryPool* pool) { - BuilderType builder(pool); + Extract extract, BuilderArgs&&... builder_args) { + BuilderType builder(std::forward(builder_args)...); PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(static_cast(literals.size()))); for (const auto& literal : literals) { if (literal.IsNull()) { @@ -484,6 +517,10 @@ Result> LiteralConverter::ConvertLiteralsToArray( case FieldType::DATE: return BuildArray( literals, [](const Literal& literal) { return literal.GetValue(); }, pool); + case FieldType::TIME: + return BuildArray( + literals, [](const Literal& literal) { return literal.GetValue(); }, + arrow::time32(arrow::TimeUnit::MILLI), pool); case FieldType::STRING: return BuildArray( literals, [](const Literal& literal) { return literal.GetValue(); }, diff --git a/src/paimon/common/predicate/literal_converter.h b/src/paimon/common/predicate/literal_converter.h index c393855b5..6b9ab52af 100644 --- a/src/paimon/common/predicate/literal_converter.h +++ b/src/paimon/common/predicate/literal_converter.h @@ -59,12 +59,12 @@ class PAIMON_EXPORT LiteralConverter { /// @param pool The pool every buffer of the result is allocated from, it must not be null. /// @return `Status::Invalid` for a field type this does not write, which is every one outside /// `BOOLEAN`, `TINYINT`, `SMALLINT`, `INT`, `BIGINT`, `FLOAT`, `DOUBLE`, `DATE`, - /// `STRING`, `BINARY`, `DECIMAL` and `TIMESTAMP`. `DECIMAL` is written with the - /// precision and the scale its literals carry, so at least one literal has to be non - /// null and every non null one has to carry the same pair. `TIMESTAMP` is written with - /// the finest time unit that keeps every value, so at least one literal has to be non - /// null to settle the unit, and a literal whose value in that unit does not fit int64 - /// fails. + /// `TIME`, `STRING`, `BINARY`, `DECIMAL` and `TIMESTAMP`. `TIME` is written as a + /// millisecond `time32`. `DECIMAL` is written with the precision and the scale its + /// literals carry, so at least one literal has to be non null and every non null one + /// has to carry the same pair. `TIMESTAMP` is written with the finest time unit that + /// keeps every value, so at least one literal has to be non null to settle the unit, + /// and a literal whose value in that unit does not fit int64 fails. static Result> ConvertLiteralsToArray( const FieldType& field_type, const std::vector& literals, arrow::MemoryPool* pool); @@ -146,6 +146,8 @@ class PAIMON_EXPORT LiteralConverter { static std::vector GetLiteralFromDateArray(const arrow::Array& array); + static Result> GetLiteralFromTimeArray(const arrow::Array& array); + static std::vector GetLiteralFromTimestampArray(const arrow::Array& array); }; } // namespace paimon diff --git a/src/paimon/common/predicate/literal_converter_test.cpp b/src/paimon/common/predicate/literal_converter_test.cpp index c862960f5..e037c7397 100644 --- a/src/paimon/common/predicate/literal_converter_test.cpp +++ b/src/paimon/common/predicate/literal_converter_test.cpp @@ -424,6 +424,38 @@ TEST_F(LiteralConverterTest, TestDateLiteral) { Literal(FieldType::DATE, -5l), Literal(FieldType::DATE)}); } +TEST_F(LiteralConverterTest, TestTimeLiteral) { + auto time_type = arrow::time32(arrow::TimeUnit::MILLI); + auto field_array = + arrow::ipc::internal::json::ArrayFromJSON(time_type, R"([0, 123, 86399999, null])") + .ValueOrDie(); + CheckResult(field_array, std::vector( + {Literal(FieldType::TIME, 0), Literal(FieldType::TIME, 123), + Literal(FieldType::TIME, 86399999), Literal(FieldType::TIME)})); + CheckLiteralsFromString( + FieldType::TIME, {"0", "00:00:00", "00:00:00.123", "23:59:59.999", "01:02:03"}, + std::vector({Literal(FieldType::TIME, 0), Literal(FieldType::TIME, 0), + Literal(FieldType::TIME, 123), Literal(FieldType::TIME, 86399999), + Literal(FieldType::TIME, 3723000)})); + ASSERT_NOK(LiteralConverter::ConvertLiteralsFromString(FieldType::TIME, "24:00:00")); + + CheckLiteralFromRow(time_type, {0, 123, 86399999, NullType()}, FieldType::TIME, + {Literal(FieldType::TIME, 0), Literal(FieldType::TIME, 123), + Literal(FieldType::TIME, 86399999), Literal(FieldType::TIME)}); + + CheckLiteralsToArray( + FieldType::TIME, + {Literal(FieldType::TIME, 0), Literal(FieldType::TIME), Literal(FieldType::TIME, 123)}, + arrow::ipc::internal::json::ArrayFromJSON(time_type, R"([0, null, 123])").ValueOrDie()); + + auto seconds_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::time32(arrow::TimeUnit::SECOND), R"([1, null])") + .ValueOrDie(); + ASSERT_NOK_WITH_MSG( + LiteralConverter::ConvertLiteralsFromArray(*seconds_array, /*own_data=*/true), + "TIME literals require time32[ms]"); +} + TEST_F(LiteralConverterTest, TestInvalidType) { auto field_array = arrow::ipc::internal::json::ArrayFromJSON(arrow::large_utf8(), R"(["apple", "苹果", null])") diff --git a/src/paimon/common/predicate/literal_test.cpp b/src/paimon/common/predicate/literal_test.cpp index a561ccdfd..0c53f6b63 100644 --- a/src/paimon/common/predicate/literal_test.cpp +++ b/src/paimon/common/predicate/literal_test.cpp @@ -138,6 +138,14 @@ TEST_F(LiteralTest, TestSimple) { ASSERT_EQ(literal1.ToString(), "10000"); ASSERT_EQ(literal2.ToString(), "20000"); } + { + Literal literal1(FieldType::TIME, 123); + Literal literal2(FieldType::TIME, 86399999); + CheckResult(literal1, literal2, FieldType::TIME); + ASSERT_EQ(literal1.ToString(), "123"); + ASSERT_EQ(literal1.HashCode(), Literal(FieldType::TIME, 123).HashCode()); + ASSERT_NOK(literal1.CompareTo(Literal(123))); + } { Literal literal1(Timestamp(1725875365442l, 120000)); Literal literal2(Timestamp(1725875365442l, 120001)); diff --git a/src/paimon/common/predicate/multi_literals_leaf_function.cpp b/src/paimon/common/predicate/multi_literals_leaf_function.cpp index a2e6d58f4..4b5cb119d 100644 --- a/src/paimon/common/predicate/multi_literals_leaf_function.cpp +++ b/src/paimon/common/predicate/multi_literals_leaf_function.cpp @@ -58,6 +58,7 @@ bool CanProbeWithIsIn(FieldType field_type) { case FieldType::FLOAT: case FieldType::DOUBLE: case FieldType::DATE: + case FieldType::TIME: case FieldType::STRING: case FieldType::BINARY: case FieldType::DECIMAL: @@ -146,6 +147,12 @@ std::shared_ptr MakeInValueSet(const std::vector& literal !IsTimestampOfUnit(data_type, LiteralConverter::MinRequiredTimeUnit(literals)))) { return nullptr; } + // A TIME value set is written in milliseconds, so only a millisecond column is probed without + // a cast. + if (field_type == FieldType::TIME && + !data_type.Equals(*arrow::time32(arrow::TimeUnit::MILLI))) { + return nullptr; + } for (const auto& literal : literals) { // A literal typed differently makes `Literal::CompareTo` fail, keep that on the row by row // path. diff --git a/src/paimon/common/predicate/multi_literals_leaf_function_test.cpp b/src/paimon/common/predicate/multi_literals_leaf_function_test.cpp index 981334eed..1e32e5fd8 100644 --- a/src/paimon/common/predicate/multi_literals_leaf_function_test.cpp +++ b/src/paimon/common/predicate/multi_literals_leaf_function_test.cpp @@ -138,6 +138,22 @@ TEST_F(MultiLiteralsLeafFunctionTest, TestDate) { ASSERT_EQ(EvalNotIn(literals, array), std::vector({0, 1, 0, 0})); } +TEST_F(MultiLiteralsLeafFunctionTest, TestTime) { + const std::vector literals = {Literal(FieldType::TIME, 0), + Literal(FieldType::TIME, 86399999)}; + auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::time32(arrow::TimeUnit::MILLI), + R"([0, 123, 86399999, null])") + .ValueOrDie(); + ASSERT_EQ(EvalIn(literals, array), std::vector({1, 0, 1, 0})); + ASSERT_EQ(EvalNotIn(literals, array), std::vector({0, 1, 0, 0})); + + auto seconds_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::time32(arrow::TimeUnit::SECOND), R"([0, 1, null])") + .ValueOrDie(); + ASSERT_NOK(In::Instance().Test(*seconds_array, literals, arrow::default_memory_pool())); + ASSERT_NOK(NotIn::Instance().Test(*seconds_array, literals, arrow::default_memory_pool())); +} + TEST_F(MultiLiteralsLeafFunctionTest, TestBoolean) { auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::boolean(), R"([true, false, null])") diff --git a/src/paimon/common/predicate/null_false_leaf_binary_function.cpp b/src/paimon/common/predicate/null_false_leaf_binary_function.cpp index b4db667e0..1c79221cd 100644 --- a/src/paimon/common/predicate/null_false_leaf_binary_function.cpp +++ b/src/paimon/common/predicate/null_false_leaf_binary_function.cpp @@ -81,6 +81,7 @@ bool CanCompareWithKernel(FieldType field_type) { case FieldType::INT: case FieldType::BIGINT: case FieldType::DATE: + case FieldType::TIME: case FieldType::STRING: case FieldType::BINARY: case FieldType::DECIMAL: @@ -142,6 +143,10 @@ bool AgreesWithScalar(const arrow::DataType& column_type, const arrow::DataType& checked_cast(column_type).unit() == checked_cast(scalar_type).unit() && checked_cast(column_type).timezone().empty(); + case arrow::Type::TIME32: + // A TIME literal is written in milliseconds, so only a millisecond column compares + // without a cast. + return column_type.Equals(scalar_type); default: return true; } diff --git a/src/paimon/common/predicate/null_false_leaf_binary_function_test.cpp b/src/paimon/common/predicate/null_false_leaf_binary_function_test.cpp index b757a460e..7e58c5f93 100644 --- a/src/paimon/common/predicate/null_false_leaf_binary_function_test.cpp +++ b/src/paimon/common/predicate/null_false_leaf_binary_function_test.cpp @@ -140,6 +140,25 @@ TEST_F(NullFalseLeafBinaryFunctionTest, TestDate) { ASSERT_EQ(Eval(GreaterOrEqual::Instance(), second, array), std::vector({0, 1, 1, 0})); } +TEST_F(NullFalseLeafBinaryFunctionTest, TestTime) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::time32(arrow::TimeUnit::MILLI), + R"([0, 123, 86399999, null])") + .ValueOrDie(); + const Literal value{FieldType::TIME, 123}; + ASSERT_EQ(Eval(Equal::Instance(), value, array), std::vector({0, 1, 0, 0})); + ASSERT_EQ(Eval(NotEqual::Instance(), value, array), std::vector({1, 0, 1, 0})); + ASSERT_EQ(Eval(LessThan::Instance(), value, array), std::vector({1, 0, 0, 0})); + ASSERT_EQ(Eval(LessOrEqual::Instance(), value, array), std::vector({1, 1, 0, 0})); + ASSERT_EQ(Eval(GreaterThan::Instance(), value, array), std::vector({0, 0, 1, 0})); + ASSERT_EQ(Eval(GreaterOrEqual::Instance(), value, array), std::vector({0, 1, 1, 0})); + + auto seconds_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::time32(arrow::TimeUnit::SECOND), R"([0, 1, null])") + .ValueOrDie(); + AssertEvalFails(Equal::Instance(), {Literal(FieldType::TIME, 1)}, seconds_array); + AssertEvalFails(LessThan::Instance(), {Literal(FieldType::TIME, 1000)}, seconds_array); +} + TEST_F(NullFalseLeafBinaryFunctionTest, TestString) { // `Literal::CompareTo` orders two strings by `std::string_view::compare`, which is the byte by // byte order a kernel compares them in, a prefix included. diff --git a/src/paimon/common/predicate/predicate_validator.h b/src/paimon/common/predicate/predicate_validator.h index 7f199f501..b794adc22 100644 --- a/src/paimon/common/predicate/predicate_validator.h +++ b/src/paimon/common/predicate/predicate_validator.h @@ -27,6 +27,7 @@ #include "arrow/util/decimal.h" #include "fmt/format.h" #include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/field_type_utils.h" #include "paimon/data/decimal.h" #include "paimon/defs.h" @@ -104,6 +105,8 @@ class PredicateValidator { if (schema_type->id() == arrow::Type::DECIMAL128) { PAIMON_RETURN_NOT_OK(ValidateDecimalLiterals( *checked_pointer_cast(schema_type), *leaf_predicate)); + } else if (schema_type->id() == arrow::Type::TIME32) { + PAIMON_RETURN_NOT_OK(ValidateTimeLiterals(*leaf_predicate)); } } else if (auto compound_predicate = std::dynamic_pointer_cast(predicate)) { @@ -138,6 +141,21 @@ class PredicateValidator { return Status::OK(); } + static Status ValidateTimeLiterals(const LeafPredicate& predicate) { + for (const Literal& literal : predicate.Literals()) { + if (literal.IsNull()) { + continue; + } + auto millis = literal.GetValue(); + if (millis < 0 || millis >= DateTimeUtils::MILLIS_PER_DAY) { + return Status::Invalid(fmt::format( + "time literal {} for field {} is out of range [0, {}) milliseconds of the day", + millis, predicate.FieldName(), DateTimeUtils::MILLIS_PER_DAY)); + } + } + return Status::OK(); + } + static Status ValidateDataTypeWithSchemaAndPredicate(const arrow::DataType& schema_type, const FieldType& field_type) { const auto kind = schema_type.id(); @@ -202,6 +220,20 @@ class PredicateValidator { } break; } + case arrow::Type::type::TIME32: { + if (field_type != FieldType::TIME) { + break; + } + // TIME literals hold milliseconds of the day, so only time32[ms] matches them. + if (checked_cast(schema_type).unit() != + arrow::TimeUnit::MILLI) { + return Status::Invalid( + fmt::format("schema type {} mismatches predicate field type TIME, TIME " + "requires time32[ms]", + schema_type.ToString())); + } + return Status::OK(); + } case arrow::Type::type::DECIMAL128: if (field_type == FieldType::DECIMAL) { return Status::OK(); diff --git a/src/paimon/common/predicate/predicate_validator_test.cpp b/src/paimon/common/predicate/predicate_validator_test.cpp index 89432b727..8b10e77b2 100644 --- a/src/paimon/common/predicate/predicate_validator_test.cpp +++ b/src/paimon/common/predicate/predicate_validator_test.cpp @@ -460,4 +460,53 @@ TEST(PredicateValidatorTest, TestValidateDecimalLiteral) { "2)"); } } + +TEST(PredicateValidatorTest, TestValidateTime) { + std::shared_ptr schema = arrow::schema(arrow::FieldVector({ + arrow::field("t", arrow::time32(arrow::TimeUnit::MILLI)), + arrow::field("i", arrow::int32()), + })); + { + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::And({ + PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"t", + FieldType::TIME, Literal(FieldType::TIME, 0)), + PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"t", FieldType::TIME, + Literal(FieldType::TIME, 86399999)), + PredicateBuilder::IsNotNull(/*field_index=*/0, /*field_name=*/"t", FieldType::TIME), + })); + ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema(*schema, predicate, + /*validate_field_idx=*/true)); + } + for (int32_t value : {-1, 86400000}) { + auto predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"t", + FieldType::TIME, Literal(FieldType::TIME, value)); + ASSERT_NOK_WITH_MSG( + PredicateValidator::ValidatePredicateWithSchema(*schema, predicate, + /*validate_field_idx=*/true), + "time literal " + std::to_string(value) + " for field t is out of range"); + } + { + auto predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"t", + FieldType::INT, Literal(1)); + ASSERT_NOK_WITH_MSG(PredicateValidator::ValidatePredicateWithSchema( + *schema, predicate, /*validate_field_idx=*/true), + "mismatches predicate field type INT"); + predicate = PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"i", FieldType::TIME, + Literal(FieldType::TIME, 1)); + ASSERT_NOK_WITH_MSG(PredicateValidator::ValidatePredicateWithSchema( + *schema, predicate, /*validate_field_idx=*/true), + "mismatches predicate field type TIME"); + } + { + std::shared_ptr seconds_schema = arrow::schema( + arrow::FieldVector({arrow::field("t", arrow::time32(arrow::TimeUnit::SECOND))})); + auto predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"t", + FieldType::TIME, Literal(FieldType::TIME, 1)); + ASSERT_NOK_WITH_MSG(PredicateValidator::ValidatePredicateWithSchema( + *seconds_schema, predicate, /*validate_field_idx=*/true), + "TIME requires time32[ms]"); + } +} } // namespace paimon::test diff --git a/src/paimon/common/types/data_type.h b/src/paimon/common/types/data_type.h index 1e49ce7e1..ceae1f1fd 100644 --- a/src/paimon/common/types/data_type.h +++ b/src/paimon/common/types/data_type.h @@ -24,6 +24,8 @@ #include #include "paimon/common/utils/jsonizable.h" +#include "paimon/result.h" +#include "paimon/visibility.h" #include "rapidjson/allocators.h" #include "rapidjson/document.h" #include "rapidjson/rapidjson.h" @@ -43,7 +45,7 @@ class DataType : public Jsonizable { // Arrow carries milliseconds, while metadata preserves the declared TIME precision. // Arrow-only schemas use Paimon's default precision of zero. - static Result GetTimePrecision(const arrow::Field& field); + PAIMON_EXPORT static Result GetTimePrecision(const arrow::Field& field); static std::unique_ptr Create( const std::shared_ptr& type, bool nullable, diff --git a/src/paimon/common/utils/binary_row_partition_computer.cpp b/src/paimon/common/utils/binary_row_partition_computer.cpp index d0be6f07a..7ade6c2a3 100644 --- a/src/paimon/common/utils/binary_row_partition_computer.cpp +++ b/src/paimon/common/utils/binary_row_partition_computer.cpp @@ -27,6 +27,7 @@ #include "fmt/ranges.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" +#include "paimon/common/types/data_type.h" #include "paimon/common/utils/string_utils.h" #include "paimon/macros.h" #include "paimon/result.h" @@ -62,12 +63,14 @@ Result> BinaryRowPartitionComputer:: for (const auto& partition_key : partition_keys) { PAIMON_ASSIGN_OR_RAISE(arrow::Type::type type_id, GetTypeFromArrowSchema(schema, partition_key)); + PAIMON_ASSIGN_OR_RAISE(int32_t time_precision, + GetTimePrecision(*schema->GetFieldByName(partition_key))); PAIMON_ASSIGN_OR_RAISE( DataConverterUtils::StrToBinaryRowConverter converter, DataConverterUtils::CreateDataToBinaryRowConverter(type_id, memory_pool.get())); PAIMON_ASSIGN_OR_RAISE(DataConverterUtils::BinaryRowFieldToStrConverter reconverter, DataConverterUtils::CreateBinaryRowFieldToStringConverter( - type_id, legacy_partition_name_enabled)); + type_id, legacy_partition_name_enabled, time_precision)); partition_converters.emplace_back(partition_key, std::move(converter), std::move(reconverter)); } @@ -107,8 +110,12 @@ Result BinaryRowPartitionComputer::ConvertToBinaryRow( // WriteTimestamp(null) for non compact precision writer.SetNullAt(field_idx); } else { - PAIMON_RETURN_NOT_OK( - partition_converter.converter(input_iter->second, field_idx, &writer)); + Status status = partition_converter.converter(input_iter->second, field_idx, &writer); + if (!status.ok()) { + return status.WithMessage(fmt::format("invalid value of partition key '{}': {}", + partition_converter.partition_key, + status.message())); + } } } writer.Complete(); @@ -187,15 +194,24 @@ Result BinaryRowPartitionComputer::GetTypeFromArrowSchema( return field->type()->id(); } +Result BinaryRowPartitionComputer::GetTimePrecision(const arrow::Field& field) { + if (field.type()->id() != arrow::Type::TIME32) { + return 0; + } + return DataType::GetTimePrecision(field); +} + Result BinaryRowPartitionComputer::PartToSimpleString( const std::shared_ptr& partition_type, const BinaryRow& partition, const std::string& delimiter, int32_t max_length, bool legacy_partition_name_enabled) { std::vector partition_converters; partition_converters.reserve(partition_type->num_fields()); for (const auto& field : partition_type->fields()) { - PAIMON_ASSIGN_OR_RAISE(DataConverterUtils::BinaryRowFieldToStrConverter converter, - DataConverterUtils::CreateBinaryRowFieldToStringConverter( - field->type()->id(), legacy_partition_name_enabled)); + PAIMON_ASSIGN_OR_RAISE(int32_t time_precision, GetTimePrecision(*field)); + PAIMON_ASSIGN_OR_RAISE( + DataConverterUtils::BinaryRowFieldToStrConverter converter, + DataConverterUtils::CreateBinaryRowFieldToStringConverter( + field->type()->id(), legacy_partition_name_enabled, time_precision)); partition_converters.emplace_back(converter); } std::vector partition_vec; diff --git a/src/paimon/common/utils/binary_row_partition_computer.h b/src/paimon/common/utils/binary_row_partition_computer.h index 08990d61c..6b4ff7d3e 100644 --- a/src/paimon/common/utils/binary_row_partition_computer.h +++ b/src/paimon/common/utils/binary_row_partition_computer.h @@ -97,6 +97,9 @@ class BinaryRowPartitionComputer { static Result GetTypeFromArrowSchema( const std::shared_ptr& schema, const std::string& field_name); + /// Returns the declared precision of a TIME field, and 0 for any other field. + static Result GetTimePrecision(const arrow::Field& field); + std::shared_ptr memory_pool_; std::vector partition_keys_; std::shared_ptr schema_; diff --git a/src/paimon/common/utils/binary_row_partition_computer_test.cpp b/src/paimon/common/utils/binary_row_partition_computer_test.cpp index 4f60781ee..60e71cbbb 100644 --- a/src/paimon/common/utils/binary_row_partition_computer_test.cpp +++ b/src/paimon/common/utils/binary_row_partition_computer_test.cpp @@ -25,9 +25,11 @@ #include #include "arrow/type.h" +#include "arrow/util/key_value_metadata.h" #include "gtest/gtest.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_string.h" +#include "paimon/common/types/data_type.h" #include "paimon/memory/memory_pool.h" #include "paimon/status.h" #include "paimon/testing/utils/binary_row_generator.h" @@ -343,6 +345,97 @@ TEST(BinaryRowPartitionComputerTest, TestToPartialBinaryRow) { ASSERT_NOK(computer->ToPartialBinaryRow({{"pt", "abc"}})); } +TEST(BinaryRowPartitionComputerTest, TestTimePartition) { + auto pool = GetDefaultPool(); + auto time_field = [](const std::string& name, const std::string& precision) { + return arrow::field(name, arrow::time32(arrow::TimeUnit::MILLI), /*nullable=*/true, + arrow::key_value_metadata({DataType::kTimePrecision}, {precision})); + }; + auto schema = + arrow::schema({arrow::field("t_default", arrow::time32(arrow::TimeUnit::MILLI)), + time_field("t3", "3"), time_field("t6", "6"), time_field("t9", "9")}); + std::vector partition_keys = {"t_default", "t3", "t6", "t9"}; + { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr computer, + BinaryRowPartitionComputer::Create(partition_keys, schema, "__DEFAULT_PARTITION__", + /*legacy_partition_name_enabled=*/false, pool)); + std::map partition_map = {{"t_default", "00:00:00"}, + {"t3", "00:00:00.123"}, + {"t6", "12:34:56.12"}, + {"t9", "23:59:59.999"}}; + ASSERT_OK_AND_ASSIGN(BinaryRow row, computer->ToBinaryRow(partition_map)); + ASSERT_EQ(0, row.GetInt(0)); + ASSERT_EQ(123, row.GetInt(1)); + ASSERT_EQ(45296120, row.GetInt(2)); + ASSERT_EQ(86399999, row.GetInt(3)); + ASSERT_OK_AND_ASSIGN(auto part_values, computer->GeneratePartitionVector(row)); + std::map actual(part_values.begin(), part_values.end()); + ASSERT_EQ(actual, partition_map); + + ASSERT_OK_AND_ASSIGN(row, computer->ToBinaryRow({{"t_default", "12:34:56.789"}, + {"t3", "01:02:03"}, + {"t6", "01:02:03.000456"}, + {"t9", "00:00:00.5"}})); + ASSERT_OK_AND_ASSIGN(part_values, computer->GeneratePartitionVector(row)); + actual = std::map(part_values.begin(), part_values.end()); + std::map expected = {{"t_default", "12:34:56"}, + {"t3", "01:02:03.0"}, + {"t6", "01:02:03.0"}, + {"t9", "00:00:00.5"}}; + ASSERT_EQ(actual, expected); + + ASSERT_OK_AND_ASSIGN(row, computer->ToBinaryRow({{"t_default", "__DEFAULT_PARTITION__"}, + {"t3", "00:00:00"}, + {"t6", "00:00:00"}, + {"t9", "00:00:00"}})); + ASSERT_TRUE(row.IsNullAt(0)); + ASSERT_NOK_WITH_MSG(computer->ToBinaryRow({{"t_default", "24:00:00"}, + {"t3", "00:00:00"}, + {"t6", "00:00:00"}, + {"t9", "00:00:00"}}), + "invalid value of partition key 't_default': failed to convert string " + "'24:00:00' to time"); + + ASSERT_OK_AND_ASSIGN(row, computer->ToBinaryRow({{"t_default", "01:02:03"}, + {"t3", "00:00:00.123"}, + {"t6", "12:00:00"}, + {"t9", "00:00:00"}})); + ASSERT_OK_AND_ASSIGN(std::string ret, + BinaryRowPartitionComputer::PartToSimpleString( + schema, row, "-", 100, /*legacy_partition_name_enabled=*/false)); + ASSERT_EQ(ret, "01:02:03-00:00:00.123-12:00:00.0-00:00:00.0"); + } + { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr computer, + BinaryRowPartitionComputer::Create(partition_keys, schema, "__DEFAULT_PARTITION__", + /*legacy_partition_name_enabled=*/true, pool)); + std::map partition_map = { + {"t_default", "0"}, {"t3", "123"}, {"t6", "45296120"}, {"t9", "86399999"}}; + ASSERT_OK_AND_ASSIGN(BinaryRow row, computer->ToBinaryRow(partition_map)); + ASSERT_OK_AND_ASSIGN(auto part_values, computer->GeneratePartitionVector(row)); + std::map actual(part_values.begin(), part_values.end()); + ASSERT_EQ(actual, partition_map); + ASSERT_OK_AND_ASSIGN(std::string ret, + BinaryRowPartitionComputer::PartToSimpleString( + schema, row, "-", 100, /*legacy_partition_name_enabled=*/true)); + ASSERT_EQ(ret, "0-123-45296120-86399999"); + } + { + auto invalid_schema = arrow::schema({time_field("t", "x")}); + ASSERT_NOK( + BinaryRowPartitionComputer::Create({"t"}, invalid_schema, "__DEFAULT_PARTITION__", + /*legacy_partition_name_enabled=*/false, pool)); + auto second_schema = + arrow::schema({arrow::field("t", arrow::time32(arrow::TimeUnit::SECOND))}); + ASSERT_NOK_WITH_MSG( + BinaryRowPartitionComputer::Create({"t"}, second_schema, "__DEFAULT_PARTITION__", + /*legacy_partition_name_enabled=*/false, pool), + "Only millisecond TIME is supported"); + } +} + TEST(BinaryRowPartitionComputerTest, TestPartToSimpleString) { auto pool = GetDefaultPool(); { diff --git a/src/paimon/common/utils/data_converter_utils.h b/src/paimon/common/utils/data_converter_utils.h index 3ffeb9864..9876192cd 100644 --- a/src/paimon/common/utils/data_converter_utils.h +++ b/src/paimon/common/utils/data_converter_utils.h @@ -35,6 +35,7 @@ #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" #include "paimon/common/data/binary_string.h" +#include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/casting/date_to_string_cast_executor.h" #include "paimon/defs.h" @@ -133,6 +134,15 @@ class DataConverterUtils { return Status::OK(); }; break; + case arrow::Type::TIME32: + converter = [](const std::string& value_str, int32_t field_idx, + BinaryRowWriter* writer) { + PAIMON_ASSIGN_OR_RAISE(int32_t time_value, + StringUtils::StringToTime(value_str)); + writer->WriteInt(field_idx, time_value); + return Status::OK(); + }; + break; default: return Status::NotImplemented( fmt::format("Do not support type {} in partition binary row", @@ -141,8 +151,9 @@ class DataConverterUtils { return converter; } + /// @param time_precision The declared precision of a TIME field, ignored for other types. static Result CreateBinaryRowFieldToStringConverter( - arrow::Type::type type, bool legacy_partition_name_enabled) { + arrow::Type::type type, bool legacy_partition_name_enabled, int32_t time_precision = 0) { BinaryRowFieldToStrConverter converter; switch (type) { case arrow::Type::BOOL: @@ -203,6 +214,22 @@ class DataConverterUtils { } break; } + case arrow::Type::TIME32: { + if (legacy_partition_name_enabled) { + // Java keeps the internal milliseconds of the day for legacy partition names. + converter = [](const BinaryRow& row, int32_t field_idx) -> Result { + return std::to_string(row.GetInt(field_idx)); + }; + } else { + // Java `TimeToStringCastRule`. + converter = [time_precision](const BinaryRow& row, + int32_t field_idx) -> Result { + return DateTimeUtils::FormatTimeMillis(row.GetInt(field_idx), + time_precision); + }; + } + break; + } default: return Status::NotImplemented( fmt::format("Do not support arrow {} in partition binary row", diff --git a/src/paimon/common/utils/data_converter_utils_test.cpp b/src/paimon/common/utils/data_converter_utils_test.cpp index 75913eccd..8b1e7fb12 100644 --- a/src/paimon/common/utils/data_converter_utils_test.cpp +++ b/src/paimon/common/utils/data_converter_utils_test.cpp @@ -44,7 +44,8 @@ TEST(DataConverterUtilsTest, TestDataToBinaryRowConverterWithLegacyPartitionName {"1234567", arrow::Type::INT64}, {"abcde", arrow::Type::STRING}, {"这是一个很长很长的中文", arrow::Type::STRING}, - {"10440", arrow::Type::DATE32}}; + {"10440", arrow::Type::DATE32}, + {"45296123", arrow::Type::TIME32}}; std::vector converters; std::vector reconverters; @@ -84,6 +85,7 @@ TEST(DataConverterUtilsTest, TestDataToBinaryRowConverterWithLegacyPartitionName ASSERT_EQ("abcde", row.GetString(9).ToString()); ASSERT_EQ("这是一个很长很长的中文", row.GetString(10).ToString()); ASSERT_EQ(10440, row.GetDate(11)); + ASSERT_EQ(45296123, row.GetInt(12)); for (size_t idx = 0; idx < data.size(); idx++) { ASSERT_OK_AND_ASSIGN(auto partition_field_str, reconverters[idx](row, idx)); @@ -105,7 +107,8 @@ TEST(DataConverterUtilsTest, TestDataToBinaryRowConverterWithNoLegacyPartitionNa {"1234567", arrow::Type::INT64}, {"abcde", arrow::Type::STRING}, {"这是一个很长很长的中文", arrow::Type::STRING}, - {"1998-08-02", arrow::Type::DATE32}}; + {"1998-08-02", arrow::Type::DATE32}, + {"12:34:56", arrow::Type::TIME32}}; std::vector converters; std::vector reconverters; @@ -140,6 +143,7 @@ TEST(DataConverterUtilsTest, TestDataToBinaryRowConverterWithNoLegacyPartitionNa ASSERT_EQ("abcde", row.GetString(9).ToString()); ASSERT_EQ("这是一个很长很长的中文", row.GetString(10).ToString()); ASSERT_EQ(10440, row.GetDate(11)); + ASSERT_EQ(45296000, row.GetInt(12)); for (size_t idx = 0; idx < data.size(); idx++) { ASSERT_OK_AND_ASSIGN(auto partition_field_str, reconverters[idx](row, idx)); diff --git a/src/paimon/common/utils/date_time_utils.h b/src/paimon/common/utils/date_time_utils.h index 49d8f2d47..71b3bc2ae 100644 --- a/src/paimon/common/utils/date_time_utils.h +++ b/src/paimon/common/utils/date_time_utils.h @@ -212,6 +212,36 @@ class DateTimeUtils { return "UNKNOWN"; } + /// Formats a TIME value, given as milliseconds of the day, as `HH:mm:ss[.fff]` like Java + /// `DateTimeUtils#formatTimestampMillis`: at most `precision` fractional digits are written, + /// trailing zeros are dropped, and a negative value wraps around midnight. + static std::string FormatTimeMillis(int32_t time, int32_t precision) { + constexpr int64_t kMillisPerHour = 3600000; + constexpr int64_t kMillisPerMinute = 60000; + constexpr int64_t kMillisPerSecond = 1000; + int64_t value = time; + while (value < 0) { + value += MILLIS_PER_DAY; + } + int64_t hour = value / kMillisPerHour; + int64_t minute = value % kMillisPerHour / kMillisPerMinute; + int64_t second = value % kMillisPerMinute / kMillisPerSecond; + int64_t milli = value % kMillisPerSecond; + std::string result = fmt::format("{:02d}:{:02d}:{:02d}", hour, minute, second); + if (precision > 0) { + result.push_back('.'); + while (precision > 0) { + result.push_back(static_cast('0' + milli / 100)); + milli = (milli % 100) * 10; + if (milli == 0) { + break; + } + --precision; + } + } + return result; + } + // there may be a precision loss for nano static Result ToUTCTimestamp(const Timestamp& timestamp) { int64_t micro_second = timestamp.ToMicrosecond(); diff --git a/src/paimon/common/utils/date_time_utils_test.cpp b/src/paimon/common/utils/date_time_utils_test.cpp index 7d32c6d9c..2e27eb194 100644 --- a/src/paimon/common/utils/date_time_utils_test.cpp +++ b/src/paimon/common/utils/date_time_utils_test.cpp @@ -181,6 +181,20 @@ TEST(DateTimeUtilsTest, TestTimestampToInteger) { } } +TEST(DateTimeUtilsTest, TestFormatTimeMillis) { + ASSERT_EQ(DateTimeUtils::FormatTimeMillis(0, 0), "00:00:00"); + ASSERT_EQ(DateTimeUtils::FormatTimeMillis(0, 3), "00:00:00.0"); + ASSERT_EQ(DateTimeUtils::FormatTimeMillis(123, 0), "00:00:00"); + ASSERT_EQ(DateTimeUtils::FormatTimeMillis(123, 3), "00:00:00.123"); + ASSERT_EQ(DateTimeUtils::FormatTimeMillis(123, 6), "00:00:00.123"); + ASSERT_EQ(DateTimeUtils::FormatTimeMillis(123, 9), "00:00:00.123"); + ASSERT_EQ(DateTimeUtils::FormatTimeMillis(123, 1), "00:00:00.1"); + ASSERT_EQ(DateTimeUtils::FormatTimeMillis(120, 3), "00:00:00.12"); + ASSERT_EQ(DateTimeUtils::FormatTimeMillis(86399999, 3), "23:59:59.999"); + ASSERT_EQ(DateTimeUtils::FormatTimeMillis(3723000, 3), "01:02:03.0"); + ASSERT_EQ(DateTimeUtils::FormatTimeMillis(-1, 3), "23:59:59.999"); +} + TEST(DateTimeUtilsTest, TestGetPrecisionFromType) { auto ts_sec_type = arrow::timestamp(arrow::TimeUnit::type::SECOND); auto ts_type = checked_pointer_cast(ts_sec_type); diff --git a/src/paimon/common/utils/field_type_utils.h b/src/paimon/common/utils/field_type_utils.h index d2786e26e..c6e259987 100644 --- a/src/paimon/common/utils/field_type_utils.h +++ b/src/paimon/common/utils/field_type_utils.h @@ -87,6 +87,8 @@ class FieldTypeUtils { return FieldType::DECIMAL; case arrow::Type::type::DATE32: return FieldType::DATE; + case arrow::Type::type::TIME32: + return FieldType::TIME; case arrow::Type::type::LIST: return FieldType::ARRAY; case arrow::Type::type::MAP: @@ -129,6 +131,8 @@ class FieldTypeUtils { return "DECIMAL"; case FieldType::DATE: return "DATE"; + case FieldType::TIME: + return "TIME"; case FieldType::ARRAY: return "ARRAY"; case FieldType::MAP: diff --git a/src/paimon/common/utils/field_type_utils_test.cpp b/src/paimon/common/utils/field_type_utils_test.cpp index c70edb092..656bc3297 100644 --- a/src/paimon/common/utils/field_type_utils_test.cpp +++ b/src/paimon/common/utils/field_type_utils_test.cpp @@ -86,6 +86,9 @@ TEST(FieldTypeUtilsTest, ConvertToFieldType) { ASSERT_OK_AND_ASSIGN(result, FieldTypeUtils::ConvertToFieldType(arrow::Type::type::TIMESTAMP)); ASSERT_EQ(result, FieldType::TIMESTAMP); + ASSERT_OK_AND_ASSIGN(result, FieldTypeUtils::ConvertToFieldType(arrow::Type::type::TIME32)); + ASSERT_EQ(result, FieldType::TIME); + ASSERT_OK_AND_ASSIGN(result, FieldTypeUtils::ConvertToFieldType(arrow::Type::type::DECIMAL128)); ASSERT_EQ(result, FieldType::DECIMAL); @@ -125,6 +128,7 @@ TEST(FieldTypeUtilsTest, FieldTypeToString) { ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::TIMESTAMP), "TIMESTAMP"); ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::DECIMAL), "DECIMAL"); ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::DATE), "DATE"); + ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::TIME), "TIME"); ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::ARRAY), "ARRAY"); ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::MAP), "MAP"); ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::STRUCT), "STRUCT"); diff --git a/src/paimon/common/utils/fields_comparator.cpp b/src/paimon/common/utils/fields_comparator.cpp index 2fee9aa15..c78bd17ae 100644 --- a/src/paimon/common/utils/fields_comparator.cpp +++ b/src/paimon/common/utils/fields_comparator.cpp @@ -114,6 +114,7 @@ Result FieldsComparator::CompareField( }); case arrow::Type::type::INT32: + case arrow::Type::type::TIME32: return FieldComparatorFunc( [field_idx](const InternalRow& lhs, const InternalRow& rhs) -> int32_t { int32_t lvalue = lhs.GetInt(field_idx); diff --git a/src/paimon/common/utils/fields_comparator_test.cpp b/src/paimon/common/utils/fields_comparator_test.cpp index 4373c1a76..5ef5c412b 100644 --- a/src/paimon/common/utils/fields_comparator_test.cpp +++ b/src/paimon/common/utils/fields_comparator_test.cpp @@ -242,6 +242,14 @@ TEST_F(FieldsComparatorTest, TestSimple) { pool.get()); CheckResult(row1, row2, {arrow::decimal128(38, 10), arrow::date32()}); } + { + BinaryRow row1 = BinaryRowGenerator::GenerateRow( + {static_cast(45296123), static_cast(123)}, pool.get()); + BinaryRow row2 = BinaryRowGenerator::GenerateRow( + {static_cast(45296123), static_cast(86399999)}, pool.get()); + CheckResult(row1, row2, + {arrow::time32(arrow::TimeUnit::MILLI), arrow::time32(arrow::TimeUnit::MILLI)}); + } } TEST_F(FieldsComparatorTest, TestFloatingPointOrder) { diff --git a/src/paimon/common/utils/string_utils.cpp b/src/paimon/common/utils/string_utils.cpp index 2e3565153..f7054ab24 100644 --- a/src/paimon/common/utils/string_utils.cpp +++ b/src/paimon/common/utils/string_utils.cpp @@ -24,6 +24,9 @@ #include #include #include +#include +#include +#include #include #include "fmt/format.h" @@ -244,6 +247,143 @@ Result StringUtils::StringToDate(const std::string& str) { return time / SECONDS_PER_DAY; } +namespace { +constexpr int32_t kMillisPerSecond = 1000; +constexpr int32_t kMillisPerMinute = 60 * kMillisPerSecond; +constexpr int32_t kMillisPerHour = 60 * kMillisPerMinute; +constexpr int32_t kMillisPerDay = 24 * kMillisPerHour; + +// Removes the characters Java `String#trim` removes from both ends. +std::string_view TrimView(std::string_view str) { + while (!str.empty() && IsTrimCharacter(static_cast(str.front()))) { + str.remove_prefix(1); + } + while (!str.empty() && IsTrimCharacter(static_cast(str.back()))) { + str.remove_suffix(1); + } + return str; +} + +bool IsDigit(char c) { + return c >= '0' && c <= '9'; +} + +// Parses a non-negative decimal component of a TIME string, surrounding whitespace allowed, like +// Java `DateTimeUtils#isInteger` followed by `Integer#parseInt`. +std::optional ParseTimeComponent(std::string_view component) { + component = TrimView(component); + if (component.empty()) { + return std::nullopt; + } + int64_t value = 0; + for (char c : component) { + if (!IsDigit(c)) { + return std::nullopt; + } + value = value * 10 + (c - '0'); + if (value > std::numeric_limits::max()) { + return std::nullopt; + } + } + return static_cast(value); +} + +// Rounds the fractional digits of a second half up to milliseconds, surrounding whitespace +// allowed, like Java `DateTimeUtils#parseFraction`. An empty fraction is 0. +std::optional ParseTimeFraction(std::string_view fraction) { + fraction = TrimView(fraction); + if (!std::all_of(fraction.begin(), fraction.end(), IsDigit)) { + return std::nullopt; + } + int32_t result = 0; + int32_t multiplier = 100; + for (size_t i = 0; i < fraction.size() && multiplier > 0; ++i, multiplier /= 10) { + result += multiplier * (fraction[i] - '0'); + } + if (fraction.size() > 3 && fraction[3] >= '5') { + ++result; + } + return result; +} + +// Parses the `hh[:mm]` of a TIME zone offset into milliseconds, like the offset part of Java +// `DateTimeUtils#parseTime`, but requiring hh <= 23 and mm <= 59. +std::optional ParseTimeOffset(std::string_view offset) { + size_t colon = offset.find(':'); + std::optional hour; + std::optional minute = 0; + if (colon == std::string_view::npos) { + hour = ParseTimeComponent(offset); + } else { + hour = ParseTimeComponent(offset.substr(0, colon)); + minute = ParseTimeComponent(offset.substr(colon + 1)); + } + if (!hour || !minute || hour.value() > 23 || minute.value() > 59) { + return std::nullopt; + } + return hour.value() * kMillisPerHour + minute.value() * kMillisPerMinute; +} +} // namespace + +Result StringUtils::StringToTime(const std::string& str) { + auto invalid = [&str]() { + return Status::Invalid( + fmt::format("failed to convert string '{}' to time, expected milliseconds of the day " + "or H[H][:m[m][:s[s][.f...]]] with an optional [+|-]hh[:mm] offset, " + "within [00:00:00, 24:00:00)", + str)); + }; + std::optional millis = StringToValue(str); + if (!millis) { + std::string_view view(str); + // Like Java `DateTimeUtils#parseTime`, a '-' offset is searched before a '+' one. + int32_t offset_sign = -1; + size_t sign_pos = view.find('-'); + if (sign_pos == std::string_view::npos) { + offset_sign = 1; + sign_pos = view.find('+'); + } + std::optional offset = 0; + if (sign_pos != std::string_view::npos) { + offset = ParseTimeOffset(view.substr(sign_pos + 1)); + view = view.substr(0, sign_pos); + } + size_t colon1 = view.find(':'); + std::optional hour; + std::optional minute = 0; + std::optional second = 0; + std::optional milli = 0; + if (colon1 == std::string_view::npos) { + hour = ParseTimeComponent(view); + } else { + hour = ParseTimeComponent(view.substr(0, colon1)); + size_t colon2 = view.find(':', colon1 + 1); + if (colon2 == std::string_view::npos) { + minute = ParseTimeComponent(view.substr(colon1 + 1)); + } else { + minute = ParseTimeComponent(view.substr(colon1 + 1, colon2 - colon1 - 1)); + size_t dot = view.find('.', colon2 + 1); + if (dot == std::string_view::npos) { + second = ParseTimeComponent(view.substr(colon2 + 1)); + } else { + second = ParseTimeComponent(view.substr(colon2 + 1, dot - colon2 - 1)); + milli = ParseTimeFraction(view.substr(dot + 1)); + } + } + } + if (!offset || !hour || !minute || !second || !milli || hour.value() > 23 || + minute.value() > 59 || second.value() > 59) { + return invalid(); + } + millis = hour.value() * kMillisPerHour + minute.value() * kMillisPerMinute + + second.value() * kMillisPerSecond + milli.value() + offset_sign * offset.value(); + } + if (millis.value() < 0 || millis.value() >= kMillisPerDay) { + return invalid(); + } + return millis.value(); +} + /// Parses a timestamp string into unix milliseconds. /// Supported formats: "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm:ss.SSS". /// Uses the default local time zone, consistent with Java Paimon behavior. diff --git a/src/paimon/common/utils/string_utils.h b/src/paimon/common/utils/string_utils.h index 91f1427c1..deafcead1 100644 --- a/src/paimon/common/utils/string_utils.h +++ b/src/paimon/common/utils/string_utils.h @@ -145,6 +145,16 @@ class PAIMON_EXPORT StringUtils { static Result StringToDate(const std::string& str); + /// Parses a TIME string into milliseconds of the day like Java `BinaryStringUtils#toTime`: + /// an integer number of milliseconds, or `H[H][:m[m][:s[s][.f...]]]` with the fraction + /// rounded half up to milliseconds and an optional `[+|-]hh[:mm]` offset added to the time. + /// + /// Unlike Java, which keeps whatever the arithmetic gives, an hour above 23, a minute or + /// second above 59, an offset above 23:59, a non-digit in the fraction and a result outside + /// [0, 86400000) are rejected, while an `hh:mm` offset after `H` or `H:m` (`12:34+01:00`), + /// which Java rejects, is accepted. + static Result StringToTime(const std::string& str); + static Result StringToTimestampMillis(const std::string& str); template diff --git a/src/paimon/common/utils/string_utils_test.cpp b/src/paimon/common/utils/string_utils_test.cpp index 89b0334f1..f554e1374 100644 --- a/src/paimon/common/utils/string_utils_test.cpp +++ b/src/paimon/common/utils/string_utils_test.cpp @@ -474,6 +474,83 @@ TEST_F(StringUtilsTest, TestStringToDate) { ASSERT_NOK(StringUtils::StringToDate("2024-00-01")); } +TEST_F(StringUtilsTest, TestStringToTime) { + ASSERT_OK_AND_ASSIGN(int32_t time, StringUtils::StringToTime("0")); + ASSERT_EQ(time, 0); + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("86399999")); + ASSERT_EQ(time, 86399999); + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("00:00:00")); + ASSERT_EQ(time, 0); + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("00:00:00.123")); + ASSERT_EQ(time, 123); + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("23:59:59.999")); + ASSERT_EQ(time, 86399999); + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("01:02:03")); + ASSERT_EQ(time, 3723000); + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("01:02")); + ASSERT_EQ(time, 3720000); + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("1")); + ASSERT_EQ(time, 1); + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("1:2:3.4")); + ASSERT_EQ(time, 3723400); + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("12:34:56.123456789")); + ASSERT_EQ(time, 45296123); + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("12:34:56.1235")); + ASSERT_EQ(time, 45296124); + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("12:34:56. 5 ")); + ASSERT_EQ(time, 45296500); + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("12:00:00.")); + ASSERT_EQ(time, 43200000); + + ASSERT_NOK_WITH_MSG(StringUtils::StringToTime(""), + "failed to convert string '' to time, expected milliseconds of the day or " + "H[H][:m[m][:s[s][.f...]]]"); + ASSERT_NOK(StringUtils::StringToTime("-1")); + ASSERT_NOK(StringUtils::StringToTime("86400000")); + ASSERT_NOK(StringUtils::StringToTime("24:00:00")); + ASSERT_NOK(StringUtils::StringToTime("12:60:00")); + ASSERT_NOK(StringUtils::StringToTime("12:00:60")); + ASSERT_NOK(StringUtils::StringToTime("12:xx:00")); + ASSERT_NOK(StringUtils::StringToTime("12:00:00.1a")); + ASSERT_NOK(StringUtils::StringToTime("23:59:59.9999")); +} + +TEST_F(StringUtilsTest, TestStringToTimeWithOffset) { + ASSERT_OK_AND_ASSIGN(int32_t time, StringUtils::StringToTime("12:34:56+01:00")); + ASSERT_EQ(time, 48896000); // 13:34:56 + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("12:34:56-01:00")); + ASSERT_EQ(time, 41696000); // 11:34:56 + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("12:34:56+01:30")); + ASSERT_EQ(time, 50696000); // 14:04:56 + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("12:34:56+01")); + ASSERT_EQ(time, 48896000); + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("12:34:56.123+01:00")); + ASSERT_EQ(time, 48896123); + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("12:34:56 + 01:00")); + ASSERT_EQ(time, 48896000); + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("12:34+01")); + ASSERT_EQ(time, 48840000); // 13:34 + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("12+01")); + ASSERT_EQ(time, 46800000); // 13:00 + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("00:00:00+00:00")); + ASSERT_EQ(time, 0); + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("01:00:00-01:00")); + ASSERT_EQ(time, 0); + ASSERT_OK_AND_ASSIGN(time, StringUtils::StringToTime("22:59:59.999+01:00")); + ASSERT_EQ(time, 86399999); + + ASSERT_NOK(StringUtils::StringToTime("23:30:00+01:00")); + ASSERT_NOK(StringUtils::StringToTime("00:30:00-01:00")); + ASSERT_NOK(StringUtils::StringToTime("12:34:56+")); + ASSERT_NOK(StringUtils::StringToTime("12:34:56+01:")); + ASSERT_NOK(StringUtils::StringToTime("12:34:56+1a")); + ASSERT_NOK(StringUtils::StringToTime("12:34:56+01:00:00")); + ASSERT_NOK(StringUtils::StringToTime("12:34:56+24:00")); + ASSERT_NOK(StringUtils::StringToTime("12:34:56+01:60")); + ASSERT_NOK(StringUtils::StringToTime("+01:00")); + ASSERT_NOK(StringUtils::StringToTime("-01:00")); +} + TEST_F(StringUtilsTest, TestStringToTimestampMillis) { TimezoneGuard tz_guard("Asia/Shanghai"); // "yyyy-MM-dd HH:mm:ss" format diff --git a/src/paimon/core/bucket/bucket_id_calculator.cpp b/src/paimon/core/bucket/bucket_id_calculator.cpp index 21a98b999..2b0ded0bb 100644 --- a/src/paimon/core/bucket/bucket_id_calculator.cpp +++ b/src/paimon/core/bucket/bucket_id_calculator.cpp @@ -140,6 +140,23 @@ static Result WriteBucketRow(int32_t col_id, }; return writer_func; } + case arrow::Type::type::TIME32: { + // Like Java, a TIME bucket key is hashed as its int32 milliseconds of the day, so + // another unit would hash to another bucket. + if (checked_cast(*field->type()).unit() != + arrow::TimeUnit::MILLI) { + return Status::Invalid( + fmt::format("Only millisecond TIME is supported as a bucket key: {}", + field->type()->ToString())); + } + const auto* typed_array = checked_cast(field.get()); + WriteFunction writer_func = [col_id, typed_array](int32_t row_id, + BinaryRowWriter* row_writer) { + CHECK_AND_SET_NULL(typed_array, row_writer, row_id, col_id); + row_writer->WriteInt(col_id, typed_array->Value(row_id)); + }; + return writer_func; + } case arrow::Type::type::STRING: { const auto* typed_array = checked_cast(field.get()); WriteFunction writer_func = [col_id, typed_array](int32_t row_id, diff --git a/src/paimon/core/bucket/bucket_id_calculator_test.cpp b/src/paimon/core/bucket/bucket_id_calculator_test.cpp index 2c3061d1c..c6c59c566 100644 --- a/src/paimon/core/bucket/bucket_id_calculator_test.cpp +++ b/src/paimon/core/bucket/bucket_id_calculator_test.cpp @@ -468,4 +468,24 @@ TEST_F(BucketIdCalculatorTest, TestCreateWithHiveBucketFunction) { } } +TEST_F(BucketIdCalculatorTest, TestTimeBucketKey) { + auto int_schema = arrow::schema(arrow::FieldVector({arrow::field("b0", arrow::int32())})); + auto time_schema = arrow::schema( + arrow::FieldVector({arrow::field("b0", arrow::time32(arrow::TimeUnit::MILLI))})); + std::string data_str = "[[0], [123], [45296123], [86399999], [null]]"; + ASSERT_OK_AND_ASSIGN( + std::vector int_buckets, + CalculateBucketIds(/*is_pk_table=*/true, /*num_buckets=*/7, int_schema, data_str)); + ASSERT_OK_AND_ASSIGN( + std::vector time_buckets, + CalculateBucketIds(/*is_pk_table=*/true, /*num_buckets=*/7, time_schema, data_str)); + ASSERT_EQ(time_buckets, int_buckets); + + auto second_schema = arrow::schema( + arrow::FieldVector({arrow::field("b0", arrow::time32(arrow::TimeUnit::SECOND))})); + ASSERT_NOK_WITH_MSG( + CalculateBucketIds(/*is_pk_table=*/true, /*num_buckets=*/7, second_schema, "[[1]]"), + "Only millisecond TIME is supported as a bucket key"); +} + } // namespace paimon::test diff --git a/src/paimon/core/bucket/bucket_select_converter.cpp b/src/paimon/core/bucket/bucket_select_converter.cpp index 5ae2b243e..1b344fb38 100644 --- a/src/paimon/core/bucket/bucket_select_converter.cpp +++ b/src/paimon/core/bucket/bucket_select_converter.cpp @@ -178,6 +178,7 @@ Status BucketSelectConverter::WriteLiteralToRow(int32_t pos, const Literal& lite break; case FieldType::INT: case FieldType::DATE: + case FieldType::TIME: writer->WriteInt(pos, literal.GetValue()); break; case FieldType::BIGINT: diff --git a/src/paimon/core/bucket/bucket_select_converter_test.cpp b/src/paimon/core/bucket/bucket_select_converter_test.cpp index 85a3f3e29..bc07ebe3d 100644 --- a/src/paimon/core/bucket/bucket_select_converter_test.cpp +++ b/src/paimon/core/bucket/bucket_select_converter_test.cpp @@ -99,6 +99,9 @@ TEST_F(BucketSelectConverterTest, PrimitiveKeyTypes) { {static_cast(1234)}); AssertDefaultBucket(FieldType::INT, Literal(static_cast(42)), arrow::int32(), {static_cast(42)}, 10); + AssertDefaultBucket(FieldType::TIME, Literal(FieldType::TIME, 45296123), + arrow::time32(arrow::TimeUnit::MILLI), {static_cast(45296123)}, + 10); AssertDefaultBucket(FieldType::BIGINT, Literal(static_cast(123456789L)), arrow::int64(), {static_cast(123456789L)}, 16); AssertDefaultBucket(FieldType::FLOAT, Literal(1.25F), arrow::float32(), {1.25F}); diff --git a/src/paimon/core/bucket/hive_bucket_function.cpp b/src/paimon/core/bucket/hive_bucket_function.cpp index e87c292d9..5d6816f5e 100644 --- a/src/paimon/core/bucket/hive_bucket_function.cpp +++ b/src/paimon/core/bucket/hive_bucket_function.cpp @@ -60,6 +60,7 @@ Result> HiveBucketFunction::Create( case FieldType::BINARY: case FieldType::DECIMAL: case FieldType::DATE: + case FieldType::TIME: break; default: return Status::Invalid(fmt::format("Unsupported type as Hive bucket key type: {}", @@ -96,6 +97,7 @@ uint32_t HiveBucketFunction::ComputeHash(const BinaryRow& row, int32_t field_ind return HiveHasher::HashInt(static_cast(row.GetShort(field_index))); case FieldType::INT: case FieldType::DATE: + case FieldType::TIME: return HiveHasher::HashInt(static_cast(row.GetInt(field_index))); case FieldType::BIGINT: return HiveHasher::HashLong(static_cast(row.GetLong(field_index))); diff --git a/src/paimon/core/bucket/hive_bucket_function_test.cpp b/src/paimon/core/bucket/hive_bucket_function_test.cpp index 43d36da58..6493edecd 100644 --- a/src/paimon/core/bucket/hive_bucket_function_test.cpp +++ b/src/paimon/core/bucket/hive_bucket_function_test.cpp @@ -440,8 +440,8 @@ TEST_F(HiveBucketFunctionTest, TestCompatibleWithJava) { ASSERT_EQ(97, func->Bucket(CreateStringRow("a"), num_buckets)); } - // Case 7: Single DATE field (same as INT) - // Java: hashInt(daysSinceEpoch) + // Case 7: Single DATE or TIME field (same as INT) + // Java: hashInt(daysSinceEpoch) or hashInt(millisOfDay) { std::vector field_infos = {HiveFieldInfo(FieldType::DATE)}; ASSERT_OK_AND_ASSIGN(auto func, HiveBucketFunction::Create(field_infos)); @@ -450,6 +450,13 @@ TEST_F(HiveBucketFunctionTest, TestCompatibleWithJava) { // date = 2000 (days), hashInt(2000) = 2000, 2000 % 128 = 80 ASSERT_EQ(80, func->Bucket(CreateIntRow(2000), num_buckets)); } + { + std::vector field_infos = {HiveFieldInfo(FieldType::TIME)}; + ASSERT_OK_AND_ASSIGN(auto func, HiveBucketFunction::Create(field_infos)); + + // time = 45296123 (12:34:56.123), hashInt(45296123) = 45296123, 45296123 % 128 = 123 + ASSERT_EQ(123, func->Bucket(CreateIntRow(45296123), num_buckets)); + } // Case 8: Multi-field row (INT, STRING, BINARY, DECIMAL) // This is the same as TestHiveBucketFunction but with num_buckets=128 diff --git a/src/paimon/core/casting/cast_executor_factory.cpp b/src/paimon/core/casting/cast_executor_factory.cpp index 8e56ca04d..b74ea20a6 100644 --- a/src/paimon/core/casting/cast_executor_factory.cpp +++ b/src/paimon/core/casting/cast_executor_factory.cpp @@ -19,8 +19,14 @@ #include "paimon/core/casting/cast_executor_factory.h" +#include #include +#include "arrow/type.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/types/data_type.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/field_type_utils.h" #include "paimon/core/casting/binary_to_blob_cast_executor.h" #include "paimon/core/casting/binary_to_string_cast_executor.h" #include "paimon/core/casting/boolean_to_decimal_cast_executor.h" @@ -40,10 +46,14 @@ #include "paimon/core/casting/string_to_date_cast_executor.h" #include "paimon/core/casting/string_to_decimal_cast_executor.h" #include "paimon/core/casting/string_to_numeric_primitive_cast_executor.h" +#include "paimon/core/casting/string_to_time_cast_executor.h" #include "paimon/core/casting/string_to_timestamp_cast_executor.h" +#include "paimon/core/casting/time_to_string_cast_executor.h" +#include "paimon/core/casting/time_to_timestamp_cast_executor.h" #include "paimon/core/casting/timestamp_to_date_cast_executor.h" #include "paimon/core/casting/timestamp_to_numeric_primitive_cast_executor.h" #include "paimon/core/casting/timestamp_to_string_cast_executor.h" +#include "paimon/core/casting/timestamp_to_time_cast_executor.h" #include "paimon/core/casting/timestamp_to_timestamp_cast_executor.h" #include "paimon/defs.h" @@ -70,6 +80,23 @@ std::shared_ptr CastExecutorFactory::GetCastExecutor(const FieldTy return src_iter->second; } +Result> CastExecutorFactory::GetCastExecutor( + const DataField& src_field, const FieldType& target) const { + PAIMON_ASSIGN_OR_RAISE(FieldType src, + FieldTypeUtils::ConvertToFieldType(src_field.Type()->id())); + if (src == FieldType::TIME && target == FieldType::STRING) { + PAIMON_ASSIGN_OR_RAISE(int32_t precision, + DataType::GetTimePrecision(*src_field.ArrowField())); + return std::shared_ptr(std::make_shared(precision)); + } + if (src == FieldType::TIMESTAMP && target == FieldType::TIME) { + const auto& src_ts_type = checked_cast(*src_field.Type()); + return std::shared_ptr(std::make_shared( + /*src_with_timezone=*/!src_ts_type.timezone().empty())); + } + return GetCastExecutor(src, target); +} + CastExecutorFactory::CastExecutorFactory() { REGISTER_CAST_EXECUTOR(TINYINT, BOOLEAN, BooleanToNumericCastExecutor); REGISTER_CAST_EXECUTOR(SMALLINT, BOOLEAN, BooleanToNumericCastExecutor); @@ -169,6 +196,10 @@ CastExecutorFactory::CastExecutorFactory() { REGISTER_CAST_EXECUTOR(BIGINT, TIMESTAMP, TimestampToNumericPrimitiveCastExecutor); REGISTER_CAST_EXECUTOR(TIMESTAMP, STRING, StringToTimestampCastExecutor); + REGISTER_CAST_EXECUTOR(TIME, STRING, StringToTimeCastExecutor); + REGISTER_CAST_EXECUTOR(STRING, TIME, TimeToStringCastExecutor); + REGISTER_CAST_EXECUTOR(TIME, TIMESTAMP, TimestampToTimeCastExecutor); + REGISTER_CAST_EXECUTOR(TIMESTAMP, TIME, TimeToTimestampCastExecutor); REGISTER_CAST_EXECUTOR(TINYINT, DECIMAL, DecimalToNumericPrimitiveCastExecutor); REGISTER_CAST_EXECUTOR(SMALLINT, DECIMAL, DecimalToNumericPrimitiveCastExecutor); diff --git a/src/paimon/core/casting/cast_executor_factory.h b/src/paimon/core/casting/cast_executor_factory.h index 50f70b0d3..351fdd99c 100644 --- a/src/paimon/core/casting/cast_executor_factory.h +++ b/src/paimon/core/casting/cast_executor_factory.h @@ -23,8 +23,10 @@ #include "paimon/core/casting/cast_executor.h" #include "paimon/defs.h" +#include "paimon/result.h" namespace paimon { +class DataField; enum class FieldType; class CastExecutorFactory { @@ -34,6 +36,13 @@ class CastExecutorFactory { std::shared_ptr GetCastExecutor(const FieldType& src, const FieldType& target) const; + /// Returns the executor casting `src_field` to `target`, or nullptr if there is none. Like + /// Java `TimeToStringCastRule`, a TIME field is cast to STRING with the precision it declares. + /// Like Java `TimestampToTimeCastRule`, a TIMESTAMP_LTZ field is cast to TIME in the local + /// time zone, also for a literal. + Result> GetCastExecutor(const DataField& src_field, + const FieldType& target) const; + private: CastExecutorFactory(); diff --git a/src/paimon/core/casting/cast_executor_factory_test.cpp b/src/paimon/core/casting/cast_executor_factory_test.cpp index 9ab7d1688..a218863a9 100644 --- a/src/paimon/core/casting/cast_executor_factory_test.cpp +++ b/src/paimon/core/casting/cast_executor_factory_test.cpp @@ -19,7 +19,16 @@ #include "paimon/core/casting/cast_executor_factory.h" +#include +#include +#include +#include + +#include "arrow/type.h" +#include "arrow/util/key_value_metadata.h" #include "gtest/gtest.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/types/data_type.h" #include "paimon/core/casting/binary_to_blob_cast_executor.h" #include "paimon/core/casting/binary_to_string_cast_executor.h" #include "paimon/core/casting/boolean_to_decimal_cast_executor.h" @@ -39,12 +48,20 @@ #include "paimon/core/casting/string_to_date_cast_executor.h" #include "paimon/core/casting/string_to_decimal_cast_executor.h" #include "paimon/core/casting/string_to_numeric_primitive_cast_executor.h" +#include "paimon/core/casting/string_to_time_cast_executor.h" #include "paimon/core/casting/string_to_timestamp_cast_executor.h" +#include "paimon/core/casting/time_to_string_cast_executor.h" +#include "paimon/core/casting/time_to_timestamp_cast_executor.h" #include "paimon/core/casting/timestamp_to_date_cast_executor.h" #include "paimon/core/casting/timestamp_to_numeric_primitive_cast_executor.h" #include "paimon/core/casting/timestamp_to_string_cast_executor.h" +#include "paimon/core/casting/timestamp_to_time_cast_executor.h" #include "paimon/core/casting/timestamp_to_timestamp_cast_executor.h" +#include "paimon/data/timestamp.h" #include "paimon/defs.h" +#include "paimon/predicate/literal.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/testing/utils/timezone_guard.h" namespace paimon::test { TEST(CastExecutorFactoryTest, TestRegister) { @@ -241,4 +258,55 @@ TEST(CastExecutorFactoryTest, TestRegister) { ASSERT_FALSE(cast_executor); } } + +TEST(CastExecutorFactoryTest, TestTimeCastExecutors) { + auto* factory = CastExecutorFactory::GetCastExecutorFactory(); + ASSERT_TRUE(std::dynamic_pointer_cast( + factory->GetCastExecutor(FieldType::STRING, FieldType::TIME))); + ASSERT_TRUE(std::dynamic_pointer_cast( + factory->GetCastExecutor(FieldType::TIME, FieldType::STRING))); + ASSERT_TRUE(std::dynamic_pointer_cast( + factory->GetCastExecutor(FieldType::TIMESTAMP, FieldType::TIME))); + ASSERT_TRUE(std::dynamic_pointer_cast( + factory->GetCastExecutor(FieldType::TIME, FieldType::TIMESTAMP))); + ASSERT_FALSE(factory->GetCastExecutor(FieldType::INT, FieldType::TIME)); + ASSERT_FALSE(factory->GetCastExecutor(FieldType::TIME, FieldType::DATE)); + + auto time_field = [](const std::string& precision) { + return DataField( + 0, arrow::field("t", arrow::time32(arrow::TimeUnit::MILLI), + /*nullable=*/true, + arrow::key_value_metadata({DataType::kTimePrecision}, {precision}))); + }; + std::vector> cases = {{"0", "01:02:03"}, + {"3", "01:02:03.0"}}; + for (const auto& [precision, expected] : cases) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, + factory->GetCastExecutor(time_field(precision), FieldType::STRING)); + ASSERT_TRUE(executor); + ASSERT_OK_AND_ASSIGN(Literal literal, + executor->Cast(Literal(FieldType::TIME, 3723000), arrow::utf8())); + ASSERT_EQ(expected, literal.GetValue()); + } + ASSERT_NOK(factory->GetCastExecutor(time_field("x"), FieldType::STRING)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, + factory->GetCastExecutor(DataField(1, arrow::field("i", arrow::int32())), + FieldType::STRING)); + ASSERT_TRUE(std::dynamic_pointer_cast(executor)); + + TimezoneGuard tz_guard("Asia/Shanghai"); + std::vector, int32_t>> ts_cases = { + {arrow::timestamp(arrow::TimeUnit::MILLI), 0}, + {arrow::timestamp(arrow::TimeUnit::MILLI, "Asia/Shanghai"), 28800000}}; + for (const auto& [type, expected] : ts_cases) { + ASSERT_OK_AND_ASSIGN( + executor, + factory->GetCastExecutor(DataField(2, arrow::field("ts", type)), FieldType::TIME)); + ASSERT_TRUE(std::dynamic_pointer_cast(executor)); + ASSERT_OK_AND_ASSIGN( + Literal literal, + executor->Cast(Literal(Timestamp(0, 0)), arrow::time32(arrow::TimeUnit::MILLI))); + ASSERT_EQ(Literal(FieldType::TIME, expected), literal) << type->ToString(); + } +} } // namespace paimon::test diff --git a/src/paimon/core/casting/cast_executor_test.cpp b/src/paimon/core/casting/cast_executor_test.cpp index 9035eb295..aeef4e0ed 100644 --- a/src/paimon/core/casting/cast_executor_test.cpp +++ b/src/paimon/core/casting/cast_executor_test.cpp @@ -57,12 +57,17 @@ #include "paimon/core/casting/string_to_date_cast_executor.h" #include "paimon/core/casting/string_to_decimal_cast_executor.h" #include "paimon/core/casting/string_to_numeric_primitive_cast_executor.h" +#include "paimon/core/casting/string_to_time_cast_executor.h" #include "paimon/core/casting/string_to_timestamp_cast_executor.h" +#include "paimon/core/casting/time_to_string_cast_executor.h" +#include "paimon/core/casting/time_to_timestamp_cast_executor.h" #include "paimon/core/casting/timestamp_to_date_cast_executor.h" #include "paimon/core/casting/timestamp_to_numeric_primitive_cast_executor.h" #include "paimon/core/casting/timestamp_to_string_cast_executor.h" +#include "paimon/core/casting/timestamp_to_time_cast_executor.h" #include "paimon/core/casting/timestamp_to_timestamp_cast_executor.h" #include "paimon/data/decimal.h" +#include "paimon/data/timestamp.h" #include "paimon/defs.h" #include "paimon/memory/memory_pool.h" #include "paimon/status.h" @@ -147,7 +152,7 @@ class CastExecutorTest : public ::testing::Test { if constexpr (std::is_same_v) { return Literal(type, data.data(), data.size()); } else if constexpr (std::is_same_v) { - if (type == FieldType::DATE) { + if (type == FieldType::DATE || type == FieldType::TIME) { return Literal(type, data); } else { return Literal(data); @@ -1632,6 +1637,136 @@ TEST_F(CastExecutorTest, TestStringToDateCastExecutorCastArray) { // invalid string refer to TestStringToDateCastExecutorCastLiteral } +TEST_F(CastExecutorTest, TestStringToTimeCastExecutorCastLiteral) { + auto cast_executor = std::make_shared(); + auto time_type = arrow::time32(arrow::TimeUnit::MILLI); + std::vector src_data = {"00:00:00", "00:00:00.123", "23:59:59.999", "1:2:3", + "45296123"}; + std::vector target_data = {0, 123, 86399999, 3723000, 45296123}; + CheckLiteralResult(cast_executor, FieldType::STRING, src_data, time_type, + target_data); + auto msg = CheckLiteralInvalidResult(cast_executor, FieldType::STRING, "24:00:00", + time_type); + ASSERT_TRUE(msg.find("failed to convert string '24:00:00' to time") != std::string::npos) + << msg; + msg = CheckLiteralInvalidResult(cast_executor, FieldType::STRING, "00:00:01", + arrow::time32(arrow::TimeUnit::SECOND)); + ASSERT_TRUE(msg.find("Only millisecond TIME is supported") != std::string::npos) << msg; +} + +TEST_F(CastExecutorTest, TestStringToTimeCastExecutorCastArray) { + auto cast_executor = std::make_shared(); + auto time_type = arrow::time32(arrow::TimeUnit::MILLI); + CheckArrayResult(cast_executor, arrow::utf8(), time_type, + R"(["00:00:00", "00:00:00.123", "23:59:59.999", "1:2:3", null])", + R"([0, 123, 86399999, 3723000, null])"); + auto msg = CheckArrayInvalidResult(cast_executor, arrow::utf8(), time_type, R"(["24:00:00"])"); + ASSERT_TRUE(msg.find("failed to convert string '24:00:00' to time") != std::string::npos) + << msg; + msg = CheckArrayInvalidResult(cast_executor, arrow::utf8(), + arrow::time32(arrow::TimeUnit::SECOND), R"(["00:00:01"])"); + ASSERT_TRUE(msg.find("Only millisecond TIME is supported") != std::string::npos) << msg; +} + +TEST_F(CastExecutorTest, TestTimeToStringCastExecutorCastLiteral) { + std::vector src_data = {0, 3723000, 45296120, 86399999}; + CheckLiteralResult( + std::make_shared(), FieldType::TIME, src_data, arrow::utf8(), + std::vector({"00:00:00", "01:02:03", "12:34:56", "23:59:59"})); + CheckLiteralResult( + std::make_shared(/*precision=*/3), FieldType::TIME, src_data, + arrow::utf8(), + std::vector({"00:00:00.0", "01:02:03.0", "12:34:56.12", "23:59:59.999"})); +} + +TEST_F(CastExecutorTest, TestTimeToStringCastExecutorCastArray) { + auto time_type = arrow::time32(arrow::TimeUnit::MILLI); + CheckArrayResult(std::make_shared(), time_type, arrow::utf8(), + R"([0, 45296123, null])", R"(["00:00:00", "12:34:56", null])"); + CheckArrayResult(std::make_shared(/*precision=*/9), time_type, + arrow::utf8(), R"([0, 45296123, 86399999, null])", + R"(["00:00:00.0", "12:34:56.123", "23:59:59.999", null])"); + auto msg = + CheckArrayInvalidResult(std::make_shared(), + arrow::time32(arrow::TimeUnit::SECOND), arrow::utf8(), R"([1])"); + ASSERT_TRUE(msg.find("Only millisecond TIME is supported") != std::string::npos) << msg; +} + +TEST_F(CastExecutorTest, TestTimestampToTimeCastExecutorCastArray) { + auto cast_executor = std::make_shared(); + auto time_type = arrow::time32(arrow::TimeUnit::MILLI); + CheckArrayResult(cast_executor, arrow::timestamp(arrow::TimeUnit::SECOND), time_type, + R"([1686227696, 0, -1, null])", R"([45296000, 0, 86399000, null])"); + CheckArrayResult(cast_executor, arrow::timestamp(arrow::TimeUnit::MILLI), time_type, + R"([1686227696123, 0, -1, null])", R"([45296123, 0, 86399999, null])"); + CheckArrayResult(cast_executor, arrow::timestamp(arrow::TimeUnit::MICRO), time_type, + R"([1686227696123456, -1, null])", R"([45296123, 86399999, null])"); + CheckArrayResult(cast_executor, arrow::timestamp(arrow::TimeUnit::NANO), time_type, + R"([1686227696123456789, -1, null])", R"([45296123, 86399999, null])"); + CheckArrayResult(cast_executor, arrow::timestamp(arrow::TimeUnit::MILLI, "Asia/Shanghai"), + time_type, R"([0, 1686227696123, null])", R"([28800000, 74096123, null])"); +} + +TEST_F(CastExecutorTest, TestTimestampToTimeCastExecutorCastLiteral) { + auto time_type = arrow::time32(arrow::TimeUnit::MILLI); + CheckLiteralResult( + std::make_shared(), FieldType::TIMESTAMP, + {Timestamp(1686227696123, 456789), Timestamp(0, 0), Timestamp(-1, 999999)}, time_type, + {45296123, 0, 86399999}); + { + TimezoneGuard tz_guard("Asia/Shanghai"); + CheckLiteralResult( + std::make_shared(/*src_with_timezone=*/true), + FieldType::TIMESTAMP, {Timestamp(0, 0), Timestamp(1686227696123, 0)}, time_type, + {28800000, 74096123}); + } + auto msg = CheckLiteralInvalidResult(std::make_shared(), + FieldType::TIMESTAMP, Timestamp(0, 0), + arrow::time32(arrow::TimeUnit::SECOND)); + ASSERT_TRUE(msg.find("Only millisecond TIME is supported") != std::string::npos) << msg; +} + +TEST_F(CastExecutorTest, TestTimeToTimestampCastExecutorCastArray) { + auto cast_executor = std::make_shared(); + auto time_type = arrow::time32(arrow::TimeUnit::MILLI); + CheckArrayResult(cast_executor, time_type, arrow::timestamp(arrow::TimeUnit::SECOND), + R"([0, 45296123, null])", R"([0, 45296, null])"); + CheckArrayResult(cast_executor, time_type, arrow::timestamp(arrow::TimeUnit::MILLI), + R"([0, 45296123, 86399999, null])", R"([0, 45296123, 86399999, null])"); + CheckArrayResult(cast_executor, time_type, arrow::timestamp(arrow::TimeUnit::MICRO), + R"([45296123, null])", R"([45296123000, null])"); + CheckArrayResult(cast_executor, time_type, arrow::timestamp(arrow::TimeUnit::NANO), + R"([45296123, null])", R"([45296123000000, null])"); + CheckArrayResult(cast_executor, time_type, + arrow::timestamp(arrow::TimeUnit::MILLI, "Asia/Shanghai"), + R"([28800000, 0, null])", R"([0, -28800000, null])"); + auto msg = CheckArrayInvalidResult(cast_executor, arrow::time32(arrow::TimeUnit::SECOND), + arrow::timestamp(arrow::TimeUnit::MILLI), R"([1])"); + ASSERT_TRUE(msg.find("Only millisecond TIME is supported") != std::string::npos) << msg; +} + +TEST_F(CastExecutorTest, TestTimeToTimestampCastExecutorCastLiteral) { + auto cast_executor = std::make_shared(); + std::vector src_data = {0, 45296123, 86399999}; + std::vector expected = {Timestamp(0, 0), Timestamp(45296123, 0), + Timestamp(86399999, 0)}; + CheckLiteralResult(cast_executor, FieldType::TIME, src_data, + arrow::timestamp(arrow::TimeUnit::MILLI), expected); + CheckLiteralResult(cast_executor, FieldType::TIME, src_data, + arrow::timestamp(arrow::TimeUnit::NANO), expected); + CheckLiteralResult( + cast_executor, FieldType::TIME, src_data, arrow::timestamp(arrow::TimeUnit::SECOND), + {Timestamp(0, 0), Timestamp(45296000, 0), Timestamp(86399000, 0)}); + { + TimezoneGuard tz_guard("Asia/Shanghai"); + CheckLiteralResult( + cast_executor, FieldType::TIME, {28800000, 0}, + arrow::timestamp(arrow::TimeUnit::MILLI, "Asia/Shanghai"), + {Timestamp(0, 0), Timestamp(-28800000, 0)}); + } + ASSERT_NOK(cast_executor->Cast(Literal(FieldType::TIME, 0), arrow::utf8())); +} + TEST_F(CastExecutorTest, TestTimestampToStringCastExecutorCastArray) { auto cast_executor = std::make_shared(); { diff --git a/src/paimon/core/casting/casted_row.cpp b/src/paimon/core/casting/casted_row.cpp index bd8b42431..dcb4ba664 100644 --- a/src/paimon/core/casting/casted_row.cpp +++ b/src/paimon/core/casting/casted_row.cpp @@ -37,15 +37,19 @@ Result> CastedRow::Create( field_getters.reserve(src_fields.size()); std::vector src_types; src_types.reserve(src_fields.size()); + std::vector> target_types; + target_types.reserve(target_fields.size()); for (size_t i = 0; i < src_fields.size(); ++i) { const auto& type = src_fields[i].Type(); PAIMON_ASSIGN_OR_RAISE(InternalRow::FieldGetterFunc getter, InternalRow::CreateFieldGetter(i, type, /*use_view=*/false)); field_getters.push_back(std::move(getter)); src_types.push_back(type->id()); + target_types.push_back(target_fields[i].Type()); } - return std::unique_ptr( - new CastedRow(cast_executors, std::move(field_getters), std::move(src_types), row)); + return std::unique_ptr(new CastedRow(cast_executors, std::move(field_getters), + std::move(src_types), std::move(target_types), + row)); } } // namespace paimon diff --git a/src/paimon/core/casting/casted_row.h b/src/paimon/core/casting/casted_row.h index 677232d23..c5bda0fb2 100644 --- a/src/paimon/core/casting/casted_row.h +++ b/src/paimon/core/casting/casted_row.h @@ -38,7 +38,6 @@ #include "paimon/common/data/internal_row.h" #include "paimon/common/types/data_field.h" #include "paimon/common/types/row_kind.h" -#include "paimon/common/utils/date_time_utils.h" #include "paimon/core/casting/cast_executor.h" #include "paimon/data/decimal.h" #include "paimon/data/timestamp.h" @@ -137,7 +136,8 @@ class CastedRow : public InternalRow { } int32_t GetInt(int32_t pos) const override { - return GetValue(pos, arrow::int32()); + // A cast to TIME checks the unit of the target type. + return GetValue(pos, target_types_[pos]); } int32_t GetDate(int32_t pos) const override { @@ -176,13 +176,8 @@ class CastedRow : public InternalRow { } Timestamp GetTimestamp(int32_t pos, int32_t precision) const override { - // timestamp does not support casting - Result> ts_type = - DateTimeUtils::GetTypeFromPrecision(precision, /*with_timezone=*/false); - if (!ts_type.ok()) { - throw std::invalid_argument(ts_type.status().ToString()); - } - return GetValue(pos, ts_type.value()); + // The target type tells TIMESTAMP_LTZ apart from TIMESTAMP, which the precision does not. + return GetValue(pos, target_types_[pos]); } std::shared_ptr GetBinary(int32_t pos) const override { @@ -214,10 +209,13 @@ class CastedRow : public InternalRow { private: CastedRow(const std::vector>& cast_executors, std::vector&& field_getters, - std::vector&& src_types, const std::shared_ptr& row) + std::vector&& src_types, + std::vector>&& target_types, + const std::shared_ptr& row) : cast_executors_(cast_executors), field_getters_(std::move(field_getters)), src_types_(std::move(src_types)), + target_types_(std::move(target_types)), row_(row) { assert(row_); } @@ -226,6 +224,7 @@ class CastedRow : public InternalRow { std::vector> cast_executors_; std::vector field_getters_; std::vector src_types_; + std::vector> target_types_; std::shared_ptr row_; }; } // namespace paimon diff --git a/src/paimon/core/casting/casted_row_test.cpp b/src/paimon/core/casting/casted_row_test.cpp index 4525e7e13..ecf57ad38 100644 --- a/src/paimon/core/casting/casted_row_test.cpp +++ b/src/paimon/core/casting/casted_row_test.cpp @@ -22,15 +22,19 @@ #include "arrow/api.h" #include "arrow/array/array_nested.h" #include "arrow/ipc/json_simple.h" +#include "arrow/util/key_value_metadata.h" #include "gtest/gtest.h" #include "paimon/common/data/columnar/columnar_row.h" #include "paimon/common/data/internal_array.h" #include "paimon/common/data/internal_map.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/types/data_type.h" #include "paimon/common/types/row_kind.h" #include "paimon/core/utils/field_mapping.h" +#include "paimon/data/timestamp.h" #include "paimon/memory/memory_pool.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/testing/utils/timezone_guard.h" namespace paimon::test { TEST(CastedRowTest, TestSimpleWithNoCasting) { @@ -141,6 +145,66 @@ TEST(CastedRowTest, TestSimpleWithCasting) { ASSERT_EQ("casted row, inner row = ColumnarRow, row_id 0", casted_row->ToString()); } +TEST(CastedRowTest, TestTimeCasting) { + TimezoneGuard tz_guard("Asia/Shanghai"); + auto pool = GetDefaultPool(); + auto time_type = arrow::time32(arrow::TimeUnit::MILLI); + auto ts_type = arrow::timestamp(arrow::TimeUnit::MILLI); + auto ltz_type = arrow::timestamp(arrow::TimeUnit::MILLI, "Asia/Shanghai"); + std::vector fields = { + DataField(0, arrow::field("time_to_string", time_type, /*nullable=*/true, + arrow::key_value_metadata({DataType::kTimePrecision}, {"3"}))), + DataField(1, arrow::field("string_to_time", arrow::utf8())), + DataField(2, arrow::field("time_to_ts", time_type)), + DataField(3, arrow::field("time_to_ltz", time_type)), + DataField(4, arrow::field("ts_to_time", ts_type)), + DataField(5, arrow::field("ltz_to_time", ltz_type))}; + auto array = arrow::ipc::internal::json::ArrayFromJSON( + DataField::ConvertDataFieldsToArrowStructType(fields), + R"([[3723000, "12:34:56.123", 45296123, 28800000, 1686227696123, 0]])") + .ValueOrDie(); + auto struct_array = std::dynamic_pointer_cast(array); + ASSERT_TRUE(struct_array); + auto row = std::make_shared(struct_array->fields(), pool, /*row_id=*/0); + + std::vector target_fields = { + DataField(0, arrow::field("time_to_string", arrow::utf8())), + DataField(1, arrow::field("string_to_time", time_type)), + DataField(2, arrow::field("time_to_ts", ts_type)), + DataField(3, arrow::field("time_to_ltz", ltz_type)), + DataField(4, arrow::field("ts_to_time", time_type)), + DataField(5, arrow::field("ltz_to_time", time_type))}; + ASSERT_OK_AND_ASSIGN(auto cast_executors, + FieldMappingBuilder::CreateDataCastExecutors(target_fields, fields)); + ASSERT_OK_AND_ASSIGN(auto casted_row, CastedRow::Create(cast_executors, /*src_fields=*/fields, + /*target_fields=*/target_fields, row)); + ASSERT_EQ(casted_row->GetString(0).ToString(), "01:02:03.0"); + ASSERT_EQ(casted_row->GetInt(1), 45296123); + ASSERT_EQ(casted_row->GetTimestamp(2, /*precision=*/3), Timestamp(45296123, 0)); + ASSERT_EQ(casted_row->GetTimestamp(3, /*precision=*/3), Timestamp(0, 0)); + ASSERT_EQ(casted_row->GetInt(4), 45296123); + ASSERT_EQ(casted_row->GetInt(5), 28800000); +} + +TEST(CastedRowTest, TestCastToNonMillisecondTime) { + auto pool = GetDefaultPool(); + std::vector fields = {DataField(0, arrow::field("f0", arrow::utf8()))}; + auto array = arrow::ipc::internal::json::ArrayFromJSON( + DataField::ConvertDataFieldsToArrowStructType(fields), R"([["00:00:01"]])") + .ValueOrDie(); + auto struct_array = std::dynamic_pointer_cast(array); + ASSERT_TRUE(struct_array); + auto row = std::make_shared(struct_array->fields(), pool, /*row_id=*/0); + + std::vector target_fields = { + DataField(0, arrow::field("f0", arrow::time32(arrow::TimeUnit::SECOND)))}; + ASSERT_OK_AND_ASSIGN(auto cast_executors, + FieldMappingBuilder::CreateDataCastExecutors(target_fields, fields)); + ASSERT_OK_AND_ASSIGN(auto casted_row, CastedRow::Create(cast_executors, /*src_fields=*/fields, + /*target_fields=*/target_fields, row)); + ASSERT_THROW(casted_row->GetInt(0), std::invalid_argument); +} + TEST(CastedRowTest, TestNestedTypeWithCasting) { auto pool = GetDefaultPool(); std::vector fields = { diff --git a/src/paimon/core/casting/string_to_time_cast_executor.cpp b/src/paimon/core/casting/string_to_time_cast_executor.cpp new file mode 100644 index 000000000..6abb275a9 --- /dev/null +++ b/src/paimon/core/casting/string_to_time_cast_executor.cpp @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/casting/string_to_time_cast_executor.h" + +#include +#include +#include + +#include "arrow/array/array_binary.h" +#include "arrow/array/builder_primitive.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/defs.h" +#include "paimon/status.h" + +namespace arrow { +class MemoryPool; +class Array; +} // namespace arrow + +namespace paimon { +Result StringToTimeCastExecutor::Cast( + const Literal& literal, const std::shared_ptr& target_type) const { + assert(literal.GetType() == FieldType::STRING); + if (!target_type->Equals(*arrow::time32(arrow::TimeUnit::MILLI))) { + return Status::Invalid( + fmt::format("Only millisecond TIME is supported: {}", target_type->ToString())); + } + if (literal.IsNull()) { + return Literal(FieldType::TIME); + } + PAIMON_ASSIGN_OR_RAISE(int32_t time, + StringUtils::StringToTime(literal.GetValue())); + return Literal(FieldType::TIME, time); +} + +Result> StringToTimeCastExecutor::Cast( + const std::shared_ptr& array, const std::shared_ptr& target_type, + arrow::MemoryPool* pool) const { + if (!target_type->Equals(*arrow::time32(arrow::TimeUnit::MILLI))) { + return Status::Invalid( + fmt::format("Only millisecond TIME is supported: {}", target_type->ToString())); + } + auto* string_array = checked_cast(array.get()); + arrow::Time32Builder time_builder(target_type, pool); + PAIMON_RETURN_NOT_OK_FROM_ARROW(time_builder.Reserve(string_array->length())); + for (int64_t i = 0; i < string_array->length(); ++i) { + if (string_array->IsNull(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(time_builder.AppendNull()); + } else { + PAIMON_ASSIGN_OR_RAISE(int32_t time, + StringUtils::StringToTime(string_array->GetString(i))); + PAIMON_RETURN_NOT_OK_FROM_ARROW(time_builder.Append(time)); + } + } + std::shared_ptr casted_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(time_builder.Finish(&casted_array)); + return casted_array; +} + +} // namespace paimon diff --git a/src/paimon/core/casting/string_to_time_cast_executor.h b/src/paimon/core/casting/string_to_time_cast_executor.h new file mode 100644 index 000000000..587f6eaa5 --- /dev/null +++ b/src/paimon/core/casting/string_to_time_cast_executor.h @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "arrow/array/array_base.h" +#include "paimon/core/casting/cast_executor.h" +#include "paimon/predicate/literal.h" +#include "paimon/result.h" + +namespace arrow { +class DataType; +class MemoryPool; +} // namespace arrow + +namespace paimon { +/// Casts STRING to TIME like Java `StringToTimeCastRule`. +class StringToTimeCastExecutor : public CastExecutor { + public: + Result Cast(const Literal& literal, + const std::shared_ptr& target_type) const override; + + Result> Cast(const std::shared_ptr& array, + const std::shared_ptr& target_type, + arrow::MemoryPool* pool) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/casting/time_to_string_cast_executor.cpp b/src/paimon/core/casting/time_to_string_cast_executor.cpp new file mode 100644 index 000000000..204d17e8a --- /dev/null +++ b/src/paimon/core/casting/time_to_string_cast_executor.cpp @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/casting/time_to_string_cast_executor.h" + +#include +#include +#include + +#include "arrow/array/array_primitive.h" +#include "arrow/array/builder_binary.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/date_time_utils.h" +#include "paimon/defs.h" +#include "paimon/status.h" + +namespace arrow { +class MemoryPool; +class Array; +} // namespace arrow + +namespace paimon { +Result TimeToStringCastExecutor::Cast( + const Literal& literal, const std::shared_ptr& target_type) const { + assert(literal.GetType() == FieldType::TIME); + if (literal.IsNull()) { + return Literal(FieldType::STRING); + } + std::string time = DateTimeUtils::FormatTimeMillis(literal.GetValue(), precision_); + return Literal(FieldType::STRING, time.data(), time.size()); +} + +Result> TimeToStringCastExecutor::Cast( + const std::shared_ptr& array, const std::shared_ptr& target_type, + arrow::MemoryPool* pool) const { + if (!array->type()->Equals(*arrow::time32(arrow::TimeUnit::MILLI))) { + return Status::Invalid( + fmt::format("Only millisecond TIME is supported: {}", array->type()->ToString())); + } + const auto& time_array = checked_cast(*array); + arrow::StringBuilder string_builder(pool); + PAIMON_RETURN_NOT_OK_FROM_ARROW(string_builder.Reserve(time_array.length())); + for (int64_t i = 0; i < time_array.length(); ++i) { + if (time_array.IsNull(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(string_builder.AppendNull()); + } else { + PAIMON_RETURN_NOT_OK_FROM_ARROW(string_builder.Append( + DateTimeUtils::FormatTimeMillis(time_array.Value(i), precision_))); + } + } + std::shared_ptr casted_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(string_builder.Finish(&casted_array)); + return casted_array; +} + +} // namespace paimon diff --git a/src/paimon/core/casting/time_to_string_cast_executor.h b/src/paimon/core/casting/time_to_string_cast_executor.h new file mode 100644 index 000000000..8893d73f1 --- /dev/null +++ b/src/paimon/core/casting/time_to_string_cast_executor.h @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "arrow/array/array_base.h" +#include "paimon/core/casting/cast_executor.h" +#include "paimon/predicate/literal.h" +#include "paimon/result.h" + +namespace arrow { +class DataType; +class MemoryPool; +} // namespace arrow + +namespace paimon { +/// Casts TIME to STRING like Java `TimeToStringCastRule`. +class TimeToStringCastExecutor : public CastExecutor { + public: + /// @param precision The declared precision of the TIME source, the most fractional digits + /// written. + explicit TimeToStringCastExecutor(int32_t precision = 0) : precision_(precision) {} + + Result Cast(const Literal& literal, + const std::shared_ptr& target_type) const override; + + Result> Cast(const std::shared_ptr& array, + const std::shared_ptr& target_type, + arrow::MemoryPool* pool) const override; + + private: + int32_t precision_; +}; + +} // namespace paimon diff --git a/src/paimon/core/casting/time_to_timestamp_cast_executor.cpp b/src/paimon/core/casting/time_to_timestamp_cast_executor.cpp new file mode 100644 index 000000000..d64c07b37 --- /dev/null +++ b/src/paimon/core/casting/time_to_timestamp_cast_executor.cpp @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/casting/time_to_timestamp_cast_executor.h" + +#include +#include + +#include "arrow/array/array_primitive.h" +#include "arrow/array/builder_primitive.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/date_time_utils.h" +#include "paimon/core/casting/casting_utils.h" +#include "paimon/data/timestamp.h" +#include "paimon/defs.h" +#include "paimon/status.h" + +namespace arrow { +class MemoryPool; +class Array; +} // namespace arrow + +namespace paimon { +Result TimeToTimestampCastExecutor::Cast( + const Literal& literal, const std::shared_ptr& target_type) const { + assert(literal.GetType() == FieldType::TIME); + if (target_type->id() != arrow::Type::type::TIMESTAMP) { + return Status::Invalid( + fmt::format("cannot cast TIME literal to {}", target_type->ToString())); + } + if (literal.IsNull()) { + return Literal(FieldType::TIMESTAMP); + } + const auto& target_ts_type = checked_cast(*target_type); + int64_t millis = literal.GetValue(); + if (target_ts_type.unit() == arrow::TimeUnit::SECOND) { + // Like the array cast, a second-precision target drops the milliseconds. + millis = millis / 1000 * 1000; + } + Timestamp timestamp = Timestamp::FromEpochMillis(millis); + if (target_ts_type.timezone().empty()) { + return Literal(timestamp); + } + // Like Java, a TIMESTAMP_LTZ value is the time on 1970-01-01 in the local time zone. + PAIMON_ASSIGN_OR_RAISE(Timestamp utc_timestamp, DateTimeUtils::ToUTCTimestamp(timestamp)); + return Literal(utc_timestamp); +} + +Result> TimeToTimestampCastExecutor::Cast( + const std::shared_ptr& array, const std::shared_ptr& target_type, + arrow::MemoryPool* pool) const { + if (!array->type()->Equals(*arrow::time32(arrow::TimeUnit::MILLI))) { + return Status::Invalid( + fmt::format("Only millisecond TIME is supported: {}", array->type()->ToString())); + } + const auto& time_array = checked_cast(*array); + auto target_ts_type = checked_pointer_cast(target_type); + const int64_t units_per_second = + DateTimeUtils::CONVERSION_FACTORS[static_cast(target_ts_type->unit())]; + // Like Java, the time of day becomes that time on 1970-01-01, in the local time zone for a + // TIMESTAMP_LTZ target. + auto local_type = std::make_shared(target_ts_type->unit()); + arrow::TimestampBuilder ts_builder(local_type, pool); + PAIMON_RETURN_NOT_OK_FROM_ARROW(ts_builder.Reserve(time_array.length())); + for (int64_t i = 0; i < time_array.length(); ++i) { + if (time_array.IsNull(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(ts_builder.AppendNull()); + } else { + PAIMON_RETURN_NOT_OK_FROM_ARROW(ts_builder.Append( + static_cast(time_array.Value(i)) * units_per_second / 1000)); + } + } + std::shared_ptr local_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(ts_builder.Finish(&local_array)); + if (target_ts_type->timezone().empty()) { + return local_array; + } + return CastingUtils::TimestampToTimestampWithTimezone(local_array, target_ts_type, pool); +} + +} // namespace paimon diff --git a/src/paimon/core/casting/time_to_timestamp_cast_executor.h b/src/paimon/core/casting/time_to_timestamp_cast_executor.h new file mode 100644 index 000000000..52d26a0d8 --- /dev/null +++ b/src/paimon/core/casting/time_to_timestamp_cast_executor.h @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "arrow/array/array_base.h" +#include "paimon/core/casting/cast_executor.h" +#include "paimon/predicate/literal.h" +#include "paimon/result.h" + +namespace arrow { +class DataType; +class MemoryPool; +} // namespace arrow + +namespace paimon { +/// Casts TIME to TIMESTAMP and TIMESTAMP_LTZ like Java `TimeToTimestampCastRule`. +class TimeToTimestampCastExecutor : public CastExecutor { + public: + Result Cast(const Literal& literal, + const std::shared_ptr& target_type) const override; + + Result> Cast(const std::shared_ptr& array, + const std::shared_ptr& target_type, + arrow::MemoryPool* pool) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/casting/timestamp_to_time_cast_executor.cpp b/src/paimon/core/casting/timestamp_to_time_cast_executor.cpp new file mode 100644 index 000000000..99ffa8fb3 --- /dev/null +++ b/src/paimon/core/casting/timestamp_to_time_cast_executor.cpp @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/casting/timestamp_to_time_cast_executor.h" + +#include +#include + +#include "arrow/array/array_primitive.h" +#include "arrow/array/builder_primitive.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/date_time_utils.h" +#include "paimon/core/casting/casting_utils.h" +#include "paimon/data/timestamp.h" +#include "paimon/defs.h" +#include "paimon/status.h" + +namespace arrow { +class MemoryPool; +class Array; +} // namespace arrow + +namespace paimon { +Result TimestampToTimeCastExecutor::Cast( + const Literal& literal, const std::shared_ptr& target_type) const { + assert(literal.GetType() == FieldType::TIMESTAMP); + if (!target_type->Equals(*arrow::time32(arrow::TimeUnit::MILLI))) { + return Status::Invalid( + fmt::format("Only millisecond TIME is supported: {}", target_type->ToString())); + } + if (literal.IsNull()) { + return Literal(FieldType::TIME); + } + auto timestamp = literal.GetValue(); + if (src_with_timezone_) { + // Like Java, a TIMESTAMP_LTZ value gives its time of day in the local time zone. + PAIMON_ASSIGN_OR_RAISE(timestamp, DateTimeUtils::ToLocalTimestamp(timestamp)); + } + // Like Java, the time is the timestamp floor-modulo one day, in milliseconds. + int64_t millis_of_day = timestamp.GetMillisecond() % DateTimeUtils::MILLIS_PER_DAY; + if (millis_of_day < 0) { + millis_of_day += DateTimeUtils::MILLIS_PER_DAY; + } + return Literal(FieldType::TIME, static_cast(millis_of_day)); +} + +Result> TimestampToTimeCastExecutor::Cast( + const std::shared_ptr& array, const std::shared_ptr& target_type, + arrow::MemoryPool* pool) const { + if (!target_type->Equals(*arrow::time32(arrow::TimeUnit::MILLI))) { + return Status::Invalid( + fmt::format("Only millisecond TIME is supported: {}", target_type->ToString())); + } + auto src_ts_type = checked_pointer_cast(array->type()); + std::shared_ptr local_array = array; + if (!src_ts_type->timezone().empty()) { + auto local_type = std::make_shared(src_ts_type->unit()); + PAIMON_ASSIGN_OR_RAISE(local_array, CastingUtils::TimestampWithTimezoneToTimestamp( + local_array, local_type, pool)); + } + const auto& ts_array = checked_cast(*local_array); + const int64_t units_per_second = + DateTimeUtils::CONVERSION_FACTORS[static_cast(src_ts_type->unit())]; + const int64_t units_per_day = DateTimeUtils::SECONDS_PER_DAY * units_per_second; + arrow::Time32Builder time_builder(target_type, pool); + PAIMON_RETURN_NOT_OK_FROM_ARROW(time_builder.Reserve(ts_array.length())); + for (int64_t i = 0; i < ts_array.length(); ++i) { + if (ts_array.IsNull(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(time_builder.AppendNull()); + continue; + } + int64_t units_of_day = ts_array.Value(i) % units_per_day; + if (units_of_day < 0) { + units_of_day += units_per_day; + } + PAIMON_RETURN_NOT_OK_FROM_ARROW( + time_builder.Append(static_cast(units_of_day * 1000 / units_per_second))); + } + std::shared_ptr casted_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(time_builder.Finish(&casted_array)); + return casted_array; +} + +} // namespace paimon diff --git a/src/paimon/core/casting/timestamp_to_time_cast_executor.h b/src/paimon/core/casting/timestamp_to_time_cast_executor.h new file mode 100644 index 000000000..5593bc670 --- /dev/null +++ b/src/paimon/core/casting/timestamp_to_time_cast_executor.h @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "arrow/array/array_base.h" +#include "paimon/core/casting/cast_executor.h" +#include "paimon/predicate/literal.h" +#include "paimon/result.h" + +namespace arrow { +class DataType; +class MemoryPool; +} // namespace arrow + +namespace paimon { +/// Casts TIMESTAMP and TIMESTAMP_LTZ to TIME like Java `TimestampToTimeCastRule`. +class TimestampToTimeCastExecutor : public CastExecutor { + public: + /// @param src_with_timezone Whether the source is a TIMESTAMP_LTZ. A literal does not carry + /// its type, so a TIMESTAMP_LTZ literal is cast in the local time + /// zone only when this is set. The array cast reads the source time + /// zone from the array type instead. + explicit TimestampToTimeCastExecutor(bool src_with_timezone = false) + : src_with_timezone_(src_with_timezone) {} + + Result Cast(const Literal& literal, + const std::shared_ptr& target_type) const override; + + Result> Cast(const std::shared_ptr& array, + const std::shared_ptr& target_type, + arrow::MemoryPool* pool) const override; + + private: + bool src_with_timezone_; +}; + +} // namespace paimon diff --git a/src/paimon/core/catalog/table_query_auth_result.cpp b/src/paimon/core/catalog/table_query_auth_result.cpp index 7806845da..15e14fe7b 100644 --- a/src/paimon/core/catalog/table_query_auth_result.cpp +++ b/src/paimon/core/catalog/table_query_auth_result.cpp @@ -78,7 +78,7 @@ struct FieldRef { int32_t index = 0; std::string name; std::shared_ptr type; - /// `UNKNOWN` for a type a predicate cannot compare, such as TIME, which a mask still copies. + /// `UNKNOWN` for a type a predicate cannot compare, which a mask still copies. FieldType field_type = FieldType::UNKNOWN; }; @@ -240,6 +240,18 @@ Result ParseDateLiteral(std::string_view text, const FieldRef& ref) { return static_cast(epoch_day); } +Result ParseTimeLiteral(std::string_view text, const FieldRef& ref) { + size_t pos = 0; + int64_t second_of_day = 0; + int64_t nanos = 0; + if (!ReadIsoTime(text, &pos, /*seconds_required=*/false, &second_of_day, &nanos) || + pos != text.size()) { + return Status::Invalid( + fmt::format("Cannot parse '{}' as a time literal of column '{}'.", text, ref.name)); + } + return static_cast(second_of_day * 1000 + nanos / 1000000); +} + /// Parses the ISO text Java `LocalDateTime.toString()` writes for a TIMESTAMP column, or the /// text `Instant.toString()` writes with a `Z` for a TIMESTAMP_LTZ one. Expanded years are /// supported; the instant must lie within the range of epoch milliseconds. @@ -379,6 +391,15 @@ Result ParseLiteral(const rapidjson::Value& value, const rapidjson::Val return Literal(FieldType::DATE, days); } break; + case FieldType::TIME: + if (value.IsString()) { + PAIMON_ASSIGN_OR_RAISE( + int32_t millis, + ParseTimeLiteral(std::string_view(value.GetString(), value.GetStringLength()), + ref)); + return Literal(FieldType::TIME, millis); + } + break; case FieldType::TIMESTAMP: if (value.IsString()) { return ParseTimestampLiteral( diff --git a/src/paimon/core/catalog/table_query_auth_result_test.cpp b/src/paimon/core/catalog/table_query_auth_result_test.cpp index 2112ad90c..ce742cbd9 100644 --- a/src/paimon/core/catalog/table_query_auth_result_test.cpp +++ b/src/paimon/core/catalog/table_query_auth_result_test.cpp @@ -607,6 +607,41 @@ TEST(TableQueryAuthResultTest, ParsesTemporalLiteralsAsJava) { {})); ASSERT_OK_AND_ASSIGN(selected, Evaluate(in_rule->RowFilter(), nanos_batch)); ASSERT_EQ(std::vector({1, 0}), selected); + + const std::string time_ref = FieldRefJson(0, "t", "TIME(3)"); + auto time_of_day = [&](const std::string& literal) -> Result { + PAIMON_ASSIGN_OR_RAISE(Literal parsed, + FilterLiteral(time_ref, "EQUAL", fmt::format("[{}]", literal))); + return parsed.GetValue(); + }; + for (const auto& [text, millis] : + std::vector>{{"00:00", 0}, + {"12:34", 45240000}, + {"12:34:56", 45296000}, + {"12:34:56.1", 45296100}, + {"12:34:56.123999999", 45296123}, + {"23:59:59.999", 86399999}}) { + ASSERT_OK_AND_ASSIGN(int32_t parsed, time_of_day(fmt::format(R"("{}")", text))); + ASSERT_EQ(millis, parsed) << text; + } + for (const std::string text : {"24:00", "12:34:56Z", "12:34:56.", "12:34.5", "1:02:03"}) { + ASSERT_NOK_WITH_MSG(time_of_day(fmt::format(R"("{}")", text)), + fmt::format("Cannot parse '{}' as a time literal", text)); + } + ASSERT_NOK_WITH_MSG(time_of_day("45296000"), "does not fit column 't' of type time32[ms]"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr time_rule, + TableQueryAuthResult::Create( + {CompoundJson("OR", LeafJson(time_ref, "IS_NULL", "[]"), + LeafJson(time_ref, "GREATER_THAN", R"(["00:00:00.5"])"))}, + {})); + auto time_batch = checked_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({arrow::field("t", arrow::time32(arrow::TimeUnit::MILLI))}), + "[[500], [501], [null]]") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(selected, Evaluate(time_rule->RowFilter(), time_batch)); + ASSERT_EQ(std::vector({0, 1, 1}), selected); } TEST(TableQueryAuthResultTest, RefusesWhatItCannotApply) { @@ -763,7 +798,7 @@ TEST(TableQueryAuthResultTest, SubstringReadsPositionsAsJava) { ASSERT_EQ(std::string::npos, status.message().find("9223372036854775807")); } -TEST(TableQueryAuthResultTest, CopiesATimeColumnThatNoFilterCanCompare) { +TEST(TableQueryAuthResultTest, CopiesATimeColumn) { const std::string src_ref = FieldRefJson(0, "src", "TIME(3)"); ASSERT_OK_AND_ASSIGN( std::shared_ptr result, @@ -786,10 +821,6 @@ TEST(TableQueryAuthResultTest, CopiesATimeColumnThatNoFilterCanCompare) { ASSERT_TRUE(masked->Equals( *arrow::ipc::internal::json::ArrayFromJSON(time_type, "[1000, null]").ValueOrDie())) << masked->ToString(); - - Status filter = TableQueryAuthResult::Create({LeafJson(src_ref, "IS_NULL", "[]")}, {}).status(); - ASSERT_TRUE(filter.IsNotImplemented()) << filter.ToString(); - ASSERT_NOK_WITH_MSG(filter, "Query auth row filter IS_NULL on column 'src' of type time32[ms]"); } TEST(TableQueryAuthResultTest, ValidatesTheRulesAgainstTheSchema) { diff --git a/src/paimon/core/io/field_mapping_reader.cpp b/src/paimon/core/io/field_mapping_reader.cpp index c7f6ba22c..e271a579c 100644 --- a/src/paimon/core/io/field_mapping_reader.cpp +++ b/src/paimon/core/io/field_mapping_reader.cpp @@ -281,6 +281,12 @@ Result> FieldMappingReader::GenerateSinglePartitio scalar = std::make_shared(value); break; } + case arrow::Type::type::TIME32: { + int32_t value = partition_.GetInt(partition_info_.value().idx_in_partition[idx]); + scalar = std::make_shared( + value, partition_info_.value().partition_read_schema[idx].Type()); + break; + } default: return Status::Invalid( fmt::format("Not support arrow type {} for partition", diff --git a/src/paimon/core/io/field_mapping_reader_test.cpp b/src/paimon/core/io/field_mapping_reader_test.cpp index 45b5da0f0..76f8f83a8 100644 --- a/src/paimon/core/io/field_mapping_reader_test.cpp +++ b/src/paimon/core/io/field_mapping_reader_test.cpp @@ -221,8 +221,9 @@ class FieldMappingReaderTest : public ::testing::Test { TEST_F(FieldMappingReaderTest, TestGenerateSinglePartitionArray) { PartitionInfo partition_info; - // read schema: p7-p0 - // partition key: p0-p7 + // read schema: p7-p0, p8 + // partition key: p0-p8 + auto time_type = arrow::time32(arrow::TimeUnit::MILLI); partition_info.partition_read_schema = {DataField(7, arrow::field("p7", arrow::date32())), DataField(6, arrow::field("p6", arrow::binary())), DataField(5, arrow::field("p5", arrow::utf8())), @@ -230,21 +231,23 @@ TEST_F(FieldMappingReaderTest, TestGenerateSinglePartitionArray) { DataField(3, arrow::field("p3", arrow::int32())), DataField(2, arrow::field("p2", arrow::int16())), DataField(1, arrow::field("p1", arrow::int8())), - DataField(0, arrow::field("p0", arrow::boolean()))}; - partition_info.idx_in_target_read_schema = {0, 1, 2, 3, 4, 5, 6, 7}; - partition_info.idx_in_partition = {7, 6, 5, 4, 3, 2, 1, 0}; + DataField(0, arrow::field("p0", arrow::boolean())), + DataField(8, arrow::field("p8", time_type))}; + partition_info.idx_in_target_read_schema = {0, 1, 2, 3, 4, 5, 6, 7, 8}; + partition_info.idx_in_partition = {7, 6, 5, 4, 3, 2, 1, 0, 8}; NonPartitionInfo non_part_info; auto field_mapping = std::make_unique(partition_info, non_part_info, /*non_exist_field_info=*/std::nullopt); auto partition = BinaryRowGenerator::GenerateRow( {false, static_cast(1), static_cast(2), static_cast(3), - static_cast(4), std::string("5"), std::make_shared("6", pool_.get()), 100}, + static_cast(4), std::string("5"), std::make_shared("6", pool_.get()), 100, + static_cast(45296123)}, pool_.get()); ASSERT_OK_AND_ASSIGN( auto mapping_reader, FieldMappingReader::Create( - /*field_count=*/8, /*reader=*/nullptr, partition, std::move(field_mapping), + /*field_count=*/9, /*reader=*/nullptr, partition, std::move(field_mapping), /*skip_map_selected_keys_filter_field_ids=*/{}, GetArrowPool(pool_))); { @@ -294,6 +297,13 @@ TEST_F(FieldMappingReaderTest, TestGenerateSinglePartitionArray) { /*idx=*/7, /*batch_size=*/1)); ASSERT_EQ(checked_cast(p0_array.get())->Value(0), false); } + { + ASSERT_OK_AND_ASSIGN(auto p8_array, mapping_reader->GenerateSinglePartitionArray( + /*idx=*/8, /*batch_size=*/2)); + ASSERT_EQ(p8_array->length(), 2); + ASSERT_TRUE(p8_array->type()->Equals(time_type)); + ASSERT_EQ(checked_cast(p8_array.get())->Value(1), 45296123); + } } TEST_F(FieldMappingReaderTest, TestReadWithOnlyPartitionField) { diff --git a/src/paimon/core/io/row_to_arrow_array_converter.h b/src/paimon/core/io/row_to_arrow_array_converter.h index 336d9857f..01be826c8 100644 --- a/src/paimon/core/io/row_to_arrow_array_converter.h +++ b/src/paimon/core/io/row_to_arrow_array_converter.h @@ -129,6 +129,7 @@ Status RowToArrowArrayConverter::Reserve(arrow::ArrayBuilder* array_builde case arrow::Type::type::INT16: case arrow::Type::type::INT32: case arrow::Type::type::DATE32: + case arrow::Type::type::TIME32: case arrow::Type::type::INT64: case arrow::Type::type::FLOAT: case arrow::Type::type::DOUBLE: @@ -207,6 +208,7 @@ Status RowToArrowArrayConverter::Accumulate(const arrow::Array* array, int case arrow::Type::type::INT16: case arrow::Type::type::INT32: case arrow::Type::type::DATE32: + case arrow::Type::type::TIME32: case arrow::Type::type::INT64: case arrow::Type::type::FLOAT: case arrow::Type::type::DOUBLE: @@ -324,6 +326,16 @@ RowToArrowArrayConverter::AppendField(bool use_view, arrow::ArrayBuilder* return field_builder->Append(value); }); } + case arrow::Type::type::TIME32: { + PAIMON_ASSIGN_OR_RAISE(arrow::Time32Builder * field_builder, + CastToTypedBuilder(array_builder)); + return RowToArrowArrayConverter::AppendValueFunc( + [field_builder](const DataGetters& data_getter, int32_t pos) -> arrow::Status { + CHECK_AND_APPEND_NULL(data_getter, field_builder, pos); + int32_t value = data_getter.GetInt(pos); + return field_builder->Append(value); + }); + } case arrow::Type::type::INT64: { PAIMON_ASSIGN_OR_RAISE(auto* field_builder, CastToTypedBuilder(array_builder)); diff --git a/src/paimon/core/io/row_to_arrow_array_converter_test.cpp b/src/paimon/core/io/row_to_arrow_array_converter_test.cpp index ca6a246a8..6e22b90c1 100644 --- a/src/paimon/core/io/row_to_arrow_array_converter_test.cpp +++ b/src/paimon/core/io/row_to_arrow_array_converter_test.cpp @@ -98,4 +98,33 @@ TEST(RowToArrowArrayConverterTest, AppendBinaryWithoutView) { ASSERT_EQ(binaries->GetString(3), binary_value); } +TEST(RowToArrowArrayConverterTest, AppendTime) { + auto pool = GetDefaultPool(); + auto arrow_pool = GetArrowPool(pool); + arrow::Time32Builder builder(arrow::time32(arrow::TimeUnit::MILLI), arrow_pool.get()); + int32_t reserve_count = 0; + ASSERT_OK_AND_ASSIGN(auto append, RowToArrowArrayConverterTestAccess::AppendField( + /*use_view=*/true, &builder, &reserve_count)); + + auto row = + BinaryRowGenerator::GenerateRow({static_cast(0), static_cast(45296123), + NullType(), static_cast(86399999)}, + pool.get()); + for (int32_t i = 0; i < row.GetFieldCount(); ++i) { + ASSERT_TRUE(append(row, i).ok()); + } + + std::shared_ptr result; + ASSERT_TRUE(builder.Finish(&result).ok()); + auto times = std::dynamic_pointer_cast(result); + ASSERT_TRUE(times); + ASSERT_TRUE(times->type()->Equals(arrow::time32(arrow::TimeUnit::MILLI))); + ASSERT_EQ(times->length(), 4); + ASSERT_EQ(times->null_count(), 1); + ASSERT_EQ(times->Value(0), 0); + ASSERT_EQ(times->Value(1), 45296123); + ASSERT_TRUE(times->IsNull(2)); + ASSERT_EQ(times->Value(3), 86399999); +} + } // namespace paimon::test diff --git a/src/paimon/core/mergetree/compact/aggregate/field_aggregate_utils.cpp b/src/paimon/core/mergetree/compact/aggregate/field_aggregate_utils.cpp index 8dc4b49ff..23341e0c4 100644 --- a/src/paimon/core/mergetree/compact/aggregate/field_aggregate_utils.cpp +++ b/src/paimon/core/mergetree/compact/aggregate/field_aggregate_utils.cpp @@ -160,6 +160,7 @@ Result FieldAggregateUtils::GetValue(const DataGetters& getters, in case arrow::Type::DATE32: return VariantType(getters.GetDate(pos)); case arrow::Type::INT32: + case arrow::Type::TIME32: return VariantType(getters.GetInt(pos)); case arrow::Type::INT64: return VariantType(getters.GetLong(pos)); @@ -210,6 +211,7 @@ Result FieldAggregateUtils::Equals(const VariantType& lhs, const VariantTy return DataDefine::GetVariantValue(lhs) == DataDefine::GetVariantValue(rhs); case arrow::Type::DATE32: + case arrow::Type::TIME32: case arrow::Type::INT32: return DataDefine::GetVariantValue(lhs) == DataDefine::GetVariantValue(rhs); @@ -259,6 +261,7 @@ bool FieldAggregateUtils::IsHashableType(const std::shared_ptr& case arrow::Type::INT16: case arrow::Type::INT32: case arrow::Type::DATE32: + case arrow::Type::TIME32: case arrow::Type::INT64: case arrow::Type::FLOAT: case arrow::Type::DOUBLE: @@ -288,6 +291,7 @@ size_t FieldAggregateUtils::Hash(const VariantType& value, std::hash{}(DataDefine::GetVariantValue(value))); case arrow::Type::INT32: case arrow::Type::DATE32: + case arrow::Type::TIME32: return CombineHash(result, std::hash{}(DataDefine::GetVariantValue(value))); case arrow::Type::INT64: diff --git a/src/paimon/core/mergetree/compact/aggregate/field_aggregate_utils_test.cpp b/src/paimon/core/mergetree/compact/aggregate/field_aggregate_utils_test.cpp index 0b0364322..842983690 100644 --- a/src/paimon/core/mergetree/compact/aggregate/field_aggregate_utils_test.cpp +++ b/src/paimon/core/mergetree/compact/aggregate/field_aggregate_utils_test.cpp @@ -60,11 +60,13 @@ TEST(FieldAggregateUtilsTest, PrimitiveExtractionEqualityAndHashing) { VariantType value; VariantType other; }; - std::vector cases = {{arrow::boolean(), true, false}, - {arrow::int8(), char{7}, char{8}}, - {arrow::int16(), int16_t{-1234}, int16_t{1234}}, - {arrow::date32(), int32_t{20000}, int32_t{20001}}, - {arrow::int64(), int64_t{1} << 40, -(int64_t{1} << 40)}}; + std::vector cases = { + {arrow::boolean(), true, false}, + {arrow::int8(), char{7}, char{8}}, + {arrow::int16(), int16_t{-1234}, int16_t{1234}}, + {arrow::date32(), int32_t{20000}, int32_t{20001}}, + {arrow::time32(arrow::TimeUnit::MILLI), int32_t{0}, int32_t{86399999}}, + {arrow::int64(), int64_t{1} << 40, -(int64_t{1} << 40)}}; for (const auto& test_case : cases) { SCOPED_TRACE(test_case.type->ToString()); GenericRow row(2); diff --git a/src/paimon/core/mergetree/compact/aggregate/field_max_agg.h b/src/paimon/core/mergetree/compact/aggregate/field_max_agg.h index f41764e4b..f1433ba22 100644 --- a/src/paimon/core/mergetree/compact/aggregate/field_max_agg.h +++ b/src/paimon/core/mergetree/compact/aggregate/field_max_agg.h @@ -75,6 +75,7 @@ class FieldMaxAgg : public FieldAggregator { case arrow::Type::type::INT16: case arrow::Type::type::INT32: case arrow::Type::type::DATE32: + case arrow::Type::type::TIME32: case arrow::Type::type::INT64: case arrow::Type::type::TIMESTAMP: case arrow::Type::type::DECIMAL128: diff --git a/src/paimon/core/mergetree/compact/aggregate/field_min_agg.h b/src/paimon/core/mergetree/compact/aggregate/field_min_agg.h index 8201a820e..29d69fbe6 100644 --- a/src/paimon/core/mergetree/compact/aggregate/field_min_agg.h +++ b/src/paimon/core/mergetree/compact/aggregate/field_min_agg.h @@ -75,6 +75,7 @@ class FieldMinAgg : public FieldAggregator { case arrow::Type::type::INT16: case arrow::Type::type::INT32: case arrow::Type::type::DATE32: + case arrow::Type::type::TIME32: case arrow::Type::type::INT64: case arrow::Type::type::TIMESTAMP: case arrow::Type::type::DECIMAL128: diff --git a/src/paimon/core/mergetree/compact/aggregate/field_min_max_agg_test.cpp b/src/paimon/core/mergetree/compact/aggregate/field_min_max_agg_test.cpp index e58df125a..6d6759a49 100644 --- a/src/paimon/core/mergetree/compact/aggregate/field_min_max_agg_test.cpp +++ b/src/paimon/core/mergetree/compact/aggregate/field_min_max_agg_test.cpp @@ -153,6 +153,8 @@ TEST(FieldMinMaxAggTest, TestVariantType) { CheckResult(arrow::int16(), static_cast(100), static_cast(15)); CheckResult(arrow::int32(), static_cast(100), static_cast(15)); CheckResult(arrow::date32(), static_cast(100), static_cast(15)); + CheckResult(arrow::time32(arrow::TimeUnit::MILLI), static_cast(86399999), + static_cast(123)); CheckResult(arrow::int64(), static_cast(100), static_cast(15)); CheckResult(arrow::float32(), static_cast(100.2), static_cast(15.1)); CheckResult(arrow::float64(), 100.23, 15.11); diff --git a/src/paimon/core/mergetree/compact/internal_row_equalizer.h b/src/paimon/core/mergetree/compact/internal_row_equalizer.h index a0a0b4c55..664a9fa71 100644 --- a/src/paimon/core/mergetree/compact/internal_row_equalizer.h +++ b/src/paimon/core/mergetree/compact/internal_row_equalizer.h @@ -97,6 +97,7 @@ class InternalRowEqualizer { return PrimitiveEqualizer( [](const DataGetters& row, int32_t pos) { return row.GetShort(pos); }); case arrow::Type::INT32: + case arrow::Type::TIME32: return PrimitiveEqualizer( [](const DataGetters& row, int32_t pos) { return row.GetInt(pos); }); case arrow::Type::DATE32: diff --git a/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp b/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp index 80bc37b55..61bb99eea 100644 --- a/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp +++ b/src/paimon/core/mergetree/compact/internal_row_equalizer_test.cpp @@ -88,6 +88,7 @@ TEST(InternalRowEqualizerTest, PrimitiveTypesAndIgnoreFields) { arrow::field("string", arrow::utf8()), arrow::field("binary", arrow::binary()), arrow::field("timestamp", arrow::timestamp(arrow::TimeUnit::MICRO)), arrow::field("decimal", arrow::decimal128(10, 2)), + arrow::field("time", arrow::time32(arrow::TimeUnit::MILLI)), arrow::field("ignored", arrow::int32())}); const float float_nan = std::numeric_limits::quiet_NaN(); @@ -106,6 +107,7 @@ TEST(InternalRowEqualizerTest, PrimitiveTypesAndIgnoreFields) { binary, Timestamp(1234, 567000), decimal, + int32_t{45296123}, int32_t{10}}; std::vector right_values = left_values; right_values.back() = int32_t{20}; @@ -116,6 +118,10 @@ TEST(InternalRowEqualizerTest, PrimitiveTypesAndIgnoreFields) { InternalRowEqualizer::Create(schema, {"ignored"})); ASSERT_EQ(0, equalizer(*left, *right)); + right->SetField(/*pos=*/12, int32_t{86399999}); + ASSERT_NE(0, equalizer(*left, *right)); + + right->SetField(/*pos=*/12, int32_t{45296123}); right->SetField(/*pos=*/3, int32_t{30}); ASSERT_NE(0, equalizer(*left, *right)); } diff --git a/src/paimon/core/mergetree/in_memory_sort_buffer.cpp b/src/paimon/core/mergetree/in_memory_sort_buffer.cpp index f0e58ef58..7e566b207 100644 --- a/src/paimon/core/mergetree/in_memory_sort_buffer.cpp +++ b/src/paimon/core/mergetree/in_memory_sort_buffer.cpp @@ -135,6 +135,7 @@ Result InMemorySortBuffer::EstimateMemoryUse(const std::shared_ptrlength() * sizeof(int32_t); case arrow::Type::type::DATE32: + case arrow::Type::type::TIME32: return null_bits_size_in_bytes + array->length() * sizeof(int32_t); case arrow::Type::type::INT64: return null_bits_size_in_bytes + array->length() * sizeof(int64_t); diff --git a/src/paimon/core/mergetree/sort_buffer_test.cpp b/src/paimon/core/mergetree/sort_buffer_test.cpp index 648a95b69..a77b98f57 100644 --- a/src/paimon/core/mergetree/sort_buffer_test.cpp +++ b/src/paimon/core/mergetree/sort_buffer_test.cpp @@ -192,6 +192,15 @@ TEST_F(SortBufferTest, TestInMemorySortBufferEstimateMemoryUse) { 1 + (13 + 3 * 4 + 1) + (3 * 4 + 1) + (3 * 4 + 1) + (3 * 8 + 1); ASSERT_EQ(memory_use, expected_memory_use); } + { + arrow::FieldVector fields = {arrow::field("f0", arrow::time32(arrow::TimeUnit::MILLI))}; + std::shared_ptr array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([[0], [null], [86399999]])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(int64_t memory_use, InMemorySortBuffer::EstimateMemoryUse(array)); + ASSERT_EQ(memory_use, 1 + (3 * 4 + 1)); + } { arrow::FieldVector fields = {arrow::field("v0", arrow::boolean()), arrow::field("v1", arrow::int8()), diff --git a/src/paimon/core/realtime/arrow_realtime_store.cpp b/src/paimon/core/realtime/arrow_realtime_store.cpp index 6c1c7a489..19f962362 100644 --- a/src/paimon/core/realtime/arrow_realtime_store.cpp +++ b/src/paimon/core/realtime/arrow_realtime_store.cpp @@ -66,6 +66,7 @@ bool SupportsMinMax(const std::shared_ptr& type) { case arrow::Type::STRING: case arrow::Type::BINARY: case arrow::Type::DATE32: + case arrow::Type::TIME32: case arrow::Type::TIMESTAMP: case arrow::Type::DECIMAL128: return true; diff --git a/src/paimon/core/realtime/arrow_realtime_store_test.cpp b/src/paimon/core/realtime/arrow_realtime_store_test.cpp index 973d460f6..7ac277fd8 100644 --- a/src/paimon/core/realtime/arrow_realtime_store_test.cpp +++ b/src/paimon/core/realtime/arrow_realtime_store_test.cpp @@ -525,6 +525,33 @@ TEST_F(ArrowRealtimeStoreTest, TestFullStatisticsPrunesNonMatchingBatch) { ASSERT_EQ(std::vector({0, 1, 10, 11}), ReadIds(unfiltered)); } +TEST_F(ArrowRealtimeStoreTest, TestFullStatisticsPrunesNonMatchingTimeBatch) { + schema_ = arrow::schema({ + DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()), + arrow::field("id", arrow::int64()), + arrow::field("t", arrow::time32(arrow::TimeUnit::MILLI)), + }); + ASSERT_OK_AND_ASSIGN(std::shared_ptr store, + CreateStore(StatisticsMode::FULL)); + ASSERT_OK(store->Write( + RealtimeWriteBatch{MakeBatch(R"([[0, 0, 0], [1, 1, 123]])"), OffsetRange(0, 2)})); + ASSERT_OK(store->Write(RealtimeWriteBatch{ + MakeBatch(R"([[2, 10, 45296123], [3, 11, 86399999]])"), OffsetRange(2, 4)})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr view, store->AcquireReadView()); + + std::unique_ptr read_schema = MakeReadSchema(schema_); + std::shared_ptr predicate = PredicateBuilder::GreaterThan( + /*field_index=*/2, /*field_name=*/"t", FieldType::TIME, Literal(FieldType::TIME, 1000)); + RealtimeQueryContext context{read_schema.get(), predicate, + view->GetOffsetRange().value_or(OffsetRange(0, 0))}; + ASSERT_OK_AND_ASSIGN(std::vector> readers, + store->CreateQueryReaders(view, context)); + ASSERT_EQ(1, readers.size()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr filtered, + ReadResultCollector::CollectResult(std::move(readers[0]))); + ASSERT_EQ(std::vector({10, 11}), ReadIds(filtered)); +} + TEST_F(ArrowRealtimeStoreTest, TestMissingStatisticsRetainsNonMatchingBatch) { ASSERT_OK(store_->Write( RealtimeWriteBatch{MakeBatch(R"([[0, 0, "a"], [1, 1, "b"]])"), OffsetRange(0, 2)})); diff --git a/src/paimon/core/realtime/index/hash/hash_realtime_index.cpp b/src/paimon/core/realtime/index/hash/hash_realtime_index.cpp index 325d2d7ec..5d830fecd 100644 --- a/src/paimon/core/realtime/index/hash/hash_realtime_index.cpp +++ b/src/paimon/core/realtime/index/hash/hash_realtime_index.cpp @@ -117,6 +117,7 @@ Result> HashRealtimeIndex::Create( case FieldType::STRING: case FieldType::BINARY: case FieldType::DATE: + case FieldType::TIME: break; default: return Status::NotImplemented("unsupported real-time hash index field type: ", diff --git a/src/paimon/core/realtime/index/realtime_index_test.cpp b/src/paimon/core/realtime/index/realtime_index_test.cpp index 027fd7a44..799e27c83 100644 --- a/src/paimon/core/realtime/index/realtime_index_test.cpp +++ b/src/paimon/core/realtime/index/realtime_index_test.cpp @@ -344,6 +344,39 @@ TEST_F(RealtimeIndexTest, FactoryAcceptsOnlyOneIndexedFieldAndReaderNeedsNoField ASSERT_EQ(RoaringBitmap64::From({30}), *candidates); } +TEST_F(RealtimeIndexTest, IndexesTimeField) { + const std::shared_ptr schema = arrow::schema( + {schema_->field(0), arrow::field("t", arrow::time32(arrow::TimeUnit::MILLI))}); + ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ScopeGuard schema_guard([&c_schema]() { ArrowSchemaRelease(&c_schema); }); + ASSERT_OK_AND_ASSIGN(std::shared_ptr index, + RealtimeIndexFactory::Get("hash", &c_schema, {}, GetDefaultPool())); + + const std::shared_ptr data = + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_(schema->fields()), + R"([[10, 0], [20, 45296123], [30, 86399999], [40, 45296123]])") + .ValueOrDie(); + ArrowArray batch; + ASSERT_TRUE(arrow::ExportArray(*data, &batch).ok()); + ScopeGuard batch_guard([&batch]() { ArrowArrayRelease(&batch); }); + ASSERT_OK(index->AddBatch(&batch)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + index->CreateReader(OffsetRange(0, 41))); + ASSERT_OK_AND_ASSIGN(std::optional candidates, + reader->VisitEqual(Literal(FieldType::TIME, 45296123))); + ASSERT_TRUE(candidates.has_value()); + ASSERT_EQ(RoaringBitmap64::From({20, 40}), *candidates); + ASSERT_OK_AND_ASSIGN(candidates, reader->VisitIn({Literal(FieldType::TIME, 0), + Literal(FieldType::TIME, 86399999)})); + ASSERT_TRUE(candidates.has_value()); + ASSERT_EQ(RoaringBitmap64::From({10, 30}), *candidates); + ASSERT_NOK_WITH_MSG(reader->VisitEqual(Literal(int32_t{45296123})), + "literal type mismatch: expected TIME, actual INT"); +} + TEST_F(RealtimeIndexTest, RejectsDecimalAndTimestampFields) { const std::vector> types = { arrow::decimal128(10, 2), diff --git a/src/paimon/core/schema/schema_manager_test.cpp b/src/paimon/core/schema/schema_manager_test.cpp index cd4823891..875372715 100644 --- a/src/paimon/core/schema/schema_manager_test.cpp +++ b/src/paimon/core/schema/schema_manager_test.cpp @@ -82,17 +82,18 @@ TEST(SchemaManagerTest, TimePrecisionRoundTrip) { } } -TEST(SchemaManagerTest, RejectTimePartitionKey) { +TEST(SchemaManagerTest, CreateTimePartitionKey) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); SchemaManager manager(std::make_shared(), dir->Str()); auto schema = arrow::schema({arrow::field("id", arrow::int32()), arrow::field("time", arrow::time32(arrow::TimeUnit::MILLI))}); - ASSERT_NOK_WITH_MSG( - manager.CreateTable(schema, {"time"}, {}, {{"file.format", "parquet"}, {"bucket", "-1"}}), - "partition field time cannot be TIME"); + ASSERT_OK_AND_ASSIGN( + auto created, + manager.CreateTable(schema, {"time"}, {}, {{"file.format", "parquet"}, {"bucket", "-1"}})); + ASSERT_EQ(created->PartitionKeys(), std::vector({"time"})); ASSERT_OK_AND_ASSIGN(auto latest, manager.Latest()); - ASSERT_FALSE(latest.has_value()); + ASSERT_TRUE(latest.has_value()); } TEST(SchemaManagerTest, ConcurrentHistoricalSchemaReads) { diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index a310ec950..b8319f6cf 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -493,10 +493,6 @@ Status SchemaValidation::ValidateNotContainSpecificType( auto it = fields_map.find(field_name); if (it != fields_map.end()) { auto field = it->second; - if (field->type()->id() == arrow::Type::TIME32) { - return Status::Invalid( - fmt::format("partition field {} cannot be TIME", field_name)); - } if (IsComplexType(field)) { return Status::Invalid( fmt::format("partition field {} cannot be TIMESTAMP/DECIMAL/BLOB", field_name)); diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index f88601da3..4694be20b 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -1255,20 +1255,22 @@ TEST(SchemaValidationTest, TestSpecificPartitionKey) { } } -TEST(SchemaValidationTest, TestTimePartitionKey) { +TEST(SchemaValidationTest, TestTimeKeys) { auto schema = arrow::schema({arrow::field("id", arrow::int32()), arrow::field("time", arrow::time32(arrow::TimeUnit::MILLI))}); - for (const std::vector& partition_keys : - {std::vector{}, std::vector{"time"}}) { + { ASSERT_OK_AND_ASSIGN(auto table_schema, - TableSchema::Create(0, schema, partition_keys, {}, + TableSchema::Create(0, schema, /*partition_keys=*/{"time"}, {}, {{"file.format", "parquet"}, {"bucket", "-1"}})); - if (partition_keys.empty()) { - ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema)); - } else { - ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), - "partition field time cannot be TIME"); - } + ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema)); + } + { + ASSERT_OK_AND_ASSIGN( + auto table_schema, + TableSchema::Create( + 0, schema, /*partition_keys=*/{}, /*primary_keys=*/{"id", "time"}, + {{"file.format", "parquet"}, {"bucket", "2"}, {"bucket-key", "time"}})); + ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema)); } } diff --git a/src/paimon/core/stats/simple_stats_collector.cpp b/src/paimon/core/stats/simple_stats_collector.cpp index c0f2c8f40..533c40985 100644 --- a/src/paimon/core/stats/simple_stats_collector.cpp +++ b/src/paimon/core/stats/simple_stats_collector.cpp @@ -99,7 +99,8 @@ Status SimpleStatsCollector::Collect(const BinaryRow& row) { } break; } - case arrow::Type::INT32: { + case arrow::Type::INT32: + case arrow::Type::TIME32: { if (column_stats_[i] == nullptr) { column_stats_[i] = ColumnStats::CreateIntColumnStats(std::nullopt, std::nullopt, std::nullopt); diff --git a/src/paimon/core/stats/simple_stats_collector_test.cpp b/src/paimon/core/stats/simple_stats_collector_test.cpp index d3a860adf..8fb21816f 100644 --- a/src/paimon/core/stats/simple_stats_collector_test.cpp +++ b/src/paimon/core/stats/simple_stats_collector_test.cpp @@ -39,10 +39,14 @@ namespace paimon::test { TEST(SimpleStatsCollectorTest, TestSimple) { arrow::FieldVector fields = { - arrow::field("f0", arrow::boolean()), arrow::field("f1", arrow::int8()), - arrow::field("f2", arrow::int16()), arrow::field("f3", arrow::int32()), - arrow::field("f4", arrow::int64()), arrow::field("f5", arrow::utf8()), + arrow::field("f0", arrow::boolean()), + arrow::field("f1", arrow::int8()), + arrow::field("f2", arrow::int16()), + arrow::field("f3", arrow::int32()), + arrow::field("f4", arrow::int64()), + arrow::field("f5", arrow::utf8()), arrow::field("f6", arrow::date32()), + arrow::field("f7", arrow::time32(arrow::TimeUnit::MILLI)), }; auto schema = arrow::schema(fields); @@ -50,21 +54,22 @@ TEST(SimpleStatsCollectorTest, TestSimple) { auto pool = GetDefaultPool(); ASSERT_OK(collector.Collect(BinaryRowGenerator::GenerateRow( {true, static_cast(1), static_cast(1), static_cast(1), - static_cast(1), std::string("abc"), 2025}, + static_cast(1), std::string("abc"), 2025, static_cast(86399999)}, pool.get()))); ASSERT_OK(collector.Collect(BinaryRowGenerator::GenerateRow( {false, static_cast(2), static_cast(2), static_cast(2), - static_cast(2), std::string("bcd"), 2026}, + static_cast(2), std::string("bcd"), 2026, static_cast(123)}, pool.get()))); ASSERT_OK_AND_ASSIGN(auto col_stats, collector.GetResult()); + ASSERT_EQ(col_stats[7]->GetFieldType(), FieldType::INT); ASSERT_OK_AND_ASSIGN(SimpleStats stats, SimpleStatsConverter::ToBinary(col_stats, pool.get())); auto expected_stats = BinaryRowGenerator::GenerateStats( {false, static_cast(1), static_cast(1), static_cast(1), - static_cast(1), std::string("abc"), 2025}, + static_cast(1), std::string("abc"), 2025, static_cast(123)}, {true, static_cast(2), static_cast(2), static_cast(2), - static_cast(2), std::string("bcd"), 2026}, - std::vector({0, 0, 0, 0, 0, 0, 0}), GetDefaultPool().get()); + static_cast(2), std::string("bcd"), 2026, static_cast(86399999)}, + std::vector({0, 0, 0, 0, 0, 0, 0, 0}), GetDefaultPool().get()); ASSERT_EQ(stats, expected_stats); } diff --git a/src/paimon/core/stats/simple_stats_evolution_test.cpp b/src/paimon/core/stats/simple_stats_evolution_test.cpp index 028134108..2d49691b8 100644 --- a/src/paimon/core/stats/simple_stats_evolution_test.cpp +++ b/src/paimon/core/stats/simple_stats_evolution_test.cpp @@ -23,12 +23,18 @@ #include #include #include +#include +#include "arrow/type.h" +#include "arrow/util/key_value_metadata.h" #include "gtest/gtest.h" #include "paimon/common/data/binary_array.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/data_define.h" #include "paimon/common/data/internal_array.h" +#include "paimon/common/data/internal_row.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/types/data_type.h" #include "paimon/common/utils/internal_row_utils.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" @@ -343,6 +349,48 @@ TEST_F(SimpleStatsEvolutionTest, TestSchemaChangeWithDenseFields) { {1l, NullType(), NullType(), NullType(), NullType(), NullType(), 0l, 4l}); } +TEST_F(SimpleStatsEvolutionTest, TestTimeSchemaChange) { + auto time_field = [](int32_t id, const std::string& name) { + return DataField( + id, arrow::field(name, arrow::time32(arrow::TimeUnit::MILLI), + /*nullable=*/true, + arrow::key_value_metadata({DataType::kTimePrecision}, {"3"}))); + }; + auto ts_type = arrow::timestamp(arrow::TimeUnit::MILLI); + std::vector data_fields = { + DataField(0, arrow::field("string_to_time", arrow::utf8())), + time_field(1, "time_to_string"), DataField(2, arrow::field("timestamp_to_time", ts_type)), + time_field(3, "time_to_timestamp")}; + std::vector table_fields = { + time_field(0, "string_to_time"), + DataField(1, arrow::field("time_to_string", arrow::utf8())), + time_field(2, "timestamp_to_time"), + DataField(3, arrow::field("time_to_timestamp", ts_type))}; + SimpleStatsEvolution evo(data_fields, table_fields, /*need_mapping=*/true, pool_); + + auto min = BinaryRowGenerator::GenerateRow( + {std::string("01:02:03"), 3723000, TimestampType(Timestamp(1686227696123l, 0), 3), 0}, + pool_.get()); + auto max = + BinaryRowGenerator::GenerateRow({std::string("12:34:56.123"), 86399999, + TimestampType(Timestamp(1686268799999l, 0), 3), 45296123}, + pool_.get()); + SimpleStats stats(min, max, BinaryArray::FromLongArray({0, 0, 0, 0}, pool_.get())); + ASSERT_OK_AND_ASSIGN(SimpleStatsEvolution::EvolutionStats result_stats, + evo.Evolution(stats, /*row_count=*/4, /*dense_fields=*/std::nullopt)); + + const InternalRow& result_min = *result_stats.min_values; + ASSERT_EQ(result_min.GetInt(0), 3723000); + ASSERT_EQ(result_min.GetString(1).ToString(), "01:02:03.0"); + ASSERT_EQ(result_min.GetInt(2), 45296123); + ASSERT_EQ(result_min.GetTimestamp(3, /*precision=*/3), Timestamp(0, 0)); + const InternalRow& result_max = *result_stats.max_values; + ASSERT_EQ(result_max.GetInt(0), 45296123); + ASSERT_EQ(result_max.GetString(1).ToString(), "23:59:59.999"); + ASSERT_EQ(result_max.GetInt(2), 86399999); + ASSERT_EQ(result_max.GetTimestamp(3, /*precision=*/3), Timestamp(45296123, 0)); +} + TEST_F(SimpleStatsEvolutionTest, TestNoSchemaChangeWithDenseFieldsWithMap) { std::string table_root = paimon::test::GetDataDir() + diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index 8db1c41cf..8cd5434f3 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -42,7 +42,6 @@ #include "paimon/common/types/data_field.h" #include "paimon/common/utils/binary_row_partition_computer.h" #include "paimon/common/utils/date_time_utils.h" -#include "paimon/common/utils/field_type_utils.h" #include "paimon/common/utils/internal_row_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/rapidjson_util.h" @@ -330,11 +329,9 @@ Result FilePath(const std::shared_ptr& path_f } Result FieldValueString(const DataField& field, const VariantType& value) { - PAIMON_ASSIGN_OR_RAISE(FieldType field_type, - FieldTypeUtils::ConvertToFieldType(field.Type()->id())); - std::shared_ptr cast_executor = - CastExecutorFactory::GetCastExecutorFactory()->GetCastExecutor(field_type, - FieldType::STRING); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr cast_executor, + CastExecutorFactory::GetCastExecutorFactory()->GetCastExecutor(field, FieldType::STRING)); if (!cast_executor) { return DataDefine::VariantValueToString(value); } diff --git a/src/paimon/core/utils/field_mapping.cpp b/src/paimon/core/utils/field_mapping.cpp index 7155c16fb..1b44926cc 100644 --- a/src/paimon/core/utils/field_mapping.cpp +++ b/src/paimon/core/utils/field_mapping.cpp @@ -208,8 +208,9 @@ Result>> FieldMappingBuilder::CreateDa PAIMON_ASSIGN_OR_RAISE(FieldType data_type, FieldTypeUtils::ConvertToFieldType(data_fields[i].Type()->id())); auto executor_factory = CastExecutorFactory::GetCastExecutorFactory(); - auto cast_executor = - executor_factory->GetCastExecutor(/*src=*/data_type, /*target=*/read_type); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr cast_executor, + executor_factory->GetCastExecutor(/*src_field=*/data_fields[i], + /*target=*/read_type)); if (!cast_executor) { return Status::Invalid( fmt::format("CreateDataCastExecutors failed: cannot find cast " diff --git a/src/paimon/core/utils/field_mapping_test.cpp b/src/paimon/core/utils/field_mapping_test.cpp index 4f66cd10b..21eb8113e 100644 --- a/src/paimon/core/utils/field_mapping_test.cpp +++ b/src/paimon/core/utils/field_mapping_test.cpp @@ -21,12 +21,19 @@ #include #include +#include +#include +#include +#include "arrow/type.h" #include "arrow/type_fwd.h" +#include "arrow/util/key_value_metadata.h" #include "gtest/gtest.h" #include "paimon/common/predicate/full_text_match_predicate.h" #include "paimon/common/predicate/leaf_predicate_impl.h" #include "paimon/common/predicate/predicate_filter.h" +#include "paimon/common/types/data_type.h" +#include "paimon/core/casting/cast_executor.h" #include "paimon/data/decimal.h" #include "paimon/defs.h" #include "paimon/predicate/full_text_match.h" @@ -737,7 +744,7 @@ TEST_F(FieldMappingTest, TestMapSelectedKeysMetadataPropagatedToDataSchema) { ASSERT_FALSE(custom_metadata_result.ok()); } -TEST_F(FieldMappingTest, TestTimeWithoutCast) { +TEST_F(FieldMappingTest, TestTimeCast) { std::vector fields = { DataField(0, arrow::field("time", arrow::time32(arrow::TimeUnit::MILLI)))}; auto schema = DataField::ConvertDataFieldsToArrowSchema(fields); @@ -753,6 +760,22 @@ TEST_F(FieldMappingTest, TestTimeWithoutCast) { std::vector int_fields = {DataField(0, arrow::field("time", arrow::int32()))}; ASSERT_NOK(FieldMappingBuilder::CreateDataCastExecutors(fields, int_fields)); ASSERT_NOK(FieldMappingBuilder::CreateDataCastExecutors(int_fields, fields)); + + std::vector time3_fields = { + DataField(0, arrow::field("time", arrow::time32(arrow::TimeUnit::MILLI), /*nullable=*/true, + arrow::key_value_metadata({DataType::kTimePrecision}, {"3"})))}; + std::vector string_fields = {DataField(0, arrow::field("time", arrow::utf8()))}; + ASSERT_OK_AND_ASSIGN(std::vector> to_string, + FieldMappingBuilder::CreateDataCastExecutors(string_fields, time3_fields)); + ASSERT_EQ(to_string.size(), 1); + ASSERT_TRUE(to_string[0]); + ASSERT_OK_AND_ASSIGN(Literal literal, + to_string[0]->Cast(Literal(FieldType::TIME, 3723000), arrow::utf8())); + ASSERT_EQ("01:02:03.0", literal.GetValue()); + ASSERT_OK_AND_ASSIGN(std::vector> to_time, + FieldMappingBuilder::CreateDataCastExecutors(fields, string_fields)); + ASSERT_EQ(to_time.size(), 1); + ASSERT_TRUE(to_time[0]); } } // namespace paimon::test diff --git a/src/paimon/format/avro/avro_direct_decoder.cpp b/src/paimon/format/avro/avro_direct_decoder.cpp index 9f2d5763a..e81310605 100644 --- a/src/paimon/format/avro/avro_direct_decoder.cpp +++ b/src/paimon/format/avro/avro_direct_decoder.cpp @@ -43,8 +43,9 @@ AvroDirectDecoder::DecodeContext::GetBuilderMetadata(const arrow::ArrayBuilder* std::shared_ptr data_type = builder->type(); BuilderMetadata metadata{data_type->id(), std::nullopt}; if (data_type->id() == arrow::Type::TIMESTAMP) { - metadata.timestamp_unit = - checked_cast(data_type.get())->unit(); + metadata.time_unit = checked_cast(data_type.get())->unit(); + } else if (data_type->id() == arrow::Type::TIME32) { + metadata.time_unit = checked_cast(data_type.get())->unit(); } return builder_metadata_.emplace(builder, metadata).first->second; } @@ -328,6 +329,19 @@ Status DecodeAvroValueToBuilder(const ::avro::NodePtr& avro_node, PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append(value)); return Status::OK(); } + case arrow::Type::TIME32: { + // Avro time-millis holds milliseconds of the day, so only a time32[ms] + // builder can take the value as is. + if (logical_type.type() != ::avro::LogicalType::Type::TIME_MILLIS || + builder_metadata.time_unit != arrow::TimeUnit::MILLI) { + return Status::TypeError( + fmt::format("Unexpected avro type [{}] with arrow type [{}].", + ::avro::toString(type), array_builder->type()->ToString())); + } + auto* builder = checked_cast(array_builder); + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append(value)); + return Status::OK(); + } default: return Status::TypeError( fmt::format("Unexpected avro type [{}] with arrow type [{}].", @@ -352,7 +366,7 @@ Status DecodeAvroValueToBuilder(const ::avro::NodePtr& avro_node, auto* builder = checked_cast(array_builder); // for arrow second, we need to convert it from avro millisecond const auto& builder_metadata = ctx->GetBuilderMetadata(builder); - if (builder_metadata.timestamp_unit == arrow::TimeUnit::type::SECOND) { + if (builder_metadata.time_unit == arrow::TimeUnit::type::SECOND) { value /= DateTimeUtils::CONVERSION_FACTORS[DateTimeUtils::MILLISECOND]; } PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append(value)); diff --git a/src/paimon/format/avro/avro_direct_decoder.h b/src/paimon/format/avro/avro_direct_decoder.h index ad5cdd9eb..bfec277e6 100644 --- a/src/paimon/format/avro/avro_direct_decoder.h +++ b/src/paimon/format/avro/avro_direct_decoder.h @@ -47,7 +47,8 @@ class AvroDirectDecoder { struct DecodeContext { struct BuilderMetadata { arrow::Type::type type; - std::optional timestamp_unit; + /// Time unit of a TIMESTAMP or TIME32 builder, empty for other types. + std::optional time_unit; }; /// Returns immutable type metadata without repeatedly copying the builder's DataType. diff --git a/src/paimon/format/avro/avro_direct_encoder.cpp b/src/paimon/format/avro/avro_direct_encoder.cpp index 465562139..94709586d 100644 --- a/src/paimon/format/avro/avro_direct_encoder.cpp +++ b/src/paimon/format/avro/avro_direct_encoder.cpp @@ -100,7 +100,8 @@ Status AvroDirectEncoder::EncodeArrowToAvro(const ::avro::NodePtr& avro_node, } case ::avro::AVRO_INT: { - // AVRO_INT can represent: int8, int16, int32, date (days since epoch) + // AVRO_INT can represent: int8, int16, int32, date (days since epoch), time + // (milliseconds of the day) switch (array.type()->id()) { case arrow::Type::INT8: { const auto& int8_array = checked_cast(array); @@ -123,10 +124,15 @@ Status AvroDirectEncoder::EncodeArrowToAvro(const ::avro::NodePtr& avro_node, encoder->encodeInt(date_array.Value(row_index)); return Status::OK(); } + case arrow::Type::TIME32: { + const auto& time_array = checked_cast(array); + encoder->encodeInt(time_array.Value(row_index)); + return Status::OK(); + } default: return Status::Invalid( fmt::format("AVRO_INT expects Int8Array or Int16Array or Int32Array or " - "Date32Array, got {}", + "Date32Array or Time32Array, got {}", array.type()->ToString())); } } diff --git a/src/paimon/format/avro/avro_direct_encoder_decoder_test.cpp b/src/paimon/format/avro/avro_direct_encoder_decoder_test.cpp index 8e65520d3..4afa9accd 100644 --- a/src/paimon/format/avro/avro_direct_encoder_decoder_test.cpp +++ b/src/paimon/format/avro/avro_direct_encoder_decoder_test.cpp @@ -265,6 +265,25 @@ TEST_F(AvroDirectEncoderDecoderTest, TestDate32Type) { CheckResult(schema_json, input_array, &builder); } +TEST_F(AvroDirectEncoderDecoderTest, TestTime32Type) { + std::string schema_json = R"({"type": "int", "logicalType": "time-millis"})"; + arrow::Time32Builder builder(arrow::time32(arrow::TimeUnit::MILLI), + arrow::default_memory_pool()); + ASSERT_TRUE(builder.Append(0).ok()); // 00:00:00 + ASSERT_TRUE(builder.Append(123).ok()); // 00:00:00.123 + ASSERT_TRUE(builder.Append(86399999).ok()); // 23:59:59.999 + std::shared_ptr input_array; + ASSERT_TRUE(builder.Finish(&input_array).ok()); + CheckResult(schema_json, input_array, &builder); + + auto avro_schema = ::avro::compileJsonSchemaFromString(R"({"type": "int"})"); + ASSERT_OK_AND_ASSIGN(auto encoded_data, EncodeData(avro_schema.root(), input_array)); + ASSERT_NOK_WITH_MSG( + DecodeWithEncodedData(avro_schema.root(), std::move(encoded_data), + /*projection=*/std::nullopt, input_array->length(), &builder), + "Unexpected avro type"); +} + TEST_F(AvroDirectEncoderDecoderTest, TestTimestampType) { // Test timestamp-millis { diff --git a/src/paimon/format/avro/avro_file_batch_reader_test.cpp b/src/paimon/format/avro/avro_file_batch_reader_test.cpp index 6a61c4c95..b59c2eac2 100644 --- a/src/paimon/format/avro/avro_file_batch_reader_test.cpp +++ b/src/paimon/format/avro/avro_file_batch_reader_test.cpp @@ -791,6 +791,31 @@ TEST_F(AvroFileBatchReaderTest, TestSetReadSchemaResetsReaderToFirstRow) { ASSERT_TRUE(projected_array->Equals(expected_projected_array)) << projected_array->ToString(); } +TEST_F(AvroFileBatchReaderTest, TestSetReadSchemaRejectsTimeOfOtherUnit) { + std::string file_path = PathUtil::JoinPath(dir_->Str(), "file.avro"); + + auto file_data_type = + arrow::struct_({arrow::field("f0", arrow::time32(arrow::TimeUnit::MILLI))}); + auto src_array = arrow::ipc::internal::json::ArrayFromJSON(file_data_type, R"([ + [1000], + [null] + ])") + .ValueOrDie(); + WriteData(src_array, file_path, /*compression=*/"null"); + + ASSERT_OK_AND_ASSIGN(auto reader_builder, file_format_->CreateReaderBuilder(/*batch_size=*/2)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_path)); + ASSERT_OK_AND_ASSIGN(auto reader, reader_builder->Build(in)); + + auto read_schema = arrow::schema({arrow::field("f0", arrow::time32(arrow::TimeUnit::SECOND))}); + std::unique_ptr c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + ASSERT_OK(reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_NOK_WITH_MSG(reader->NextBatch(), + "Unexpected avro type [int] with arrow type [time32[s]]"); +} + TEST_F(AvroFileBatchReaderTest, TestGetNumberOfRows) { std::string file_path = PathUtil::JoinPath(dir_->Str(), "file.avro"); diff --git a/src/paimon/format/avro/avro_schema_converter.cpp b/src/paimon/format/avro/avro_schema_converter.cpp index 3d6550bf4..1dafc502d 100644 --- a/src/paimon/format/avro/avro_schema_converter.cpp +++ b/src/paimon/format/avro/avro_schema_converter.cpp @@ -31,6 +31,7 @@ #include "avro/ValidSchema.hh" #include "fmt/format.h" #include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/data_type.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/format/avro/avro_file_format_factory.h" @@ -114,6 +115,11 @@ Result> AvroSchemaConverter::GetArrowType( return Status::TypeError("invalid avro date stored as ", toString(type)); } return arrow::date32(); + case ::avro::LogicalType::Type::TIME_MILLIS: + if (type != ::avro::AVRO_INT) { + return Status::TypeError("invalid avro time stored as ", toString(type)); + } + return arrow::time32(arrow::TimeUnit::MILLI); case ::avro::LogicalType::Type::DECIMAL: if (type != ::avro::AVRO_BYTES) { return Status::TypeError("invalid avro decimal stored as ", toString(type)); @@ -277,6 +283,21 @@ Result<::avro::Schema> AvroSchemaConverter::ArrowTypeToAvroSchema( date_schema.root()->setLogicalType(date_type); return nullable ? NullableSchema(date_schema) : date_schema; } + case arrow::Type::type::TIME32: { + // Like Java `AvroSchemaConverter`, TIME is an Avro int of milliseconds of the day, and + // a precision above 3 is rejected. + PAIMON_ASSIGN_OR_RAISE(int32_t precision, DataType::GetTimePrecision(*field)); + if (precision > 3) { + return Status::Invalid(fmt::format( + "Avro does not support TIME type with precision: {}, it only supports " + "precision up to 3.", + precision)); + } + ::avro::IntSchema time_schema; + time_schema.root()->setLogicalType( + ::avro::LogicalType(::avro::LogicalType::TIME_MILLIS)); + return nullable ? NullableSchema(time_schema) : time_schema; + } case arrow::Type::type::TIMESTAMP: { const auto& arrow_timestamp_type = checked_pointer_cast(arrow_type); diff --git a/src/paimon/format/avro/avro_schema_converter_test.cpp b/src/paimon/format/avro/avro_schema_converter_test.cpp index e30de486b..4d8fc8256 100644 --- a/src/paimon/format/avro/avro_schema_converter_test.cpp +++ b/src/paimon/format/avro/avro_schema_converter_test.cpp @@ -19,10 +19,13 @@ #include "paimon/format/avro/avro_schema_converter.h" #include "arrow/api.h" +#include "arrow/util/key_value_metadata.h" #include "avro/Compiler.hh" +#include "avro/LogicalType.hh" #include "avro/ValidSchema.hh" #include "gtest/gtest.h" #include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/data_type.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/core/manifest/manifest_file_meta.h" #include "paimon/core/utils/versioned_object_serializer.h" @@ -247,6 +250,33 @@ TEST(AvroSchemaConverterTest, TestAvroSchemaToArrowDataTypeWithTimestampType) { ASSERT_TRUE(arrow_type->Equals(arrow::struct_(expected_fields))) << arrow_type->ToString(); } +TEST(AvroSchemaConverterTest, TestTimeType) { + auto time_schema = [](const std::string& precision) { + return arrow::schema( + {arrow::field("t", arrow::time32(arrow::TimeUnit::MILLI), /*nullable=*/false, + arrow::key_value_metadata({DataType::kTimePrecision}, {precision}))}); + }; + auto expected_type = + arrow::struct_({arrow::field("t", arrow::time32(arrow::TimeUnit::MILLI), false)}); + for (const std::string precision : {"0", "3"}) { + ASSERT_OK_AND_ASSIGN(::avro::ValidSchema avro_schema, + AvroSchemaConverter::ArrowSchemaToAvroSchema(time_schema(precision))); + const ::avro::NodePtr& time_node = avro_schema.root()->leafAt(0); + ASSERT_EQ(time_node->type(), ::avro::AVRO_INT); + ASSERT_EQ(time_node->logicalType().type(), ::avro::LogicalType::TIME_MILLIS); + ASSERT_OK_AND_ASSIGN(std::shared_ptr arrow_type, + AvroSchemaConverter::AvroSchemaToArrowDataType(avro_schema)); + ASSERT_TRUE(arrow_type->Equals(expected_type)) << arrow_type->ToString(); + } + for (const std::string precision : {"6", "9"}) { + ASSERT_NOK_WITH_MSG(AvroSchemaConverter::ArrowSchemaToAvroSchema(time_schema(precision)), + "Avro does not support TIME type with precision: " + precision); + } + ASSERT_NOK_WITH_MSG(AvroSchemaConverter::ArrowSchemaToAvroSchema(arrow::schema( + {arrow::field("t", arrow::time32(arrow::TimeUnit::SECOND))})), + "Only millisecond TIME is supported"); +} + TEST(AvroSchemaConverterTest, TestVariantNotSupported) { auto arrow_schema = arrow::schema({arrow::field("id", arrow::int32()), VariantTypeUtils::ToArrowField("v")}); diff --git a/src/paimon/format/avro/avro_stats_extractor.cpp b/src/paimon/format/avro/avro_stats_extractor.cpp index 5dfa82c01..cc1517569 100644 --- a/src/paimon/format/avro/avro_stats_extractor.cpp +++ b/src/paimon/format/avro/avro_stats_extractor.cpp @@ -101,6 +101,8 @@ Result> AvroStatsExtractor::FetchColumnStatistics( case arrow::Type::type::DATE32: return ColumnStats::CreateDateColumnStats(std::nullopt, std::nullopt, std::nullopt); + case arrow::Type::type::TIME32: + return ColumnStats::CreateIntColumnStats(std::nullopt, std::nullopt, std::nullopt); case arrow::Type::type::TIMESTAMP: { auto ts_type = checked_pointer_cast<::arrow::TimestampType>(type); int32_t precision = DateTimeUtils::GetPrecisionFromType(ts_type); diff --git a/src/paimon/format/avro/avro_stats_extractor_test.cpp b/src/paimon/format/avro/avro_stats_extractor_test.cpp index 76f19ff1a..4d5425e22 100644 --- a/src/paimon/format/avro/avro_stats_extractor_test.cpp +++ b/src/paimon/format/avro/avro_stats_extractor_test.cpp @@ -100,13 +100,14 @@ TEST_F(AvroStatsExtractorTest, TestPrimitiveStatsExtractor) { arrow::field("f18", arrow::timestamp(arrow::TimeUnit::MILLI, timezone)), arrow::field("f19", arrow::timestamp(arrow::TimeUnit::MICRO, timezone)), arrow::field("f20", arrow::timestamp(arrow::TimeUnit::NANO, timezone)), + arrow::field("f21", arrow::time32(arrow::TimeUnit::MILLI)), }; auto schema = std::make_shared(fields); auto array = std::dynamic_pointer_cast( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ - [1, 11, 111, 1111, 1.1, 1.11, "Hello", "你好", 1234, "2033-05-18 03:33:20.0", "1.22", true, "2033-05-18 03:33:20", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20.0"], - [2, 22, 222, 2222, 2.2, 2.22, "World", "世界", -1234, "1899-01-01 00:59:20.001001001", "2.22", false, "1899-01-01 00:59:20", "1899-01-01 00:59:20", "1899-01-01 00:59:20", "1899-01-01 00:59:20.001001001","1899-01-01 00:59:20", "1899-01-01 00:59:20", "1899-01-01 00:59:20", "1899-01-01 00:59:20.001001001"], - [null, null, 0, null, null, 0, null, null, null, null, null, null, null, null, null, null, null, null, null, null] + [1, 11, 111, 1111, 1.1, 1.11, "Hello", "你好", 1234, "2033-05-18 03:33:20.0", "1.22", true, "2033-05-18 03:33:20", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20.0", 0], + [2, 22, 222, 2222, 2.2, 2.22, "World", "世界", -1234, "1899-01-01 00:59:20.001001001", "2.22", false, "1899-01-01 00:59:20", "1899-01-01 00:59:20", "1899-01-01 00:59:20", "1899-01-01 00:59:20.001001001","1899-01-01 00:59:20", "1899-01-01 00:59:20", "1899-01-01 00:59:20", "1899-01-01 00:59:20.001001001", 86399999], + [null, null, 0, null, null, 0, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null] ])") .ValueOrDie()); auto src_chunk_array = std::make_shared(arrow::ArrayVector({array})); @@ -126,10 +127,11 @@ TEST_F(AvroStatsExtractorTest, TestPrimitiveStatsExtractor) { const auto& column_stats = stats_with_info.first; const auto& file_stats = stats_with_info.second; - ASSERT_EQ(column_stats.size(), 20); + ASSERT_EQ(column_stats.size(), 21); for (const auto& stats : column_stats) { ASSERT_EQ(stats->ToString(), "min null, max null, null count null"); } + ASSERT_EQ(column_stats[20]->GetFieldType(), FieldType::INT); ASSERT_EQ(3, file_stats.GetRowCount()); } diff --git a/src/paimon/format/mosaic/mosaic_file_format_test.cpp b/src/paimon/format/mosaic/mosaic_file_format_test.cpp index ed94feda0..7ae06127e 100644 --- a/src/paimon/format/mosaic/mosaic_file_format_test.cpp +++ b/src/paimon/format/mosaic/mosaic_file_format_test.cpp @@ -373,6 +373,38 @@ TEST_F(MosaicFileFormatTest, ExtractStatistics) { } } +TEST_F(MosaicFileFormatTest, WriteThenReadAndExtractTimeStatistics) { + std::map options = { + {"file.format", "mosaic"}, + {Options::FILE_BLOCK_SIZE, "1 B"}, + {MOSAIC_STATS_COLUMNS, "t"}, + }; + ASSERT_OK_AND_ASSIGN(std::shared_ptr configured_format, + FileFormatFactory::Get("mosaic", options)); + std::string path = PathUtil::JoinPath(directory_->Str(), "time.mosaic"); + arrow::FieldVector fields = { + arrow::field("t", arrow::time32(arrow::TimeUnit::MILLI)), + }; + std::shared_ptr schema = arrow::schema(fields); + std::shared_ptr data = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [45296123], [0], [null], [86399999], [123] + ])") + .ValueOrDie(); + ASSERT_OK(WriteFile(path, schema, data, /*batch_size=*/2, configured_format)); + AssertReadWithBatchSizes(path, schema, data, {1, 2, 5}); + + ::ArrowSchema ffi_schema = {}; + ASSERT_TRUE(arrow::ExportSchema(*schema, &ffi_schema).ok()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr extractor, + configured_format->CreateStatsExtractor(&ffi_schema)); + ASSERT_OK_AND_ASSIGN(auto result, extractor->ExtractWithFileInfo(file_system_, path, pool_)); + ASSERT_EQ(result.second.GetRowCount(), 5); + ASSERT_EQ(result.first.size(), 1); + ASSERT_EQ(result.first[0]->GetFieldType(), FieldType::INT); + ASSERT_EQ(result.first[0]->ToString(), "min 0, max 86399999, null count 1"); +} + TEST_F(MosaicFileFormatTest, RowGroupPredicateFiltering) { std::map options = { {"file.format", "mosaic"}, diff --git a/src/paimon/format/mosaic/mosaic_stats.cpp b/src/paimon/format/mosaic/mosaic_stats.cpp index 17b788200..8d9dbab13 100644 --- a/src/paimon/format/mosaic/mosaic_stats.cpp +++ b/src/paimon/format/mosaic/mosaic_stats.cpp @@ -196,6 +196,13 @@ Result> ConvertFieldStatistics( CollectMinMax(stats, DecodeBigEndian)); return ColumnStats::CreateDateColumnStats(min_max.first, min_max.second, null_count); } + case arrow::Type::TIME32: { + // Like Java MosaicObjects, TIME statistics are the int milliseconds of the day, and + // like the other formats they are int statistics. + PAIMON_ASSIGN_OR_RAISE(OptionalMinMax min_max, + CollectMinMax(stats, DecodeBigEndian)); + return ColumnStats::CreateIntColumnStats(min_max.first, min_max.second, null_count); + } case arrow::Type::TIMESTAMP: { auto timestamp_type = checked_pointer_cast(type); auto decoder = [×tamp_type](const std::vector& bytes) { diff --git a/src/paimon/format/orc/orc_adapter.cpp b/src/paimon/format/orc/orc_adapter.cpp index dc99c5e68..67652d531 100644 --- a/src/paimon/format/orc/orc_adapter.cpp +++ b/src/paimon/format/orc/orc_adapter.cpp @@ -897,6 +897,13 @@ Result> MakeArrowBuilder( return MakeOrcBackedPrimitiveBuilder, ::orc::IntVectorBatch, int32_t>( type, column_vector_batch, pool); + case arrow::Type::type::TIME32: + if (checked_cast(*type).unit() != arrow::TimeUnit::MILLI) { + return Status::TypeError("Only millisecond TIME is supported: ", type->ToString()); + } + return MakeOrcBackedPrimitiveBuilder, + ::orc::IntVectorBatch, int32_t>( + type, column_vector_batch, pool); case arrow::Type::type::INT64: return MakeOrcBackedPrimitiveBuilder, @@ -1328,6 +1335,14 @@ arrow::Status WriteBatch(const arrow::Array& array, ::orc::ColumnVectorBatch* co case arrow::Type::type::INT32: return ShallowCopyGenericBatch( array, column_vector_batch); + case arrow::Type::type::TIME32: + if (checked_cast(*array.type()).unit() != + arrow::TimeUnit::MILLI) { + return arrow::Status::TypeError("Only millisecond TIME is supported: ", + array.type()->ToString()); + } + return ShallowCopyGenericBatch( + array, column_vector_batch); case arrow::Type::type::INT64: return ShallowCopyGenericBatch( array, column_vector_batch); @@ -1395,6 +1410,13 @@ arrow::Result> GetOrcType(const arrow::DataType& ty return ::orc::createPrimitiveType(::orc::TypeKind::SHORT); case arrow::Type::type::INT32: return ::orc::createPrimitiveType(::orc::TypeKind::INT); + case arrow::Type::type::TIME32: + // Java `OrcTypeUtil` maps TIME to an ORC int of milliseconds of the day. + if (checked_cast(type).unit() != arrow::TimeUnit::MILLI) { + return arrow::Status::TypeError("Only millisecond TIME is supported: ", + type.ToString()); + } + return ::orc::createPrimitiveType(::orc::TypeKind::INT); case arrow::Type::type::INT64: return ::orc::createPrimitiveType(::orc::TypeKind::LONG); case arrow::Type::type::FLOAT: diff --git a/src/paimon/format/orc/orc_adapter_test.cpp b/src/paimon/format/orc/orc_adapter_test.cpp index a59feee9a..620707c70 100644 --- a/src/paimon/format/orc/orc_adapter_test.cpp +++ b/src/paimon/format/orc/orc_adapter_test.cpp @@ -645,6 +645,37 @@ TEST_P(OrcAdapterTest, TestDecimalAndTimestamp) { ASSERT_TRUE(src_array->Equals(target_array)); } +TEST_P(OrcAdapterTest, TestTime) { + arrow::FieldVector fields = {arrow::field("f0", arrow::time32(arrow::TimeUnit::MILLI))}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr<::orc::Type> orc_type, + OrcAdapter::GetOrcType(arrow::Schema(fields))); + ASSERT_EQ("struct", orc_type->toString()); + + auto arrow_type = arrow::struct_(fields); + auto src_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow_type, R"([[0], [123], [86399999], [null]])") + .ValueOrDie(); + auto read_batch = GenerateOrcReadBatch(src_array); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr target_array, + OrcAdapter::AppendBatch(arrow_type, read_batch.get(), arrow::default_memory_pool())); + ASSERT_TRUE(target_array->Equals(src_array)) << target_array->ToString(); + + arrow::FieldVector second_fields = {arrow::field("f0", arrow::time32(arrow::TimeUnit::SECOND))}; + ASSERT_NOK_WITH_MSG(OrcAdapter::GetOrcType(arrow::Schema(second_fields)), + "Only millisecond TIME is supported"); + auto second_type = arrow::struct_(second_fields); + ASSERT_NOK_WITH_MSG( + OrcAdapter::AppendBatch(second_type, read_batch.get(), arrow::default_memory_pool()), + "Only millisecond TIME is supported"); + auto second_array = + arrow::ipc::internal::json::ArrayFromJSON(second_type, R"([[1]])").ValueOrDie(); + std::unique_ptr<::orc::ColumnVectorBatch> write_batch = orc_type->createRowBatch( + /*size=*/1, *::orc::getDefaultPool(), /*encoded=*/false, /*useTightNumericVector=*/true); + ASSERT_NOK_WITH_MSG(OrcAdapter::WriteBatch(second_array, write_batch.get()), + "Only millisecond TIME is supported"); +} + TEST_F(OrcAdapterTest, TestBridgeForMapType) { // map type will lose meta of key and value field after c <-> c++ bridge auto key_field = DataField::ConvertDataFieldToArrowField( diff --git a/src/paimon/format/orc/orc_stats_extractor_test.cpp b/src/paimon/format/orc/orc_stats_extractor_test.cpp index ffebb3b55..9b997c75a 100644 --- a/src/paimon/format/orc/orc_stats_extractor_test.cpp +++ b/src/paimon/format/orc/orc_stats_extractor_test.cpp @@ -520,4 +520,10 @@ TEST_F(OrcStatsExtractorTest, TestExtractStatsTimestampType) { } } +TEST_F(OrcStatsExtractorTest, TestExtractStatsTimeType) { + arrow::FieldVector fields = {arrow::field("t", arrow::time32(arrow::TimeUnit::MILLI))}; + CheckStats(fields, R"([[0], [123], [86399999], [null]])", {"min 0, max 86399999, null count 1"}, + /*expect_row_count=*/4); +} + } // namespace paimon::orc::test diff --git a/src/paimon/format/orc/predicate_converter.cpp b/src/paimon/format/orc/predicate_converter.cpp index d756e1331..33eb08afb 100644 --- a/src/paimon/format/orc/predicate_converter.cpp +++ b/src/paimon/format/orc/predicate_converter.cpp @@ -230,6 +230,7 @@ Result<::orc::PredicateDataType> PredicateConverter::ConvertToOrcPredicateType( case FieldType::SMALLINT: case FieldType::INT: case FieldType::BIGINT: + case FieldType::TIME: return ::orc::PredicateDataType::LONG; case FieldType::FLOAT: case FieldType::DOUBLE: @@ -271,6 +272,7 @@ Result> PredicateConverter::ConvertToOrcLiterals( orc_literals.emplace_back(static_cast(literal.GetValue())); break; case FieldType::INT: + case FieldType::TIME: orc_literals.emplace_back(static_cast(literal.GetValue())); break; case FieldType::BIGINT: diff --git a/src/paimon/format/orc/predicate_converter_test.cpp b/src/paimon/format/orc/predicate_converter_test.cpp index f9ec55577..70b2abae2 100644 --- a/src/paimon/format/orc/predicate_converter_test.cpp +++ b/src/paimon/format/orc/predicate_converter_test.cpp @@ -71,6 +71,13 @@ TEST(PredicateConverterTest, TestSimple) { ASSERT_OK_AND_ASSIGN(auto search_arg, PredicateConverter::Convert(*orc_type, predicate)); ASSERT_EQ("leaf-0 = (column(id=4) = 10), expr = leaf-0", search_arg->toString()); } + { + auto predicate = + PredicateBuilder::LessThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::TIME, + Literal(FieldType::TIME, 45296123)); + ASSERT_OK_AND_ASSIGN(auto search_arg, PredicateConverter::Convert(*orc_type, predicate)); + ASSERT_EQ("leaf-0 = (column(id=4) < 45296123), expr = leaf-0", search_arg->toString()); + } { // Noted: equal null convert to is null in orc, which is different from java paimon, // where any field value equal/not equal null return false diff --git a/src/paimon/format/parquet/column_index_filter.cpp b/src/paimon/format/parquet/column_index_filter.cpp index 74dc37fee..a0b808493 100644 --- a/src/paimon/format/parquet/column_index_filter.cpp +++ b/src/paimon/format/parquet/column_index_filter.cpp @@ -541,7 +541,8 @@ std::optional ColumnIndexFilter::CompareEncodedWithLiteral( case FieldType::TINYINT: case FieldType::SMALLINT: case FieldType::INT: - case FieldType::DATE: { + case FieldType::DATE: + case FieldType::TIME: { if (encoded.size() < sizeof(int32_t)) { return std::nullopt; } diff --git a/src/paimon/format/parquet/column_index_filter_test.cpp b/src/paimon/format/parquet/column_index_filter_test.cpp index 0f3621c16..9b1dac212 100644 --- a/src/paimon/format/parquet/column_index_filter_test.cpp +++ b/src/paimon/format/parquet/column_index_filter_test.cpp @@ -849,6 +849,56 @@ TEST_F(ColumnIndexFilterTest, DecimalWithoutSchemaKeepsAllPages) { EXPECT_EQ(row_count, ranges.RowCount()); } +TEST_F(ColumnIndexFilterTest, TimePages) { + auto time_type = arrow::time32(arrow::TimeUnit::MILLI); + arrow::Time32Builder builder(time_type, arrow::default_memory_pool()); + for (int32_t millis : {0, 45296123, 86399999}) { + for (int32_t i = 0; i < 10; ++i) { + ASSERT_TRUE(builder.Append(millis).ok()); + } + } + std::shared_ptr values = builder.Finish().ValueOrDie(); + auto data = arrow::StructArray::Make({values}, {arrow::field("value", time_type)}).ValueOrDie(); + std::string file_name = dir_->Str() + "/time.parquet"; + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/30); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); + auto in_stream = std::make_shared(in, length, arrow_pool_); + auto reader = ::parquet::ParquetFileReader::Open(in_stream); + ASSERT_TRUE(reader); + auto page_index_reader = reader->GetPageIndexReader(); + ASSERT_TRUE(page_index_reader); + int64_t row_count = reader->metadata()->RowGroup(0)->num_rows(); + auto filter = [&](const std::shared_ptr& predicate) { + return ColumnIndexFilter::CalculateRowRanges(predicate, page_index_reader->RowGroup(0), + {{"value", 0}}, reader->metadata()->schema(), + row_count); + }; + + auto equal = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"value", FieldType::TIME, + Literal(FieldType::TIME, 45296123)); + ASSERT_OK_AND_ASSIGN(auto ranges, filter(equal)); + ASSERT_EQ(1, ranges.GetRanges().size()); + EXPECT_EQ(10, ranges.GetRanges()[0].from); + EXPECT_EQ(19, ranges.GetRanges()[0].to); + + auto less_than = PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"value", + FieldType::TIME, Literal(FieldType::TIME, 1000)); + ASSERT_OK_AND_ASSIGN(ranges, filter(less_than)); + ASSERT_EQ(1, ranges.GetRanges().size()); + EXPECT_EQ(0, ranges.GetRanges()[0].from); + EXPECT_EQ(9, ranges.GetRanges()[0].to); + + auto greater_or_equal = + PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"value", FieldType::TIME, + Literal(FieldType::TIME, 86399999)); + ASSERT_OK_AND_ASSIGN(ranges, filter(greater_or_equal)); + ASSERT_EQ(1, ranges.GetRanges().size()); + EXPECT_EQ(20, ranges.GetRanges()[0].from); + EXPECT_EQ(29, ranges.GetRanges()[0].to); +} + /// Predicates referencing fields absent from the data file are stripped upstream /// by FieldMappingBuilder, so reaching ColumnIndexFilter with such a predicate is /// a contract violation and surfaces as an error. diff --git a/src/paimon/format/parquet/parquet_stats_extractor.cpp b/src/paimon/format/parquet/parquet_stats_extractor.cpp index ddca47a69..dc9b92b57 100644 --- a/src/paimon/format/parquet/parquet_stats_extractor.cpp +++ b/src/paimon/format/parquet/parquet_stats_extractor.cpp @@ -143,6 +143,12 @@ Result> ConvertStatsToColumnStats( auto [min, max] = CollectMinMaxStats<::parquet::Int32Statistics>(typed_stats); return ColumnStats::CreateDateColumnStats(min, max, null_count); } + case arrow::Type::TIME32: { + // Like Java, TIME statistics are int statistics. + auto typed_stats = checked_pointer_cast<::parquet::Int32Statistics>(stats); + auto [min, max] = CollectMinMaxStats<::parquet::Int32Statistics>(typed_stats); + return ColumnStats::CreateIntColumnStats(min, max, null_count); + } case arrow::Type::TIMESTAMP: { auto timestamp_type = checked_pointer_cast<::arrow::TimestampType>(data_type); if (timestamp_type->unit() == arrow::TimeUnit::type::NANO) { diff --git a/src/paimon/format/parquet/parquet_stats_extractor_test.cpp b/src/paimon/format/parquet/parquet_stats_extractor_test.cpp index 11fe8f05f..7e4bfdb9a 100644 --- a/src/paimon/format/parquet/parquet_stats_extractor_test.cpp +++ b/src/paimon/format/parquet/parquet_stats_extractor_test.cpp @@ -238,6 +238,12 @@ TEST_F(ParquetStatsExtractorTest, TestExtractStatsSimpleType) { CheckStats(fields, data_str, expected_stats_str, /*expect_row_count=*/4); } +TEST_F(ParquetStatsExtractorTest, TestExtractStatsTimeType) { + arrow::FieldVector fields = {arrow::field("t", arrow::time32(arrow::TimeUnit::MILLI))}; + CheckStats(fields, R"([[0], [123], [86399999], [null]])", {"min 0, max 86399999, null count 1"}, + /*expect_row_count=*/4, {FieldType::INT}); +} + TEST_F(ParquetStatsExtractorTest, TestExtractStatsComplexType) { arrow::FieldVector fields = { arrow::field("f1", arrow::map(arrow::int8(), arrow::int16())), diff --git a/src/paimon/format/parquet/predicate_converter.cpp b/src/paimon/format/parquet/predicate_converter.cpp index e81a675e9..be933cfae 100644 --- a/src/paimon/format/parquet/predicate_converter.cpp +++ b/src/paimon/format/parquet/predicate_converter.cpp @@ -342,6 +342,9 @@ Result PredicateConverter::ConvertToArrowLiteral( case FieldType::DATE: return arrow::compute::literal(std::make_shared( static_cast(literal.GetValue()))); + case FieldType::TIME: + return arrow::compute::literal(std::make_shared( + literal.GetValue(), arrow::TimeUnit::MILLI)); case FieldType::BIGINT: return arrow::compute::literal(std::make_shared( static_cast(literal.GetValue()))); diff --git a/src/paimon/format/parquet/predicate_converter_test.cpp b/src/paimon/format/parquet/predicate_converter_test.cpp index ba7664c4e..a2bfa7098 100644 --- a/src/paimon/format/parquet/predicate_converter_test.cpp +++ b/src/paimon/format/parquet/predicate_converter_test.cpp @@ -43,7 +43,8 @@ Literal BigIntLiteral(int64_t value) { } // namespace TEST(PredicateConverterTest, TestSimple) { - // "struct"; + // "struct"; { auto predicate = PredicateBuilder::IsNull(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT); @@ -79,6 +80,14 @@ TEST(PredicateConverterTest, TestSimple) { predicate, /*predicate_node_count_limit=*/100)); ASSERT_EQ("(f6 == 1970-01-11)", expression.ToString()); } + { + auto predicate = + PredicateBuilder::Equal(/*field_index=*/8, /*field_name=*/"f8", FieldType::TIME, + Literal(FieldType::TIME, 45296123)); + ASSERT_OK_AND_ASSIGN(auto expression, PredicateConverter::Convert( + predicate, /*predicate_node_count_limit=*/100)); + ASSERT_EQ("(f8 == 12:34:56.123)", expression.ToString()); + } { auto predicate = PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, BigIntLiteral(5)); diff --git a/src/paimon/testing/utils/read_result_collector.h b/src/paimon/testing/utils/read_result_collector.h index 219869777..89f1c04a7 100644 --- a/src/paimon/testing/utils/read_result_collector.h +++ b/src/paimon/testing/utils/read_result_collector.h @@ -26,6 +26,7 @@ #include #include "arrow/api.h" +#include "arrow/array/concatenate.h" #include "arrow/c/bridge.h" #include "arrow/compute/api.h" #include "paimon/common/reader/reader_utils.h" @@ -181,17 +182,23 @@ class ReadResultCollector { static Result> SortArray( const std::shared_ptr& array, const std::shared_ptr& schema) { + if (array->num_chunks() == 0) { + return array; + } std::vector sort_keys; for (const auto& name : schema->field_names()) { sort_keys.emplace_back(name, arrow::compute::SortOrder::Ascending); } auto sort_options = arrow::compute::SortOptions(sort_keys); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr concatenated, + arrow::Concatenate(array->chunks())); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - auto sorted_indices, arrow::compute::SortIndices(arrow::Datum(array), sort_options)); + std::shared_ptr sorted_indices, + arrow::compute::SortIndices(arrow::Datum(concatenated), sort_options)); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - auto sorted_batch, - arrow::compute::Take(arrow::Datum(array), arrow::Datum(sorted_indices))); - return sorted_batch.chunked_array(); + arrow::Datum sorted_batch, + arrow::compute::Take(arrow::Datum(concatenated), arrow::Datum(sorted_indices))); + return std::make_shared(sorted_batch.make_array()); } private: diff --git a/test/inte/paimon_read_compat_inte_test.cpp b/test/inte/paimon_read_compat_inte_test.cpp index 5761720d4..88240489f 100644 --- a/test/inte/paimon_read_compat_inte_test.cpp +++ b/test/inte/paimon_read_compat_inte_test.cpp @@ -408,18 +408,23 @@ TEST_P(PaimonReadCompatInteTest, ReadsCompatibleTypeValues) { TEST_P(PaimonReadCompatInteTest, ReadsTimeValues) { const CompatibilityParam& param = GetParam(); - if (param.file_format != "parquet") { - GTEST_SKIP() << "TIME reading is only supported for Parquet"; - } - for (int32_t precision : {0, 3, 6, 9}) { + std::vector precisions = + param.file_format == "avro" ? std::vector{0, 3} : std::vector{0, 3, 6, 9}; + for (int32_t precision : precisions) { + SCOPED_TRACE(precision); std::string field_name = "f_time_" + std::to_string(precision); ASSERT_OK_AND_ASSIGN(auto result, ReadTable(param, param.writer_prefix + "_time_types", {field_name})); auto rows = GetOnlyStructChunk(result); ASSERT_TRUE(rows); - ASSERT_EQ(rows->length(), 2); + std::string expected = precision == 0 ? "45296000, null" : "45296123, null"; + if (param.file_format == "orc") { + expected += ", 0"; + } else if (param.file_format == "avro") { + expected += ", null"; + } AssertFieldEqualsJson(rows, field_name, arrow::time32(arrow::TimeUnit::MILLI), - precision == 0 ? "[45296000, null]" : "[45296123, null]"); + "[" + expected + "]"); } } @@ -588,65 +593,4 @@ INSTANTIATE_TEST_SUITE_P(FormatsAndWriters, PaimonReadCompatInteTest, return info.param.file_format + "_" + info.param.writer_prefix; }); -struct UnsupportedReadCase { - std::string name; - std::string table_suffix; - std::string field_name; - std::string expected_error; -}; - -struct UnsupportedReadParam { - std::string file_format; - std::string writer_prefix; - UnsupportedReadCase read_case; -}; - -class PaimonUnsupportedTypeInteTest : public ::testing::TestWithParam {}; - -TEST_P(PaimonUnsupportedTypeInteTest, ReportsExpectedError) { - const UnsupportedReadParam& test_case = GetParam(); - ASSERT_NOK_WITH_MSG(ReadTable(test_case.file_format, - test_case.writer_prefix + "_" + test_case.read_case.table_suffix, - {test_case.read_case.field_name}), - test_case.read_case.expected_error); -} - -std::vector UnsupportedReadParams() { - const std::vector read_cases = { - {"TimePrecision0", "time_types", "f_time_0", ""}, - {"TimePrecision3", "time_types", "f_time_3", ""}, - {"TimePrecision6", "time_types", "f_time_6", ""}, - {"TimePrecision9", "time_types", "f_time_9", ""}, - }; - std::vector result; - for (const CompatibilityParam& param : CompatibilityParams()) { - for (const UnsupportedReadCase& read_case : read_cases) { - bool is_time = read_case.table_suffix == "time_types"; - if (is_time && param.file_format == "parquet") { - continue; - } - // Java Avro cannot create TIME(6/9), so its negative table contains TIME(0/3) only. - if (param.file_format == "avro" && - (read_case.name == "TimePrecision6" || read_case.name == "TimePrecision9")) { - continue; - } - auto expected_case = read_case; - if (is_time) { - expected_case.expected_error = param.file_format == "orc" - ? "Unknown or unsupported Arrow type: time32[ms]" - : "invalid avro logical type"; - } - result.push_back({param.file_format, param.writer_prefix, expected_case}); - } - } - return result; -} - -INSTANTIATE_TEST_SUITE_P(UnsupportedTypes, PaimonUnsupportedTypeInteTest, - ::testing::ValuesIn(UnsupportedReadParams()), - [](const ::testing::TestParamInfo& info) { - return info.param.file_format + "_" + info.param.writer_prefix + "_" + - info.param.read_case.name; - }); - } // namespace paimon::test diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 7bdb500b2..a8d7e7f82 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -35,6 +35,7 @@ #include "arrow/array/array_base.h" #include "arrow/c/abi.h" #include "arrow/ipc/json_simple.h" +#include "arrow/util/key_value_metadata.h" #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" @@ -45,6 +46,7 @@ #include "paimon/common/reader/concat_batch_reader.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/types/data_type.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/read_ahead_cache.h" @@ -1308,6 +1310,45 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedTable) { ASSERT_EQ(max_value_stats_array->GetString(0), "{dt=20260527, pk=a, v=1}"); } +TEST(SystemTableReadInteTest, TestReadFilesSystemTableWithTimeColumns) { + auto time_field = [](const std::string& name, const std::string& precision) { + return arrow::field(name, arrow::time32(arrow::TimeUnit::MILLI), /*nullable=*/false, + arrow::key_value_metadata({DataType::kTimePrecision}, {precision})); + }; + arrow::FieldVector fields = {time_field("dt", "3"), time_field("pk", "0"), + time_field("v", "9")}; + std::map options = { + {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, {Options::BUCKET, "1"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), arrow::schema(fields), + /*partition_keys=*/{"dt"}, + /*primary_keys=*/{"dt", "pk"}, options, + /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[3723000, 45296000, 45296123]])", + /*partition_map=*/{{"dt", "01:02:03.0"}}, + /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + ASSERT_OK_AND_ASSIGN(auto files_result, ReadSystemTable(table_path + "$files", options)); + auto files_array = SingleStructChunk(files_result); + ASSERT_EQ(files_array->length(), 1); + auto partition_array = std::dynamic_pointer_cast(files_array->field(0)); + auto min_key_array = std::dynamic_pointer_cast(files_array->field(8)); + auto min_value_stats_array = + std::dynamic_pointer_cast(files_array->field(11)); + ASSERT_TRUE(partition_array); + ASSERT_TRUE(min_key_array); + ASSERT_TRUE(min_value_stats_array); + ASSERT_EQ(partition_array->GetString(0), "{01:02:03.0}"); + ASSERT_EQ(min_key_array->GetString(0), "[12:34:56]"); + ASSERT_EQ(min_value_stats_array->GetString(0), "{dt=01:02:03.0, pk=12:34:56, v=12:34:56.123}"); +} + TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedPartialWrite) { arrow::FieldVector fields = { arrow::field("dt", arrow::utf8()), diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 50772b438..c211f4425 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -30,11 +31,14 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "arrow/type.h" +#include "arrow/util/key_value_metadata.h" #include "fmt/format.h" #include "gtest/gtest.h" +#include "paimon/bucket/bucket_id_calculator.h" #include "paimon/commit_context.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/reader/reader_utils.h" +#include "paimon/common/types/data_type.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/path_util.h" @@ -42,6 +46,7 @@ #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/schema/schema_manager.h" +#include "paimon/core/snapshot.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/data/shredding/map_shared_shredding_schema_utils.h" #include "paimon/defs.h" @@ -49,6 +54,7 @@ #include "paimon/file_store_write.h" #include "paimon/format/file_format_factory.h" #include "paimon/fs/file_system.h" +#include "paimon/memory/memory_pool.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/predicate/vector_search.h" @@ -155,6 +161,15 @@ class WriteAndReadInteTest Status CompactAndCommit(const std::string& table_path, const std::map& options, int64_t commit_identifier) const { + return CompactBucketsAndCommit(table_path, options, /*partition=*/{}, /*buckets=*/{0}, + /*full_compaction=*/true, commit_identifier); + } + + Status CompactBucketsAndCommit(const std::string& table_path, + const std::map& options, + const std::map& partition, + const std::set& buckets, bool full_compaction, + int64_t commit_identifier) const { WriteContextBuilder write_context_builder(table_path, "commit_user"); write_context_builder.WithTempDirectory(LookupTempDirectory()); PAIMON_ASSIGN_OR_RAISE( @@ -162,8 +177,9 @@ class WriteAndReadInteTest write_context_builder.SetOptions(options).WithStreamingMode(true).Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_write, FileStoreWrite::Create(std::move(write_context))); - PAIMON_RETURN_NOT_OK(file_store_write->Compact(/*partition=*/{}, /*bucket=*/0, - /*full_compaction=*/true)); + for (int32_t bucket : buckets) { + PAIMON_RETURN_NOT_OK(file_store_write->Compact(partition, bucket, full_compaction)); + } PAIMON_ASSIGN_OR_RAISE( std::vector> compact_messages, file_store_write->PrepareCommit(/*wait_compaction=*/true, commit_identifier)); @@ -5985,53 +6001,6 @@ TEST(VortexPrimaryKeyWriteInteTest, TestPKWriteFinalize) { } #endif -#if defined(PAIMON_ENABLE_LANCE) || defined(PAIMON_ENABLE_VORTEX) -// Regression: append-table finalization always extracts statistics, and TIME has no statistics -// factory of its own, so the extractor must still yield one entry for a TIME column. -TEST(TimeColumnWriteInteTest, TestAppendTimeColumnFinalize) { - std::vector file_formats; -#ifdef PAIMON_ENABLE_LANCE - file_formats.push_back("lance"); -#endif -#ifdef PAIMON_ENABLE_VORTEX - file_formats.push_back("vortex"); -#endif - for (const auto& file_format : file_formats) { - SCOPED_TRACE(file_format); - auto dir = UniqueTestDirectory::Create("local"); - ASSERT_NE(dir, nullptr); - arrow::FieldVector fields = {arrow::field("f_time", arrow::time32(arrow::TimeUnit::MILLI)), - arrow::field("f1", arrow::int32())}; - auto schema = arrow::schema(fields); - std::map options = {{Options::FILE_FORMAT, file_format}, - {Options::BUCKET, "-1"}, - {Options::FILE_SYSTEM, "local"}}; - ASSERT_OK_AND_ASSIGN(auto helper, - TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, options, - /*is_streaming_mode=*/false)); - int64_t commit_identifier = 0; - ASSERT_OK_AND_ASSIGN( - std::unique_ptr batch, - TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[3723123, 1],[null, 2]])", - /*partition_map=*/{}, /*bucket=*/0, {})); - ASSERT_OK_AND_ASSIGN(auto commit_msgs, - helper->WriteAndCommit(std::move(batch), commit_identifier++, - /*expected_commit_messages=*/std::nullopt)); - arrow::FieldVector fields_with_row_kind = fields; - fields_with_row_kind.insert(fields_with_row_kind.begin(), - arrow::field("_VALUE_KIND", arrow::int8())); - ASSERT_OK_AND_ASSIGN( - std::vector> splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - ASSERT_OK_AND_ASSIGN( - bool success, helper->ReadAndCheckResult(arrow::struct_(fields_with_row_kind), splits, - R"([[0, 3723123, 1],[0, null, 2]])")); - ASSERT_TRUE(success); - } -} -#endif - #ifdef PAIMON_ENABLE_LANCE class LanceTimestampLtzInteTest : public ::testing::Test, @@ -6165,7 +6134,585 @@ INSTANTIATE_TEST_SUITE_P(TimezoneAndPrimaryKey, LanceTimestampLtzInteTest, ::testing::Bool())); #endif +class TimeWriteAndReadInteTest : public WriteAndReadInteTest {}; + +std::vector> GetTestValuesForTimeWriteAndReadInteTest() { + std::vector> values = + GetTestValuesForWriteAndReadInteTest(); +#ifdef PAIMON_ENABLE_VORTEX + values.emplace_back("vortex", "local"); +#endif + return values; +} + +arrow::FieldVector ReadFieldsWithRowKind(const arrow::FieldVector& fields) { + arrow::FieldVector result = {arrow::field("_VALUE_KIND", arrow::int8())}; + for (const auto& field : fields) { + result.push_back(field->WithNullable(true)); + } + return result; +} + +TEST_P(TimeWriteAndReadInteTest, TestPKTableWithTimeColumns) { + auto [file_format, file_system] = GetParam(); + auto time_field = [](const std::string& name, const std::string& precision, bool nullable) { + return arrow::field(name, arrow::time32(arrow::TimeUnit::MILLI), nullable, + arrow::key_value_metadata({DataType::kTimePrecision}, {precision})); + }; + arrow::FieldVector fields = { + time_field("part", "3", /*nullable=*/false), + time_field("pk", "0", /*nullable=*/false), + time_field("v_time", file_format == "avro" ? "3" : "9", /*nullable=*/true), + arrow::field("v", arrow::int32()), + }; + constexpr int32_t kNumBuckets = 4; + const bool has_key_stats = file_format == "parquet" || file_format == "orc"; + std::map options = { + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, std::to_string(kNumBuckets)}, + {Options::FILE_SYSTEM, file_system}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN( + std::unique_ptr helper, + TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{"part"}, + /*primary_keys=*/{"part", "pk"}, options, + /*is_streaming_mode=*/true, /*ignore_if_exists=*/false, + LookupTempDirectory())); + const std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + + const std::vector keys = {0, 1000, 3600000, 7200000, 45296000, 86399000}; + std::shared_ptr key_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[1]}), + fmt::format("[[{}]]", fmt::join(keys, "],["))) + .ValueOrDie(); + ::ArrowArray c_keys; + ::ArrowSchema c_key_schema; + ASSERT_TRUE(arrow::ExportArray(*key_array, &c_keys, &c_key_schema).ok()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr calculator, + BucketIdCalculator::Create(/*is_pk_table=*/true, kNumBuckets, GetDefaultPool())); + std::vector bucket_ids(keys.size()); + ASSERT_OK(calculator->CalculateBucketIds(&c_keys, &c_key_schema, bucket_ids.data())); + std::map bucket_of_key; + std::set buckets; + for (size_t i = 0; i < keys.size(); ++i) { + bucket_of_key[keys[i]] = bucket_ids[i]; + buckets.insert(bucket_ids[i]); + } + ASSERT_GT(buckets.size(), 1); + + const std::map partition = {{"part", "12:34:56.123"}}; + int64_t commit_identifier = 0; + struct Row { + int32_t pk; + std::string v_time; + int32_t v; + RecordBatch::RowKind kind; + }; + auto write_and_commit = [&](const std::vector& rows) -> Status { + std::map, std::vector>> + rows_of_bucket; + for (const Row& row : rows) { + auto& [values, kinds] = rows_of_bucket[bucket_of_key.at(row.pk)]; + values.push_back(fmt::format("[45296123, {}, {}, {}]", row.pk, row.v_time, row.v)); + kinds.push_back(row.kind); + } + std::vector> batches; + for (const auto& [bucket, values_and_kinds] : rows_of_bucket) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch, + TestHelper::MakeRecordBatch( + arrow::struct_(fields), + fmt::format("[{}]", fmt::join(values_and_kinds.first, ",")), + partition, bucket, values_and_kinds.second)); + batches.push_back(std::move(batch)); + } + return helper + ->WriteAndCommit(std::move(batches), commit_identifier++, + /*expected_commit_messages=*/std::nullopt) + .status(); + }; + + auto create_plan = + [&](const std::shared_ptr& predicate) -> Result> { + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.SetOptions(options).AddOption(Options::SCAN_MODE, + StartupMode::LatestFull().ToString()); + if (predicate) { + scan_context_builder.SetPredicate(predicate); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, + scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context))); + return table_scan->CreatePlan(); + }; + auto planned_buckets = [](const Plan& plan) -> Result> { + std::set result; + for (const auto& split : plan.Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + if (!data_split) { + return Status::Invalid("split cannot be cast to DataSplitImpl"); + } + result.insert(data_split->Bucket()); + } + return result; + }; + arrow::FieldVector fields_with_row_kind = ReadFieldsWithRowKind(fields); + const std::shared_ptr result_schema = arrow::schema(fields_with_row_kind); + auto read = [&](const Plan& plan, const std::shared_ptr& predicate) + -> Result> { + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options); + if (predicate) { + read_context_builder.SetPredicate(predicate).EnablePredicateFilter(true); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, + read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, + table_read->CreateReader(plan.Splits())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr actual, + ReadResultCollector::CollectResult(std::move(batch_reader))); + return ReadResultCollector::SortArray(actual, result_schema); + }; + std::set live_buckets; + for (const auto& [key, bucket] : bucket_of_key) { + if (key != 1000) { + live_buckets.insert(bucket); + } + } + auto check_rows = [&](const std::shared_ptr& predicate, + const std::string& expected_rows, std::set* filtered_buckets) { + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_with_row_kind), + expected_rows) + .ValueOrDie()); + for (bool enable_scan_predicate : {false, true}) { + if (enable_scan_predicate && !predicate) { + break; + } + SCOPED_TRACE(enable_scan_predicate ? "with scan predicate" : "without scan predicate"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, + create_plan(enable_scan_predicate ? predicate : nullptr)); + ASSERT_OK_AND_ASSIGN(std::set scanned_buckets, planned_buckets(*plan)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + read(*plan, predicate)); + ASSERT_TRUE(expected->Equals(actual)) + << "actual: " << actual->ToString() << "\nexpected: " << expected->ToString(); + if (enable_scan_predicate) { + *filtered_buckets = std::move(scanned_buckets); + } else { + ASSERT_TRUE(std::includes(scanned_buckets.begin(), scanned_buckets.end(), + live_buckets.begin(), live_buckets.end())); + } + } + }; + auto pk_equal = [](int32_t pk) { + return PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"pk", FieldType::TIME, + Literal(FieldType::TIME, pk)); + }; + auto check_table = [&](const std::string& all_rows, const std::string& point_rows, + const std::string& in_rows) { + std::set filtered_buckets; + { + SCOPED_TRACE("all rows"); + check_rows(/*predicate=*/nullptr, all_rows, /*filtered_buckets=*/nullptr); + } + { + SCOPED_TRACE("pk = 12:34:56"); + check_rows(pk_equal(45296000), point_rows, &filtered_buckets); + ASSERT_EQ(std::set({bucket_of_key.at(45296000)}), filtered_buckets); + } + { + SCOPED_TRACE("pk IN (02:00:00, 23:59:59)"); + check_rows(PredicateBuilder::In( + /*field_index=*/1, /*field_name=*/"pk", FieldType::TIME, + {Literal(FieldType::TIME, 7200000), Literal(FieldType::TIME, 86399000)}), + in_rows, &filtered_buckets); + ASSERT_EQ(1, filtered_buckets.count(bucket_of_key.at(7200000))); + ASSERT_EQ(1, filtered_buckets.count(bucket_of_key.at(86399000))); + } + if (has_key_stats) { + SCOPED_TRACE("pk > 23:59:59"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, + create_plan(PredicateBuilder::GreaterThan( + /*field_index=*/1, /*field_name=*/"pk", FieldType::TIME, + Literal(FieldType::TIME, 86399000)))); + ASSERT_TRUE(plan->Splits().empty()); + } + }; + + const RecordBatch::RowKind insert_kind = RecordBatch::RowKind::INSERT; + const RecordBatch::RowKind update_kind = RecordBatch::RowKind::UPDATE_AFTER; + const RecordBatch::RowKind delete_kind = RecordBatch::RowKind::DELETE; + ASSERT_OK(write_and_commit({{0, "0", 1, insert_kind}, + {1000, "null", 6, insert_kind}, + {3600000, "123", 2, insert_kind}, + {7200000, "null", 4, insert_kind}, + {45296000, "45296123", 5, insert_kind}, + {86399000, "86399999", 3, insert_kind}})); + ASSERT_OK(write_and_commit({{3600000, "45296123", 20, update_kind}, + {7200000, "null", 40, update_kind}, + {1000, "null", 6, delete_kind}})); + const std::string point_rows = R"([[0, 45296123, 45296000, 45296123, 5]])"; + const std::string in_rows = R"([ + [0, 45296123, 7200000, null, 40], + [0, 45296123, 86399000, 86399999, 3] + ])"; + const std::string upserted_rows = R"([ + [0, 45296123, 0, 0, 1], + [0, 45296123, 3600000, 45296123, 20], + [0, 45296123, 7200000, null, 40], + [0, 45296123, 45296000, 45296123, 5], + [0, 45296123, 86399000, 86399999, 3] + ])"; + { + SCOPED_TRACE("after upsert and delete"); + check_table(upserted_rows, point_rows, in_rows); + } + + helper.reset(); + ASSERT_OK(CompactBucketsAndCommit(table_path, options, partition, buckets, + /*full_compaction=*/true, commit_identifier++)); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(table_path, options, + /*is_streaming_mode=*/true, LookupTempDirectory())); + ASSERT_OK_AND_ASSIGN(std::optional compacted_snapshot, helper->LatestSnapshot()); + ASSERT_TRUE(compacted_snapshot); + ASSERT_EQ(Snapshot::CommitKind::Compact(), compacted_snapshot->GetCommitKind()); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN(auto compacted_files, CurrentDataFiles(options)); + ASSERT_FALSE(compacted_files.empty()); + for (const auto& [bucket_path, file] : compacted_files) { + ASSERT_EQ(core_options.GetNumLevels() - 1, file->level) << file->file_name; + } + { + SCOPED_TRACE("after full compaction"); + check_table(upserted_rows, point_rows, in_rows); + } + + ASSERT_OK(write_and_commit({{0, "0", 100, update_kind}})); + ASSERT_OK(write_and_commit({{0, "null", 101, update_kind}})); + const std::string final_rows = R"([ + [0, 45296123, 0, null, 101], + [0, 45296123, 3600000, 45296123, 20], + [0, 45296123, 7200000, null, 40], + [0, 45296123, 45296000, 45296123, 5], + [0, 45296123, 86399000, 86399999, 3] + ])"; + { + SCOPED_TRACE("after updates over the compacted files"); + check_table(final_rows, point_rows, in_rows); + SCOPED_TRACE("pk = 00:00:00"); + std::set filtered_buckets; + check_rows(pk_equal(0), R"([[0, 45296123, 0, null, 101]])", &filtered_buckets); + ASSERT_EQ(std::set({bucket_of_key.at(0)}), filtered_buckets); + } +} + +TEST_P(TimeWriteAndReadInteTest, TestAppendTableWithTimePartitionAndValuePredicates) { + auto [file_format, file_system] = GetParam(); + arrow::FieldVector fields = { + arrow::field("part", arrow::time32(arrow::TimeUnit::MILLI), /*nullable=*/false, + arrow::key_value_metadata({DataType::kTimePrecision}, {"3"})), + arrow::field("v_time", arrow::time32(arrow::TimeUnit::MILLI)), + arrow::field("v", arrow::int32()), + }; + std::map options = { + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN( + std::unique_ptr helper, + TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{"part"}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + const std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + + constexpr int32_t kEarlyPart = 3723000; + constexpr int32_t kLatePart = 45296123; + std::vector> batches; + ASSERT_OK_AND_ASSIGN(std::unique_ptr early_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([ + [3723000, 0, 1], + [3723000, null, 2], + [3723000, 45296123, 3] + ])", + {{"part", "01:02:03.0"}}, /*bucket=*/0, {})); + batches.push_back(std::move(early_batch)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr late_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([ + [45296123, null, 4], + [45296123, 3723000, 5], + [45296123, 86399999, 6] + ])", + {{"part", "12:34:56.123"}}, /*bucket=*/0, {})); + batches.push_back(std::move(late_batch)); + ASSERT_OK(helper->WriteAndCommit(std::move(batches), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector fields_with_row_kind = ReadFieldsWithRowKind(fields); + const std::shared_ptr result_schema = arrow::schema(fields_with_row_kind); + auto check_rows = [&](const std::shared_ptr& predicate, + const std::string& expected_rows, + const std::set& expected_partitions) { + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_with_row_kind), + expected_rows) + .ValueOrDie()); + for (bool enable_scan_predicate : {false, true}) { + SCOPED_TRACE(enable_scan_predicate ? "with scan predicate" : "without scan predicate"); + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.SetOptions(options).AddOption( + Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + if (enable_scan_predicate) { + scan_context_builder.SetPredicate(predicate); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); + std::set planned_partitions; + for (const auto& split : plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + planned_partitions.insert(data_split->Partition().GetInt(0)); + } + ASSERT_EQ(enable_scan_predicate ? expected_partitions + : std::set({kEarlyPart, kLatePart}), + planned_partitions); + + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options).SetPredicate(predicate).EnablePredicateFilter( + true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, + TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + table_read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + ReadResultCollector::CollectResult(std::move(batch_reader))); + ASSERT_OK_AND_ASSIGN(actual, ReadResultCollector::SortArray(actual, result_schema)); + ASSERT_TRUE(expected->Equals(actual)) + << "actual: " << actual->ToString() << "\nexpected: " << expected->ToString(); + } + }; + auto time_literal = [](int32_t millis) { return Literal(FieldType::TIME, millis); }; + auto part_equal = [&](int32_t millis) { + return PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"part", FieldType::TIME, + time_literal(millis)); + }; + auto v_time_is_null = + PredicateBuilder::IsNull(/*field_index=*/1, /*field_name=*/"v_time", FieldType::TIME); + auto v_time_is_not_null = PredicateBuilder::IsNotNull(/*field_index=*/1, + /*field_name=*/"v_time", FieldType::TIME); + { + SCOPED_TRACE("part = 01:02:03 AND v_time IS NULL"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({part_equal(kEarlyPart), v_time_is_null})); + check_rows(predicate, R"([[0, 3723000, null, 2]])", {kEarlyPart}); + } + { + SCOPED_TRACE("part > 01:02:03 AND v_time IS NULL"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({PredicateBuilder::GreaterThan( + /*field_index=*/0, /*field_name=*/"part", + FieldType::TIME, time_literal(kEarlyPart)), + v_time_is_null})); + check_rows(predicate, R"([[0, 45296123, null, 4]])", {kLatePart}); + } + { + SCOPED_TRACE("part = 12:34:56.123 AND v_time >= 12:00:00"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({part_equal(kLatePart), + PredicateBuilder::GreaterOrEqual( + /*field_index=*/1, /*field_name=*/"v_time", + FieldType::TIME, time_literal(43200000))})); + check_rows(predicate, R"([[0, 45296123, 86399999, 6]])", {kLatePart}); + } + { + SCOPED_TRACE("v_time IS NOT NULL AND v_time < 12:00:00"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({v_time_is_not_null, + PredicateBuilder::LessThan( + /*field_index=*/1, /*field_name=*/"v_time", + FieldType::TIME, time_literal(43200000))})); + check_rows(predicate, R"([ + [0, 3723000, 0, 1], + [0, 45296123, 3723000, 5] + ])", + {kEarlyPart, kLatePart}); + } + { + SCOPED_TRACE("part = 01:02:03 OR v_time IS NULL"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::Or({part_equal(kEarlyPart), v_time_is_null})); + check_rows(predicate, R"([ + [0, 3723000, 0, 1], + [0, 3723000, 45296123, 3], + [0, 3723000, null, 2], + [0, 45296123, null, 4] + ])", + {kEarlyPart, kLatePart}); + } +} + +TEST_P(TimeWriteAndReadInteTest, TestAppendTableWithTimeSchemaEvolution) { + auto [file_format, file_system] = GetParam(); + auto time_field = [](const std::string& name) { + return arrow::field(name, arrow::time32(arrow::TimeUnit::MILLI), /*nullable=*/true, + arrow::key_value_metadata({DataType::kTimePrecision}, {"3"})); + }; + arrow::FieldVector fields_v0 = { + arrow::field("string_to_time", arrow::utf8()), + time_field("time_to_string"), + arrow::field("timestamp_to_time", arrow::timestamp(arrow::TimeUnit::MILLI)), + time_field("time_to_timestamp"), + }; + std::map options = { + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN( + std::unique_ptr helper, + TestHelper::Create(test_dir_, arrow::schema(fields_v0), /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields_v0), R"([ + ["12:34:56.123", 3723000, 1686227696123, 45296123], + [null, null, null, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector fields_v1 = { + time_field("string_to_time"), + arrow::field("time_to_string", arrow::utf8()), + time_field("timestamp_to_time"), + arrow::field("time_to_timestamp", arrow::timestamp(arrow::TimeUnit::MILLI)), + }; + ASSERT_OK(WriteNextSchema({DataField(0, fields_v1[0]), DataField(1, fields_v1[1]), + DataField(2, fields_v1[2]), DataField(3, fields_v1[3])}, + /*highest_field_id=*/3, options)); + + arrow::FieldVector fields_with_row_kind = fields_v1; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(fields_with_row_kind), splits, + R"([ + [0, 45296123, "01:02:03.0", 45296123, 45296123], + [0, null, null, null, null] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestPKTableWithTimeKeyAndDeletionVectors) { + auto [file_format, file_system] = GetParam(); + arrow::FieldVector fields = { + arrow::field("pk", arrow::time32(arrow::TimeUnit::MILLI), /*nullable=*/false, + arrow::key_value_metadata({DataType::kTimePrecision}, {"3"})), + arrow::field("v", arrow::int32()), + }; + std::map options = { + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::DELETION_VECTORS_ENABLED, "true"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + const std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, + CreateLookupTestHelper(arrow::schema(fields), options)); + + constexpr int32_t kNumRows = 10000; + auto value_of = [](int32_t i) { + return static_cast(static_cast(i) * 2654435761U & 0x7fffffffU); + }; + std::vector rows; + rows.reserve(kNumRows); + for (int32_t i = 0; i < kNumRows; ++i) { + rows.push_back(fmt::format("[{}, {}]", i * 1000, value_of(i))); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr insert_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + fmt::format("[{}]", fmt::join(rows, ",")), + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(insert_batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + helper.reset(); + ASSERT_OK(CompactBucketsAndCommit(table_path, options, /*partition=*/{}, /*buckets=*/{0}, + /*full_compaction=*/true, /*commit_identifier=*/1)); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN(auto compacted_files, CurrentDataFiles(options)); + ASSERT_EQ(1, compacted_files.size()); + ASSERT_EQ(core_options.GetNumLevels() - 1, compacted_files[0].second->level); + + ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr update_batch, + TestHelper::MakeRecordBatch( + arrow::struct_(fields), R"([[5000000, -1], [1000, null]])", + /*partition_map=*/{}, /*bucket=*/0, + {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE})); + ASSERT_OK(helper->WriteAndCommit(std::move(update_batch), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + helper.reset(); + ASSERT_OK(CompactBucketsAndCommit(table_path, options, /*partition=*/{}, /*buckets=*/{0}, + /*full_compaction=*/false, /*commit_identifier=*/3)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, InnerScan(options)); + bool has_deletion_vector = false; + for (const auto& split : plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + for (const std::optional& deletion_file : data_split->DeletionFiles()) { + has_deletion_vector = has_deletion_vector || deletion_file.has_value(); + } + } + ASSERT_TRUE(has_deletion_vector); + + rows.clear(); + for (int32_t i = 0; i < kNumRows; ++i) { + if (i == 1) { + continue; + } + rows.push_back(fmt::format("[0, {}, {}]", i * 1000, i == 5000 ? -1 : value_of(i))); + } + arrow::FieldVector fields_with_row_kind = ReadFieldsWithRowKind(fields); + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_with_row_kind), + fmt::format("[{}]", fmt::join(rows, ","))) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(helper, CreateLookupTestHelper(table_path, options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + helper->ReadResult(plan->Splits())); + ASSERT_OK_AND_ASSIGN( + actual, ReadResultCollector::SortArray(actual, arrow::schema(fields_with_row_kind))); + ASSERT_TRUE(expected->Equals(actual)); +} + INSTANTIATE_TEST_SUITE_P(FileFormatAndFileSystem, WriteAndReadInteTest, ::testing::ValuesIn(GetTestValuesForWriteAndReadInteTest())); +INSTANTIATE_TEST_SUITE_P(FileFormatAndFileSystem, TimeWriteAndReadInteTest, + ::testing::ValuesIn(GetTestValuesForTimeWriteAndReadInteTest())); + } // namespace paimon::test