diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java index 2874849bb56a..ef421706d006 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java @@ -77,6 +77,7 @@ import static org.apache.paimon.data.columnar.ColumnVectorUtils.createParquetWritableColumnVector; import static org.apache.paimon.format.parquet.ParquetSchemaConverter.PAIMON_SCHEMA; +import static org.apache.paimon.format.parquet.ParquetSchemaConverter.isThreeLevelList; import static org.apache.paimon.format.parquet.ParquetSchemaConverter.parquetListElementType; import static org.apache.paimon.format.parquet.ParquetSchemaConverter.parquetMapKeyValueType; import static org.apache.paimon.format.parquet.reader.ParquetReaderUtil.buildFieldsList; @@ -336,11 +337,10 @@ private Type clipParquetType(DataType readType, Type parquetType) { // There are two representations for array type in parquet. // See link: // https://impala.apache.org/docs/build/html/topics/impala_parquet_array_resolution.html. - int level = arrayGroup.getType(0) instanceof GroupType ? 3 : 2; Type elementType = clipParquetType(elementReadType, parquetListElementType(arrayGroup)); - if (level == 3) { + if (isThreeLevelList(arrayGroup)) { // In case that the name in middle level is not "list". Type groupMiddle = new GroupType( diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java index 912baec0721c..45b7e6673f84 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java @@ -50,6 +50,7 @@ import java.util.List; import java.util.stream.Collectors; +import static org.apache.paimon.utils.Preconditions.checkArgument; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64; @@ -63,6 +64,8 @@ public class ParquetSchemaConverter { public static final String MAP_KEY_NAME = "key"; public static final String MAP_VALUE_NAME = "value"; public static final String LIST_ELEMENT_NAME = "element"; + private static final String LIST_WRAPPER_NAME = "list"; + private static final String LEGACY_LIST_ARRAY_NAME = "array"; /** * Whether {@code type} is an unsigned integer. Such a column stores a signed value whose bits @@ -490,20 +493,107 @@ public static DataField convertToPaimonField(Type parquetType) { return new DataField(parquetType.getId().intValue(), parquetType.getName(), paimonDataType); } + /** Returns true if the given group is annotated as a Parquet LIST logical type. */ + public static boolean isList(GroupType listType) { + return listType.getLogicalTypeAnnotation() + instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation; + } + + /** + * Returns true if the given group is a three-level Parquet list. + * + *

In a three-level list the immediate repeated child is a wrapper group whose single + * non-repeated child is the actual element type. This covers the canonical layout ({@code list + * -> element}) as well as legacy wrappers such as Hive's {@code bag} layout. + * + *

This corresponds to the Parquet spec's backward-compatibility Rule 5: a repeated + * group that contains exactly one non-repeated child is a wrapper, unless it matches one of + * Rules 1-4. + * + *

The compatibility encodings that are not three-level are: + * + *

+ * + *

See the Parquet spec: LogicalTypes#Backward-compatibility-rules + */ + public static boolean isThreeLevelList(GroupType listType) { + if (!isList(listType)) { + return false; + } + + // A list must have exactly one repeated child (the middle level). + if (listType.getFieldCount() != 1) { + return false; + } + Type middle = listType.getType(0); + if (middle.isPrimitive() || middle.getRepetition() != Type.Repetition.REPEATED) { + return false; + } + GroupType repeatedGroup = middle.asGroupType(); + + // Rule 5: the repeated group is a wrapper containing exactly one non-repeated child. + if (repeatedGroup.getFieldCount() != 1 + || repeatedGroup.getType(0).getRepetition() == Type.Repetition.REPEATED) { + return false; + } + + // Rule 4: legacy "array" and "_tuple" encodings are not wrappers; the repeated + // group itself is the element type. + return !LEGACY_LIST_ARRAY_NAME.equals(repeatedGroup.getName()) + && !(listType.getName() + "_tuple").equals(repeatedGroup.getName()); + } + + /** + * Returns true if the given group follows the canonical three-level Parquet list layout ({@code + * list -> element}). + * + *

The canonical layout is described in the Parquet spec: LogicalTypes#Lists + */ + public static boolean isCanonicalList(Type type) { + if (type.isPrimitive()) { + return false; + } + + GroupType listGroup = type.asGroupType(); + if (!isThreeLevelList(listGroup)) { + return false; + } + + Type middle = listGroup.getType(0); + Type element = middle.asGroupType().getType(0); + return LIST_WRAPPER_NAME.equals(middle.getName()) + && LIST_ELEMENT_NAME.equals(element.getName()); + } + + /** + * Returns the element type of the given LIST-annotated group according to the Parquet spec's + * backward-compatibility rules for lists. + * + *

For a three-level list (Rule 5) the returned type is the single child of the repeated + * wrapper. For Rules 1-4 the repeated field itself is returned because it is the element type. + */ public static Type parquetListElementType(GroupType listType) { - int level = listType.getType(0) instanceof GroupType ? 3 : 2; - if (level == 3) { - // Level 3 representation of list type. - // List type should only have one middle group type, which is repeated, and one element - // type, which is optional. + checkArgument( + listType.getLogicalTypeAnnotation() + instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation, + "Expected LIST-annotated group but got: %s", + listType); + + if (isThreeLevelList(listType)) { return listType.getType(0).asGroupType().getType(0); - } else if (level == 2) { - // Level 2 representation of list type - return listType.getType(0); - } else { - throw new UnsupportedOperationException( - "Parquet list type only have two level representation and three level representation."); } + + return listType.getType(0); } public static Pair parquetMapKeyValueType(GroupType mapType) { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingTypePruner.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingTypePruner.java index f9e5075342e1..62f4e8ae8bfe 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingTypePruner.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingTypePruner.java @@ -25,7 +25,6 @@ import org.apache.paimon.types.RowType; import org.apache.parquet.schema.GroupType; -import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.Type; import javax.annotation.Nullable; @@ -38,6 +37,7 @@ import java.util.Map; import java.util.Set; +import static org.apache.paimon.format.parquet.ParquetSchemaConverter.isCanonicalList; import static org.apache.paimon.format.parquet.ParquetSchemaConverter.parquetListElementType; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -53,9 +53,6 @@ * objectSchemaMap}. */ public class VariantShreddingTypePruner { - private static final String LIST_WRAPPER_NAME = "list"; - private static final String LIST_ELEMENT_NAME = "element"; - @Nullable private final PathNode root; VariantShreddingTypePruner(RowType variantRowType) { @@ -206,48 +203,6 @@ private GroupType clipListShreddingRow(GroupType group, PathNode node, ListThe canonical layout is described in the Parquet spec: LogicalTypes#Lists - */ - private static boolean isCanonicalList(Type type) { - if (type.isPrimitive()) { - return false; - } - - GroupType listGroup = type.asGroupType(); - // 1. Must be a LIST logical type. - if (!(listGroup.getLogicalTypeAnnotation() - instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation)) { - return false; - } - - // 2. LIST group must have exactly one child named "list". - if (listGroup.getFieldCount() != 1) { - return false; - } - Type middle = listGroup.getType(0); - if (!LIST_WRAPPER_NAME.equals(middle.getName())) { - return false; - } - - // 3. The child must be a repeated group. - if (middle.isPrimitive() || middle.getRepetition() != Type.Repetition.REPEATED) { - return false; - } - GroupType repeatedWrapper = middle.asGroupType(); - - // 4. The repeated wrapper must contain exactly one child named "element". - if (repeatedWrapper.getFieldCount() != 1) { - return false; - } - - Type element = repeatedWrapper.getType(0); - return LIST_ELEMENT_NAME.equals(element.getName()); - } - /** Returns true if the given group is a plain struct (not a Parquet list or map). */ private static boolean isObjectGroup(Type type) { if (type.isPrimitive()) { diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java index f40550831dff..134af54a8a25 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java @@ -26,6 +26,7 @@ import org.apache.paimon.types.RowType; import org.apache.parquet.schema.ColumnOrder; +import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.Type; @@ -39,6 +40,7 @@ import static org.apache.paimon.format.parquet.ParquetSchemaConverter.convertToParquetMessageType; import static org.apache.paimon.types.DataTypesTest.assertThat; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64; /** Test for {@link ParquetSchemaConverter}. */ @@ -206,4 +208,169 @@ public void testGeographyLogicalTypeDefaults() { assertThat(expected).isEqualTo(convertToPaimonRowType(messageType)); } + + // Rule 5: canonical three-level list (list -> element) with a primitive element. + @Test + public void testParquetListElementTypeThreeLevelPrimitive() { + GroupType list = + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField( + Types.buildGroup(Type.Repetition.REPEATED) + .optional(INT32) + .named("element") + .named("list")) + .named("arr"); + + Type element = ParquetSchemaConverter.parquetListElementType(list); + assertThat(element.isPrimitive()).isEqualTo(true); + assertThat(element.getName()).isEqualTo("element"); + } + + // Rule 5: canonical three-level list (list -> element) with a group element. + @Test + public void testParquetListElementTypeThreeLevelGroupElement() { + GroupType elementStruct = + Types.buildGroup(Type.Repetition.OPTIONAL) + .optional(INT32) + .named("x") + .optional(INT32) + .named("y") + .named("element"); + GroupType list = + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField( + Types.buildGroup(Type.Repetition.REPEATED) + .addField(elementStruct) + .named("list")) + .named("arr"); + + Type element = ParquetSchemaConverter.parquetListElementType(list); + assertThat(element.isPrimitive()).isEqualTo(false); + assertThat(element.getName()).isEqualTo("element"); + assertThat(element.asGroupType().getFieldCount()).isEqualTo(2); + } + + // Rule 1: a repeated primitive field is itself the element type. + @Test + public void testParquetListElementTypeTwoLevelPrimitive() { + GroupType list = + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .repeated(INT32) + .named("element") + .named("arr"); + + Type element = ParquetSchemaConverter.parquetListElementType(list); + assertThat(element.isPrimitive()).isEqualTo(true); + assertThat(element.getName()).isEqualTo("element"); + } + + // Rule 2: a repeated group with multiple fields is itself the element type. + @Test + public void testParquetListElementTypeTwoLevelGroupElement() { + GroupType elementStruct = + Types.buildGroup(Type.Repetition.REPEATED) + .optional(INT32) + .named("x") + .optional(INT32) + .named("y") + .named("element"); + GroupType list = + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField(elementStruct) + .named("arr"); + + Type element = ParquetSchemaConverter.parquetListElementType(list); + assertThat(element.isPrimitive()).isEqualTo(false); + assertThat(element.getName()).isEqualTo("element"); + assertThat(element.asGroupType().getFieldCount()).isEqualTo(2); + } + + // Rule 3: a repeated group with a single repeated field is the element type. + @Test + public void testParquetListElementTypeLegacyNestedRepeatedWrapper() { + GroupType wrapper = + Types.buildGroup(Type.Repetition.REPEATED) + .repeated(INT32) + .named("array") + .named("array"); + GroupType list = + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField(wrapper) + .named("arr"); + + Type element = ParquetSchemaConverter.parquetListElementType(list); + assertThat(element.isPrimitive()).isEqualTo(false); + assertThat(element.getName()).isEqualTo("array"); + assertThat(element.asGroupType().getFieldCount()).isEqualTo(1); + assertThat(element.asGroupType().getType(0).getName()).isEqualTo("array"); + } + + // Rule 4: a repeated group named "array" with one field is the element type. + @Test + public void testParquetListElementTypeLegacyArrayWrapper() { + GroupType arrayGroup = + Types.buildGroup(Type.Repetition.REPEATED) + .optional(INT32) + .named("foo") + .named("array"); + GroupType list = + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField(arrayGroup) + .named("arr"); + + Type element = ParquetSchemaConverter.parquetListElementType(list); + assertThat(element.isPrimitive()).isEqualTo(false); + assertThat(element.getName()).isEqualTo("array"); + } + + // Rule 4: a repeated group named "_tuple" with one field is the element type. + @Test + public void testParquetListElementTypeLegacyListTupleWrapper() { + GroupType tupleGroup = + Types.buildGroup(Type.Repetition.REPEATED) + .required(BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("str") + .named("my_list_tuple"); + GroupType list = + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField(tupleGroup) + .named("my_list"); + + Type element = ParquetSchemaConverter.parquetListElementType(list); + assertThat(element.isPrimitive()).isEqualTo(false); + assertThat(element.getName()).isEqualTo("my_list_tuple"); + assertThat(element.asGroupType().getFieldCount()).isEqualTo(1); + assertThat(element.asGroupType().getType(0).getName()).isEqualTo("str"); + } + + // Rule 5: a repeated group with a single non-repeated field that is neither "array" nor + // "_tuple" unwraps to the single child (e.g. Hive's bag/array_element encoding). + @Test + public void testParquetListElementTypeLegacyBagWrapper() { + GroupType bagGroup = + Types.buildGroup(Type.Repetition.REPEATED) + .optional(INT32) + .named("array_element") + .named("bag"); + GroupType list = + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField(bagGroup) + .named("arr"); + + Assertions.assertThat(ParquetSchemaConverter.isThreeLevelList(list)).isTrue(); + Assertions.assertThat(ParquetSchemaConverter.isCanonicalList(list)).isFalse(); + + Type element = ParquetSchemaConverter.parquetListElementType(list); + assertThat(element.isPrimitive()).isEqualTo(true); + assertThat(element.getName()).isEqualTo("array_element"); + } }