diff --git a/.codespell_ignore b/.codespell_ignore index d76687440..455b8f3c8 100644 --- a/.codespell_ignore +++ b/.codespell_ignore @@ -11,3 +11,4 @@ NotIn Collet convertor inout +nd diff --git a/LICENSE b/LICENSE index e986c187b..94de8eef0 100644 --- a/LICENSE +++ b/LICENSE @@ -353,6 +353,7 @@ This product includes code from Apache Arrow. * third_party/versions.txt * Upstream source patches: * cmake_modules/arrow.diff + * cmake_modules/arrow_geospatial.diff Copyright: 2016-2024 The Apache Software Foundation. Home page: https://arrow.apache.org/ diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index dffebe650..2379a4593 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -2022,12 +2022,15 @@ macro(build_arrow) set(ARROW_CONFIGURE SOURCE_SUBDIR "cpp" CMAKE_ARGS ${ARROW_CMAKE_ARGS}) set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/arrow.diff") + set(GEOSPATIAL_PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/arrow_geospatial.diff") externalproject_add(arrow_ep URL ${ARROW_SOURCE_URL} URL_HASH "SHA256=${PAIMON_ARROW_BUILD_SHA256_CHECKSUM}" LOG_PATCH ON PATCH_COMMAND ${CMAKE_COMMAND} -E chdir bash -c "[ -f .patched ] && echo ' patch already applied, ignore...' || patch -s -N -p1 -i '${PATCH_FILE}' && touch .patched" + COMMAND ${CMAKE_COMMAND} -E chdir bash -c + "[ -f .geospatial-patched ] || (patch -s -N -p1 -i '${GEOSPATIAL_PATCH_FILE}' && touch .geospatial-patched)" GIT_SUBMODULES "" GIT_SUBMODULES_RECURSE FALSE ${ARROW_CONFIGURE} UPDATE_DISCONNECTED 1 BUILD_BYPRODUCTS "${ARROW_STATIC_LIB}" diff --git a/cmake_modules/arrow_geospatial.diff b/cmake_modules/arrow_geospatial.diff new file mode 100644 index 000000000..520857831 --- /dev/null +++ b/cmake_modules/arrow_geospatial.diff @@ -0,0 +1,1384 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Backport GEOMETRY/GEOGRAPHY logical types from Apache Arrow 21.0.0. +# Binary/WKB native schema mapping and null-count-only statistics retain C++17 support. +diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake +--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake ++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake +@@ -410,7 +410,7 @@ + set(ARROW_WITH_AZURE_SDK ON) + endif() + +-if(ARROW_JSON) ++if(ARROW_JSON OR ARROW_PARQUET) + set(ARROW_WITH_RAPIDJSON ON) + endif() + +diff --git a/cpp/src/arrow/c/bridge.cc b/cpp/src/arrow/c/bridge.cc +--- a/cpp/src/arrow/c/bridge.cc ++++ b/cpp/src/arrow/c/bridge.cc +@@ -1298,13 +1298,10 @@ + } + + bool keys_sorted = (c_struct_->flags & ARROW_FLAG_MAP_KEYS_SORTED); +- bool values_nullable = value_type->field(1)->nullable(); +- // Some implementations of Arrow (such as Rust) use a non-standard field name +- // for key ("keys") and value ("values") fields. For simplicity, we override +- // them on import. +- auto values_field = +- ::arrow::field("value", value_type->field(1)->type(), values_nullable); +- type_ = map(value_type->field(0)->type(), values_field, keys_sorted); ++ // Normalize the C Data Interface child names while keeping logical type metadata. ++ auto key = value_type->field(0)->WithName("key")->WithNullable(false); ++ auto item = value_type->field(1)->WithName("value"); ++ type_ = std::make_shared(std::move(key), std::move(item), keys_sorted); + return Status::OK(); + } + +diff --git a/cpp/src/generated/parquet_types.cpp b/cpp/src/generated/parquet_types.cpp +--- a/cpp/src/generated/parquet_types.cpp ++++ b/cpp/src/generated/parquet_types.cpp +@@ -372,6 +372,41 @@ + } + } + ++int _kEdgeInterpolationAlgorithmValues[] = { ++ EdgeInterpolationAlgorithm::SPHERICAL, ++ EdgeInterpolationAlgorithm::VINCENTY, ++ EdgeInterpolationAlgorithm::THOMAS, ++ EdgeInterpolationAlgorithm::ANDOYER, ++ EdgeInterpolationAlgorithm::KARNEY ++}; ++const char* _kEdgeInterpolationAlgorithmNames[] = { ++ "SPHERICAL", ++ "VINCENTY", ++ "THOMAS", ++ "ANDOYER", ++ "KARNEY" ++}; ++const std::map _EdgeInterpolationAlgorithm_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(5, _kEdgeInterpolationAlgorithmValues, _kEdgeInterpolationAlgorithmNames), ::apache::thrift::TEnumIterator(-1, nullptr, nullptr)); ++ ++std::ostream& operator<<(std::ostream& out, const EdgeInterpolationAlgorithm::type& val) { ++ std::map::const_iterator it = _EdgeInterpolationAlgorithm_VALUES_TO_NAMES.find(val); ++ if (it != _EdgeInterpolationAlgorithm_VALUES_TO_NAMES.end()) { ++ out << it->second; ++ } else { ++ out << static_cast(val); ++ } ++ return out; ++} ++ ++std::string to_string(const EdgeInterpolationAlgorithm::type& val) { ++ std::map::const_iterator it = _EdgeInterpolationAlgorithm_VALUES_TO_NAMES.find(val); ++ if (it != _EdgeInterpolationAlgorithm_VALUES_TO_NAMES.end()) { ++ return std::string(it->second); ++ } else { ++ return std::to_string(static_cast(val)); ++ } ++} ++ + int _kEncodingValues[] = { + /** + * Default encoding. +@@ -1562,6 +1597,140 @@ + } + + ++GeometryType::~GeometryType() noexcept { ++} ++ ++GeometryType::GeometryType() noexcept ++ : crs() { ++} ++ ++void GeometryType::__set_crs(const std::string& val) { ++ this->crs = val; ++__isset.crs = true; ++} ++std::ostream& operator<<(std::ostream& out, const GeometryType& obj) ++{ ++ obj.printTo(out); ++ return out; ++} ++ ++ ++void swap(GeometryType &a, GeometryType &b) { ++ using ::std::swap; ++ swap(a.crs, b.crs); ++ swap(a.__isset, b.__isset); ++} ++ ++bool GeometryType::operator==(const GeometryType & rhs) const ++{ ++ if (__isset.crs != rhs.__isset.crs) ++ return false; ++ else if (__isset.crs && !(crs == rhs.crs)) ++ return false; ++ return true; ++} ++ ++GeometryType::GeometryType(const GeometryType& other110) { ++ crs = other110.crs; ++ __isset = other110.__isset; ++} ++GeometryType::GeometryType(GeometryType&& other111) noexcept { ++ crs = std::move(other111.crs); ++ __isset = other111.__isset; ++} ++GeometryType& GeometryType::operator=(const GeometryType& other112) { ++ crs = other112.crs; ++ __isset = other112.__isset; ++ return *this; ++} ++GeometryType& GeometryType::operator=(GeometryType&& other113) noexcept { ++ crs = std::move(other113.crs); ++ __isset = other113.__isset; ++ return *this; ++} ++void GeometryType::printTo(std::ostream& out) const { ++ using ::apache::thrift::to_string; ++ out << "GeometryType("; ++ out << "crs="; (__isset.crs ? (out << to_string(crs)) : (out << "")); ++ out << ")"; ++} ++ ++ ++GeographyType::~GeographyType() noexcept { ++} ++ ++GeographyType::GeographyType() noexcept ++ : crs(), ++ algorithm(static_cast(0)) { ++} ++ ++void GeographyType::__set_crs(const std::string& val) { ++ this->crs = val; ++__isset.crs = true; ++} ++ ++void GeographyType::__set_algorithm(const EdgeInterpolationAlgorithm::type val) { ++ this->algorithm = val; ++__isset.algorithm = true; ++} ++std::ostream& operator<<(std::ostream& out, const GeographyType& obj) ++{ ++ obj.printTo(out); ++ return out; ++} ++ ++ ++void swap(GeographyType &a, GeographyType &b) { ++ using ::std::swap; ++ swap(a.crs, b.crs); ++ swap(a.algorithm, b.algorithm); ++ swap(a.__isset, b.__isset); ++} ++ ++bool GeographyType::operator==(const GeographyType & rhs) const ++{ ++ if (__isset.crs != rhs.__isset.crs) ++ return false; ++ else if (__isset.crs && !(crs == rhs.crs)) ++ return false; ++ if (__isset.algorithm != rhs.__isset.algorithm) ++ return false; ++ else if (__isset.algorithm && !(algorithm == rhs.algorithm)) ++ return false; ++ return true; ++} ++ ++GeographyType::GeographyType(const GeographyType& other115) { ++ crs = other115.crs; ++ algorithm = other115.algorithm; ++ __isset = other115.__isset; ++} ++GeographyType::GeographyType(GeographyType&& other116) noexcept { ++ crs = std::move(other116.crs); ++ algorithm = other116.algorithm; ++ __isset = other116.__isset; ++} ++GeographyType& GeographyType::operator=(const GeographyType& other117) { ++ crs = other117.crs; ++ algorithm = other117.algorithm; ++ __isset = other117.__isset; ++ return *this; ++} ++GeographyType& GeographyType::operator=(GeographyType&& other118) noexcept { ++ crs = std::move(other118.crs); ++ algorithm = other118.algorithm; ++ __isset = other118.__isset; ++ return *this; ++} ++void GeographyType::printTo(std::ostream& out) const { ++ using ::apache::thrift::to_string; ++ out << "GeographyType("; ++ out << "crs="; (__isset.crs ? (out << to_string(crs)) : (out << "")); ++ out << ", " << "algorithm="; (__isset.algorithm ? (out << to_string(algorithm)) : (out << "")); ++ out << ")"; ++} ++ ++ + LogicalType::~LogicalType() noexcept { + } + +@@ -1631,6 +1800,15 @@ + __isset.UUID = true; + } + ++void LogicalType::__set_GEOMETRY(const GeometryType& val) { ++ this->GEOMETRY = val; ++__isset.GEOMETRY = true; ++} ++ ++void LogicalType::__set_GEOGRAPHY(const GeographyType& val) { ++ this->GEOGRAPHY = val; ++__isset.GEOGRAPHY = true; ++} + void LogicalType::__set_FLOAT16(const Float16Type& val) { + this->FLOAT16 = val; + __isset.FLOAT16 = true; +@@ -1658,10 +1836,12 @@ + swap(a.BSON, b.BSON); + swap(a.UUID, b.UUID); + swap(a.FLOAT16, b.FLOAT16); ++ swap(a.GEOMETRY, b.GEOMETRY); ++ swap(a.GEOGRAPHY, b.GEOGRAPHY); + swap(a.__isset, b.__isset); + } + +-LogicalType::LogicalType(const LogicalType& other92) noexcept { ++LogicalType::LogicalType(const LogicalType& other92) { + STRING = other92.STRING; + MAP = other92.MAP; + LIST = other92.LIST; +@@ -1676,6 +1856,8 @@ + BSON = other92.BSON; + UUID = other92.UUID; + FLOAT16 = other92.FLOAT16; ++ GEOMETRY = other92.GEOMETRY; ++ GEOGRAPHY = other92.GEOGRAPHY; + __isset = other92.__isset; + } + LogicalType::LogicalType(LogicalType&& other93) noexcept { +@@ -1693,9 +1875,11 @@ + BSON = std::move(other93.BSON); + UUID = std::move(other93.UUID); + FLOAT16 = std::move(other93.FLOAT16); ++ GEOMETRY = std::move(other93.GEOMETRY); ++ GEOGRAPHY = std::move(other93.GEOGRAPHY); + __isset = other93.__isset; + } +-LogicalType& LogicalType::operator=(const LogicalType& other94) noexcept { ++LogicalType& LogicalType::operator=(const LogicalType& other94) { + STRING = other94.STRING; + MAP = other94.MAP; + LIST = other94.LIST; +@@ -1710,6 +1894,8 @@ + BSON = other94.BSON; + UUID = other94.UUID; + FLOAT16 = other94.FLOAT16; ++ GEOMETRY = other94.GEOMETRY; ++ GEOGRAPHY = other94.GEOGRAPHY; + __isset = other94.__isset; + return *this; + } +@@ -1728,6 +1914,8 @@ + BSON = std::move(other95.BSON); + UUID = std::move(other95.UUID); + FLOAT16 = std::move(other95.FLOAT16); ++ GEOMETRY = std::move(other95.GEOMETRY); ++ GEOGRAPHY = std::move(other95.GEOGRAPHY); + __isset = other95.__isset; + return *this; + } +@@ -1748,6 +1936,8 @@ + out << ", " << "BSON="; (__isset.BSON ? (out << to_string(BSON)) : (out << "")); + out << ", " << "UUID="; (__isset.UUID ? (out << to_string(UUID)) : (out << "")); + out << ", " << "FLOAT16="; (__isset.FLOAT16 ? (out << to_string(FLOAT16)) : (out << "")); ++ out << ", " << "GEOMETRY="; (__isset.GEOMETRY ? (out << to_string(GEOMETRY)) : (out << "")); ++ out << ", " << "GEOGRAPHY="; (__isset.GEOGRAPHY ? (out << to_string(GEOGRAPHY)) : (out << "")); + out << ")"; + } + +diff --git a/cpp/src/generated/parquet_types.h b/cpp/src/generated/parquet_types.h +--- a/cpp/src/generated/parquet_types.h ++++ b/cpp/src/generated/parquet_types.h +@@ -217,6 +217,22 @@ + * enums are also used to specify the encoding of definition and repetition levels. + * See the accompanying doc for the details of the more complicated encodings. + */ ++struct EdgeInterpolationAlgorithm { ++ enum type { ++ SPHERICAL = 0, ++ VINCENTY = 1, ++ THOMAS = 2, ++ ANDOYER = 3, ++ KARNEY = 4 ++ }; ++}; ++ ++extern const std::map _EdgeInterpolationAlgorithm_VALUES_TO_NAMES; ++ ++std::ostream& operator<<(std::ostream& out, const EdgeInterpolationAlgorithm::type& val); ++ ++std::string to_string(const EdgeInterpolationAlgorithm::type& val); ++ + struct Encoding { + enum type { + /** +@@ -1467,8 +1483,127 @@ + + std::ostream& operator<<(std::ostream& out, const BsonType& obj); + ++typedef struct _GeometryType__isset { ++ _GeometryType__isset() : crs(false) {} ++ bool crs :1; ++} _GeometryType__isset; ++ ++/** ++ * Embedded Geometry logical type annotation ++ * ++ * Geospatial features in the Well-Known Binary (WKB) format and edges interpolation ++ * is always linear/planar. ++ * ++ * A custom CRS can be set by the crs field. If unset, it defaults to "OGC:CRS84", ++ * which means that the geometries must be stored in longitude, latitude based on ++ * the WGS84 datum. ++ * ++ * Allowed for physical type: BYTE_ARRAY. ++ * ++ * See Geospatial.md for details. ++ */ ++class GeometryType { ++ public: ++ ++ GeometryType(const GeometryType&); ++ GeometryType(GeometryType&&) noexcept; ++ GeometryType& operator=(const GeometryType&); ++ GeometryType& operator=(GeometryType&&) noexcept; ++ GeometryType() noexcept; ++ ++ virtual ~GeometryType() noexcept; ++ std::string crs; ++ ++ _GeometryType__isset __isset; ++ ++ void __set_crs(const std::string& val); ++ ++ bool operator == (const GeometryType & rhs) const; ++ bool operator != (const GeometryType &rhs) const { ++ return !(*this == rhs); ++ } ++ ++ bool operator < (const GeometryType & ) const; ++ ++ template ++ uint32_t read(Protocol_* iprot); ++ template ++ uint32_t write(Protocol_* oprot) const; ++ ++ virtual void printTo(std::ostream& out) const; ++}; ++ ++void swap(GeometryType &a, GeometryType &b); ++ ++std::ostream& operator<<(std::ostream& out, const GeometryType& obj); ++ ++typedef struct _GeographyType__isset { ++ _GeographyType__isset() : crs(false), algorithm(false) {} ++ bool crs :1; ++ bool algorithm :1; ++} _GeographyType__isset; ++ ++/** ++ * Embedded Geography logical type annotation ++ * ++ * Geospatial features in the WKB format with an explicit (non-linear/non-planar) ++ * edges interpolation algorithm. ++ * ++ * A custom geographic CRS can be set by the crs field, where longitudes are ++ * bound by [-180, 180] and latitudes are bound by [-90, 90]. If unset, the CRS ++ * defaults to "OGC:CRS84". ++ * ++ * An optional algorithm can be set to correctly interpret edges interpolation ++ * of the geometries. If unset, the algorithm defaults to SPHERICAL. ++ * ++ * Allowed for physical type: BYTE_ARRAY. ++ * ++ * See Geospatial.md for details. ++ */ ++class GeographyType { ++ public: ++ ++ GeographyType(const GeographyType&); ++ GeographyType(GeographyType&&) noexcept; ++ GeographyType& operator=(const GeographyType&); ++ GeographyType& operator=(GeographyType&&) noexcept; ++ GeographyType() noexcept; ++ ++ virtual ~GeographyType() noexcept; ++ std::string crs; ++ /** ++ * ++ * @see EdgeInterpolationAlgorithm ++ */ ++ EdgeInterpolationAlgorithm::type algorithm; ++ ++ _GeographyType__isset __isset; ++ ++ void __set_crs(const std::string& val); ++ ++ void __set_algorithm(const EdgeInterpolationAlgorithm::type val); ++ ++ bool operator == (const GeographyType & rhs) const; ++ bool operator != (const GeographyType &rhs) const { ++ return !(*this == rhs); ++ } ++ ++ bool operator < (const GeographyType & ) const; ++ ++ template ++ uint32_t read(Protocol_* iprot); ++ template ++ uint32_t write(Protocol_* oprot) const; ++ ++ virtual void printTo(std::ostream& out) const; ++}; ++ ++void swap(GeographyType &a, GeographyType &b); ++ ++std::ostream& operator<<(std::ostream& out, const GeographyType& obj); ++ + typedef struct _LogicalType__isset { +- _LogicalType__isset() : STRING(false), MAP(false), LIST(false), ENUM(false), DECIMAL(false), DATE(false), TIME(false), TIMESTAMP(false), INTEGER(false), UNKNOWN(false), JSON(false), BSON(false), UUID(false), FLOAT16(false) {} ++ _LogicalType__isset() : STRING(false), MAP(false), LIST(false), ENUM(false), DECIMAL(false), DATE(false), TIME(false), TIMESTAMP(false), INTEGER(false), UNKNOWN(false), JSON(false), BSON(false), UUID(false), FLOAT16(false), GEOMETRY(false), GEOGRAPHY(false) {} + bool STRING :1; + bool MAP :1; + bool LIST :1; +@@ -1483,6 +1618,8 @@ + bool BSON :1; + bool UUID :1; + bool FLOAT16 :1; ++ bool GEOMETRY :1; ++ bool GEOGRAPHY :1; + } _LogicalType__isset; + + /** +@@ -1495,9 +1632,9 @@ + class LogicalType { + public: + +- LogicalType(const LogicalType&) noexcept; ++ LogicalType(const LogicalType&); + LogicalType(LogicalType&&) noexcept; +- LogicalType& operator=(const LogicalType&) noexcept; ++ LogicalType& operator=(const LogicalType&); + LogicalType& operator=(LogicalType&&) noexcept; + LogicalType() noexcept { + } +@@ -1517,6 +1654,8 @@ + BsonType BSON; + UUIDType UUID; + Float16Type FLOAT16; ++ GeometryType GEOMETRY; ++ GeographyType GEOGRAPHY; + + _LogicalType__isset __isset; + +@@ -1547,6 +1686,8 @@ + void __set_UUID(const UUIDType& val); + + void __set_FLOAT16(const Float16Type& val); ++ void __set_GEOMETRY(const GeometryType& val); ++ void __set_GEOGRAPHY(const GeographyType& val); + + bool operator == (const LogicalType & rhs) const + { +@@ -1605,6 +1746,14 @@ + if (__isset.FLOAT16 != rhs.__isset.FLOAT16) + return false; + else if (__isset.FLOAT16 && !(FLOAT16 == rhs.FLOAT16)) ++ return false; ++ if (__isset.GEOMETRY != rhs.__isset.GEOMETRY) ++ return false; ++ else if (__isset.GEOMETRY && !(GEOMETRY == rhs.GEOMETRY)) ++ return false; ++ if (__isset.GEOGRAPHY != rhs.__isset.GEOGRAPHY) ++ return false; ++ else if (__isset.GEOGRAPHY && !(GEOGRAPHY == rhs.GEOGRAPHY)) + return false; + return true; + } +diff --git a/cpp/src/generated/parquet_types.tcc b/cpp/src/generated/parquet_types.tcc +--- a/cpp/src/generated/parquet_types.tcc ++++ b/cpp/src/generated/parquet_types.tcc +@@ -1189,6 +1189,137 @@ + } + + template ++uint32_t GeometryType::read(Protocol_* iprot) { ++ ++ ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); ++ uint32_t xfer = 0; ++ std::string fname; ++ ::apache::thrift::protocol::TType ftype; ++ int16_t fid; ++ ++ xfer += iprot->readStructBegin(fname); ++ ++ using ::apache::thrift::protocol::TProtocolException; ++ ++ ++ while (true) ++ { ++ xfer += iprot->readFieldBegin(fname, ftype, fid); ++ if (ftype == ::apache::thrift::protocol::T_STOP) { ++ break; ++ } ++ switch (fid) ++ { ++ case 1: ++ if (ftype == ::apache::thrift::protocol::T_STRING) { ++ xfer += iprot->readString(this->crs); ++ this->__isset.crs = true; ++ } else { ++ xfer += iprot->skip(ftype); ++ } ++ break; ++ default: ++ xfer += iprot->skip(ftype); ++ break; ++ } ++ xfer += iprot->readFieldEnd(); ++ } ++ ++ xfer += iprot->readStructEnd(); ++ ++ return xfer; ++} ++ ++template ++uint32_t GeometryType::write(Protocol_* oprot) const { ++ uint32_t xfer = 0; ++ ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); ++ xfer += oprot->writeStructBegin("GeometryType"); ++ ++ if (this->__isset.crs) { ++ xfer += oprot->writeFieldBegin("crs", ::apache::thrift::protocol::T_STRING, 1); ++ xfer += oprot->writeString(this->crs); ++ xfer += oprot->writeFieldEnd(); ++ } ++ xfer += oprot->writeFieldStop(); ++ xfer += oprot->writeStructEnd(); ++ return xfer; ++} ++ ++template ++uint32_t GeographyType::read(Protocol_* iprot) { ++ ++ ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); ++ uint32_t xfer = 0; ++ std::string fname; ++ ::apache::thrift::protocol::TType ftype; ++ int16_t fid; ++ ++ xfer += iprot->readStructBegin(fname); ++ ++ using ::apache::thrift::protocol::TProtocolException; ++ ++ ++ while (true) ++ { ++ xfer += iprot->readFieldBegin(fname, ftype, fid); ++ if (ftype == ::apache::thrift::protocol::T_STOP) { ++ break; ++ } ++ switch (fid) ++ { ++ case 1: ++ if (ftype == ::apache::thrift::protocol::T_STRING) { ++ xfer += iprot->readString(this->crs); ++ this->__isset.crs = true; ++ } else { ++ xfer += iprot->skip(ftype); ++ } ++ break; ++ case 2: ++ if (ftype == ::apache::thrift::protocol::T_I32) { ++ int32_t ecast114; ++ xfer += iprot->readI32(ecast114); ++ this->algorithm = static_cast(ecast114); ++ this->__isset.algorithm = true; ++ } else { ++ xfer += iprot->skip(ftype); ++ } ++ break; ++ default: ++ xfer += iprot->skip(ftype); ++ break; ++ } ++ xfer += iprot->readFieldEnd(); ++ } ++ ++ xfer += iprot->readStructEnd(); ++ ++ return xfer; ++} ++ ++template ++uint32_t GeographyType::write(Protocol_* oprot) const { ++ uint32_t xfer = 0; ++ ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); ++ xfer += oprot->writeStructBegin("GeographyType"); ++ ++ if (this->__isset.crs) { ++ xfer += oprot->writeFieldBegin("crs", ::apache::thrift::protocol::T_STRING, 1); ++ xfer += oprot->writeString(this->crs); ++ xfer += oprot->writeFieldEnd(); ++ } ++ if (this->__isset.algorithm) { ++ xfer += oprot->writeFieldBegin("algorithm", ::apache::thrift::protocol::T_I32, 2); ++ xfer += oprot->writeI32(static_cast(this->algorithm)); ++ xfer += oprot->writeFieldEnd(); ++ } ++ xfer += oprot->writeFieldStop(); ++ xfer += oprot->writeStructEnd(); ++ return xfer; ++} ++ ++template + uint32_t LogicalType::read(Protocol_* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); +@@ -1322,6 +1453,22 @@ + xfer += iprot->skip(ftype); + } + break; ++ case 17: ++ if (ftype == ::apache::thrift::protocol::T_STRUCT) { ++ xfer += this->GEOMETRY.read(iprot); ++ this->__isset.GEOMETRY = true; ++ } else { ++ xfer += iprot->skip(ftype); ++ } ++ break; ++ case 18: ++ if (ftype == ::apache::thrift::protocol::T_STRUCT) { ++ xfer += this->GEOGRAPHY.read(iprot); ++ this->__isset.GEOGRAPHY = true; ++ } else { ++ xfer += iprot->skip(ftype); ++ } ++ break; + default: + xfer += iprot->skip(ftype); + break; +@@ -1408,6 +1555,16 @@ + if (this->__isset.FLOAT16) { + xfer += oprot->writeFieldBegin("FLOAT16", ::apache::thrift::protocol::T_STRUCT, 15); + xfer += this->FLOAT16.write(oprot); ++ xfer += oprot->writeFieldEnd(); ++ } ++ if (this->__isset.GEOMETRY) { ++ xfer += oprot->writeFieldBegin("GEOMETRY", ::apache::thrift::protocol::T_STRUCT, 17); ++ xfer += this->GEOMETRY.write(oprot); ++ xfer += oprot->writeFieldEnd(); ++ } ++ if (this->__isset.GEOGRAPHY) { ++ xfer += oprot->writeFieldBegin("GEOGRAPHY", ::apache::thrift::protocol::T_STRUCT, 18); ++ xfer += this->GEOGRAPHY.write(oprot); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); +diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt +--- a/cpp/src/parquet/CMakeLists.txt ++++ b/cpp/src/parquet/CMakeLists.txt +@@ -340,6 +340,7 @@ + + # Thrift requires these definitions for some types that we use + foreach(LIB_TARGET ${PARQUET_LIBRARIES}) ++ target_link_libraries(${LIB_TARGET} PRIVATE "$") + target_compile_definitions(${LIB_TARGET} + PRIVATE PARQUET_EXPORTING + PRIVATE HAVE_INTTYPES_H +diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc +--- a/cpp/src/parquet/arrow/schema.cc ++++ b/cpp/src/parquet/arrow/schema.cc +@@ -28,6 +28,10 @@ + #include "arrow/type.h" + #include "arrow/util/base64.h" + #include "arrow/util/checked_cast.h" ++#include "arrow/json/rapidjson_defs.h" ++#include ++#include ++#include + #include "arrow/util/key_value_metadata.h" + #include "arrow/util/logging.h" + #include "arrow/util/string.h" +@@ -276,10 +280,86 @@ + } + } + ++constexpr char kExtensionName[] = "ARROW:extension:name"; ++constexpr char kExtensionMetadata[] = "ARROW:extension:metadata"; ++constexpr char kWkbExtension[] = "geoarrow.wkb"; ++ ++::arrow::Result> GeospatialLogicalType( ++ const Field& field) { ++ const auto& metadata = field.metadata(); ++ int index = metadata ? metadata->FindKey(kExtensionName) : -1; ++ if (index < 0 || metadata->value(index) != kWkbExtension) { ++ return LogicalType::None(); ++ } ++ if (field.type()->id() != ArrowTypeId::BINARY) { ++ return Status::Invalid("geoarrow.wkb requires Binary storage"); ++ } ++ ++ std::string crs = "OGC:CRS84"; ++ std::string edges = "planar"; ++ if (metadata->Contains(kExtensionMetadata)) { ++ ASSIGN_OR_RAISE(auto json, metadata->Get(kExtensionMetadata)); ++ ::arrow::rapidjson::Document doc; ++ doc.Parse(json.data(), json.size()); ++ if (doc.HasParseError() || !doc.IsObject()) { ++ return Status::Invalid("Invalid geoarrow.wkb metadata"); ++ } ++ if (doc.HasMember("crs")) { ++ if (!doc["crs"].IsString() || doc["crs"].GetStringLength() == 0) { ++ return Status::Invalid("geoarrow.wkb requires a nonempty CRS"); ++ } ++ crs.assign(doc["crs"].GetString(), doc["crs"].GetStringLength()); ++ } ++ if (doc.HasMember("edges")) { ++ if (!doc["edges"].IsString()) { ++ return Status::Invalid("geoarrow.wkb edges must be a string"); ++ } ++ edges.assign(doc["edges"].GetString(), doc["edges"].GetStringLength()); ++ } ++ } ++ if (edges == "planar") return LogicalType::Geometry(std::move(crs)); ++ ++ using Algorithm = LogicalType::EdgeInterpolationAlgorithm; ++ static constexpr std::pair algorithms[] = { ++ {"spherical", Algorithm::SPHERICAL}, {"vincenty", Algorithm::VINCENTY}, ++ {"thomas", Algorithm::THOMAS}, {"andoyer", Algorithm::ANDOYER}, ++ {"karney", Algorithm::KARNEY}}; ++ for (const auto& [name, algorithm] : algorithms) { ++ if (::arrow::internal::AsciiEqualsCaseInsensitive(edges, name)) { ++ return LogicalType::Geography(std::move(crs), algorithm); ++ } ++ } ++ return Status::Invalid("Invalid geoarrow.wkb edge interpolation algorithm: ", edges); ++} ++ ++std::shared_ptr GeospatialMetadata(const LogicalType& type) { ++ if (!type.is_geometry() && !type.is_geography()) return nullptr; ++ namespace rj = ::arrow::rapidjson; ++ std::string_view crs; ++ rj::StringBuffer buffer; ++ rj::Writer writer(buffer); ++ writer.StartObject(); ++ if (type.is_geography()) { ++ const auto& geography = checked_cast(type); ++ crs = geography.crs(); ++ writer.Key("edges"); ++ auto edges = geography.algorithm_name(); ++ writer.String(edges.data(), static_cast(edges.size())); ++ } else { ++ crs = checked_cast(type).crs(); ++ } ++ if (crs.empty()) crs = "OGC:CRS84"; ++ writer.Key("crs"); ++ writer.String(crs.data(), static_cast(crs.size())); ++ writer.EndObject(); ++ return ::arrow::key_value_metadata({kExtensionName, kExtensionMetadata}, ++ {kWkbExtension, buffer.GetString()}); ++} ++ + Status FieldToNode(const std::string& name, const std::shared_ptr& field, + const WriterProperties& properties, + const ArrowWriterProperties& arrow_properties, NodePtr* out) { +- std::shared_ptr logical_type = LogicalType::None(); ++ ASSIGN_OR_RAISE(auto logical_type, GeospatialLogicalType(*field)); + ParquetType::type type; + Repetition::type repetition = RepetitionFromNullable(field->nullable()); + int field_id = FieldIdFromMetadata(field->metadata()); +@@ -497,7 +577,8 @@ + Status PopulateLeaf(int column_index, const std::shared_ptr& field, + LevelInfo current_levels, SchemaTreeContext* ctx, + const SchemaField* parent, SchemaField* out) { +- out->field = field; ++ auto geo_metadata = GeospatialMetadata(*ctx->schema->Column(column_index)->logical_type()); ++ out->field = geo_metadata ? field->WithMergedMetadata(geo_metadata) : field; + out->column_index = column_index; + out->level_info = current_levels; + ctx->RecordLeaf(out); +@@ -772,7 +853,7 @@ + RETURN_NOT_OK(PopulateLeaf(column_index, child_field, current_levels, ctx, out, + &out->children[0])); + +- out->field = ::arrow::field(node.name(), ::arrow::list(child_field), ++ out->field = ::arrow::field(node.name(), ::arrow::list(out->children[0].field), + /*nullable=*/false, FieldIdMetadata(node.field_id())); + out->level_info = current_levels; + // At this point current_levels has consider this list the ancestor so restore +diff --git a/cpp/src/parquet/arrow/schema_internal.cc b/cpp/src/parquet/arrow/schema_internal.cc +--- a/cpp/src/parquet/arrow/schema_internal.cc ++++ b/cpp/src/parquet/arrow/schema_internal.cc +@@ -116,6 +116,8 @@ + case LogicalType::Type::NONE: + case LogicalType::Type::ENUM: + case LogicalType::Type::JSON: ++ case LogicalType::Type::GEOMETRY: ++ case LogicalType::Type::GEOGRAPHY: + case LogicalType::Type::BSON: + return ::arrow::binary(); + default: +diff --git a/cpp/src/parquet/column_writer.cc b/cpp/src/parquet/column_writer.cc +--- a/cpp/src/parquet/column_writer.cc ++++ b/cpp/src/parquet/column_writer.cc +@@ -1224,7 +1224,8 @@ + current_dict_encoder_ = dynamic_cast*>(current_encoder_.get()); + + if (properties->statistics_enabled(descr_->path()) && +- (SortOrder::UNKNOWN != descr_->sort_order())) { ++ (SortOrder::UNKNOWN != descr_->sort_order() || ++ descr_->logical_type()->is_geometry() || descr_->logical_type()->is_geography())) { + page_statistics_ = MakeStatistics(descr_, allocator_); + chunk_statistics_ = MakeStatistics(descr_, allocator_); + } +diff --git a/cpp/src/parquet/metadata.cc b/cpp/src/parquet/metadata.cc +--- a/cpp/src/parquet/metadata.cc ++++ b/cpp/src/parquet/metadata.cc +@@ -255,13 +255,16 @@ + DCHECK(writer_version_ != nullptr); + // If the column statistics don't exist or column sort order is unknown + // we cannot use the column stats ++ const auto& logical_type = descr_->logical_type(); ++ const bool geospatial = logical_type->is_geometry() || logical_type->is_geography(); + if (!column_metadata_->__isset.statistics || +- descr_->sort_order() == SortOrder::UNKNOWN) { ++ (descr_->sort_order() == SortOrder::UNKNOWN && !geospatial)) { + return false; + } + if (possible_stats_ == nullptr) { + possible_stats_ = MakeColumnStats(*column_metadata_, descr_); + } ++ if (geospatial) return possible_stats_->HasNullCount(); + EncodedStatistics encodedStatistics = possible_stats_->Encode(); + return writer_version_->HasCorrectStatistics(type(), encodedStatistics, + descr_->sort_order()); +diff --git a/cpp/src/parquet/parquet.thrift b/cpp/src/parquet/parquet.thrift +--- a/cpp/src/parquet/parquet.thrift ++++ b/cpp/src/parquet/parquet.thrift +@@ -374,6 +374,54 @@ + struct BsonType { + } + ++enum EdgeInterpolationAlgorithm { ++ SPHERICAL = 0; ++ VINCENTY = 1; ++ THOMAS = 2; ++ ANDOYER = 3; ++ KARNEY = 4; ++} ++ ++/** ++ * Embedded Geometry logical type annotation ++ * ++ * Geospatial features in the Well-Known Binary (WKB) format and edges interpolation ++ * is always linear/planar. ++ * ++ * A custom CRS can be set by the crs field. If unset, it defaults to "OGC:CRS84", ++ * which means that the geometries must be stored in longitude, latitude based on ++ * the WGS84 datum. ++ * ++ * Allowed for physical type: BYTE_ARRAY. ++ * ++ * See Geospatial.md for details. ++ */ ++struct GeometryType { ++ 1: optional string crs; ++} ++ ++/** ++ * Embedded Geography logical type annotation ++ * ++ * Geospatial features in the WKB format with an explicit (non-linear/non-planar) ++ * edges interpolation algorithm. ++ * ++ * A custom geographic CRS can be set by the crs field, where longitudes are ++ * bound by [-180, 180] and latitudes are bound by [-90, 90]. If unset, the CRS ++ * defaults to "OGC:CRS84". ++ * ++ * An optional algorithm can be set to correctly interpret edges interpolation ++ * of the geometries. If unset, the algorithm defaults to SPHERICAL. ++ * ++ * Allowed for physical type: BYTE_ARRAY. ++ * ++ * See Geospatial.md for details. ++ */ ++struct GeographyType { ++ 1: optional string crs; ++ 2: optional EdgeInterpolationAlgorithm algorithm; ++} ++ + /** + * LogicalType annotations to replace ConvertedType. + * +@@ -403,6 +451,8 @@ + 12: JsonType JSON // use ConvertedType JSON + 13: BsonType BSON // use ConvertedType BSON + 14: UUIDType UUID // no compatible ConvertedType ++ 17: GeometryType GEOMETRY ++ 18: GeographyType GEOGRAPHY + 15: Float16Type FLOAT16 // no compatible ConvertedType + } + +diff --git a/cpp/src/parquet/statistics.cc b/cpp/src/parquet/statistics.cc +--- a/cpp/src/parquet/statistics.cc ++++ b/cpp/src/parquet/statistics.cc +@@ -569,7 +569,10 @@ + min_buffer_(AllocateBuffer(pool_, 0)), + max_buffer_(AllocateBuffer(pool_, 0)), + logical_type_(LogicalTypeId(descr_)) { +- comparator_ = MakeComparator(descr); ++ if (logical_type_ != LogicalType::Type::GEOMETRY && ++ logical_type_ != LogicalType::Type::GEOGRAPHY) { ++ comparator_ = MakeComparator(descr); ++ } + TypedStatisticsImpl::Reset(); + } + +@@ -606,13 +609,13 @@ + has_distinct_count_ = false; + } + +- if (!encoded_min.empty()) { ++ if (comparator_ && !encoded_min.empty()) { + PlainDecode(encoded_min, &min_); + } +- if (!encoded_max.empty()) { ++ if (comparator_ && !encoded_max.empty()) { + PlainDecode(encoded_max, &max_); + } +- has_min_max_ = has_min_max; ++ has_min_max_ = comparator_ && has_min_max; + } + + bool HasDistinctCount() const override { return has_distinct_count_; }; +@@ -667,6 +670,8 @@ + } + + void SetMinMax(const T& arg_min, const T& arg_max) override { ++ if (logical_type_ == LogicalType::Type::GEOMETRY || ++ logical_type_ == LogicalType::Type::GEOGRAPHY) return; + SetMinMaxPair({arg_min, arg_max}); + } + +@@ -707,7 +712,7 @@ + IncrementNumValues(values.length() - values.null_count()); + } + +- if (values.null_count() == values.length()) { ++ if (!comparator_ || values.null_count() == values.length()) { + return; + } + +@@ -863,7 +868,7 @@ + IncrementNullCount(null_count); + IncrementNumValues(num_values); + +- if (num_values == 0) return; ++ if (!comparator_ || num_values == 0) return; + SetMinMaxPair(comparator_->GetMinMax(values, num_values)); + } + +@@ -878,7 +883,7 @@ + IncrementNullCount(null_count); + IncrementNumValues(num_values); + +- if (num_values == 0) return; ++ if (!comparator_ || num_values == 0) return; + SetMinMaxPair(comparator_->GetMinMaxSpaced(values, num_spaced_values, valid_bits, + valid_bits_offset)); + } +diff --git a/cpp/src/parquet/types.cc b/cpp/src/parquet/types.cc +--- a/cpp/src/parquet/types.cc ++++ b/cpp/src/parquet/types.cc +@@ -22,6 +22,10 @@ + #include + + #include "arrow/util/checked_cast.h" ++#include "arrow/json/rapidjson_defs.h" ++#include ++#include ++#include + #include "arrow/util/compression.h" + #include "arrow/util/logging.h" + +@@ -463,6 +467,32 @@ + return UUIDLogicalType::Make(); + } else if (type.__isset.FLOAT16) { + return Float16LogicalType::Make(); ++ } else if (type.__isset.GEOMETRY) { ++ std::string crs; ++ if (type.GEOMETRY.__isset.crs) { ++ crs = type.GEOMETRY.crs; ++ } ++ ++ return GeometryLogicalType::Make(std::move(crs)); ++ } else if (type.__isset.GEOGRAPHY) { ++ std::string crs; ++ if (type.GEOGRAPHY.__isset.crs) { ++ crs = type.GEOGRAPHY.crs; ++ } ++ ++ LogicalType::EdgeInterpolationAlgorithm algorithm; ++ if (!type.GEOGRAPHY.__isset.algorithm) { ++ algorithm = LogicalType::EdgeInterpolationAlgorithm::SPHERICAL; ++ } else { ++ int32_t value = static_cast(type.GEOGRAPHY.algorithm); ++ if (value < 0 || value > 4) { ++ throw ParquetException("Unrecognized geography edge interpolation algorithm"); ++ } ++ algorithm = static_cast(value + 1); ++ } ++ ++ return GeographyLogicalType::Make(std::move(crs), algorithm); ++ + } else { + throw ParquetException("Metadata contains Thrift LogicalType that is not recognized"); + } +@@ -518,6 +548,15 @@ + + std::shared_ptr LogicalType::Float16() { + return Float16LogicalType::Make(); ++} ++ ++std::shared_ptr LogicalType::Geometry(std::string crs) { ++ return GeometryLogicalType::Make(std::move(crs)); ++} ++ ++std::shared_ptr LogicalType::Geography( ++ std::string crs, LogicalType::EdgeInterpolationAlgorithm algorithm) { ++ return GeographyLogicalType::Make(std::move(crs), algorithm); + } + + std::shared_ptr LogicalType::None() { return NoLogicalType::Make(); } +@@ -602,6 +641,8 @@ + class BSON; + class UUID; + class Float16; ++ class Geometry; ++ class Geography; + class No; + class Undefined; + +@@ -673,6 +714,12 @@ + bool LogicalType::is_UUID() const { return impl_->type() == LogicalType::Type::UUID; } + bool LogicalType::is_float16() const { + return impl_->type() == LogicalType::Type::FLOAT16; ++} ++bool LogicalType::is_geometry() const { ++ return impl_->type() == LogicalType::Type::GEOMETRY; ++} ++bool LogicalType::is_geography() const { ++ return impl_->type() == LogicalType::Type::GEOGRAPHY; + } + bool LogicalType::is_none() const { return impl_->type() == LogicalType::Type::NONE; } + bool LogicalType::is_valid() const { +@@ -1603,6 +1650,198 @@ + + GENERATE_MAKE(Float16) + ++namespace { ++void WriteCrsKeyAndValue(const std::string_view crs, std::ostream& json) { ++ // There is no restriction on the crs value here, and it may contain quotes ++ // or backslashes that would result in invalid JSON if unescaped. ++ namespace rj = ::arrow::rapidjson; ++ rj::StringBuffer buffer; ++ rj::Writer writer(buffer); ++ rj::Value v; ++ v.SetString(crs.data(), static_cast(crs.size())); ++ v.Accept(writer); ++ json << R"(, "crs": )" << buffer.GetString(); ++} ++} // namespace ++ ++class LogicalType::Impl::Geometry final : public LogicalType::Impl::Incompatible, ++ public LogicalType::Impl::SimpleApplicable { ++ public: ++ friend class GeometryLogicalType; ++ ++ std::string ToString() const override; ++ std::string ToJSON() const override; ++ format::LogicalType ToThrift() const override; ++ bool Equals(const LogicalType& other) const override; ++ ++ const std::string& crs() const { return crs_; } ++ ++ private: ++ explicit Geometry(std::string crs) ++ : LogicalType::Impl(LogicalType::Type::GEOMETRY, SortOrder::UNKNOWN), ++ LogicalType::Impl::SimpleApplicable(parquet::Type::BYTE_ARRAY), ++ crs_(std::move(crs)) {} ++ ++ std::string crs_; ++}; ++ ++std::string LogicalType::Impl::Geometry::ToString() const { ++ std::stringstream type; ++ type << "Geometry(crs=" << crs_ << ")"; ++ return type.str(); ++} ++ ++std::string LogicalType::Impl::Geometry::ToJSON() const { ++ std::stringstream json; ++ json << R"({"Type": "Geometry")"; ++ ++ if (!crs_.empty()) { ++ WriteCrsKeyAndValue(crs_, json); ++ } ++ ++ json << "}"; ++ return json.str(); ++} ++ ++format::LogicalType LogicalType::Impl::Geometry::ToThrift() const { ++ format::LogicalType type; ++ format::GeometryType geometry_type; ++ ++ // Canonically export crs of "" as an unset CRS ++ if (!crs_.empty()) { ++ geometry_type.__set_crs(crs_); ++ } ++ ++ type.__set_GEOMETRY(geometry_type); ++ return type; ++} ++ ++bool LogicalType::Impl::Geometry::Equals(const LogicalType& other) const { ++ if (other.is_geometry()) { ++ const auto& other_geometry = checked_cast(other); ++ return crs() == other_geometry.crs(); ++ } else { ++ return false; ++ } ++} ++ ++const std::string& GeometryLogicalType::crs() const { ++ return (dynamic_cast(*impl_)).crs(); ++} ++ ++std::shared_ptr GeometryLogicalType::Make(std::string crs) { ++ auto logical_type = std::shared_ptr(new GeometryLogicalType()); ++ logical_type->impl_.reset(new LogicalType::Impl::Geometry(std::move(crs))); ++ return logical_type; ++} ++ ++class LogicalType::Impl::Geography final : public LogicalType::Impl::Incompatible, ++ public LogicalType::Impl::SimpleApplicable { ++ public: ++ friend class GeographyLogicalType; ++ ++ std::string ToString() const override; ++ std::string ToJSON() const override; ++ format::LogicalType ToThrift() const override; ++ bool Equals(const LogicalType& other) const override; ++ ++ const std::string& crs() const { return crs_; } ++ LogicalType::EdgeInterpolationAlgorithm algorithm() const { return algorithm_; } ++ ++ std::string_view algorithm_name() const { ++ switch (algorithm_) { ++ case LogicalType::EdgeInterpolationAlgorithm::SPHERICAL: ++ return "spherical"; ++ case LogicalType::EdgeInterpolationAlgorithm::VINCENTY: ++ return "vincenty"; ++ case LogicalType::EdgeInterpolationAlgorithm::THOMAS: ++ return "thomas"; ++ case LogicalType::EdgeInterpolationAlgorithm::ANDOYER: ++ return "andoyer"; ++ case LogicalType::EdgeInterpolationAlgorithm::KARNEY: ++ return "karney"; ++ default: ++ return "unknown"; ++ } ++ } ++ ++ private: ++ Geography(std::string crs, LogicalType::EdgeInterpolationAlgorithm algorithm) ++ : LogicalType::Impl(LogicalType::Type::GEOGRAPHY, SortOrder::UNKNOWN), ++ LogicalType::Impl::SimpleApplicable(parquet::Type::BYTE_ARRAY), ++ crs_(std::move(crs)), ++ algorithm_(algorithm) {} ++ ++ std::string crs_; ++ LogicalType::EdgeInterpolationAlgorithm algorithm_; ++}; ++ ++std::string LogicalType::Impl::Geography::ToString() const { ++ std::stringstream type; ++ type << "Geography(crs=" << crs_ << ", algorithm=" << algorithm_name() << ")"; ++ return type.str(); ++} ++ ++std::string LogicalType::Impl::Geography::ToJSON() const { ++ std::stringstream json; ++ json << R"({"Type": "Geography")"; ++ ++ if (!crs_.empty()) { ++ WriteCrsKeyAndValue(crs_, json); ++ } ++ ++ if (algorithm_ != LogicalType::EdgeInterpolationAlgorithm::SPHERICAL) { ++ json << R"(, "algorithm": ")" << algorithm_name() << R"(")"; ++ } ++ ++ json << "}"; ++ return json.str(); ++} ++ ++format::LogicalType LogicalType::Impl::Geography::ToThrift() const { ++ format::LogicalType type; ++ format::GeographyType geography_type; ++ ++ // Canonically export crs of "" as an unset CRS ++ if (!crs_.empty()) { ++ geography_type.__set_crs(crs_); ++ } ++ ++ geography_type.__set_algorithm(static_cast( ++ static_cast(algorithm_) - 1)); ++ ++ type.__set_GEOGRAPHY(geography_type); ++ return type; ++} ++ ++bool LogicalType::Impl::Geography::Equals(const LogicalType& other) const { ++ if (other.is_geography()) { ++ const auto& other_geography = checked_cast(other); ++ return crs() == other_geography.crs() && algorithm() == other_geography.algorithm(); ++ } else { ++ return false; ++ } ++} ++ ++const std::string& GeographyLogicalType::crs() const { ++ return (dynamic_cast(*impl_)).crs(); ++} ++ ++LogicalType::EdgeInterpolationAlgorithm GeographyLogicalType::algorithm() const { ++ return (dynamic_cast(*impl_)).algorithm(); ++} ++ ++std::string_view GeographyLogicalType::algorithm_name() const { ++ return (dynamic_cast(*impl_)).algorithm_name(); ++} ++ ++std::shared_ptr GeographyLogicalType::Make( ++ std::string crs, LogicalType::EdgeInterpolationAlgorithm algorithm) { ++ auto logical_type = std::shared_ptr(new GeographyLogicalType()); ++ logical_type->impl_.reset(new LogicalType::Impl::Geography(std::move(crs), algorithm)); ++ return logical_type; ++} ++ + class LogicalType::Impl::No final : public LogicalType::Impl::SimpleCompatible, + public LogicalType::Impl::UniversalApplicable { + public: +diff --git a/cpp/src/parquet/types.h b/cpp/src/parquet/types.h +--- a/cpp/src/parquet/types.h ++++ b/cpp/src/parquet/types.h +@@ -158,12 +158,23 @@ + BSON, + UUID, + FLOAT16, ++ GEOMETRY, ++ GEOGRAPHY, + NONE // Not a real logical type; should always be last element + }; + }; + + struct TimeUnit { + enum unit { UNKNOWN = 0, MILLIS = 1, MICROS, NANOS }; ++ }; ++ ++ enum class EdgeInterpolationAlgorithm { ++ UNKNOWN = 0, ++ SPHERICAL = 1, ++ VINCENTY = 2, ++ THOMAS = 3, ++ ANDOYER = 4, ++ KARNEY = 5 + }; + + /// \brief If possible, return a logical type equivalent to the given legacy +@@ -212,6 +223,12 @@ + static std::shared_ptr BSON(); + static std::shared_ptr UUID(); + static std::shared_ptr Float16(); ++ ++ static std::shared_ptr Geometry(std::string crs = ""); ++ ++ static std::shared_ptr Geography( ++ std::string crs = "", LogicalType::EdgeInterpolationAlgorithm algorithm = ++ EdgeInterpolationAlgorithm::SPHERICAL); + + /// \brief Create a placeholder for when no logical type is specified + static std::shared_ptr None(); +@@ -266,6 +283,8 @@ + bool is_BSON() const; + bool is_UUID() const; + bool is_float16() const; ++ bool is_geometry() const; ++ bool is_geography() const; + bool is_none() const; + /// \brief Return true if this logical type is of a known type. + bool is_valid() const; +@@ -444,6 +463,30 @@ + + private: + Float16LogicalType() = default; ++}; ++ ++class PARQUET_EXPORT GeometryLogicalType : public LogicalType { ++ public: ++ static std::shared_ptr Make(std::string crs = ""); ++ ++ const std::string& crs() const; ++ ++ private: ++ GeometryLogicalType() = default; ++}; ++ ++class PARQUET_EXPORT GeographyLogicalType : public LogicalType { ++ public: ++ static std::shared_ptr Make( ++ std::string crs = "", LogicalType::EdgeInterpolationAlgorithm algorithm = ++ EdgeInterpolationAlgorithm::SPHERICAL); ++ ++ const std::string& crs() const; ++ LogicalType::EdgeInterpolationAlgorithm algorithm() const; ++ std::string_view algorithm_name() const; ++ ++ private: ++ GeographyLogicalType() = default; + }; + + /// \brief Allowed for any physical type. diff --git a/docs/source/user_guide/data_types.rst b/docs/source/user_guide/data_types.rst index f9a9e3baa..41105dd61 100644 --- a/docs/source/user_guide/data_types.rst +++ b/docs/source/user_guide/data_types.rst @@ -97,6 +97,14 @@ and `Arrow DataTypes physical_type = MapSharedShreddingUtils::BuildSpecificPhysicalStructType( - logical_map_type->item_type(), selected_physical_column_ids, - logical_map_type->item_field()->nullable(), include_overflow); + logical_map_type->item_field(), selected_physical_column_ids, include_overflow); auto physical_read_field = logical_map_field->WithType(physical_type); std::shared_ptr read_plan = std::make_shared( logical_map_field, physical_read_field, ResolveSelectedKeyIds(meta, selected_keys)); @@ -267,8 +266,7 @@ MapSharedShreddingReadPlanFactory::CreateSharedSelectedKeysReadPlan( } std::shared_ptr physical_type = MapSharedShreddingUtils::BuildSpecificPhysicalStructType( - value_field->type(), selected_physical_column_ids, value_field->nullable(), - include_overflow); + value_field, selected_physical_column_ids, include_overflow); auto physical_read_field = selected_keys_field->WithType(physical_type); std::shared_ptr read_plan = std::make_shared(selected_keys_field, physical_read_field, diff --git a/src/paimon/common/data/shredding/map_shared_shredding_schema_utils.cpp b/src/paimon/common/data/shredding/map_shared_shredding_schema_utils.cpp index e15a4ac5f..367550db2 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_schema_utils.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_schema_utils.cpp @@ -99,8 +99,8 @@ Result> MapSharedShreddingAccessBuilder::Bui encoded_keys.push_back(','); } encoded_keys.append(impl_->keys[i]); - fields.push_back(arrow::field(impl_->keys[i], impl_->map_type->item_type(), - /*nullable=*/true)); + fields.push_back( + impl_->map_type->item_field()->WithName(impl_->keys[i])->WithNullable(true)); } auto metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {encoded_keys}); auto access_field = impl_->map_field->WithType(arrow::struct_(std::move(fields))) diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp b/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp index c345f3146..ea5f8c3e0 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp @@ -83,36 +83,34 @@ Result> MapSharedShreddingUtils::DetectShreddingColumns // ---- Schema conversion ---- std::shared_ptr MapSharedShreddingUtils::BuildSpecificPhysicalStructType( - const std::shared_ptr& value_type, const std::set& physical_col_ids, - bool value_nullable, bool include_overflow) { + const std::shared_ptr& value_field, const std::set& physical_col_ids, + bool include_overflow) { std::vector sorted_cols(physical_col_ids.begin(), physical_col_ids.end()); - return InnerBuildSpecificPhysicalStructType(value_type, sorted_cols, value_nullable, - include_overflow); + return InnerBuildSpecificPhysicalStructType(value_field, sorted_cols, include_overflow); } std::shared_ptr MapSharedShreddingUtils::BuildPhysicalStructType( - const std::shared_ptr& value_type, int32_t num_columns, bool value_nullable) { + const std::shared_ptr& value_field, int32_t num_columns) { std::vector sorted_cols(num_columns); std::iota(sorted_cols.begin(), sorted_cols.end(), 0); - return InnerBuildSpecificPhysicalStructType(value_type, sorted_cols, value_nullable, + return InnerBuildSpecificPhysicalStructType(value_field, sorted_cols, /*include_overflow=*/true); } std::shared_ptr MapSharedShreddingUtils::InnerBuildSpecificPhysicalStructType( - const std::shared_ptr& value_type, const std::vector& sorted_cols, - bool value_nullable, bool include_overflow) { + const std::shared_ptr& value_field, const std::vector& sorted_cols, + bool include_overflow) { arrow::FieldVector struct_fields; struct_fields.reserve(sorted_cols.size() + 2); struct_fields.push_back( arrow::field(MapSharedShreddingDefine::kFieldMapping, arrow::list(arrow::int32()), true)); for (const auto& col : sorted_cols) { - struct_fields.push_back(arrow::field(MapSharedShreddingDefine::PhysicalColumnName(col), - value_type, value_nullable)); + struct_fields.push_back( + value_field->WithName(MapSharedShreddingDefine::PhysicalColumnName(col))); } if (include_overflow) { - struct_fields.push_back(arrow::field( - MapSharedShreddingDefine::kOverflow, - arrow::map(arrow::int32(), arrow::field("value", value_type, value_nullable)), true)); + auto overflow_type = arrow::map(arrow::int32(), value_field->WithName("value")); + struct_fields.push_back(arrow::field(MapSharedShreddingDefine::kOverflow, overflow_type)); } return arrow::struct_(std::move(struct_fields)); } @@ -133,9 +131,7 @@ Result> MapSharedShreddingUtils::LogicalToPhysica field->name(), field->type()->name())); } auto map_type = checked_pointer_cast(field->type()); - auto value_type = map_type->item_type(); - bool value_nullable = map_type->item_field()->nullable(); - auto physical_type = BuildPhysicalStructType(value_type, it->second, value_nullable); + auto physical_type = BuildPhysicalStructType(map_type->item_field(), it->second); auto physical_field = field->WithType(physical_type); physical_fields.push_back(physical_field); } else { diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils.h b/src/paimon/common/data/shredding/map_shared_shredding_utils.h index 00351b65b..f3b310494 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_utils.h +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils.h @@ -86,13 +86,12 @@ class MapSharedShreddingUtils { const std::map& field_to_num_columns); /// Builds the physical Arrow type for one shredding MAP column with physical_col_ids. - /// @param value_type The value type of the original MAP. + /// @param value_field The value field of the original MAP. /// @param physical_col_ids The set of physical column ids to include. - /// @param value_nullable Whether the MAP's value field is nullable. /// @param include_overflow Whether to include __overflow column. static std::shared_ptr BuildSpecificPhysicalStructType( - const std::shared_ptr& value_type, - const std::set& physical_col_ids, bool value_nullable, bool include_overflow); + const std::shared_ptr& value_field, const std::set& physical_col_ids, + bool include_overflow); // ---- Metadata serialization ---- @@ -140,21 +139,18 @@ class MapSharedShreddingUtils { const MapSharedShreddingFieldMeta& meta, const std::string& name); /// Builds the physical Arrow type for one shredding MAP column. - /// @param value_type The value type of the original MAP. + /// @param value_field The value field of the original MAP. /// @param num_columns Number of physical columns K. - /// @param value_nullable Whether the MAP's value field is nullable. static std::shared_ptr BuildPhysicalStructType( - const std::shared_ptr& value_type, int32_t num_columns, - bool value_nullable); + const std::shared_ptr& value_field, int32_t num_columns); /// Builds the physical Arrow type for one shredding MAP column with sorted_cols. - /// @param value_type The value type of the original MAP. + /// @param value_field The value field of the original MAP. /// @param sorted_cols The vector of physical column ids to include. - /// @param value_nullable Whether the MAP's value field is nullable. /// @param include_overflow Whether to include __overflow column. static std::shared_ptr InnerBuildSpecificPhysicalStructType( - const std::shared_ptr& value_type, const std::vector& sorted_cols, - bool value_nullable, bool include_overflow); + const std::shared_ptr& value_field, const std::vector& sorted_cols, + bool include_overflow); }; } // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp index 85467e48f..029825582 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp @@ -24,6 +24,7 @@ #include "arrow/type.h" #include "arrow/util/key_value_metadata.h" #include "gtest/gtest.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/core/core_options.h" #include "paimon/testing/utils/testharness.h" @@ -184,27 +185,23 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNoShreddingColumns) { } TEST(MapSharedShreddingUtilsTest, BuildSpecificPhysicalStructTypeWithOverflow) { - auto actual = MapSharedShreddingUtils::BuildSpecificPhysicalStructType( - arrow::int64(), /*physical_col_ids=*/{3, 1}, /*value_nullable=*/false, - /*include_overflow=*/true); - - auto expected = arrow::struct_({ - arrow::field("__field_mapping", arrow::list(arrow::int32()), true), - arrow::field("__col_1", arrow::int64(), false), - arrow::field("__col_3", arrow::int64(), false), - arrow::field("__overflow", - arrow::map(arrow::int32(), arrow::field("value", arrow::int64(), false)), - true), - }); - ASSERT_TRUE(actual->Equals(*expected)) << "Expected:\n" - << expected->ToString() << "\nActual:\n" - << actual->ToString(); + ASSERT_OK_AND_ASSIGN(auto geo, + GeospatialType::GeographyField("place", "EPSG:4326", "karney", false)); + for (const auto& item : {arrow::field("value", arrow::int64(), false), geo}) { + auto actual = MapSharedShreddingUtils::BuildSpecificPhysicalStructType(item, {3, 1}, true); + auto expected = arrow::struct_({ + arrow::field("__field_mapping", arrow::list(arrow::int32())), + item->WithName("__col_1"), + item->WithName("__col_3"), + arrow::field("__overflow", arrow::map(arrow::int32(), item->WithName("value"))), + }); + ASSERT_TRUE(actual->Equals(*expected, true)) << actual->ToString(); + } } TEST(MapSharedShreddingUtilsTest, BuildSpecificPhysicalStructTypeWithoutOverflow) { auto actual = MapSharedShreddingUtils::BuildSpecificPhysicalStructType( - arrow::utf8(), /*physical_col_ids=*/{3}, /*value_nullable=*/true, - /*include_overflow=*/false); + arrow::field("value", arrow::utf8()), {3}, false); auto expected = arrow::struct_({ arrow::field("__field_mapping", arrow::list(arrow::int32()), true), diff --git a/src/paimon/common/predicate/leaf_predicate_impl.h b/src/paimon/common/predicate/leaf_predicate_impl.h index a301e19c8..e0063da13 100644 --- a/src/paimon/common/predicate/leaf_predicate_impl.h +++ b/src/paimon/common/predicate/leaf_predicate_impl.h @@ -27,6 +27,7 @@ #include "paimon/common/predicate/leaf_function.h" #include "paimon/common/predicate/literal_converter.h" #include "paimon/common/predicate/predicate_filter.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/predicate/leaf_predicate.h" namespace paimon { @@ -44,21 +45,39 @@ class LeafPredicateImpl : public LeafPredicate, public PredicateFilter { Result> Test(const arrow::Array& array, arrow::MemoryPool* pool) const override { const auto& struct_array = checked_cast(array); - if (field_index_ >= struct_array.num_fields()) { + if (field_index_ < 0 || field_index_ >= struct_array.num_fields()) { return Status::Invalid( fmt::format("field index {} exceed field count {} in struct array", field_index_, struct_array.num_fields())); } + const auto function = leaf_function_.GetType(); + if (function != Function::Type::IS_NULL && function != Function::Type::IS_NOT_NULL && + GeospatialType::IsGeospatial(*struct_array.type()->field(field_index_))) { + return Status::NotImplemented("GEOMETRY/GEOGRAPHY only supports null predicates"); + } const auto& field_array = struct_array.field(field_index_); return leaf_function_.Test(*field_array, literals_, pool); } Result Test(const std::shared_ptr& schema, const InternalRow& row) const override { - if (field_index_ >= row.GetFieldCount()) { + if (field_index_ < 0 || field_index_ >= row.GetFieldCount()) { return Status::Invalid(fmt::format("field index {} exceed field count {} in row", field_index_, row.GetFieldCount())); } + // Partition filters can omit the schema for types without precision or scale parameters. + if (schema && field_index_ >= schema->num_fields()) { + return Status::Invalid("Predicate field index exceeds schema field count"); + } + if (schema && GeospatialType::IsGeospatial(*schema->field(field_index_))) { + if (leaf_function_.GetType() == Function::Type::IS_NULL) { + return row.IsNullAt(field_index_); + } + if (leaf_function_.GetType() == Function::Type::IS_NOT_NULL) { + return !row.IsNullAt(field_index_); + } + return Status::NotImplemented("GEOMETRY/GEOGRAPHY only supports null predicates"); + } PAIMON_ASSIGN_OR_RAISE(Literal value, LiteralConverter::ConvertLiteralsFromRow( schema, row, field_index_, field_type_)); return leaf_function_.Test(value, literals_); @@ -67,6 +86,22 @@ class LeafPredicateImpl : public LeafPredicate, public PredicateFilter { Result Test(const std::shared_ptr& schema, int64_t row_count, const InternalRow& min_values, const InternalRow& max_values, const InternalArray& null_counts) const override { + if (field_index_ < 0 || (schema && field_index_ >= schema->num_fields()) || + field_index_ >= null_counts.Size()) { + return Status::Invalid("Predicate field index exceeds statistics field count"); + } + if (schema && GeospatialType::IsGeospatial(*schema->field(field_index_))) { + if (null_counts.IsNullAt(field_index_)) { + return true; + } + if (leaf_function_.GetType() == Function::Type::IS_NULL) { + return null_counts.GetLong(field_index_) > 0; + } + if (leaf_function_.GetType() == Function::Type::IS_NOT_NULL) { + return null_counts.GetLong(field_index_) < row_count; + } + return true; + } PAIMON_ASSIGN_OR_RAISE(Literal min_value, LiteralConverter::ConvertLiteralsFromRow(schema, min_values, field_index_, field_type_)); diff --git a/src/paimon/common/predicate/predicate_validator.h b/src/paimon/common/predicate/predicate_validator.h index 00241723f..31dbb1a8c 100644 --- a/src/paimon/common/predicate/predicate_validator.h +++ b/src/paimon/common/predicate/predicate_validator.h @@ -107,6 +107,21 @@ class PredicateValidator { field_name, schema_field_idx, leaf_predicate->FieldIndex())); } // check field type (schema vs. predicate) + if (GeospatialType::IsGeospatial(*schema.field(schema_field_idx))) { + PAIMON_ASSIGN_OR_RAISE(FieldType type, FieldTypeUtils::ConvertToFieldType( + schema.field(schema_field_idx))); + if (type != leaf_predicate->GetFieldType()) { + return Status::Invalid("Geospatial predicate type mismatch for field ", + field_name); + } + auto function = leaf_predicate->GetFunction().GetType(); + if (function != Function::Type::IS_NULL && + function != Function::Type::IS_NOT_NULL) { + return Status::NotImplemented( + "GEOMETRY/GEOGRAPHY only supports null predicates"); + } + return Status::OK(); + } const std::shared_ptr& schema_type = schema.field(schema_field_idx)->type(); PAIMON_RETURN_NOT_OK(ValidateDataTypeWithSchemaAndPredicate( diff --git a/src/paimon/common/predicate/predicate_validator_test.cpp b/src/paimon/common/predicate/predicate_validator_test.cpp index 487482db9..649043a57 100644 --- a/src/paimon/common/predicate/predicate_validator_test.cpp +++ b/src/paimon/common/predicate/predicate_validator_test.cpp @@ -20,6 +20,7 @@ #include "arrow/type_fwd.h" #include "gtest/gtest.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/data/decimal.h" #include "paimon/data/timestamp.h" #include "paimon/predicate/full_text_match.h" @@ -32,6 +33,40 @@ class Schema; } // namespace arrow namespace paimon::test { + +TEST(PredicateValidatorTest, GeospatialNullPredicates) { + ASSERT_OK_AND_ASSIGN(auto geom, GeospatialType::GeometryField("g")); + ASSERT_OK_AND_ASSIGN(auto geog, GeospatialType::GeographyField("g")); + for (const auto& field : {geom, geog}) { + SCOPED_TRACE(field->ToString(true)); + auto schema = arrow::schema({field}); + ASSERT_OK_AND_ASSIGN(auto type, GeospatialType::FromField(*field)); + ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema( + *schema, PredicateBuilder::IsNull(0, "g", type.Type()), true)); + ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema( + *schema, PredicateBuilder::IsNotNull(0, "g", type.Type()), true)); + ASSERT_NOK(PredicateValidator::ValidatePredicateWithSchema( + *schema, PredicateBuilder::IsNull(0, "g", FieldType::BINARY), true)); + for (FieldType predicate_type : {FieldType::BINARY, type.Type()}) { + auto wkb = Literal(predicate_type, "wkb", 3, false); + const char* expected_error = predicate_type == FieldType::BINARY + ? "Geospatial predicate type mismatch" + : "only supports null predicates"; + for (const auto& comparison : {PredicateBuilder::Equal(0, "g", predicate_type, wkb), + PredicateBuilder::LessThan(0, "g", predicate_type, wkb), + PredicateBuilder::In(0, "g", predicate_type, {wkb})}) { + ASSERT_NOK_WITH_MSG( + PredicateValidator::ValidatePredicateWithSchema(*schema, comparison, true), + expected_error); + if (predicate_type == FieldType::BINARY) { + ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema( + *arrow::schema({arrow::field("g", arrow::binary())}), comparison, true)); + } + } + } + } +} + TEST(PredicateValidatorTest, TestFullTextMatchRequiresProjectedStringField) { auto schema = arrow::schema({arrow::field("id", arrow::int32()), arrow::field("text", arrow::utf8())}); diff --git a/src/paimon/common/reader/blob_view_resolving_batch_reader.cpp b/src/paimon/common/reader/blob_view_resolving_batch_reader.cpp index 37d05020b..b9121b32d 100644 --- a/src/paimon/common/reader/blob_view_resolving_batch_reader.cpp +++ b/src/paimon/common/reader/blob_view_resolving_batch_reader.cpp @@ -68,12 +68,9 @@ Result BlobViewResolvingBatchReader::NextBatch() { const auto struct_type = struct_array->struct_type(); arrow::ArrayVector new_fields = struct_array->fields(); - std::vector field_names; - field_names.reserve(struct_type->num_fields()); for (int32_t field_idx = 0; field_idx < struct_type->num_fields(); ++field_idx) { const auto& field = struct_type->field(field_idx); - field_names.push_back(field->name()); if (read_blob_view_fields_.find(field->name()) == read_blob_view_fields_.end()) { continue; } @@ -87,7 +84,7 @@ Result BlobViewResolvingBatchReader::NextBatch() { PAIMON_ASSIGN_OR_RAISE(new_fields[field_idx], ResolveBinaryColumn(large_binary_array)); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr resolved_struct_array, - arrow::StructArray::Make(new_fields, field_names)); + arrow::StructArray::Make(new_fields, struct_type->fields())); PAIMON_RETURN_NOT_OK_FROM_ARROW( arrow::ExportArray(*resolved_struct_array, c_array.get(), c_schema.get())); PAIMON_RETURN_NOT_OK(AddArrowArrayLifetime(c_array.get(), c_schema.get(), arrow_pool_)); diff --git a/src/paimon/common/reader/blob_view_resolving_batch_reader_test.cpp b/src/paimon/common/reader/blob_view_resolving_batch_reader_test.cpp index 7b257dfab..150c5f1da 100644 --- a/src/paimon/common/reader/blob_view_resolving_batch_reader_test.cpp +++ b/src/paimon/common/reader/blob_view_resolving_batch_reader_test.cpp @@ -36,6 +36,7 @@ #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/blob_view_struct.h" #include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/data/blob_descriptor.h" @@ -174,6 +175,17 @@ TEST_F(BlobViewResolvingBatchReaderTest, TestResolvesBlobViewColumn) { MakeBlobViewStructBytes("db", "tbl", /*field_id=*/3, /*row_id=*/200)); std::shared_ptr src_struct = BuildStructArray({row0_view, row1_view}, {true, true}); + ASSERT_OK_AND_ASSIGN(auto geography, + GeospatialType::GeographyField("location", "EPSG:4326", "karney")); + arrow::BinaryBuilder wkb_builder; + ASSERT_TRUE(wkb_builder.Append(std::string("\x01\x07\0\0\0\0\0\0\0", 9)).ok()); + ASSERT_TRUE(wkb_builder.AppendNull().ok()); + std::shared_ptr wkb; + ASSERT_TRUE(wkb_builder.Finish(&wkb).ok()); + auto combined = arrow::StructArray::Make({src_struct->field(0), wkb}, + {src_struct->type()->field(0), geography}); + ASSERT_TRUE(combined.ok()); + src_struct = *combined; ASSERT_OK_AND_ASSIGN(auto expected_row0_descriptor, MakeBlobDescriptorBytes("/path/a", /*offset=*/0, /*length=*/8)); @@ -195,8 +207,12 @@ TEST_F(BlobViewResolvingBatchReaderTest, TestResolvesBlobViewColumn) { auto reader = std::make_unique( std::move(inner_reader), std::vector{"blob_col"}, std::move(resolver), GetArrowPool(pool_)); - ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(reader))); + ASSERT_OK_AND_ASSIGN(auto result_array, + ReadResultCollector::CollectResult(std::move(reader), true)); + ASSERT_TRUE(result_array->type()->Equals(*src_struct->type(), true)); auto struct_array = std::dynamic_pointer_cast(result_array->chunk(0)); + ASSERT_TRUE(struct_array); + ASSERT_TRUE(struct_array->field(1)->Equals(wkb)); auto result_blob_column = std::dynamic_pointer_cast(struct_array->field(0)); diff --git a/src/paimon/common/reader/data_evolution_file_reader.cpp b/src/paimon/common/reader/data_evolution_file_reader.cpp index 5cace4181..5154b3c58 100644 --- a/src/paimon/common/reader/data_evolution_file_reader.cpp +++ b/src/paimon/common/reader/data_evolution_file_reader.cpp @@ -101,6 +101,7 @@ Result DataEvolutionFileReader::NextBatchWithB } int32_t read_field_count = read_schema_->num_fields(); arrow::ArrayVector target_sub_array_vec; + arrow::FieldVector fields = read_schema_->fields(); target_sub_array_vec.reserve(read_field_count); for (int32_t i = 0; i < read_field_count; i++) { if (reader_offsets_[i] == -1) { @@ -112,11 +113,12 @@ Result DataEvolutionFileReader::NextBatchWithB const auto& sub_array = array_for_each_reader[reader_offsets_[i]]; assert(sub_array->num_fields() > field_offsets_[i]); // Each file is already aligned to its read schema by its FieldMappingReader. - target_sub_array_vec.push_back(sub_array->field(field_offsets_[i])); + auto column = sub_array->field(field_offsets_[i]); + fields[i] = fields[i]->WithType(column->type()); + target_sub_array_vec.push_back(std::move(column)); } - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - std::shared_ptr target_array, - arrow::StructArray::Make(target_sub_array_vec, read_schema_->field_names())); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr target_array, + arrow::StructArray::Make(target_sub_array_vec, fields)); std::unique_ptr target_c_arrow_array = std::make_unique(); std::unique_ptr target_c_schema = std::make_unique(); PAIMON_RETURN_NOT_OK_FROM_ARROW( diff --git a/src/paimon/common/reader/data_evolution_file_reader_test.cpp b/src/paimon/common/reader/data_evolution_file_reader_test.cpp index 3a98d42ab..de4b64bbe 100644 --- a/src/paimon/common/reader/data_evolution_file_reader_test.cpp +++ b/src/paimon/common/reader/data_evolution_file_reader_test.cpp @@ -28,6 +28,8 @@ #include "arrow/ipc/api.h" #include "arrow/util/range.h" #include "gtest/gtest.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/memory/memory_pool.h" #include "paimon/testing/mock/mock_file_batch_reader.h" @@ -87,8 +89,9 @@ class DataEvolutionFileReaderTest : public ::testing::Test, // check result array ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( - std::move(data_evolution_file_reader))); + std::move(data_evolution_file_reader), true)); auto expected_chunk_array = std::make_shared(expected_array); + ASSERT_TRUE(result_array->type()->Equals(*expected_array->type(), true)); ASSERT_TRUE(result_array->Equals(expected_chunk_array)); } } @@ -327,11 +330,17 @@ TEST_P(DataEvolutionFileReaderTest, TestSimple) { } TEST_P(DataEvolutionFileReaderTest, TestWithNonExistField) { + ASSERT_OK_AND_ASSIGN(auto geometry, GeospatialType::GeometryField("f2", "EPSG:3857", false)); + ASSERT_OK_AND_ASSIGN(auto geography, + GeospatialType::GeographyField("f4", "EPSG:4326", "karney")); arrow::FieldVector read_fields = { - arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::int32()), - arrow::field("f2", arrow::utf8()), arrow::field("f3", arrow::int32()), - arrow::field("f4", arrow::utf8()), arrow::field("f5", arrow::int32()), - arrow::field("non-field", arrow::int32()), + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::int32()), + DataField::ConvertDataFieldToArrowField(DataField(2, geometry)), + arrow::field("f3", arrow::int32()), + DataField::ConvertDataFieldToArrowField(DataField(4, geography)), + arrow::field("f5", arrow::int32()), + geography->WithName("non-field"), }; auto read_schema = arrow::schema(read_fields); diff --git a/src/paimon/common/table/special_fields.h b/src/paimon/common/table/special_fields.h index 73562140b..5305fc8c9 100644 --- a/src/paimon/common/table/special_fields.h +++ b/src/paimon/common/table/special_fields.h @@ -96,8 +96,15 @@ struct SpecialFields { /// Match Java's Parquet array-element IDs. Depth starts at 1 and resets at each ROW. static constexpr int32_t GetArrayElementFieldId(int32_t array_field_id, int32_t depth) { - return STRUCTURED_TYPE_FIELD_ID_BASE + array_field_id * STRUCTURED_TYPE_FIELD_DEPTH_LIMIT + - depth; + return static_cast( + STRUCTURED_TYPE_FIELD_ID_BASE + + static_cast(array_field_id) * STRUCTURED_TYPE_FIELD_DEPTH_LIMIT + depth); + } + + static constexpr int32_t GetMapKeyFieldId(int32_t map_field_id, int32_t depth) { + return static_cast( + STRUCTURED_TYPE_FIELD_ID_BASE - + static_cast(map_field_id) * STRUCTURED_TYPE_FIELD_DEPTH_LIMIT - depth); } // `_REALTIME_OFFSET` is persisted as a regular TableSchema field in realtime deduplicate diff --git a/src/paimon/common/types/data_field.cpp b/src/paimon/common/types/data_field.cpp index 8323fcf0b..968293ad3 100644 --- a/src/paimon/common/types/data_field.cpp +++ b/src/paimon/common/types/data_field.cpp @@ -29,6 +29,7 @@ #include "fmt/format.h" #include "paimon/common/types/data_type.h" #include "paimon/common/types/data_type_json_parser.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/object_utils.h" #include "paimon/common/utils/rapidjson_util.h" @@ -42,7 +43,7 @@ namespace paimon { bool DataField::operator==(const DataField& other) const { return id_ == other.id_ && field_->Equals(other.field_, /*check_metadata=*/false) && - description_ == other.description_; + GeospatialType::Equals(*field_, *other.field_) && description_ == other.description_; } rapidjson::Value DataField::ToJson(rapidjson::Document::AllocatorType* allocator) const diff --git a/src/paimon/common/types/data_type.cpp b/src/paimon/common/types/data_type.cpp index f5e562955..76df1ef17 100644 --- a/src/paimon/common/types/data_type.cpp +++ b/src/paimon/common/types/data_type.cpp @@ -27,6 +27,7 @@ #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/types/array_type.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/types/map_type.h" #include "paimon/common/types/row_type.h" #include "paimon/common/types/vector_type.h" @@ -130,6 +131,13 @@ std::string DataType::DataTypeToString(const std::shared_ptr& t case arrow::Type::type::STRING: return "STRING"; case arrow::Type::type::BINARY: + if (GeospatialType::IsGeospatial(metadata_)) { + auto geo = GeospatialType::FromField(*arrow::field("", type, nullable_, metadata_)); + if (!geo.ok()) { + throw std::invalid_argument(geo.status().ToString()); + } + return geo.value().ToString(); + } return "BYTES"; case arrow::Type::type::DATE32: return "DATE"; diff --git a/src/paimon/common/types/data_type_json_parser.cpp b/src/paimon/common/types/data_type_json_parser.cpp index c05110f43..82e3be146 100644 --- a/src/paimon/common/types/data_type_json_parser.cpp +++ b/src/paimon/common/types/data_type_json_parser.cpp @@ -35,6 +35,8 @@ #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/types/data_field.h" #include "paimon/common/types/data_type.h" +#include "paimon/common/types/geospatial_type.h" +#include "paimon/common/types/map_type.h" #include "paimon/common/types/vector_type.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/date_time_utils.h" @@ -91,6 +93,7 @@ struct Token { struct AtomicTypeAttributes { bool is_blob = false; bool is_variant = false; + std::shared_ptr geospatial_field; std::optional time_precision; }; @@ -151,6 +154,8 @@ enum class Keyword : int32_t { ROW, BLOB, VARIANT, + GEOMETRY, + GEOGRAPHY, VECTOR, // NULL is keyword in c++ NULL_, @@ -201,6 +206,8 @@ const std::map& Keywords() { {"ROW", Keyword::ROW}, {"BLOB", Keyword::BLOB}, {"VARIANT", Keyword::VARIANT}, + {"GEOMETRY", Keyword::GEOMETRY}, + {"GEOGRAPHY", Keyword::GEOGRAPHY}, {"VECTOR", Keyword::VECTOR}, {"NULL", Keyword::NULL_}, {"RAW", Keyword::RAW}, @@ -256,6 +263,9 @@ class TokenParser { Result> ParseTimestampType(); Result> ParseTimestampLtzType(); Result> ParseVectorType(); + Result ParseGeospatialParameter(); + Result> ParseGeospatialType(bool geography, + AtomicTypeAttributes* attributes); Result ParseOptionalPrecision(int32_t default_precision); private: @@ -487,6 +497,9 @@ Result> TokenParser::ParseTypeByKeyword( AtomicTypeAttributes* attributes) { PAIMON_RETURN_NOT_OK(NextToken(TokenType::KEYWORD)); switch (TokenAsKeyword()) { + case Keyword::GEOMETRY: + case Keyword::GEOGRAPHY: + return ParseGeospatialType(TokenAsKeyword() == Keyword::GEOGRAPHY, attributes); case Keyword::CHAR: case Keyword::VARCHAR: return ParseStringType(); @@ -664,6 +677,38 @@ Result> TokenParser::ParseVectorType() { length.value()); } +Result TokenParser::ParseGeospatialParameter() { + PAIMON_RETURN_NOT_OK(NextToken()); + const auto& token = GetToken(); + if (token.type == TokenType::KEYWORD) { + return input_string_.substr(token.cursor_position + 1 - token.value.size(), + token.value.size()); + } + if (token.type == TokenType::IDENTIFIER || token.type == TokenType::LITERAL_STRING) { + return token.value; + } + return Status::Invalid("Geospatial type parameter expected"); +} + +Result> TokenParser::ParseGeospatialType( + bool geography, AtomicTypeAttributes* attributes) { + std::string crs = GeospatialType::kDefaultCrs; + std::string edges = GeospatialType::kDefaultEdges; + if (HasNextToken({TokenType::BEGIN_PARAMETER})) { + PAIMON_RETURN_NOT_OK(NextToken(TokenType::BEGIN_PARAMETER)); + PAIMON_ASSIGN_OR_RAISE(crs, ParseGeospatialParameter()); + if (geography && HasNextToken({TokenType::LIST_SEPARATOR})) { + PAIMON_RETURN_NOT_OK(NextToken(TokenType::LIST_SEPARATOR)); + PAIMON_ASSIGN_OR_RAISE(edges, ParseGeospatialParameter()); + } + PAIMON_RETURN_NOT_OK(NextToken(TokenType::END_PARAMETER)); + } + PAIMON_ASSIGN_OR_RAISE(attributes->geospatial_field, + geography ? GeospatialType::GeographyField("", crs, edges) + : GeospatialType::GeometryField("", crs)); + return arrow::binary(); +} + Result TokenParser::ParseOptionalPrecision(int32_t default_precision) { auto precision = default_precision; if (HasNextToken({TokenType::BEGIN_PARAMETER})) { @@ -770,7 +815,9 @@ Result> DataTypeJsonParser::ParseAtomicTypeField( const std::string type_str(type_json_value.GetString(), type_json_value.GetStringLength()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr type, ParseAtomicType(type_str, &nullable, &attributes)); - if (attributes.is_blob) { + if (attributes.geospatial_field) { + return attributes.geospatial_field->WithName(name)->WithNullable(nullable); + } else if (attributes.is_blob) { return BlobUtils::ToArrowField(name, nullable); } else if (attributes.is_variant) { return VariantTypeUtils::ToArrowField(name, nullable); @@ -860,15 +907,15 @@ Result> DataTypeJsonParser::ParseMapType( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key, ParseType("key", type_json_value["key"], field_id_assigner)); - // NOTE: Unlike Java Paimon, this C++ implementation does not support nullable keys in - // MapType. This is a limitation of Apache Arrow, which does not allow null keys in its - // MapType. As a result, we explicitly set `nullable = false` for the map key, regardless of - // the original schema. Please be aware of this behavioral difference when migrating or - // interoperating with Java Paimon. - key = key->WithNullable(false); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr value, ParseType("value", type_json_value["value"], field_id_assigner)); - return arrow::field(name, std::make_shared(key, value), nullable); + std::shared_ptr metadata; + if (key->nullable() && + (GeospatialType::ContainsGeospatial(*key) || GeospatialType::ContainsGeospatial(*value))) { + metadata = arrow::key_value_metadata({MapType::KEY_NULLABLE}, {"true"}); + } + return arrow::field(name, std::make_shared(key->WithNullable(false), value), + nullable, metadata); } Result> DataTypeJsonParser::ParseRowType( diff --git a/src/paimon/common/types/geospatial_type.cpp b/src/paimon/common/types/geospatial_type.cpp new file mode 100644 index 000000000..05f5311d1 --- /dev/null +++ b/src/paimon/common/types/geospatial_type.cpp @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/types/geospatial_type.h" + +#include +#include +#include + +#include "arrow/util/key_value_metadata.h" +#include "fmt/format.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/types/map_type.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/string_utils.h" +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" +#include "re2/re2.h" + +namespace paimon { +namespace { + +bool CrsEqualsIgnoreCase(const std::string& left, const std::string& right) { + if (StringUtils::EqualsIgnoreCase(left, right)) { + return true; + } + auto ascii = [](uint8_t c) { return c < 128; }; + if (std::all_of(left.begin(), left.end(), ascii) && + std::all_of(right.begin(), right.end(), ascii)) { + return false; + } + // Java's equalsIgnoreCase groups dotted and dotless I with ASCII I. RE2's + // Unicode simple case folding handles the remaining characters. + auto normalize_i = [](const std::string& crs) { + return StringUtils::Replace(StringUtils::Replace(crs, u8"İ", "i"), u8"ı", "i"); + }; + re2::RE2::Options options; + options.set_literal(true); + options.set_case_sensitive(false); + return re2::RE2::FullMatch(normalize_i(right), re2::RE2(normalize_i(left), options)); +} + +const std::string* FieldId(const arrow::Field& field) { + const auto& metadata = field.metadata(); + int32_t index = metadata ? metadata->FindKey(DataField::FIELD_ID) : -1; + return index < 0 ? nullptr : &metadata->value(index); +} + +Result LogicalTypeName(const arrow::Field& field) { + if (GeospatialType::IsGeospatial(field)) { + PAIMON_ASSIGN_OR_RAISE(GeospatialType type, GeospatialType::FromField(field)); + return type.ToString(); + } + return field.type()->id() == arrow::Type::BINARY ? "BYTES" : field.type()->ToString(); +} + +} // namespace + +bool GeospatialType::IsGeospatial(const std::shared_ptr& metadata) { + int32_t index = metadata ? metadata->FindKey(kExtensionNameKey) : -1; + return index >= 0 && metadata->value(index) == kExtensionName; +} + +bool GeospatialType::IsGeospatial(const arrow::Field& field) { + return IsGeospatial(field.metadata()); +} + +bool GeospatialType::ContainsGeospatial(const arrow::Field& field) { + return IsGeospatial(field) || ContainsGeospatial(field.type()->fields()); +} + +bool GeospatialType::ContainsGeospatial(const arrow::FieldVector& fields) { + return std::any_of(fields.begin(), fields.end(), + [](const auto& field) { return ContainsGeospatial(*field); }); +} + +Result GeospatialType::Create(const std::string& crs, const std::string& edges) { + if (crs.empty()) { + return Status::Invalid("Invalid CRS: empty string"); + } + std::string normalized = StringUtils::ToLowerCase(edges); + if (!normalized.empty() && normalized != "spherical" && normalized != "vincenty" && + normalized != "thomas" && normalized != "andoyer" && normalized != "karney") { + return Status::Invalid("Invalid edge interpolation algorithm: ", edges); + } + return GeospatialType(crs, normalized); +} + +Status GeospatialType::ValidateFileFormat(const arrow::Schema& schema, const std::string& format) { + if (!StringUtils::EqualsIgnoreCase(format, "parquet") && ContainsGeospatial(schema.fields())) { + return Status::Invalid("GEOMETRY/GEOGRAPHY only supports parquet data files: ", format); + } + return Status::OK(); +} + +Result GeospatialType::FromField(const arrow::Field& field) { + if (!IsGeospatial(field) || field.type()->id() != arrow::Type::BINARY) { + return Status::Invalid("geoarrow.wkb requires a Binary field: ", field.name()); + } + std::string crs = kDefaultCrs; + std::string edges; + if (field.metadata()->Contains(kExtensionMetadataKey)) { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::string json, + field.metadata()->Get(kExtensionMetadataKey)); + rapidjson::Document doc; + doc.Parse(json.data(), json.size()); + if (doc.HasParseError() || !doc.IsObject()) { + return Status::Invalid("Invalid geoarrow.wkb metadata for field ", field.name()); + } + if (doc.HasMember("crs")) { + if (!doc["crs"].IsString()) { + return Status::Invalid("geoarrow.wkb CRS must be a string"); + } + crs.assign(doc["crs"].GetString(), doc["crs"].GetStringLength()); + } + if (doc.HasMember("edges")) { + if (!doc["edges"].IsString() || doc["edges"].GetStringLength() == 0) { + return Status::Invalid("Invalid edge interpolation algorithm in geoarrow.wkb"); + } + edges.assign(doc["edges"].GetString(), doc["edges"].GetStringLength()); + if (edges == "planar") { + edges.clear(); + } + } + } + return Create(crs, edges); +} + +Result> GeospatialType::GeometryField(const std::string& name, + const std::string& crs, + bool nullable) { + return MakeField(name, crs, "", nullable); +} + +Result> GeospatialType::GeographyField(const std::string& name, + const std::string& crs, + const std::string& edges, + bool nullable) { + if (edges.empty()) { + return Status::Invalid("Invalid edge interpolation algorithm: empty string"); + } + return MakeField(name, crs, edges, nullable); +} + +Result> GeospatialType::MakeField(const std::string& name, + const std::string& crs, + const std::string& edges, + bool nullable) { + PAIMON_ASSIGN_OR_RAISE(GeospatialType type, Create(crs, edges)); + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + writer.StartObject(); + if (!type.edges_.empty()) { + writer.Key("edges"); + writer.String(type.edges_.data(), static_cast(type.edges_.size())); + } + writer.Key("crs"); + writer.String(type.crs_.data(), static_cast(type.crs_.size())); + writer.EndObject(); + return arrow::field(name, arrow::binary(), nullable, + arrow::KeyValueMetadata::Make({kExtensionNameKey, kExtensionMetadataKey}, + {kExtensionName, buffer.GetString()})); +} + +bool GeospatialType::Equals(const arrow::Field& left, const arrow::Field& right) { + bool left_geo = IsGeospatial(left); + bool right_geo = IsGeospatial(right); + if (left_geo || right_geo) { + if (!left_geo || !right_geo) { + return false; + } + auto lhs = FromField(left); + auto rhs = FromField(right); + return lhs.ok() && rhs.ok() && lhs.value().Edges() == rhs.value().Edges() && + CrsEqualsIgnoreCase(lhs.value().Crs(), rhs.value().Crs()); + } + if (left.type()->id() == arrow::Type::MAP && right.type()->id() == arrow::Type::MAP && + MapType::HasNullableKey(left.metadata()) != MapType::HasNullableKey(right.metadata())) { + return false; + } + if (left.type()->num_fields() != right.type()->num_fields()) { + return !ContainsGeospatial(left) && !ContainsGeospatial(right); + } + for (int32_t i = 0; i < left.type()->num_fields(); ++i) { + if (!Equals(*left.type()->field(i), *right.type()->field(i))) { + return false; + } + } + return true; +} + +Status GeospatialType::ValidateEvolution(const arrow::Field& source, const arrow::Field& target) { + if (IsGeospatial(source) || IsGeospatial(target)) { + if (Equals(source, target)) { + return Status::OK(); + } + PAIMON_ASSIGN_OR_RAISE(std::string source_type, LogicalTypeName(source)); + PAIMON_ASSIGN_OR_RAISE(std::string target_type, LogicalTypeName(target)); + return Status::Invalid("Incompatible geospatial type change for field ", target.name(), + ": ", source_type, " -> ", target_type); + } + if (!ContainsGeospatial(source) && !ContainsGeospatial(target)) { + return Status::OK(); + } + if (source.type()->id() != target.type()->id()) { + return Status::Invalid("Incompatible geospatial type change for field ", target.name(), + ": ", source.type()->ToString(), " -> ", target.type()->ToString()); + } + if (source.type()->id() == arrow::Type::STRUCT) { + const auto& row = checked_cast(*source.type()); + std::unordered_map fields_by_id; + for (const auto& field : row.fields()) { + if (const auto* id = FieldId(*field)) { + fields_by_id.emplace(*id, field.get()); + } + } + for (const auto& target_child : target.type()->fields()) { + const arrow::Field* source_child = row.GetFieldByName(target_child->name()).get(); + if (const auto* id = FieldId(*target_child)) { + auto it = fields_by_id.find(*id); + if (it != fields_by_id.end()) { + source_child = it->second; + } else if (source_child && FieldId(*source_child)) { + // A new field reuses the old name with a different ID. + continue; + } + } + if (source_child) { + PAIMON_RETURN_NOT_OK(ValidateEvolution(*source_child, *target_child)); + } + } + } else if (source.type()->id() == arrow::Type::MAP) { + const auto& source_map = checked_cast(*source.type()); + const auto& target_map = checked_cast(*target.type()); + PAIMON_RETURN_NOT_OK(ValidateEvolution(*source_map.key_field(), *target_map.key_field())); + return ValidateEvolution(*source_map.item_field(), *target_map.item_field()); + } else { + for (int32_t i = 0; i < source.type()->num_fields(); ++i) { + PAIMON_RETURN_NOT_OK( + ValidateEvolution(*source.type()->field(i), *target.type()->field(i))); + } + } + return Status::OK(); +} + +std::string GeospatialType::ToString() const { + // Match GeometryType.formatCrs, including Java's non-breaking space exceptions. + static const re2::RE2 delimiters(R"([<>(),.\x27\x60\x09-\x0d\x1c-\x20\x{1680}\x{2000}-\x{2006})" + R"(\x{2008}-\x{200a}\x{2028}\x{2029}\x{205f}\x{3000}])"); + static const re2::RE2 leading_digit(R"(^\p{Nd})"); + // Java checks the first UTF-16 code unit, which excludes supplementary digits. + bool quote = + re2::RE2::PartialMatch(crs_, delimiters) || + (static_cast(crs_.front()) < 0xf0 && re2::RE2::PartialMatch(crs_, leading_digit)); + std::string crs = quote ? fmt::format("'{}'", StringUtils::Replace(crs_, "'", "''")) : crs_; + return edges_.empty() ? fmt::format("GEOMETRY({})", crs) + : fmt::format("GEOGRAPHY({}, {})", crs, edges_); +} + +} // namespace paimon diff --git a/src/paimon/common/types/geospatial_type.h b/src/paimon/common/types/geospatial_type.h new file mode 100644 index 000000000..0ac4e8029 --- /dev/null +++ b/src/paimon/common/types/geospatial_type.h @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "arrow/type.h" +#include "paimon/defs.h" +#include "paimon/result.h" + +namespace paimon { + +/// Paimon GEOMETRY or GEOGRAPHY carried by an Arrow Binary field with GeoArrow WKB metadata. +class PAIMON_EXPORT GeospatialType { + public: + static constexpr char kExtensionNameKey[] = "ARROW:extension:name"; + static constexpr char kExtensionMetadataKey[] = "ARROW:extension:metadata"; + static constexpr char kExtensionName[] = "geoarrow.wkb"; + static constexpr char kDefaultCrs[] = "OGC:CRS84"; + static constexpr char kDefaultEdges[] = "spherical"; + + static bool IsGeospatial(const std::shared_ptr& metadata); + static bool IsGeospatial(const arrow::Field& field); + static bool ContainsGeospatial(const arrow::Field& field); + static bool ContainsGeospatial(const arrow::FieldVector& fields); + static Status ValidateFileFormat(const arrow::Schema& schema, const std::string& format); + static Result FromField(const arrow::Field& field); + static Result> GeometryField(const std::string& name, + const std::string& crs = kDefaultCrs, + bool nullable = true); + static Result> GeographyField( + const std::string& name, const std::string& crs = kDefaultCrs, + const std::string& edges = kDefaultEdges, bool nullable = true); + + /// Compares geospatial attributes of corresponding fields, including logical MAP key + /// nullability. + static bool Equals(const arrow::Field& left, const arrow::Field& right); + /// Matches projected ROW fields by ID, falling back to names when IDs are unavailable. + static Status ValidateEvolution(const arrow::Field& source, const arrow::Field& target); + + FieldType Type() const { + return edges_.empty() ? FieldType::GEOMETRY : FieldType::GEOGRAPHY; + } + const std::string& Crs() const { + return crs_; + } + const std::string& Edges() const { + return edges_; + } + std::string ToString() const; + + private: + GeospatialType(const std::string& crs, const std::string& edges) : crs_(crs), edges_(edges) {} + static Result Create(const std::string& crs, const std::string& edges); + static Result> MakeField(const std::string& name, + const std::string& crs, + const std::string& edges, bool nullable); + + std::string crs_; + std::string edges_; +}; + +} // namespace paimon diff --git a/src/paimon/common/types/geospatial_type_test.cpp b/src/paimon/common/types/geospatial_type_test.cpp new file mode 100644 index 000000000..d5aa74782 --- /dev/null +++ b/src/paimon/common/types/geospatial_type_test.cpp @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/types/geospatial_type.h" + +#include +#include + +#include "arrow/c/bridge.h" +#include "arrow/util/key_value_metadata.h" +#include "gtest/gtest.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/types/data_type.h" +#include "paimon/common/types/data_type_json_parser.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +TEST(GeospatialTypeTest, SchemaRoundTrip) { + const std::pair cases[] = { + {"GEOMETRY", "GEOMETRY(OGC:CRS84)"}, + {"GEOMETRY(EPSG:3857) NOT NULL", "GEOMETRY(EPSG:3857) NOT NULL"}, + {"geography", "GEOGRAPHY(OGC:CRS84, spherical)"}, + {"GEOGRAPHY(EPSG:4326)", "GEOGRAPHY(EPSG:4326, spherical)"}, + {"GEOGRAPHY('a custom, ''quoted'' CRS', KARNEY) NOT NULL", + "GEOGRAPHY('a custom, ''quoted'' CRS', karney) NOT NULL"}, + {"GEOGRAPHY('123', THOMAS)", "GEOGRAPHY('123', thomas)"}, + {"GEOGRAPHY(EPSG:4326, ANDOYER)", "GEOGRAPHY(EPSG:4326, andoyer)"}, + {"GEOGRAPHY(EPSG:4326, VINCENTY)", "GEOGRAPHY(EPSG:4326, vincenty)"}, + {"GEOMETRY('int')", "GEOMETRY(int)"}, + {u8"GEOMETRY('Référence')", u8"GEOMETRY(Référence)"}, + {u8"GEOMETRY('地方\u3000坐标')", u8"GEOMETRY('地方\u3000坐标')"}, + {u8"GEOMETRY('A\u00a0B')", u8"GEOMETRY(A\u00a0B)"}, + {u8"GEOMETRY('\u0661CRS')", u8"GEOMETRY('\u0661CRS')"}}; + for (const auto& [sql, canonical] : cases) { + SCOPED_TRACE(sql); + rapidjson::Document input; + input.SetString(sql, input.GetAllocator()); + ASSERT_OK_AND_ASSIGN(auto field, DataTypeJsonParser::ParseType("geo", input)); + ASSERT_EQ(field->type()->id(), arrow::Type::BINARY); + auto type = DataType::Create(field->type(), field->nullable(), field->metadata()); + ASSERT_OK_AND_ASSIGN(auto json, type->ToJsonString()); + rapidjson::Document output; + output.Parse(json.c_str()); + ASSERT_TRUE(output.IsString()); + ASSERT_STREQ(output.GetString(), canonical); + ASSERT_OK_AND_ASSIGN(auto restored, DataTypeJsonParser::ParseType("geo", output)); + ASSERT_TRUE(field->Equals(restored, true)); + } +} + +TEST(GeospatialTypeTest, RejectsInvalidSql) { + for (const auto& sql : + {"GEOMETRY()", "GEOMETRY('')", "GEOMETRY(123)", "GEOMETRY(a, spherical)", + "GEOGRAPHY(a, rhumb)", "GEOGRAPHY(a, '')", "GEOGRAPHY(a, spherical, karney)", + "GEOGRAPHY(a", "GEOGRAPHY('unterminated)"}) { + SCOPED_TRACE(sql); + rapidjson::Document input; + input.SetString(sql, input.GetAllocator()); + ASSERT_NOK(DataTypeJsonParser::ParseType("geo", input)); + } +} + +TEST(GeospatialTypeTest, GeoArrowWkbMetadata) { + auto wkb = arrow::field("g", arrow::binary(), true, + arrow::key_value_metadata({"ARROW:extension:name"}, {"geoarrow.wkb"})); + auto with_json = [&](const char* json) { + return wkb->WithMergedMetadata( + arrow::key_value_metadata({"ARROW:extension:metadata"}, {json})); + }; + for (const auto& field : {wkb, with_json("{}"), with_json(R"({"edges":"planar"})")}) { + ASSERT_OK_AND_ASSIGN(auto type, GeospatialType::FromField(*field)); + ASSERT_EQ(type.Type(), FieldType::GEOMETRY); + ASSERT_EQ(type.Crs(), "OGC:CRS84"); + } + auto java_field = with_json(R"({"edges":"karney","crs":"EPSG:4326"})"); + ASSERT_OK_AND_ASSIGN(auto cpp_field, + GeospatialType::GeographyField("g", "EPSG:4326", "karney")); + ASSERT_TRUE(cpp_field->Equals(java_field, true)); + ASSERT_OK_AND_ASSIGN(auto geography, GeospatialType::FromField(*java_field)); + ASSERT_EQ(geography.Type(), FieldType::GEOGRAPHY); + ASSERT_EQ(geography.Crs(), "EPSG:4326"); + ASSERT_EQ(geography.Edges(), "karney"); + for (const auto* json : {"[]", "invalid", R"({"crs":null})", R"({"crs":""})", R"({"edges":5})", + R"({"edges":""})", R"({"edges":"rhumb"})"}) { + SCOPED_TRACE(json); + auto field = with_json(json); + ASSERT_NOK(GeospatialType::FromField(*field)); + ASSERT_NOK(DataField(0, field).ToJsonString()); + } + ASSERT_NOK(GeospatialType::FromField(*wkb->WithType(arrow::utf8()))); + ASSERT_NOK(GeospatialType::GeometryField("g", "")); + ASSERT_NOK(GeospatialType::GeographyField("g", "")); + ASSERT_NOK(GeospatialType::GeographyField("g", "EPSG:4326", "")); + ASSERT_NOK(GeospatialType::GeographyField("g", "EPSG:4326", "planar")); +} + +TEST(GeospatialTypeTest, NestedMetadataAndIdsSurviveCDataBridge) { + const std::string json = R"json({ + "id":7,"name":"r","type":{"type":"ROW NOT NULL","fields":[ + {"id":8,"name":"a","type":{"type":"ARRAY","element":"GEOMETRY('a, b') NOT NULL"}}, + {"id":9,"name":"m","type":{"type":"MAP","key":"GEOMETRY(EPSG:4326)", + "value":"GEOGRAPHY(EPSG:4326, karney)"}}, + {"id":10,"name":"required","type":{"type":"MAP NOT NULL", + "key":"GEOGRAPHY NOT NULL","value":"BYTES"}} + ]} + })json"; + ASSERT_OK_AND_ASSIGN(auto data_field, DataField::FromJsonString(json)); + auto field = DataField::ConvertDataFieldToArrowField(data_field); + ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportField(*field, &c_schema).ok()); + auto imported = arrow::ImportField(&c_schema); + ASSERT_TRUE(imported.ok()); + ASSERT_TRUE(field->Equals(*imported, true)); + ASSERT_OK_AND_ASSIGN(auto restored, DataField::ConvertArrowFieldToDataField(*imported)); + ASSERT_EQ(data_field, restored); + ASSERT_OK_AND_ASSIGN(auto serialized, restored.ToJsonString()); + rapidjson::Document document; + document.Parse(serialized.c_str()); + ASSERT_EQ(document["id"].GetInt(), 7); + ASSERT_STREQ(document["type"]["type"].GetString(), "ROW NOT NULL"); + const auto& fields = document["type"]["fields"]; + ASSERT_EQ(fields.Size(), 3u); + ASSERT_EQ(fields[0]["id"].GetInt(), 8); + ASSERT_STREQ(fields[0]["type"]["element"].GetString(), "GEOMETRY('a, b') NOT NULL"); + ASSERT_EQ(fields[1]["id"].GetInt(), 9); + ASSERT_STREQ(fields[1]["type"]["key"].GetString(), "GEOMETRY(EPSG:4326)"); + ASSERT_EQ(fields[2]["id"].GetInt(), 10); + ASSERT_STREQ(fields[2]["type"]["key"].GetString(), "GEOGRAPHY(OGC:CRS84, spherical) NOT NULL"); + ASSERT_OK_AND_ASSIGN(auto roundtrip, DataField::FromJsonString(serialized)); + ASSERT_EQ(data_field, roundtrip); +} + +TEST(GeospatialTypeTest, LogicalEqualityAndEvolution) { + ASSERT_OK_AND_ASSIGN(auto geom, GeospatialType::GeometryField("g", "EPSG:4326")); + ASSERT_OK_AND_ASSIGN(auto same, GeospatialType::GeometryField("g", "epsg:4326")); + ASSERT_EQ(DataField(1, geom), DataField(1, same)); + ASSERT_OK(GeospatialType::ValidateEvolution(*geom, *same)); + ASSERT_OK_AND_ASSIGN(auto upper, GeospatialType::GeometryField("g", u8"RÉF:İ")); + ASSERT_OK_AND_ASSIGN(auto lower, GeospatialType::GeometryField("g", u8"réf:ı")); + ASSERT_EQ(DataField(1, upper), DataField(1, lower)); + ASSERT_OK_AND_ASSIGN(auto crs, GeospatialType::GeometryField("g", "EPSG:3857")); + ASSERT_OK_AND_ASSIGN(auto geog, GeospatialType::GeographyField("g", "EPSG:4326")); + ASSERT_OK_AND_ASSIGN(auto edges, GeospatialType::GeographyField("g", "EPSG:4326", "karney")); + for (const auto& other : {arrow::field("g", arrow::binary()), crs, geog, edges}) { + ASSERT_FALSE(DataField(1, geom) == DataField(1, other)); + ASSERT_NOK(GeospatialType::ValidateEvolution(*geom, *other)); + auto lhs = arrow::field("r", arrow::struct_({geom})); + auto rhs = arrow::field("r", arrow::struct_({other})); + ASSERT_FALSE(DataField(2, lhs) == DataField(2, rhs)); + ASSERT_NOK(GeospatialType::ValidateEvolution(*lhs, *rhs)); + } + ASSERT_NOK_WITH_MSG(GeospatialType::ValidateEvolution(*geog, *edges), + "GEOGRAPHY(EPSG:4326, spherical) -> GEOGRAPHY(EPSG:4326, karney)"); + auto plain = arrow::field("bytes", arrow::binary()); + ASSERT_OK_AND_ASSIGN(auto json, DataType::Create(plain->type(), true, nullptr)->ToJsonString()); + ASSERT_EQ(json, "\"BYTES\""); +} + +TEST(GeospatialTypeTest, NestedEvolutionMatchesFieldIds) { + ASSERT_OK_AND_ASSIGN(auto geom, GeospatialType::GeometryField("shape")); + ASSERT_OK_AND_ASSIGN(auto geog, GeospatialType::GeographyField("position")); + auto shape = DataField::ConvertDataFieldToArrowField(DataField(1, geom)); + auto position = DataField::ConvertDataFieldToArrowField(DataField(2, geog)); + auto source = arrow::field("r", arrow::struct_({shape, position})); + auto renamed = shape->WithName("renamed"); + auto replacement = DataField::ConvertDataFieldToArrowField( + DataField(3, arrow::field("shape", arrow::binary()))); + for (const auto& fields : + {arrow::FieldVector{position, renamed}, arrow::FieldVector{renamed}, + arrow::FieldVector{replacement, position}, arrow::FieldVector{geom, geog}}) { + auto target = arrow::field("r", arrow::struct_(fields)); + ASSERT_OK(GeospatialType::ValidateEvolution(*source, *target)); + } + ASSERT_OK(GeospatialType::ValidateEvolution(*arrow::field("r", arrow::struct_({geom, geog})), + *source)); + auto incompatible = renamed->WithType(arrow::utf8()); + for (const auto& type : {arrow::struct_({position, incompatible}), + arrow::struct_({shape, position->RemoveMetadata()})}) { + ASSERT_NOK(GeospatialType::ValidateEvolution(*source, *source->WithType(type))); + } + auto bytes = arrow::field("value", arrow::binary()); + const std::pair, std::shared_ptr> types[] = { + {arrow::list(geom), arrow::list(bytes)}, + {std::make_shared(geom->WithNullable(false), bytes), + arrow::map(arrow::binary(), bytes)}, + {arrow::map(arrow::int32(), geog), arrow::map(arrow::int32(), bytes)}}; + for (const auto& [source_type, target_type] : types) { + auto left = arrow::field("container", source_type); + auto right = arrow::field("container", target_type); + ASSERT_NOK(GeospatialType::ValidateEvolution(*left, *right)); + ASSERT_NOK(GeospatialType::ValidateEvolution(*right, *left)); + } +} + +} // namespace paimon::test diff --git a/src/paimon/common/types/map_type.h b/src/paimon/common/types/map_type.h index 3e9cd01a9..4a8825a4d 100644 --- a/src/paimon/common/types/map_type.h +++ b/src/paimon/common/types/map_type.h @@ -29,16 +29,16 @@ namespace paimon { -// NOTE: -// In contrast to Java Paimon, this C++ implementation does NOT allow nullable keys in MapType. -// This restriction is due to the limitations of Apache Arrow, which does not support nullable map -// keys. -// -// When using this class, ensure that the key type is always non-nullable. -// Attempting to use a nullable key will result in incorrect behavior or runtime errors. class MapType : public DataType { public: static constexpr char TYPE[] = "MAP"; + // Java allows nullable MAP keys in table schemas. Arrow requires physical keys to be non-null. + static constexpr char KEY_NULLABLE[] = "paimon.map_key_nullable"; + + static bool HasNullableKey(const std::shared_ptr& metadata) { + int32_t index = metadata ? metadata->FindKey(KEY_NULLABLE) : -1; + return index >= 0 && metadata->value(index) == "true"; + } MapType(const std::shared_ptr& type, bool nullable, const std::shared_ptr& metadata) @@ -53,10 +53,11 @@ class MapType : public DataType { *allocator); auto type = checked_cast(type_.get()); auto key_field = type->key_field(); + bool key_nullable = key_field->nullable() || HasNullableKey(metadata_); // The key and value metadata is load-bearing: it is what marks an extension type such as // VARIANT. std::shared_ptr key_data_type = - DataType::Create(key_field->type(), key_field->nullable(), key_field->metadata()); + DataType::Create(key_field->type(), key_nullable, key_field->metadata()); obj.AddMember(rapidjson::StringRef("key"), RapidJsonUtil::SerializeValue(*key_data_type, allocator).Move(), *allocator); auto value_field = type->item_field(); diff --git a/src/paimon/common/utils/arrow/arrow_utils.cpp b/src/paimon/common/utils/arrow/arrow_utils.cpp index 77e321682..e33832d89 100644 --- a/src/paimon/common/utils/arrow/arrow_utils.cpp +++ b/src/paimon/common/utils/arrow/arrow_utils.cpp @@ -38,6 +38,7 @@ #include "arrow/util/bitmap_ops.h" #include "arrow/util/compression.h" #include "fmt/format.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/string_utils.h" @@ -674,6 +675,10 @@ bool ArrowUtils::EqualsIgnoreNullable(const std::shared_ptr& ty if (field->name() != other_field->name()) { return false; } + if ((GeospatialType::IsGeospatial(*field) || GeospatialType::IsGeospatial(*other_field)) && + !GeospatialType::Equals(*field, *other_field)) { + return false; + } if (!EqualsIgnoreNullable(field->type(), other_field->type())) { return false; } diff --git a/src/paimon/common/utils/arrow/arrow_utils_test.cpp b/src/paimon/common/utils/arrow/arrow_utils_test.cpp index 6bbd0f625..071be1d19 100644 --- a/src/paimon/common/utils/arrow/arrow_utils_test.cpp +++ b/src/paimon/common/utils/arrow/arrow_utils_test.cpp @@ -32,6 +32,7 @@ #include "arrow/util/byte_size.h" #include "gtest/gtest.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/status.h" #include "paimon/testing/utils/testharness.h" @@ -1027,6 +1028,38 @@ TEST(ArrowUtilsTest, TestDictionaryArrayMemoryUsage) { } TEST(ArrowUtilsTest, TestEqualsIgnoreNullable) { + { + ASSERT_OK_AND_ASSIGN(auto geometry, GeospatialType::GeometryField("g", "EPSG:4326")); + ASSERT_OK_AND_ASSIGN(auto geography, GeospatialType::GeographyField("g", "EPSG:4326")); + ASSERT_OK_AND_ASSIGN(auto other_crs, GeospatialType::GeometryField("g", "EPSG:3857")); + ASSERT_OK_AND_ASSIGN(auto other_edges, + GeospatialType::GeographyField("g", "EPSG:4326", "karney")); + auto nested_types = [](const std::shared_ptr& field) { + return std::vector>{ + arrow::struct_({field}), arrow::list(field), + std::make_shared(field->WithNullable(false), + arrow::field("value", arrow::int32())), + arrow::map(arrow::int32(), field)}; + }; + for (const auto& field : {geometry, geography}) { + auto source_types = nested_types(field); + ASSERT_OK_AND_ASSIGN( + auto same, + field == geometry + ? GeospatialType::GeometryField("g", "epsg:4326", false) + : GeospatialType::GeographyField("g", "epsg:4326", "spherical", false)); + for (const auto& other : {same, arrow::field("g", arrow::binary()), other_crs, + other_edges, field == geometry ? geography : geometry}) { + auto target_types = nested_types(other); + for (size_t i = 0; i < source_types.size(); ++i) { + ASSERT_EQ(ArrowUtils::EqualsIgnoreNullable(source_types[i], target_types[i]), + other == same) + << source_types[i]->ToString(true) << " -> " + << target_types[i]->ToString(true); + } + } + } + } { // test simple ASSERT_FALSE(ArrowUtils::EqualsIgnoreNullable(arrow::int32(), arrow::int64())); diff --git a/src/paimon/common/utils/field_type_utils.h b/src/paimon/common/utils/field_type_utils.h index c6e259987..59159af99 100644 --- a/src/paimon/common/utils/field_type_utils.h +++ b/src/paimon/common/utils/field_type_utils.h @@ -27,6 +27,7 @@ #include "arrow/type_fwd.h" #include "fmt/format.h" #include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/defs.h" #include "paimon/result.h" #include "paimon/status.h" @@ -53,6 +54,10 @@ class FieldTypeUtils { /// Converts an arrow field to a `FieldType`, disambiguating metadata-marked extension types /// (a VARIANT field is physically a STRUCT with the variant metadata marker). static Result ConvertToFieldType(const std::shared_ptr& field) { + if (GeospatialType::IsGeospatial(*field)) { + PAIMON_ASSIGN_OR_RAISE(GeospatialType type, GeospatialType::FromField(*field)); + return type.Type(); + } if (VariantTypeUtils::IsVariantField(field)) { return FieldType::VARIANT; } @@ -105,6 +110,10 @@ class FieldTypeUtils { static std::string FieldTypeToString(const FieldType& type) { switch (type) { + case FieldType::GEOMETRY: + return "GEOMETRY"; + case FieldType::GEOGRAPHY: + return "GEOGRAPHY"; case FieldType::BOOLEAN: return "BOOLEAN"; case FieldType::TINYINT: diff --git a/src/paimon/core/casting/cast_executor_factory.cpp b/src/paimon/core/casting/cast_executor_factory.cpp index b74ea20a6..df9b9de3a 100644 --- a/src/paimon/core/casting/cast_executor_factory.cpp +++ b/src/paimon/core/casting/cast_executor_factory.cpp @@ -83,7 +83,7 @@ std::shared_ptr CastExecutorFactory::GetCastExecutor(const FieldTy Result> CastExecutorFactory::GetCastExecutor( const DataField& src_field, const FieldType& target) const { PAIMON_ASSIGN_OR_RAISE(FieldType src, - FieldTypeUtils::ConvertToFieldType(src_field.Type()->id())); + FieldTypeUtils::ConvertToFieldType(src_field.ArrowField())); if (src == FieldType::TIME && target == FieldType::STRING) { PAIMON_ASSIGN_OR_RAISE(int32_t precision, DataType::GetTimePrecision(*src_field.ArrowField())); diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 3a87f6616..ae82a32ef 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -1796,9 +1796,13 @@ bool CoreOptions::CompactionForceUpLevel0() const { } std::map CoreOptions::GetFieldsSequenceGroups() const { - auto raw_options = impl_->raw_options; + return ParseFieldsSequenceGroups(impl_->raw_options); +} + +std::map CoreOptions::ParseFieldsSequenceGroups( + const std::map& options) { std::map sequence_groups; - for (const auto& [key, value] : raw_options) { + for (const auto& [key, value] : options) { if (StringUtils::StartsWith(key, Options::FIELDS_PREFIX, /*start_pos=*/0) && StringUtils::EndsWith(key, Options::SEQUENCE_GROUP)) { std::string seq_fields_str = diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index c478b2c1c..8cec088cc 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -234,6 +234,8 @@ class PAIMON_EXPORT CoreOptions { int32_t FileIndexFormatVersion() const; std::map GetFieldsSequenceGroups() const; + static std::map ParseFieldsSequenceGroups( + const std::map& options); bool PartialUpdateRemoveRecordOnDelete() const; bool AggregationRemoveRecordOnDelete() const; bool TableReadSequenceNumberEnabled() const; diff --git a/src/paimon/core/io/complete_row_tracking_fields_reader.cpp b/src/paimon/core/io/complete_row_tracking_fields_reader.cpp index e9d703bc8..d8a9d7a2e 100644 --- a/src/paimon/core/io/complete_row_tracking_fields_reader.cpp +++ b/src/paimon/core/io/complete_row_tracking_fields_reader.cpp @@ -127,7 +127,8 @@ CompleteRowTrackingFieldsBatchReader::NextBatchWithBitmap() { // re-construct result array with read schema arrow::ArrayVector sub_array_vec; sub_array_vec.reserve(read_schema_->num_fields()); - for (const auto& field : read_schema_->fields()) { + arrow::FieldVector fields = read_schema_->fields(); + for (auto& field : fields) { const auto& field_name = field->name(); if (field_name == row_id_field_name) { sub_array_vec.push_back(row_id_array); @@ -141,12 +142,12 @@ CompleteRowTrackingFieldsBatchReader::NextBatchWithBitmap() { "Data file does not contain field {} in CompleteRowTrackingFieldsBatchReader", field_name)); } + field = field->WithType(sub_array->type()); sub_array_vec.push_back(sub_array); } } - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - std::shared_ptr target_array, - arrow::StructArray::Make(sub_array_vec, read_schema_->field_names())); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr target_array, + arrow::StructArray::Make(sub_array_vec, fields)); PAIMON_RETURN_NOT_OK_FROM_ARROW( arrow::ExportArray(*target_array, c_array.get(), c_schema.get())); PAIMON_RETURN_NOT_OK(AddArrowArrayLifetime(c_array.get(), c_schema.get(), arrow_pool_)); diff --git a/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp b/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp index 04f05c30a..e9af35006 100644 --- a/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp +++ b/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp @@ -29,6 +29,8 @@ #include "arrow/array/array_nested.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/memory/memory_pool.h" #include "paimon/status.h" @@ -60,8 +62,9 @@ class CompleteRowTrackingFieldsBatchReaderTest : public testing::Test { /*selection_bitmap=*/std::nullopt)); ASSERT_OK_AND_ASSIGN(auto result_with_special_fields, paimon::test::ReadResultCollector::CollectResult( - std::move(complete_row_tracking_fields_reader))); + std::move(complete_row_tracking_fields_reader), true)); auto expected_chunk_array = std::make_shared(expected_array); + ASSERT_TRUE(result_with_special_fields->type()->Equals(*expected_array->type(), true)); ASSERT_TRUE(result_with_special_fields->Equals(expected_chunk_array)); } } @@ -350,6 +353,29 @@ TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestNestedType) { CheckResult(src_array, /*first_row_id=*/100, /*snapshot_id=*/8, read_schema, target_array); } +TEST_F(CompleteRowTrackingFieldsBatchReaderTest, GeospatialMetadataSurvivesRowTracking) { + ASSERT_OK_AND_ASSIGN(auto geometry, GeospatialType::GeometryField("shape", "EPSG:3857")); + geometry = DataField::ConvertDataFieldToArrowField(DataField(0, geometry)); + ASSERT_OK_AND_ASSIGN(auto geography, + GeospatialType::GeographyField("location", "EPSG:4326", "karney")); + auto nested = arrow::field("nested", arrow::struct_({geography}), false); + auto file_type = arrow::struct_({geometry, nested}); + auto read_schema = arrow::schema({nested, arrow::field("_ROW_ID", arrow::int64()), geometry, + arrow::field("_SEQUENCE_NUMBER", arrow::int64())}); + auto source = arrow::ipc::internal::json::ArrayFromJSON(file_type, R"([ + ["\u0001\u0007\u0000\u0000\u0000\u0000\u0000\u0000\u0000", [null]], + [null, ["\u0001\u0007\u0000\u0000\u0000\u0000\u0000\u0000\u0000"]] + ])"); + ASSERT_TRUE(source.ok()); + auto expected = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(read_schema->fields()), R"([ + [[null], 100, "\u0001\u0007\u0000\u0000\u0000\u0000\u0000\u0000\u0000", 4], + [["\u0001\u0007\u0000\u0000\u0000\u0000\u0000\u0000\u0000"], 101, null, 4] + ])"); + ASSERT_TRUE(expected.ok()); + CheckResult(*source, 100, 4, read_schema, *expected); +} + TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestInvalidWithReadNonExistField) { arrow::FieldVector fields = { arrow::field("f0", arrow::int32()), diff --git a/src/paimon/core/io/field_mapping_reader.cpp b/src/paimon/core/io/field_mapping_reader.cpp index e271a579c..b7e900137 100644 --- a/src/paimon/core/io/field_mapping_reader.cpp +++ b/src/paimon/core/io/field_mapping_reader.cpp @@ -181,13 +181,13 @@ Result FieldMappingReader::NextBatchWithBitmap arrow::ImportArray(c_array.get(), c_schema.get())); arrow::ArrayVector target_array(field_count_); - std::vector target_field_names(field_count_); + arrow::FieldVector target_fields(field_count_); // mapping non-partition array PAIMON_ASSIGN_OR_RAISE(std::shared_ptr casted_non_partition_array, CastNonPartitionArrayIfNeed(non_partition_array)); PAIMON_RETURN_NOT_OK(MappingFields( casted_non_partition_array, non_partition_info_.non_partition_read_schema, - non_partition_info_.idx_in_target_read_schema, &target_array, &target_field_names)); + non_partition_info_.idx_in_target_read_schema, &target_array, &target_fields)); // mapping partition array if (partition_info_ != std::nullopt) { @@ -198,7 +198,7 @@ Result FieldMappingReader::NextBatchWithBitmap auto trim_partition_array = partition_array_->Slice(0, non_partition_array->length()); PAIMON_RETURN_NOT_OK(MappingFields( trim_partition_array, partition_info_.value().partition_read_schema, - partition_info_.value().idx_in_target_read_schema, &target_array, &target_field_names)); + partition_info_.value().idx_in_target_read_schema, &target_array, &target_fields)); } // mapping non-exist array if (non_exist_field_info_ != std::nullopt) { @@ -210,12 +210,12 @@ Result FieldMappingReader::NextBatchWithBitmap PAIMON_RETURN_NOT_OK(MappingFields(trim_non_exist_array, non_exist_field_info_.value().non_exist_read_schema, non_exist_field_info_.value().idx_in_target_read_schema, - &target_array, &target_field_names)); + &target_array, &target_fields)); } // construct target array PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, - arrow::StructArray::Make(target_array, target_field_names)); + arrow::StructArray::Make(target_array, target_fields)); std::unique_ptr target_c_arrow_array = std::make_unique(); std::unique_ptr target_c_schema = std::make_unique(); PAIMON_RETURN_NOT_OK_FROM_ARROW( @@ -339,7 +339,7 @@ Status FieldMappingReader::MappingFields(const std::shared_ptr& da const std::vector& read_fields_of_data_array, const std::vector& idx_in_target_schema, arrow::ArrayVector* target_array, - std::vector* target_field_names) { + arrow::FieldVector* target_fields) { auto* struct_array = checked_cast(data_array.get()); assert(struct_array->fields().size() == idx_in_target_schema.size()); for (size_t i = 0; i < idx_in_target_schema.size(); i++) { @@ -353,8 +353,9 @@ Status FieldMappingReader::MappingFields(const std::shared_ptr& da field_array, read_field.ArrowField(), arrow_pool_.get())); } + (*target_fields)[idx_in_target_schema[i]] = + read_field.ArrowField()->WithType(field_array->type()); (*target_array)[idx_in_target_schema[i]] = std::move(field_array); - (*target_field_names)[idx_in_target_schema[i]] = read_field.Name(); } return Status::OK(); } diff --git a/src/paimon/core/io/field_mapping_reader.h b/src/paimon/core/io/field_mapping_reader.h index fa976b2f6..b2eb3ab33 100644 --- a/src/paimon/core/io/field_mapping_reader.h +++ b/src/paimon/core/io/field_mapping_reader.h @@ -114,8 +114,7 @@ class FieldMappingReader : public FileBatchReader { Status MappingFields(const std::shared_ptr& src_array, const std::vector& read_fields_of_data_array, const std::vector& idx_in_target_schema, - arrow::ArrayVector* target_array, - std::vector* target_field_names); + arrow::ArrayVector* target_array, arrow::FieldVector* target_fields); private: bool need_mapping_ = false; diff --git a/src/paimon/core/operation/data_evolution_file_store_scan.cpp b/src/paimon/core/operation/data_evolution_file_store_scan.cpp index 3ef72899d..70dccad68 100644 --- a/src/paimon/core/operation/data_evolution_file_store_scan.cpp +++ b/src/paimon/core/operation/data_evolution_file_store_scan.cpp @@ -25,6 +25,7 @@ #include "paimon/common/reader/data_evolution_array.h" #include "paimon/common/reader/data_evolution_row.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/object_utils.h" #include "paimon/common/utils/range_helper.h" #include "paimon/core/utils/data_evolution_utils.h" @@ -183,6 +184,8 @@ DataEvolutionFileStoreScan::EvolutionStats( if (file_meta->first_row_id == group_start && file_meta->row_count == group_row_count && table_field.Type()->Equals(write_field.Type()) && + GeospatialType::Equals(*table_field.ArrowField(), + *write_field.ArrowField()) && stats_iter != stats_field_map.end()) { row_offsets[table_field_idx] = entry_idx; field_offsets[table_field_idx] = stats_iter->second; diff --git a/src/paimon/core/operation/file_store_scan.cpp b/src/paimon/core/operation/file_store_scan.cpp index a40be6742..3335c76ae 100644 --- a/src/paimon/core/operation/file_store_scan.cpp +++ b/src/paimon/core/operation/file_store_scan.cpp @@ -36,6 +36,7 @@ #include "paimon/common/io/cache/cache_key.h" #include "paimon/common/predicate/literal_converter.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/field_type_utils.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/core/io/data_file_meta.h" @@ -89,7 +90,10 @@ Result> FileStoreScan::ReconstructPredicateWithNonCas continue; } // Exclude fields requiring casting to avoid false negatives in stats filtering. - if (!data_iter->second.second.Type()->Equals(table_iter->second.Type())) { + const auto& data_field = data_iter->second.second; + const auto& table_field = table_iter->second; + if (!data_field.Type()->Equals(table_field.Type()) || + !GeospatialType::Equals(*data_field.ArrowField(), *table_field.ArrowField())) { excluded_field_names.insert(field_name); } } diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index be0f7a917..3f573c28d 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -37,6 +37,7 @@ #include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" @@ -137,6 +138,8 @@ Result> InternalReadContext::AlignReadFieldWithTab PAIMON_RETURN_NOT_OK( NestedProjectionUtils::ValidateMapSharedShreddingAccessField(read_field).status()); auto read_struct = checked_pointer_cast(read_field->type()); + PAIMON_RETURN_NOT_OK( + GeospatialType::ValidateEvolution(*table_map->item_field(), *read_struct->field(0))); const auto& selected_value_type = read_struct->field(0)->type(); if (!selected_value_type->Equals(table_map->item_type())) { return Status::Invalid(fmt::format( @@ -202,12 +205,14 @@ Result> InternalReadContext::AlignReadFieldWithTab PAIMON_ASSIGN_OR_RAISE( std::shared_ptr rebased_item_field, AlignReadFieldWithTableFieldIds(read_map->item_field(), table_map->item_field())); - auto rebased_type = arrow::map(rebased_key_field->type(), rebased_item_field); + auto rebased_type = std::make_shared(rebased_key_field, rebased_item_field, + read_map->keys_sorted()); auto aligned_field = table_field->WithType(rebased_type); return DataField::MergeFieldMetadataByWhitelist(aligned_field, read_field, kReadMetadataWhitelist); } + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateEvolution(*table_field, *read_field)); if (!read_field->type()->Equals(table_field->type())) { return Status::Invalid(fmt::format( "Read schema field '{}' type {} does not match table field type {}", read_field->name(), diff --git a/src/paimon/core/realtime/arrow_realtime_store.cpp b/src/paimon/core/realtime/arrow_realtime_store.cpp index 298ac476f..042ef2265 100644 --- a/src/paimon/core/realtime/arrow_realtime_store.cpp +++ b/src/paimon/core/realtime/arrow_realtime_store.cpp @@ -35,6 +35,7 @@ #include "paimon/common/predicate/predicate_filter.h" #include "paimon/common/reader/reader_utils.h" #include "paimon/common/table/special_fields.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -56,8 +57,8 @@ namespace paimon { namespace { -bool SupportsMinMax(const std::shared_ptr& type) { - switch (type->id()) { +bool SupportsMinMax(const arrow::Field& field) { + switch (field.type()->id()) { case arrow::Type::BOOL: case arrow::Type::INT8: case arrow::Type::INT16: @@ -66,12 +67,13 @@ bool SupportsMinMax(const std::shared_ptr& type) { case arrow::Type::FLOAT: case arrow::Type::DOUBLE: case arrow::Type::STRING: - case arrow::Type::BINARY: case arrow::Type::DATE32: case arrow::Type::TIME32: case arrow::Type::TIMESTAMP: case arrow::Type::DECIMAL128: return true; + case arrow::Type::BINARY: + return !GeospatialType::IsGeospatial(field); default: return false; } @@ -912,9 +914,10 @@ Result> ArrowRealtimeStore::C aggregate_options.min_count = 1; arrow::compute::ExecContext exec_context(arrow_pool_.get()); - for (const std::shared_ptr& field : data->fields()) { + for (int32_t i = 0; i < data->num_fields(); ++i) { + const auto& field = data->field(i); null_count_builder.UnsafeAppend(field->null_count()); - if (!SupportsMinMax(field->type())) { + if (!SupportsMinMax(*write_schema_->field(i))) { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( std::shared_ptr min_value, arrow::MakeArrayOfNull(field->type(), /*length=*/1, arrow_pool_.get())); diff --git a/src/paimon/core/realtime/arrow_realtime_store_test.cpp b/src/paimon/core/realtime/arrow_realtime_store_test.cpp index fe00d43b9..625479443 100644 --- a/src/paimon/core/realtime/arrow_realtime_store_test.cpp +++ b/src/paimon/core/realtime/arrow_realtime_store_test.cpp @@ -29,6 +29,7 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "paimon/common/table/special_fields.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/core/realtime/realtime_offset_batch_reader.h" @@ -639,6 +640,38 @@ TEST_F(ArrowRealtimeStoreTest, TestFullStatisticsPrunesNonMatchingBatch) { ASSERT_EQ(std::vector({0, 1, 10, 11}), ReadIds(unfiltered)); } +TEST_F(ArrowRealtimeStoreTest, GeospatialStatisticsKeepNullCountsWithoutBounds) { + ASSERT_OK_AND_ASSIGN(auto geometry, GeospatialType::GeometryField("geo")); + ASSERT_OK_AND_ASSIGN(auto geography, GeospatialType::GeographyField("geo")); + for (const auto& field : {geometry, geography}) { + SCOPED_TRACE(field->ToString(true)); + schema_ = arrow::schema( + {DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()), + arrow::field("id", arrow::int64()), field, arrow::field("bytes", arrow::binary())}); + ASSERT_OK_AND_ASSIGN(auto store, CreateStore(StatisticsMode::FULL)); + auto data = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema_->fields()), R"([ + [0, 0, "\u0001\u0007\u0000\u0000\u0000\u0000\u0000\u0000\u0000", "a"], + [1, 1, null, "z"] + ])"); + ASSERT_TRUE(data.ok()) << data.status().ToString(); + ASSERT_OK_AND_ASSIGN(auto statistics, store->CollectStatistics( + checked_pointer_cast(*data))); + ASSERT_TRUE(statistics); + ASSERT_TRUE(statistics->min_values->field(2)->IsNull(0)); + ASSERT_TRUE(statistics->max_values->field(2)->IsNull(0)); + auto null_counts = checked_pointer_cast(statistics->null_counts); + ASSERT_EQ(null_counts->Value(2), 1); + ASSERT_EQ(null_counts->Value(3), 0); + ASSERT_EQ( + checked_pointer_cast(statistics->min_values->field(3))->GetView(0), + "a"); + ASSERT_EQ( + checked_pointer_cast(statistics->max_values->field(3))->GetView(0), + "z"); + } +} + TEST_F(ArrowRealtimeStoreTest, TestFullStatisticsPrunesNonMatchingTimeBatch) { schema_ = arrow::schema({ DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()), diff --git a/src/paimon/core/schema/arrow_schema_validator.cpp b/src/paimon/core/schema/arrow_schema_validator.cpp index c0447c31b..7ae187f64 100644 --- a/src/paimon/core/schema/arrow_schema_validator.cpp +++ b/src/paimon/core/schema/arrow_schema_validator.cpp @@ -29,6 +29,7 @@ #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/types/data_field.h" #include "paimon/common/types/data_type.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/types/vector_type.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/decimal_utils.h" @@ -199,6 +200,9 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr& Status ArrowSchemaValidator::ValidateField(const std::shared_ptr& field, bool allow_blob) { + if (GeospatialType::IsGeospatial(*field)) { + PAIMON_RETURN_NOT_OK(GeospatialType::FromField(*field)); + } const auto kind = field->type()->id(); switch (kind) { case arrow::Type::type::BOOL: diff --git a/src/paimon/core/schema/arrow_schema_validator_test.cpp b/src/paimon/core/schema/arrow_schema_validator_test.cpp index 46f30d3c7..cbc61f6c6 100644 --- a/src/paimon/core/schema/arrow_schema_validator_test.cpp +++ b/src/paimon/core/schema/arrow_schema_validator_test.cpp @@ -30,6 +30,7 @@ #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/types/data_field.h" #include "paimon/common/types/data_type.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/testing/utils/testharness.h" @@ -110,6 +111,32 @@ TEST(ArrowSchemaValidatorTest, TestInvalidTimePrecision) { ASSERT_NOK_WITH_MSG(seconds->ToJsonString(), "Only millisecond TIME is supported"); } +TEST(ArrowSchemaValidatorTest, GeospatialMetadataValidation) { + ASSERT_OK_AND_ASSIGN(auto geometry, GeospatialType::GeometryField("geo")); + ASSERT_OK_AND_ASSIGN(auto geography, GeospatialType::GeographyField("geo")); + for (const auto& valid : {geometry, geography}) { + auto invalid_metadata = valid->WithMergedMetadata(arrow::KeyValueMetadata::Make( + {GeospatialType::kExtensionMetadataKey}, {R"({"crs":null})"})); + for (const auto& leaf : {valid, valid->WithType(arrow::utf8()), invalid_metadata}) { + for (const auto& field : + {leaf, arrow::field("row", arrow::struct_({leaf})), + arrow::field("array", arrow::list(leaf)), + arrow::field("map", arrow::map(arrow::int32(), leaf)), + arrow::field("map", std::make_shared( + leaf->WithNullable(false), + arrow::field("value", arrow::int32())))}) { + SCOPED_TRACE(field->ToString(true)); + auto status = ArrowSchemaValidator::ValidateSchema(*arrow::schema({field})); + if (leaf == valid) { + ASSERT_OK(status); + } else { + ASSERT_NOK_WITH_MSG(status, "geoarrow.wkb"); + } + } + } + } +} + TEST(ArrowSchemaValidatorTest, TestValidateNoRedundantFields) { auto col1_field = arrow::field("col1", arrow::int64()); auto col2_field = arrow::field("col2", arrow::int32()); diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index b8319f6cf..c494d5476 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -44,6 +44,7 @@ #include "paimon/common/options/memory_size.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/arrow/vector_utils.h" #include "paimon/common/utils/binary_row_partition_computer.h" @@ -233,10 +234,82 @@ bool SchemaValidation::IsComplexType(const std::shared_ptr& field) BlobUtils::IsBlobField(field)); } +Status SchemaValidation::ValidateGeospatialKeys(const std::vector& fields, + const std::vector& primary_keys, + const std::vector& partition_keys, + const std::vector& bucket_keys) { + if (primary_keys.empty() && partition_keys.empty() && bucket_keys.empty()) { + return Status::OK(); + } + for (const auto& field : fields) { + if (GeospatialType::ContainsGeospatial(*field.ArrowField())) { + for (const auto* keys : {&primary_keys, &partition_keys, &bucket_keys}) { + if (std::find(keys->begin(), keys->end(), field.Name()) != keys->end()) { + return Status::Invalid( + "GEOMETRY/GEOGRAPHY field cannot be a primary, partition or bucket key: ", + field.Name()); + } + } + } + } + return Status::OK(); +} + +Status SchemaValidation::ValidateGeospatialOptions( + const std::vector& fields, const std::map& options) { + std::set geospatial_names; + for (const auto& field : fields) { + if (GeospatialType::ContainsGeospatial(*field.ArrowField())) { + geospatial_names.insert(field.Name()); + } + } + if (geospatial_names.empty()) { + return Status::OK(); + } + + auto validate_format = [](const std::string& key, const std::string& format) -> Status { + if (!StringUtils::EqualsIgnoreCase(format, "parquet")) { + return Status::Invalid(fmt::format( + "GEOMETRY/GEOGRAPHY only supports parquet data files, but {} is {}.", key, format)); + } + return Status::OK(); + }; + for (const char* key : {Options::FILE_FORMAT, Options::CHANGELOG_FILE_FORMAT}) { + auto it = options.find(key); + if (it != options.end()) { + PAIMON_RETURN_NOT_OK(validate_format(key, it->second)); + } + } + PAIMON_RETURN_NOT_OK( + ValidatePerLevelOption(options, Options::FILE_FORMAT_PER_LEVEL, validate_format)); + auto validate_ordering = [&](const std::string& columns, const std::string& role) -> Status { + for (auto name : StringUtils::Split(columns, Options::FIELDS_SEPARATOR)) { + StringUtils::Trim(&name); + if (geospatial_names.count(name)) { + return Status::Invalid( + fmt::format("GEOMETRY/GEOGRAPHY field '{}' cannot be used in {}.", name, role)); + } + } + return Status::OK(); + }; + for (const char* key : + {Options::BUCKET_KEY, Options::SEQUENCE_FIELD, "clustering.columns", "sort.columns"}) { + auto it = options.find(key); + if (it != options.end()) { + PAIMON_RETURN_NOT_OK(validate_ordering(it->second, key)); + } + } + for (const auto& group : CoreOptions::ParseFieldsSequenceGroups(options)) { + PAIMON_RETURN_NOT_OK(validate_ordering(group.first, "a sequence group ordering key")); + } + return Status::OK(); +} + Status SchemaValidation::ValidateGenericSchema(const std::vector& fields, const std::vector& bucket_keys, const std::vector& primary_keys, const std::vector& partition_keys) { + PAIMON_RETURN_NOT_OK(ValidateGeospatialKeys(fields, primary_keys, partition_keys, bucket_keys)); std::vector field_names; field_names.reserve(fields.size()); for (const DataField& field : fields) { @@ -326,6 +399,13 @@ Status SchemaValidation::ValidateFormatTableSchema( // kept; this only has to fail for a format nothing here can read. PAIMON_RETURN_NOT_OK(FormatTable::ParseFormat(file_format)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_schema, schema.GetArrowSchema()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema, + arrow::ImportSchema(c_schema.get())); + PAIMON_ASSIGN_OR_RAISE(std::vector fields, + DataField::ConvertArrowSchemaToDataFields(arrow_schema)); + PAIMON_RETURN_NOT_OK(ValidateGeospatialOptions(fields, effective_options)); + // The remaining options are read through `CoreOptions`, so a default only ever changes in one // place. `target-file-row-num` is validated there, so it needs no check of its own here. PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, @@ -377,9 +457,6 @@ Status SchemaValidation::ValidateFormatTableSchema( // building the computer that does the round trip, not by listing the types it accepts, which // would be a second list to keep in step. if (!schema.PartitionKeys().empty()) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_schema, schema.GetArrowSchema()); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema, - arrow::ImportSchema(c_schema.get())); Result> partition_computer = BinaryRowPartitionComputer::Create( schema.PartitionKeys(), arrow_schema, core_options.GetPartitionDefaultName(), @@ -401,6 +478,7 @@ Status SchemaValidation::ValidateFormatTableSchema( Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) { PAIMON_RETURN_NOT_OK(ValidateGenericTableSchema(schema)); + PAIMON_RETURN_NOT_OK(ValidateGeospatialOptions(schema.Fields(), schema.Options())); PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(schema.Options())); PAIMON_RETURN_NOT_OK(ValidateBucket(schema, options)); diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h index 0ad605c5a..22b7510f4 100644 --- a/src/paimon/core/schema/schema_validation.h +++ b/src/paimon/core/schema/schema_validation.h @@ -52,6 +52,15 @@ class SchemaValidation { static Status ValidateTableSchema(const TableSchema& schema); + static Status ValidateGeospatialKeys(const std::vector& fields, + const std::vector& primary_keys, + const std::vector& partition_keys, + const std::vector& bucket_keys); + + /// Checks geospatial format and ordering restrictions against the effective table options. + static Status ValidateGeospatialOptions(const std::vector& fields, + const std::map& options); + /// Validates real-time options after operation-specific options have been merged with the /// table schema options. static Status ValidateRealtime(const TableSchema& schema, const CoreOptions& options); diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 4694be20b..210792db0 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -28,6 +28,7 @@ #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/table/special_fields.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" @@ -2158,4 +2159,53 @@ TEST(SchemaValidationTest, TestMapSharedShreddingRejectsPostponeBucketMode) { "MAP shared-shredding currently does not support postpone bucket mode."); } +TEST(SchemaValidationTest, GeospatialRestrictions) { + ASSERT_OK_AND_ASSIGN(auto geo, GeospatialType::GeographyField("geo")); + ASSERT_OK_AND_ASSIGN(auto geom, GeospatialType::GeometryField("geo")); + for (const auto& field : + {geo, geom, arrow::field("geo", arrow::struct_({geo->WithName("nested")})), + arrow::field("geo", arrow::list(geom)), + arrow::field("geo", + std::make_shared(geom->WithNullable(false), + arrow::field("value", arrow::binary()))), + arrow::field("geo", arrow::map(arrow::int32(), geo))}) { + SCOPED_TRACE(field->ToString(true)); + auto arrow_schema = arrow::schema({arrow::field("id", arrow::int32()), field}); + auto validate = [&](const std::vector& partitions, + const std::vector& primary, + const std::map& options) -> Status { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr schema, + TableSchema::Create(0, arrow_schema, partitions, primary, options)); + return SchemaValidation::ValidateTableSchema(*schema); + }; + ASSERT_OK(validate({}, {}, {{Options::FILE_FORMAT, "parquet"}})); + ASSERT_NOK_WITH_MSG(validate({"geo"}, {}, {}), "GEOMETRY/GEOGRAPHY"); + ASSERT_NOK_WITH_MSG(validate({}, {"geo"}, {}), "GEOMETRY/GEOGRAPHY"); + ASSERT_NOK_WITH_MSG( + validate({}, {}, {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "geo"}}), + "GEOMETRY/GEOGRAPHY"); + ASSERT_NOK_WITH_MSG(validate({}, {"id"}, {{Options::SEQUENCE_FIELD, "geo"}}), + "GEOMETRY/GEOGRAPHY"); + ASSERT_NOK_WITH_MSG(validate({}, {"id"}, {{"fields.geo.sequence-group", "id"}}), + "GEOMETRY/GEOGRAPHY"); + for (const char* key : {"clustering.columns", "sort.columns"}) { + ASSERT_OK(validate({}, {}, {{key, "id"}})); + ASSERT_NOK_WITH_MSG(validate({}, {}, {{key, "id, geo"}}), "GEOMETRY/GEOGRAPHY"); + } + for (const auto& option : + {std::make_pair(Options::FILE_FORMAT, "orc"), + std::make_pair(Options::FILE_FORMAT, "avro"), + std::make_pair(Options::FILE_FORMAT, "blob"), + std::make_pair(Options::FILE_FORMAT, "lance"), + std::make_pair(Options::FILE_FORMAT, "mosaic"), + std::make_pair(Options::FILE_FORMAT, "vortex"), + std::make_pair(Options::FILE_FORMAT_PER_LEVEL, "0:parquet,1:orc"), + std::make_pair(Options::CHANGELOG_FILE_FORMAT, "avro")}) { + ASSERT_NOK_WITH_MSG(validate({}, {}, {{option.first, option.second}}), + "GEOMETRY/GEOGRAPHY only supports parquet"); + } + } +} + } // namespace paimon::test diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index 1c0cfd333..17c7c6ccc 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -39,6 +39,7 @@ #include "paimon/common/utils/rapidjson_util.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/schema/arrow_schema_validator.h" +#include "paimon/core/schema/schema_validation.h" #include "paimon/defs.h" #include "paimon/status.h" #include "paimon/utils/special_field_ids.h" @@ -148,7 +149,8 @@ Result> TableSchema::AssignFieldIdsRecursively( // in-memory schema and the same schema reloaded from JSON remain equivalent. auto new_map_type = std::make_shared(key_field, value_field, /*keys_sorted=*/false); - return arrow::field(field->name(), new_map_type, field->nullable(), metadata); + auto map_field = field->WithType(new_map_type); + return metadata ? map_field->WithMergedMetadata(metadata) : map_field; } return metadata ? field->WithMergedMetadata(metadata) : field; } @@ -218,6 +220,8 @@ Status TableSchema::DeriveBucketInfo() { if (bucket_keys.empty()) { bucket_keys = keys; } + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateGeospatialKeys(fields_, primary_keys_, + partition_keys_, bucket_keys)); bucket_keys_ = bucket_keys; PAIMON_ASSIGN_OR_RAISE(num_bucket_, OptionsUtils::GetValueFromMap(Options(), Options::BUCKET, -1)); diff --git a/src/paimon/core/schema/table_schema_test.cpp b/src/paimon/core/schema/table_schema_test.cpp index 73909d418..753c390be 100644 --- a/src/paimon/core/schema/table_schema_test.cpp +++ b/src/paimon/core/schema/table_schema_test.cpp @@ -26,6 +26,7 @@ #include "gtest/gtest.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/string_utils.h" @@ -141,18 +142,23 @@ TEST_F(TableSchemaTest, TestCreateWithAllFieldsHaveFieldId) { ASSERT_EQ(table_schema->Options(), options); } -TEST_F(TableSchemaTest, TestGetFieldTypeForVariant) { +TEST_F(TableSchemaTest, GetFieldTypeDistinguishesLogicalTypes) { + ASSERT_OK_AND_ASSIGN(auto geometry, GeospatialType::GeometryField("shape")); + ASSERT_OK_AND_ASSIGN(auto geography, GeospatialType::GeographyField("position")); arrow::FieldVector fields = {arrow::field("id", arrow::int32()), - VariantTypeUtils::ToArrowField("v")}; + VariantTypeUtils::ToArrowField("v"), geometry, geography, + arrow::field("bytes", arrow::binary())}; ASSERT_OK_AND_ASSIGN(auto table_schema, - TableSchema::Create(/*schema_id=*/0, arrow::schema(fields), - /*partition_keys=*/{}, /*primary_keys=*/{}, - /*options=*/{})); - ASSERT_OK_AND_ASSIGN(FieldType id_type, table_schema->GetFieldType("id")); - ASSERT_EQ(id_type, FieldType::INT); - // The variant marker must disambiguate the physical struct into FieldType::VARIANT. - ASSERT_OK_AND_ASSIGN(FieldType variant_type, table_schema->GetFieldType("v")); - ASSERT_EQ(variant_type, FieldType::VARIANT); + TableSchema::Create(0, arrow::schema(fields), {}, {}, {})); + const std::pair expected[] = {{"id", FieldType::INT}, + {"v", FieldType::VARIANT}, + {"shape", FieldType::GEOMETRY}, + {"position", FieldType::GEOGRAPHY}, + {"bytes", FieldType::BINARY}}; + for (const auto& [name, type] : expected) { + ASSERT_OK_AND_ASSIGN(FieldType actual, table_schema->GetFieldType(name)); + ASSERT_EQ(actual, type) << name; + } } TEST_F(TableSchemaTest, TestComputeHighestFieldId) { diff --git a/src/paimon/core/stats/simple_stats_collector.cpp b/src/paimon/core/stats/simple_stats_collector.cpp index 533c40985..43e738d53 100644 --- a/src/paimon/core/stats/simple_stats_collector.cpp +++ b/src/paimon/core/stats/simple_stats_collector.cpp @@ -28,6 +28,7 @@ #include "fmt/format.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_string.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/format/column_stats.h" namespace paimon { @@ -137,18 +138,18 @@ Status SimpleStatsCollector::Collect(const BinaryRow& row) { case arrow::Type::STRING: case arrow::Type::BINARY: { if (column_stats_[i] == nullptr) { - column_stats_[i] = ColumnStats::CreateStringColumnStats( - std::nullopt, std::nullopt, std::nullopt); + column_stats_[i] = + ColumnStats::CreateStringColumnStats(std::nullopt, std::nullopt, 0); } auto typed_stats = dynamic_cast(column_stats_[i].get()); if (typed_stats == nullptr) { assert(false); return Status::Invalid("cast typed stats failed"); } - if (!row.IsNullAt(i)) { - typed_stats->Collect(row.GetString(i).ToString()); - } else { + if (row.IsNullAt(i)) { typed_stats->Collect(std::nullopt); + } else if (!GeospatialType::IsGeospatial(*field)) { + typed_stats->Collect(row.GetString(i).ToString()); } break; } diff --git a/src/paimon/core/stats/simple_stats_collector_test.cpp b/src/paimon/core/stats/simple_stats_collector_test.cpp index 8fb21816f..85c926dc0 100644 --- a/src/paimon/core/stats/simple_stats_collector_test.cpp +++ b/src/paimon/core/stats/simple_stats_collector_test.cpp @@ -29,9 +29,12 @@ #include "paimon/common/data/binary_array.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/data_define.h" +#include "paimon/common/predicate/predicate_filter.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/core/stats/simple_stats.h" #include "paimon/core/stats/simple_stats_converter.h" #include "paimon/memory/memory_pool.h" +#include "paimon/predicate/predicate_builder.h" #include "paimon/testing/utils/binary_row_generator.h" #include "paimon/testing/utils/testharness.h" @@ -72,6 +75,17 @@ TEST(SimpleStatsCollectorTest, TestSimple) { std::vector({0, 0, 0, 0, 0, 0, 0, 0}), GetDefaultPool().get()); ASSERT_EQ(stats, expected_stats); + for (int64_t value : {1, 3}) { + auto predicate = std::dynamic_pointer_cast( + PredicateBuilder::Equal(4, "f4", FieldType::BIGINT, Literal(value))); + ASSERT_TRUE(predicate); + ASSERT_OK_AND_ASSIGN(bool row_matches, predicate->Test(nullptr, stats.MinValues())); + ASSERT_EQ(row_matches, value == 1); + ASSERT_OK_AND_ASSIGN( + bool stats_match, + predicate->Test(nullptr, 2, stats.MinValues(), stats.MaxValues(), stats.NullCounts())); + ASSERT_EQ(stats_match, value == 1); + } } TEST(SimpleStatsCollectorTest, TestNull) { @@ -110,4 +124,78 @@ TEST(SimpleStatsCollectorTest, TestInvalidPartition) { ASSERT_NOK_WITH_MSG(collector.Collect(BinaryRow::EmptyRow()), "partition schema not equal"); } +TEST(SimpleStatsCollectorTest, GeospatialNullCountsWithoutBounds) { + ASSERT_OK_AND_ASSIGN(auto geometry, GeospatialType::GeometryField("geo")); + ASSERT_OK_AND_ASSIGN(auto geography, GeospatialType::GeographyField("geo")); + auto pool = GetDefaultPool(); + for (const auto& geo : {geometry, geography}) { + SCOPED_TRACE(geo->ToString(true)); + auto schema = arrow::schema({geo, arrow::field("bytes", arrow::binary())}); + ASSERT_OK_AND_ASSIGN(auto type, GeospatialType::FromField(*geo)); + auto is_null = std::dynamic_pointer_cast( + PredicateBuilder::IsNull(0, "geo", type.Type())); + auto is_not_null = std::dynamic_pointer_cast( + PredicateBuilder::IsNotNull(0, "geo", type.Type())); + ASSERT_TRUE(is_null); + ASSERT_TRUE(is_not_null); + for (int64_t null_count : {0, 1, 3}) { + SCOPED_TRACE(null_count); + SimpleStatsCollector collector(schema); + for (int64_t i = 0; i < 3; ++i) { + auto row = + i < null_count + ? BinaryRowGenerator::GenerateRow({NullType(), NullType()}, pool.get()) + : BinaryRowGenerator::GenerateRow( + {std::string(i == 2 ? "a" : "z"), std::string(i == 2 ? "a" : "z")}, + pool.get()); + ASSERT_OK(collector.Collect(row)); + ASSERT_OK_AND_ASSIGN(bool row_is_null, is_null->Test(schema, row)); + ASSERT_EQ(row_is_null, i < null_count); + ASSERT_OK_AND_ASSIGN(bool row_is_not_null, is_not_null->Test(schema, row)); + ASSERT_EQ(row_is_not_null, i >= null_count); + } + ASSERT_OK_AND_ASSIGN(auto columns, collector.GetResult()); + ASSERT_OK_AND_ASSIGN(auto stats, SimpleStatsConverter::ToBinary(columns, pool.get())); + ASSERT_TRUE(stats.MinValues().IsNullAt(0)); + ASSERT_TRUE(stats.MaxValues().IsNullAt(0)); + for (int32_t i = 0; i < 2; ++i) { + ASSERT_FALSE(stats.NullCounts().IsNullAt(i)); + ASSERT_EQ(stats.NullCounts().GetLong(i), null_count); + } + if (null_count == 3) { + ASSERT_TRUE(stats.MinValues().IsNullAt(1)); + ASSERT_TRUE(stats.MaxValues().IsNullAt(1)); + } else { + ASSERT_EQ(stats.MinValues().GetString(1).ToString(), "a"); + ASSERT_EQ(stats.MaxValues().GetString(1).ToString(), "z"); + } + ASSERT_OK_AND_ASSIGN( + bool contains_null, + is_null->Test(schema, 3, stats.MinValues(), stats.MaxValues(), stats.NullCounts())); + ASSERT_EQ(contains_null, null_count > 0); + ASSERT_OK_AND_ASSIGN(bool contains_non_null, + is_not_null->Test(schema, 3, stats.MinValues(), stats.MaxValues(), + stats.NullCounts())); + ASSERT_EQ(contains_non_null, null_count < 3); + } + auto unknown_counts = BinaryRowGenerator::FromLongArrayWithNull({-1, -1}, pool.get()); + auto binary_bounds = + BinaryRowGenerator::GenerateRow({std::string("a"), std::string("a")}, pool.get()); + auto comparison = std::dynamic_pointer_cast(PredicateBuilder::Equal( + 0, "geo", FieldType::BINARY, Literal(FieldType::BINARY, "z", 1))); + ASSERT_TRUE(comparison); + ASSERT_NOK_WITH_MSG(comparison->Test(schema, binary_bounds), + "only supports null predicates"); + for (const auto& predicate : {is_null, is_not_null, comparison}) { + ASSERT_OK_AND_ASSIGN(bool matches, predicate->Test(schema, 3, binary_bounds, + binary_bounds, unknown_counts)); + ASSERT_TRUE(matches); + } + auto known_counts = BinaryArray::FromLongArray({0, 0}, pool.get()); + ASSERT_OK_AND_ASSIGN( + bool matches, comparison->Test(schema, 3, binary_bounds, binary_bounds, known_counts)); + ASSERT_TRUE(matches); + } +} + } // namespace paimon::test diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 5b3eeb6f4..de74bb359 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -298,6 +298,8 @@ Result FileStoreTable::ResolveOptions( if (type != bound_type) { return Status::Invalid("context type contradicts the loaded table"); } + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateGeospatialOptions( + checked_cast(*schema_).Fields(), options)); return CoreOptions::FromMap(options, file_system_, fs_scheme_map_); } diff --git a/src/paimon/core/table/system/system_table_schema.cpp b/src/paimon/core/table/system/system_table_schema.cpp index 62c823ffc..d2e4a5b46 100644 --- a/src/paimon/core/table/system/system_table_schema.cpp +++ b/src/paimon/core/table/system/system_table_schema.cpp @@ -55,7 +55,7 @@ Result SystemTableSchema::GetFieldType(const std::string& field_name) if (!field) { return Status::NotExist("field ", field_name, " not exist in system table schema"); } - return FieldTypeUtils::ConvertToFieldType(field->type()->id()); + return FieldTypeUtils::ConvertToFieldType(field); } std::optional SystemTableSchema::Comment() const { diff --git a/src/paimon/core/utils/field_mapping.cpp b/src/paimon/core/utils/field_mapping.cpp index 1b44926cc..1b48f96b9 100644 --- a/src/paimon/core/utils/field_mapping.cpp +++ b/src/paimon/core/utils/field_mapping.cpp @@ -27,6 +27,7 @@ #include "paimon/common/predicate/compound_predicate_impl.h" #include "paimon/common/predicate/full_text_match_predicate.h" #include "paimon/common/predicate/leaf_predicate_impl.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/field_type_utils.h" #include "paimon/common/utils/object_utils.h" @@ -133,6 +134,10 @@ Result FieldMappingBuilder::CreateExistFieldInfo( read_field.ArrowField(), data_field.Type())); pruned_type = std::move(map_access_data_type); } else { + // Pruning can substitute the projected LIST/MAP type, so compare the original + // fields before their child metadata is replaced. + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateEvolution(*data_field.ArrowField(), + *read_field.ArrowField())); PAIMON_ASSIGN_OR_RAISE(pruned_type, NestedProjectionUtils::PruneDataType( read_field.Type(), data_field.Type())); } @@ -194,6 +199,8 @@ Result>> FieldMappingBuilder::CreateDa std::vector> cast_executors; cast_executors.reserve(read_fields.size()); for (size_t i = 0; i < read_fields.size(); i++) { + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateEvolution(*data_fields[i].ArrowField(), + *read_fields[i].ArrowField())); if (!read_fields[i].Type()->Equals(data_fields[i].Type())) { auto read_type_id = read_fields[i].Type()->id(); if (read_type_id == arrow::Type::STRUCT || read_type_id == arrow::Type::LIST || @@ -204,9 +211,9 @@ Result>> FieldMappingBuilder::CreateDa continue; } PAIMON_ASSIGN_OR_RAISE(FieldType read_type, - FieldTypeUtils::ConvertToFieldType(read_fields[i].Type()->id())); + FieldTypeUtils::ConvertToFieldType(read_fields[i].ArrowField())); PAIMON_ASSIGN_OR_RAISE(FieldType data_type, - FieldTypeUtils::ConvertToFieldType(data_fields[i].Type()->id())); + FieldTypeUtils::ConvertToFieldType(data_fields[i].ArrowField())); auto executor_factory = CastExecutorFactory::GetCastExecutorFactory(); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr cast_executor, executor_factory->GetCastExecutor(/*src_field=*/data_fields[i], @@ -306,12 +313,15 @@ Result>> FieldMappingBuilder::Reconstru return std::optional>(); } std::vector new_literals; - PAIMON_ASSIGN_OR_RAISE(FieldType read_type, FieldTypeUtils::ConvertToFieldType( - read_field_iter->second.Type()->id())); - PAIMON_ASSIGN_OR_RAISE(FieldType data_type, FieldTypeUtils::ConvertToFieldType( - data_field_iter->second.second.Type()->id())); - if (read_type != data_type || - (!read_field_iter->second.Type()->Equals(data_field_iter->second.second.Type()))) { + const auto& read_field = read_field_iter->second; + const auto& [data_field_idx, data_field] = data_field_iter->second; + PAIMON_RETURN_NOT_OK( + GeospatialType::ValidateEvolution(*data_field.ArrowField(), *read_field.ArrowField())); + PAIMON_ASSIGN_OR_RAISE(FieldType read_type, + FieldTypeUtils::ConvertToFieldType(read_field.ArrowField())); + PAIMON_ASSIGN_OR_RAISE(FieldType data_type, + FieldTypeUtils::ConvertToFieldType(data_field.ArrowField())); + if (read_type != data_type || !read_field.Type()->Equals(data_field.Type())) { // read type and data type are different, only push down integer predicates if (FieldTypeUtils::IsIntegerNumeric(read_type) && FieldTypeUtils::IsIntegerNumeric(data_type) && @@ -320,17 +330,16 @@ Result>> FieldMappingBuilder::Reconstru auto cast_executor = executor_factory->GetCastExecutor(/*src=*/read_type, /*target=*/data_type); if (!cast_executor) { - return Status::Invalid(fmt::format( - "ReconstructLeafPredicate failed: cannot find cast " - "executor for field {} from type {} to {}", - read_field_iter->second.Name(), FieldTypeUtils::FieldTypeToString(read_type), - FieldTypeUtils::FieldTypeToString(data_type))); + return Status::Invalid( + fmt::format("ReconstructLeafPredicate failed: cannot find cast " + "executor for field {} from type {} to {}", + read_field.Name(), FieldTypeUtils::FieldTypeToString(read_type), + FieldTypeUtils::FieldTypeToString(data_type))); } new_literals.reserve(leaf_predicate->Literals().size()); for (const auto& literal : leaf_predicate->Literals()) { - PAIMON_ASSIGN_OR_RAISE( - Literal casted, - cast_executor->Cast(literal, data_field_iter->second.second.Type())); + PAIMON_ASSIGN_OR_RAISE(Literal casted, + cast_executor->Cast(literal, data_field.Type())); // ignore if any literal is overflowed. if (CastingUtils::IsIntegerLiteralCastedOverflow(literal, casted)) { return std::optional>(); @@ -344,7 +353,6 @@ Result>> FieldMappingBuilder::Reconstru new_literals = leaf_predicate->Literals(); } - const auto& [data_field_idx, data_field] = data_field_iter->second; return std::optional>(std::make_shared( leaf_predicate->GetLeafFunction(), /*data_field_idx=*/data_field_idx, /*data_field_name=*/data_field.Name(), /*data_field_type=*/data_type, new_literals)); diff --git a/src/paimon/core/utils/field_mapping_test.cpp b/src/paimon/core/utils/field_mapping_test.cpp index 21eb8113e..be57d397b 100644 --- a/src/paimon/core/utils/field_mapping_test.cpp +++ b/src/paimon/core/utils/field_mapping_test.cpp @@ -33,7 +33,9 @@ #include "paimon/common/predicate/leaf_predicate_impl.h" #include "paimon/common/predicate/predicate_filter.h" #include "paimon/common/types/data_type.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/core/casting/cast_executor.h" +#include "paimon/core/casting/cast_executor_factory.h" #include "paimon/data/decimal.h" #include "paimon/defs.h" #include "paimon/predicate/full_text_match.h" @@ -778,4 +780,44 @@ TEST_F(FieldMappingTest, TestTimeCast) { ASSERT_TRUE(to_time[0]); } +TEST_F(FieldMappingTest, GeospatialCastRestrictions) { + ASSERT_OK_AND_ASSIGN(auto geom, GeospatialType::GeometryField("geo")); + ASSERT_OK_AND_ASSIGN(auto geog, GeospatialType::GeographyField("geo")); + ASSERT_OK_AND_ASSIGN(auto crs, GeospatialType::GeometryField("geo", "EPSG:3857")); + ASSERT_OK_AND_ASSIGN(auto same, GeospatialType::GeometryField("geo", "ogc:crs84")); + ASSERT_OK( + FieldMappingBuilder::CreateDataCastExecutors({DataField(0, same)}, {DataField(0, geom)})); + for (const auto& other : + {geog, crs, arrow::field("geo", arrow::binary()), arrow::field("geo", arrow::utf8())}) { + ASSERT_NOK_WITH_MSG(FieldMappingBuilder::CreateDataCastExecutors({DataField(0, other)}, + {DataField(0, geom)}), + "Incompatible geospatial type change"); + std::vector source = {DataField(0, arrow::field("arr", arrow::list(other)))}; + std::vector target = {DataField(0, arrow::field("arr", arrow::list(geom)))}; + ASSERT_NOK(FieldMappingBuilder::CreateDataCastExecutors(source, target)); + ASSERT_OK_AND_ASSIGN(auto builder, + FieldMappingBuilder::Create( + DataField::ConvertDataFieldsToArrowSchema(target), {}, nullptr)); + ASSERT_NOK_WITH_MSG(builder->CreateFieldMapping(source), "Incompatible geospatial type"); + } + ASSERT_OK_AND_ASSIGN(auto cast, CastExecutorFactory::GetCastExecutorFactory()->GetCastExecutor( + DataField(0, geom), FieldType::STRING)); + ASSERT_FALSE(cast); +} + +TEST_F(FieldMappingTest, GeospatialNullPredicateSurvivesRename) { + ASSERT_OK_AND_ASSIGN(auto field, GeospatialType::GeographyField("location")); + auto predicate = PredicateBuilder::IsNull(0, "location", FieldType::GEOGRAPHY); + ASSERT_OK_AND_ASSIGN(auto reconstructed, + FieldMappingBuilder::ReconstructPredicateWithDataFields( + predicate, {{"location", DataField(7, field)}}, + {{7, {3, DataField(7, field->WithName("previous_location"))}}})); + ASSERT_TRUE(reconstructed); + auto leaf = std::dynamic_pointer_cast(reconstructed.value()); + ASSERT_TRUE(leaf); + ASSERT_EQ(leaf->FieldName(), "previous_location"); + ASSERT_EQ(leaf->FieldIndex(), 3); + ASSERT_EQ(leaf->GetFieldType(), FieldType::GEOGRAPHY); +} + } // namespace paimon::test diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp index ff0690292..e4d466a3c 100644 --- a/src/paimon/core/utils/nested_projection_utils.cpp +++ b/src/paimon/core/utils/nested_projection_utils.cpp @@ -36,6 +36,7 @@ #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/variant/variant_access_utils.h" #include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/vector_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/string_utils.h" @@ -179,7 +180,9 @@ Result EqualWithFieldIds(const std::shared_ptr& a, for (int32_t i = 0; i < a->num_fields(); ++i) { const auto& fa = a->field(i); const auto& fb = b->field(i); - if (fa->nullable() != fb->nullable()) { + if (fa->nullable() != fb->nullable() || + ((GeospatialType::IsGeospatial(*fa) || GeospatialType::IsGeospatial(*fb)) && + !GeospatialType::Equals(*fa, *fb))) { return false; } if (a->id() == arrow::Type::STRUCT) { @@ -485,9 +488,11 @@ Result> NestedProjectionUtils::ValidateMapSharedShreddi fmt::format("selected-key metadata size {} does not match STRUCT field count {} for {}", selected_keys.size(), struct_type->num_fields(), field->name())); } - const auto& value_type = struct_type->field(0)->type(); + const auto& value_field = struct_type->field(0); + const auto& value_type = value_field->type(); for (int32_t i = 1; i < struct_type->num_fields(); ++i) { - if (!struct_type->field(i)->type()->Equals(value_type)) { + if (!struct_type->field(i)->type()->Equals(value_type) || + !GeospatialType::Equals(*struct_type->field(i), *value_field)) { return Status::Invalid(fmt::format( "selected-key MAP fields must have the same value type, but {} and {} differ", value_type->ToString(), struct_type->field(i)->type()->ToString())); @@ -515,6 +520,8 @@ NestedProjectionUtils::BuildMapSharedShreddingAccessDataType( arrow::FieldVector data_children; data_children.reserve(read_struct->num_fields()); for (const auto& read_child : read_struct->fields()) { + PAIMON_RETURN_NOT_OK( + GeospatialType::ValidateEvolution(*data_map->item_field(), *read_child)); data_children.push_back(read_child->WithType(data_map->item_type())); } return arrow::struct_(std::move(data_children)); diff --git a/src/paimon/format/avro/avro_file_batch_reader.cpp b/src/paimon/format/avro/avro_file_batch_reader.cpp index de3c2837b..e3e39f2fe 100644 --- a/src/paimon/format/avro/avro_file_batch_reader.cpp +++ b/src/paimon/format/avro/avro_file_batch_reader.cpp @@ -26,6 +26,7 @@ #include "arrow/c/bridge.h" #include "fmt/format.h" #include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -258,6 +259,7 @@ Status AvroFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, // TODO(menglingda.mld): support predicate PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_read_schema, arrow::ImportSchema(read_schema)); + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateFileFormat(*arrow_read_schema, "avro")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, ArrowUtils::DataTypeToSchema(file_data_type_)); PAIMON_ASSIGN_OR_RAISE( diff --git a/src/paimon/format/avro/avro_file_format.cpp b/src/paimon/format/avro/avro_file_format.cpp index e69b8e032..03dc155b8 100644 --- a/src/paimon/format/avro/avro_file_format.cpp +++ b/src/paimon/format/avro/avro_file_format.cpp @@ -22,7 +22,7 @@ #include #include "arrow/c/bridge.h" -#include "arrow/c/helpers.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/format/avro/avro_reader_builder.h" #include "paimon/format/avro/avro_stats_extractor.h" @@ -51,12 +51,15 @@ Result> AvroFileFormat::CreateWriterBuilder( } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr typed_schema, arrow::ImportSchema(schema)); + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateFileFormat(*typed_schema, "avro")); return std::make_unique(typed_schema, batch_size, options_); } Result> AvroFileFormat::CreateStatsExtractor( ::ArrowSchema* schema) const { - ArrowSchemaRelease(schema); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr typed_schema, + arrow::ImportSchema(schema)); + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateFileFormat(*typed_schema, "avro")); return std::make_unique(options_); } diff --git a/src/paimon/format/blob/blob_file_batch_reader.cpp b/src/paimon/format/blob/blob_file_batch_reader.cpp index b7c1b961a..df502ace1 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader.cpp @@ -38,6 +38,7 @@ #include "paimon/common/data/blob_utils.h" #include "paimon/common/io/offset_input_stream.h" #include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" @@ -402,6 +403,7 @@ Status BlobFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema, arrow::ImportSchema(read_schema)); + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateFileFormat(*arrow_schema, "blob")); if (arrow_schema->num_fields() != 1) { return Status::Invalid( fmt::format("read schema field number {} is not 1", arrow_schema->num_fields())); diff --git a/src/paimon/format/blob/blob_file_format.h b/src/paimon/format/blob/blob_file_format.h index 486cd0400..c2a008d6e 100644 --- a/src/paimon/format/blob/blob_file_format.h +++ b/src/paimon/format/blob/blob_file_format.h @@ -23,6 +23,7 @@ #include #include "arrow/c/bridge.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/format/blob/blob_reader_builder.h" #include "paimon/format/blob/blob_stats_extractor.h" @@ -56,6 +57,7 @@ class BlobFileFormat : public FileFormat { int32_t batch_size) const override { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr typed_schema, arrow::ImportSchema(schema)); + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateFileFormat(*typed_schema, Identifier())); auto data_type = arrow::struct_(typed_schema->fields()); return std::make_unique(data_type, options_); } @@ -64,6 +66,7 @@ class BlobFileFormat : public FileFormat { ::ArrowSchema* schema) const override { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr typed_schema, arrow::ImportSchema(schema)); + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateFileFormat(*typed_schema, Identifier())); return std::make_unique(typed_schema); } diff --git a/src/paimon/format/lance/lance_file_batch_reader.cpp b/src/paimon/format/lance/lance_file_batch_reader.cpp index c2606afa8..072c4ef9a 100644 --- a/src/paimon/format/lance/lance_file_batch_reader.cpp +++ b/src/paimon/format/lance/lance_file_batch_reader.cpp @@ -26,6 +26,7 @@ #include "arrow/c/helpers.h" #include "fmt/format.h" #include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/scope_guard.h" @@ -191,7 +192,10 @@ Status LanceFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, return Status::Invalid("Lance read schema is nullptr"); } (void)predicate; - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(read_schema_, arrow::ImportSchema(read_schema)); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, + arrow::ImportSchema(read_schema)); + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateFileFormat(*schema, "lance")); + read_schema_ = std::move(schema); projection_names_.clear(); for (const std::shared_ptr& field : read_schema_->fields()) { diff --git a/src/paimon/format/lance/lance_file_format.cpp b/src/paimon/format/lance/lance_file_format.cpp index 801546398..6c9c543fe 100644 --- a/src/paimon/format/lance/lance_file_format.cpp +++ b/src/paimon/format/lance/lance_file_format.cpp @@ -20,6 +20,7 @@ #include "paimon/format/lance/lance_file_format.h" #include "arrow/c/bridge.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/format/lance/lance_reader_builder.h" #include "paimon/format/lance/lance_stats_extractor.h" @@ -40,6 +41,7 @@ Result> LanceFileFormat::CreateWriterBuilder( } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr typed_schema, arrow::ImportSchema(schema)); + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateFileFormat(*typed_schema, "lance")); return std::make_unique(typed_schema, options_); } @@ -50,6 +52,7 @@ Result> LanceFileFormat::CreateStatsExtrac } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr typed_schema, arrow::ImportSchema(schema)); + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateFileFormat(*typed_schema, "lance")); return std::make_unique(typed_schema, options_); } diff --git a/src/paimon/format/mosaic/mosaic_file_batch_reader.cpp b/src/paimon/format/mosaic/mosaic_file_batch_reader.cpp index ed508a954..61d720ab5 100644 --- a/src/paimon/format/mosaic/mosaic_file_batch_reader.cpp +++ b/src/paimon/format/mosaic/mosaic_file_batch_reader.cpp @@ -29,6 +29,7 @@ #include "fmt/format.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/predicate/predicate_filter.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -218,6 +219,7 @@ Status MosaicFileBatchReader::SetReadSchema( (void)selection_bitmap; PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, arrow::ImportSchema(read_schema)); + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateFileFormat(*schema, "mosaic")); std::vector names; std::vector name_pointers; names.reserve(schema->num_fields()); diff --git a/src/paimon/format/mosaic/mosaic_file_format.cpp b/src/paimon/format/mosaic/mosaic_file_format.cpp index 937a72624..5d840fb09 100644 --- a/src/paimon/format/mosaic/mosaic_file_format.cpp +++ b/src/paimon/format/mosaic/mosaic_file_format.cpp @@ -21,6 +21,7 @@ #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/format/mosaic/mosaic_reader_builder.h" #include "paimon/format/mosaic/mosaic_stats_extractor.h" @@ -41,6 +42,7 @@ Result> MosaicFileFormat::CreateWriterBuilder( } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr typed_schema, arrow::ImportSchema(schema)); + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateFileFormat(*typed_schema, "mosaic")); return std::make_unique(typed_schema, options_); } @@ -51,6 +53,7 @@ Result> MosaicFileFormat::CreateStatsExtra } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr typed_schema, arrow::ImportSchema(schema)); + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateFileFormat(*typed_schema, "mosaic")); return std::make_unique(typed_schema); } diff --git a/src/paimon/format/orc/orc_adapter_test.cpp b/src/paimon/format/orc/orc_adapter_test.cpp index 620707c70..f30061970 100644 --- a/src/paimon/format/orc/orc_adapter_test.cpp +++ b/src/paimon/format/orc/orc_adapter_test.cpp @@ -677,7 +677,6 @@ TEST_P(OrcAdapterTest, TestTime) { } TEST_F(OrcAdapterTest, TestBridgeForMapType) { - // map type will lose meta of key and value field after c <-> c++ bridge auto key_field = DataField::ConvertDataFieldToArrowField( DataField(100, arrow::field("key", arrow::int32()))); auto value_field = DataField::ConvertDataFieldToArrowField( @@ -692,8 +691,7 @@ TEST_F(OrcAdapterTest, TestBridgeForMapType) { // from c to c++ auto new_arrow_schema = arrow::ImportSchema(c_schema.get()).ValueOrDie(); ASSERT_TRUE(new_arrow_schema->Equals(arrow_schema, /*check_metadata=*/false)); - // check meta failed - ASSERT_FALSE(new_arrow_schema->Equals(arrow_schema, /*check_metadata=*/true)); + ASSERT_TRUE(new_arrow_schema->Equals(arrow_schema, true)); } TEST_F(OrcAdapterTest, TestDataBufferSetData) { diff --git a/src/paimon/format/orc/orc_file_batch_reader.cpp b/src/paimon/format/orc/orc_file_batch_reader.cpp index 526ff7cd2..7473caffd 100644 --- a/src/paimon/format/orc/orc_file_batch_reader.cpp +++ b/src/paimon/format/orc/orc_file_batch_reader.cpp @@ -32,6 +32,7 @@ #include "fmt/format.h" #include "orc/OrcFile.hh" #include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/options_utils.h" @@ -178,6 +179,7 @@ Status OrcFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema, arrow::ImportSchema(read_schema)); + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateFileFormat(*arrow_schema, "orc")); if (ArrowSchemaValidator::ContainTimestampWithTimezone( *arrow::struct_(arrow_schema->fields()))) { PAIMON_ASSIGN_OR_RAISE(bool ltz_legacy, OptionsUtils::GetValueFromMap( diff --git a/src/paimon/format/orc/orc_file_format.h b/src/paimon/format/orc/orc_file_format.h index d75c67bf7..1ea148ecd 100644 --- a/src/paimon/format/orc/orc_file_format.h +++ b/src/paimon/format/orc/orc_file_format.h @@ -24,6 +24,7 @@ #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/format/file_format.h" #include "paimon/format/orc/orc_reader_builder.h" @@ -49,6 +50,7 @@ class OrcFileFormat : public FileFormat { int32_t batch_size) const override { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr typed_schema, arrow::ImportSchema(schema)); + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateFileFormat(*typed_schema, "orc")); return std::make_unique(typed_schema, batch_size, options_); } @@ -56,6 +58,7 @@ class OrcFileFormat : public FileFormat { ::ArrowSchema* schema) const override { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr typed_schema, arrow::ImportSchema(schema)); + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateFileFormat(*typed_schema, "orc")); return std::make_unique(typed_schema); } diff --git a/src/paimon/format/parquet/CMakeLists.txt b/src/paimon/format/parquet/CMakeLists.txt index 937e7124a..9cded3398 100644 --- a/src/paimon/format/parquet/CMakeLists.txt +++ b/src/paimon/format/parquet/CMakeLists.txt @@ -65,6 +65,7 @@ if(PAIMON_BUILD_TESTS) predicate_pushdown_test.cpp column_index_filter_test.cpp variant_parquet_test.cpp + geospatial_parquet_test.cpp STATIC_LINK_LIBS paimon_shared test_utils_static diff --git a/src/paimon/format/parquet/column_index_filter.cpp b/src/paimon/format/parquet/column_index_filter.cpp index a0b808493..0a344f1ac 100644 --- a/src/paimon/format/parquet/column_index_filter.cpp +++ b/src/paimon/format/parquet/column_index_filter.cpp @@ -85,6 +85,17 @@ Result ColumnIndexFilter::VisitLeafPredicate( auto function_type = function.GetType(); int32_t column_index = it->second; + FieldType field_type = leaf_predicate->GetFieldType(); + const auto* column = schema && column_index >= 0 && column_index < schema->num_columns() + ? schema->Column(column_index) + : nullptr; + bool geospatial = field_type == FieldType::GEOMETRY || field_type == FieldType::GEOGRAPHY || + (column && (column->logical_type()->is_geometry() || + column->logical_type()->is_geography())); + if (geospatial && function_type != Function::Type::IS_NULL && + function_type != Function::Type::IS_NOT_NULL) { + return RowRanges::CreateSingle(row_group_row_count); + } auto column_index_ptr = rg_page_index_reader->GetColumnIndex(column_index); auto offset_index_ptr = rg_page_index_reader->GetOffsetIndex(column_index); @@ -94,14 +105,13 @@ Result ColumnIndexFilter::VisitLeafPredicate( } const auto& literals = leaf_predicate->Literals(); - FieldType field_type = leaf_predicate->GetFieldType(); // The physical type decides how the encoded page statistics are laid out. It cannot // be inferred from the length of the encoded bounds: a DECIMAL stored as a 4-byte // FIXED_LEN_BYTE_ARRAY (precision 7-9) is indistinguishable from an INT32 one by // length alone, yet is big-endian rather than little-endian. ::parquet::Type::type physical_type = ::parquet::Type::UNDEFINED; - if (schema && column_index < schema->num_columns()) { - physical_type = schema->Column(column_index)->physical_type(); + if (column) { + physical_type = column->physical_type(); } if (function_type != Function::Type::IS_NULL && function_type != Function::Type::IS_NOT_NULL && diff --git a/src/paimon/format/parquet/column_index_filter_test.cpp b/src/paimon/format/parquet/column_index_filter_test.cpp index 9b1dac212..5f761d935 100644 --- a/src/paimon/format/parquet/column_index_filter_test.cpp +++ b/src/paimon/format/parquet/column_index_filter_test.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include "arrow/api.h" @@ -378,6 +379,46 @@ class TestRowGroupPageIndexReader : public ::parquet::RowGroupPageIndexReader { std::shared_ptr<::parquet::OffsetIndex> offset_index_; }; +TEST_F(ColumnIndexFilterTest, GeospatialColumnsIgnoreBinaryPageBounds) { + using ::parquet::LogicalType; + using ::parquet::Repetition; + auto index = std::make_shared(); + index->null_pages_[0] = true; + index->null_counts_[0] = 10; + index->null_counts_[1] = 1; + index->non_null_pages_.erase(index->non_null_pages_.begin()); + auto page_reader = std::make_shared( + index, page_index_reader_->RowGroup(0)->GetOffsetIndex(0)); + auto comparison = + PredicateBuilder::Equal(0, "val", FieldType::BINARY, Literal(FieldType::BINARY, "z", 1)); + const std::pair> types[] = { + {FieldType::BINARY, LogicalType::None()}, + {FieldType::GEOMETRY, LogicalType::Geometry()}, + {FieldType::GEOGRAPHY, LogicalType::Geography()}}; + for (const auto& [type, logical] : types) { + auto leaf = ::parquet::schema::PrimitiveNode::Make("val", Repetition::OPTIONAL, logical, + ::parquet::Type::BYTE_ARRAY); + ::parquet::SchemaDescriptor schema; + schema.Init(::parquet::schema::GroupNode::Make("root", Repetition::REQUIRED, {leaf})); + ASSERT_OK_AND_ASSIGN(auto matches, ColumnIndexFilter::CalculateRowRanges( + comparison, page_reader, column_name_to_index_, + &schema, row_group_row_count_)); + ASSERT_EQ(matches.RowCount(), type == FieldType::BINARY ? 0 : row_group_row_count_); + ASSERT_OK_AND_ASSIGN(matches, ColumnIndexFilter::CalculateRowRanges( + PredicateBuilder::IsNull(0, "val", type), page_reader, + column_name_to_index_, &schema, row_group_row_count_)); + ASSERT_EQ(matches.GetRanges().size(), 1u); + ASSERT_EQ(matches.GetRanges()[0].from, 0); + ASSERT_EQ(matches.GetRanges()[0].to, 19); + ASSERT_OK_AND_ASSIGN(matches, ColumnIndexFilter::CalculateRowRanges( + PredicateBuilder::IsNotNull(0, "val", type), page_reader, + column_name_to_index_, &schema, row_group_row_count_)); + ASSERT_EQ(matches.GetRanges().size(), 1u); + ASSERT_EQ(matches.GetRanges()[0].from, 10); + ASSERT_EQ(matches.GetRanges()[0].to, row_group_row_count_ - 1); + } +} + TEST_F(ColumnIndexFilterTest, InWithNullKeepsPagesWithPossibleNulls) { auto index = std::make_shared(); index->null_pages_[0] = true; diff --git a/src/paimon/format/parquet/geospatial_parquet_test.cpp b/src/paimon/format/parquet/geospatial_parquet_test.cpp new file mode 100644 index 000000000..4882510d4 --- /dev/null +++ b/src/paimon/format/parquet/geospatial_parquet_test.cpp @@ -0,0 +1,378 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/io/memory.h" +#include "gtest/gtest.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/types/geospatial_type.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/format/column_stats.h" +#include "paimon/format/format_writer.h" +#include "paimon/format/parquet/parquet_file_format.h" +#include "paimon/format/parquet/parquet_stats_extractor.h" +#include "paimon/format/reader_builder.h" +#include "paimon/format/writer_builder.h" +#include "paimon/format/written_file_stats_provider.h" +#include "paimon/fs/file_system.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/file_batch_reader.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" +#include "parquet/arrow/schema.h" +#include "parquet/arrow/writer.h" +#include "parquet/file_reader.h" +#include "parquet/metadata.h" +#include "parquet/properties.h" +#include "parquet/statistics.h" +#include "rapidjson/document.h" + +namespace paimon::parquet::test { +namespace { + +constexpr std::string_view kEmptyCollection("\x01\x07\x00\x00\x00\x00\x00\x00\x00", 9); +constexpr std::string_view kEmptyPoint( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf8\x7f" + "\x00\x00\x00\x00\x00\x00\xf8\x7f", + 21); +constexpr std::string_view kPoint( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0\x3f" + "\x00\x00\x00\x00\x00\x00\x00\x40", + 21); + +Result> MakeData() { + arrow::BinaryBuilder builder; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(kEmptyCollection)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(kPoint)); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr values, builder.Finish()); + arrow::FieldVector fields; + arrow::ArrayVector arrays; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr geom, + GeospatialType::GeometryField("geom", "EPSG:3857")); + fields.push_back(geom); + arrays.push_back(values); + for (const char* edges : {"spherical", "vincenty", "thomas", "andoyer", "karney"}) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr geog, + GeospatialType::GeographyField(edges, "EPSG:4326", edges)); + fields.push_back(geog); + arrays.push_back(values); + } + auto child = geom->WithName("g")->WithMergedMetadata( + arrow::key_value_metadata({DataField::FIELD_ID}, {"20"})); + auto other_child = fields[5]->WithName("h")->WithMergedMetadata( + arrow::key_value_metadata({DataField::FIELD_ID}, {"21"})); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr nested, + arrow::StructArray::Make({values, values}, {child, other_child})); + fields.push_back(arrow::field("nested", nested->type())); + arrays.push_back(nested); + arrow::Int32Builder offsets_builder; + PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.AppendValues({0, 1, 2, 3})); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr offsets, + offsets_builder.Finish()); + auto list_type = arrow::list(geom->WithName("item")); + auto list = + std::make_shared(list_type, 3, offsets->data()->buffers[1], values); + fields.push_back(arrow::field("arr", list_type)); + arrays.push_back(list); + arrow::BinaryBuilder key_builder; + for (int32_t i = 0; i < 3; ++i) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(key_builder.Append(kEmptyPoint)); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr keys, key_builder.Finish()); + auto map_type = std::make_shared(geom->WithName("key")->WithNullable(false), + fields[5]->WithName("value")); + auto map = + std::make_shared(map_type, 3, offsets->data()->buffers[1], keys, values); + fields.push_back(arrow::field("mapping", map_type)); + arrays.push_back(map); + fields.push_back(arrow::field("bytes", arrow::binary())); + arrays.push_back(values); + for (int32_t i = 0; i < static_cast(fields.size()); ++i) { + fields[i] = fields[i]->WithMergedMetadata( + arrow::key_value_metadata({DataField::FIELD_ID}, {std::to_string(i)})); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr result, + arrow::StructArray::Make(arrays, fields)); + return result; +} + +} // namespace + +class GeospatialParquetTest : public ::testing::Test { + public: + void SetUp() override { + dir_ = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir_); + fs_ = dir_->GetFileSystem(); + ASSERT_OK_AND_ASSIGN(data_, MakeData()); + } + + void Write(const std::string& path) { + ParquetFileFormat format({}); + ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(data_->type()->fields()), &schema).ok()); + ASSERT_OK_AND_ASSIGN(auto builder, format.CreateWriterBuilder(&schema, 1024)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr output, fs_->Create(path, true)); + ASSERT_OK_AND_ASSIGN(auto writer, builder->Build(output, "uncompressed")); + ArrowArray array; + ASSERT_TRUE(arrow::ExportArray(*data_, &array).ok()); + ASSERT_OK(writer->AddBatch(&array)); + ASSERT_OK(writer->Finish()); + ASSERT_OK(output->Close()); + auto stats_provider = dynamic_cast(writer.get()); + ASSERT_TRUE(stats_provider); + ASSERT_OK_AND_ASSIGN(auto stats, stats_provider->ExtractWrittenFileStats(GetDefaultPool())); + ASSERT_EQ(stats.size(), data_->num_fields()); + for (int32_t i = 0; i < 6; ++i) { + ASSERT_EQ(stats[i]->NullCount(), 1); + auto string_stats = dynamic_cast(stats[i].get()); + ASSERT_TRUE(string_stats); + ASSERT_FALSE(string_stats->Min()); + ASSERT_FALSE(string_stats->Max()); + } + } + + void Check(const std::string& path) { + auto native = ::parquet::ParquetFileReader::OpenFile(path, false); + auto metadata = native->metadata(); + ASSERT_EQ(metadata->num_rows(), 3); + ASSERT_EQ(metadata->schema()->num_columns(), 12); + const std::vector field_ids = {0, + 1, + 2, + 3, + 4, + 5, + 20, + 21, + SpecialFields::GetArrayElementFieldId(7, 1), + SpecialFields::GetMapKeyFieldId(8, 1), + SpecialFields::GetArrayElementFieldId(8, 1), + 9}; + for (int32_t i = 0; i < metadata->schema()->num_columns(); ++i) { + ASSERT_EQ(metadata->schema()->Column(i)->schema_node()->field_id(), field_ids[i]); + } + for (int32_t i = 0; i < 11; ++i) { + const auto* column = metadata->schema()->Column(i); + const auto& logical = column->logical_type(); + ASSERT_EQ(column->physical_type(), ::parquet::Type::BYTE_ARRAY); + ASSERT_EQ(column->sort_order(), ::parquet::SortOrder::UNKNOWN); + if ((i >= 1 && i <= 5) || i == 7 || i == 10) { + ASSERT_TRUE(logical->is_geography()); + const auto& geography = + checked_cast(*logical); + ASSERT_EQ(geography.crs(), "EPSG:4326"); + ASSERT_EQ(static_cast(geography.algorithm()), i <= 5 ? i : 5); + } else { + ASSERT_TRUE(logical->is_geometry()); + const auto& geometry = + checked_cast(*logical); + ASSERT_EQ(geometry.crs(), "EPSG:3857"); + } + auto stats = metadata->RowGroup(0)->ColumnChunk(i)->statistics(); + ASSERT_TRUE(stats); + ASSERT_TRUE(stats->HasNullCount()); + ASSERT_EQ(stats->null_count(), i == 9 ? 0 : 1); + ASSERT_FALSE(stats->HasMinMax()); + } + ASSERT_TRUE(metadata->schema()->Column(11)->logical_type()->is_none()); + auto bytes_stats = metadata->RowGroup(0)->ColumnChunk(11)->statistics(); + ASSERT_TRUE(bytes_stats); + ASSERT_TRUE(bytes_stats->HasMinMax()); + for (bool projected : {false, true}) { + auto read_schema = arrow::schema(data_->type()->fields()); + std::shared_ptr expected = data_; + if (projected) { + auto nested = checked_pointer_cast(data_->field(6)); + auto projected_row = + arrow::StructArray::Make({nested->field(0)}, {nested->type()->field(0)}); + ASSERT_TRUE(projected_row.ok()); + read_schema = + arrow::schema({read_schema->field(6)->WithType((*projected_row)->type()), + read_schema->field(7), read_schema->field(8)}); + auto projected_data = arrow::StructArray::Make( + {*projected_row, data_->field(7), data_->field(8)}, read_schema->fields()); + ASSERT_TRUE(projected_data.ok()); + expected = *projected_data; + } + ParquetFileFormat format({}); + ASSERT_OK_AND_ASSIGN(auto builder, format.CreateReaderBuilder(2)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr input, fs_->Open(path)); + ASSERT_OK_AND_ASSIGN(auto reader, builder->Build(input)); + ASSERT_OK_AND_ASSIGN(auto c_file_schema, reader->GetFileSchema()); + auto file_schema = arrow::ImportSchema(c_file_schema.get()); + ASSERT_TRUE(file_schema.ok()); + ASSERT_TRUE( + GeospatialType::Equals(*arrow::field("", arrow::struct_((*file_schema)->fields())), + *arrow::field("", data_->type()))); + ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*read_schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto result, paimon::test::ReadResultCollector::CollectResult( + std::move(reader), true)); + ASSERT_TRUE(result->type()->Equals(*arrow::struct_(read_schema->fields()), true)) + << result->type()->ToString(true) << "\nExpected: " << read_schema->ToString(true); + ASSERT_TRUE(result->Equals(arrow::ChunkedArray(expected))) << result->ToString(); + } + } + + protected: + std::unique_ptr dir_; + std::shared_ptr fs_; + std::shared_ptr data_; +}; + +TEST_F(GeospatialParquetTest, NativeAnnotationsNestedValuesAndStatistics) { + std::string path = dir_->Str() + "/geospatial.parquet"; + ASSERT_NO_FATAL_FAILURE(Write(path)); + ASSERT_NO_FATAL_FAILURE(Check(path)); +} + +TEST_F(GeospatialParquetTest, NativeSchemaDefaultsAndEscapedCrs) { + for (const std::string crs : {"", R"(a "quoted"\CRS)"}) { + for (const auto& logical : + {::parquet::LogicalType::Geometry(crs), ::parquet::LogicalType::Geography(crs)}) { + for (bool nullable : {false, true}) { + SCOPED_TRACE(logical->ToString()); + SCOPED_TRACE(nullable); + auto node = ::parquet::schema::PrimitiveNode::Make( + "geo", + nullable ? ::parquet::Repetition::OPTIONAL : ::parquet::Repetition::REQUIRED, + logical, ::parquet::Type::BYTE_ARRAY, -1, 42); + ::parquet::SchemaDescriptor native_schema; + native_schema.Init(::parquet::schema::GroupNode::Make( + "root", ::parquet::Repetition::REQUIRED, {node})); + std::shared_ptr schema; + ASSERT_TRUE(::parquet::arrow::FromParquetSchema(&native_schema, &schema).ok()); + ASSERT_EQ(schema->num_fields(), 1); + ASSERT_EQ(schema->field(0)->nullable(), nullable); + ASSERT_OK_AND_ASSIGN(auto type, GeospatialType::FromField(*schema->field(0))); + ASSERT_EQ(type.Type(), + logical->is_geometry() ? FieldType::GEOMETRY : FieldType::GEOGRAPHY); + ASSERT_EQ(type.Crs(), crs.empty() ? "OGC:CRS84" : crs); + ASSERT_EQ(type.Edges(), logical->is_geometry() ? "" : "spherical"); + std::shared_ptr<::parquet::SchemaDescriptor> roundtrip; + ASSERT_TRUE(::parquet::arrow::ToParquetSchema( + schema.get(), *::parquet::default_writer_properties(), &roundtrip) + .ok()); + const auto* column = roundtrip->Column(0); + ASSERT_EQ(column->schema_node()->field_id(), 42); + ASSERT_EQ(column->max_definition_level(), nullable ? 1 : 0); + ASSERT_EQ(column->sort_order(), ::parquet::SortOrder::UNKNOWN); + rapidjson::Document json; + json.Parse(column->logical_type()->ToJSON().c_str()); + ASSERT_FALSE(json.HasParseError()); + ASSERT_TRUE(json.HasMember("crs")); + ASSERT_TRUE(json["crs"].IsString()); + ASSERT_EQ(json["crs"].GetString(), type.Crs()); + std::shared_ptr restored; + ASSERT_TRUE(::parquet::arrow::FromParquetSchema(roundtrip.get(), &restored).ok()); + ASSERT_TRUE(schema->Equals(*restored, true)); + } + } + } +} + +TEST_F(GeospatialParquetTest, MissingRowGroupStatisticsKeepsFileNullCountUnknown) { + auto schema = arrow::schema({data_->type()->field(0), data_->type()->field(1)}); + auto table = arrow::Table::Make(schema, arrow::ArrayVector{data_->field(0), data_->field(1)}); + ParquetStatsExtractor extractor(schema); + const std::vector>>> groups = { + {{true, 1}, {true, 2}, {false, std::nullopt}, {true, std::nullopt}}, + {{false, std::nullopt}, {true, std::nullopt}}}; + for (const auto& sequence : groups) { + std::shared_ptr<::parquet::FileMetaData> combined; + for (const auto& [write_stats, expected_file_null_count] : sequence) { + ::parquet::WriterProperties::Builder properties; + if (!write_stats) { + properties.disable_statistics(); + } + auto sink = arrow::io::BufferOutputStream::Create().ValueOrDie(); + ASSERT_TRUE(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), sink, + table->num_rows(), properties.build()) + .ok()); + auto buffer = sink->Finish().ValueOrDie(); + auto reader = ::parquet::ParquetFileReader::Open( + std::make_shared(buffer)); + if (combined) { + combined->AppendRowGroups(*reader->metadata()); + } else { + combined = reader->metadata(); + } + ASSERT_OK_AND_ASSIGN(auto stats, + extractor.ExtractFromMetadata(*combined, GetDefaultPool())); + ASSERT_EQ(stats.first.size(), 2u); + for (const auto& column : stats.first) { + ASSERT_EQ(column->NullCount(), expected_file_null_count); + auto string_stats = dynamic_cast(column.get()); + ASSERT_TRUE(string_stats); + ASSERT_FALSE(string_stats->Min()); + ASSERT_FALSE(string_stats->Max()); + } + } + } +} + +TEST_F(GeospatialParquetTest, IncompatibleReadSchema) { + std::string path = dir_->Str() + "/geospatial.parquet"; + ASSERT_NO_FATAL_FAILURE(Write(path)); + ParquetFileFormat format({}); + ASSERT_OK_AND_ASSIGN(auto builder, format.CreateReaderBuilder(2)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr input, fs_->Open(path)); + ASSERT_OK_AND_ASSIGN(auto reader, builder->Build(input)); + ASSERT_OK_AND_ASSIGN(auto different_crs, GeospatialType::GeometryField("geom", "EPSG:4326")); + const auto& map_type = checked_cast(*data_->field(8)->type()); + ASSERT_OK_AND_ASSIGN(auto different_edges, + GeospatialType::GeographyField("value", "EPSG:4326", "thomas")); + for (const auto& field : + {different_crs, data_->type()->field(0)->WithName("bytes"), + arrow::field("geom", arrow::binary()), arrow::field("arr", arrow::list(arrow::binary())), + arrow::field("mapping", arrow::map(arrow::binary(), map_type.item_field())), + arrow::field("mapping", + std::make_shared(map_type.key_field(), different_edges)), + arrow::field("nested", arrow::struct_({arrow::field("h", arrow::binary())}))}) { + ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*arrow::schema({field}), &schema).ok()); + ASSERT_NOK_WITH_MSG(reader->SetReadSchema(&schema, nullptr, std::nullopt), + "Incompatible geospatial type change"); + } +} + +TEST_F(GeospatialParquetTest, JavaInteroperability) { + const char* directory = std::getenv("PAIMON_GEOSPATIAL_INTEROP_DIR"); + if (!directory) { + GTEST_SKIP() << "Run test/inte/geospatial/run.sh to exercise Paimon Java interoperability"; + } + ASSERT_NO_FATAL_FAILURE(Check(std::string(directory) + "/java.parquet")); + ASSERT_NO_FATAL_FAILURE(Write(std::string(directory) + "/cpp.parquet")); +} + +} // namespace paimon::parquet::test diff --git a/src/paimon/format/parquet/parquet_field_id_converter.cpp b/src/paimon/format/parquet/parquet_field_id_converter.cpp index adb271734..377637d3b 100644 --- a/src/paimon/format/parquet/parquet_field_id_converter.cpp +++ b/src/paimon/format/parquet/parquet_field_id_converter.cpp @@ -18,15 +18,17 @@ #include "paimon/format/parquet/parquet_field_id_converter.h" -#include #include #include #include #include "arrow/api.h" +#include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/string_utils.h" namespace paimon::parquet { @@ -37,11 +39,14 @@ const char ParquetFieldIdConverter::PARQUET_FIELD_ID[] = "PARQUET:field_id"; Result> ParquetFieldIdConverter::AddParquetIdsFromPaimonIds( const std::shared_ptr& schema) { + // Match Java's Parquet IDs for LIST elements and MAP keys and values. + auto mode = GeospatialType::ContainsGeospatial(schema->fields()) + ? IdConvertType::PAIMON_TO_PARQUET_WITH_ELEMENT_IDS + : IdConvertType::PAIMON_TO_PARQUET_ID; std::vector> new_fields; for (const auto& field : schema->fields()) { - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - std::shared_ptr new_field, - ProcessField(field, ParquetFieldIdConverter::IdConvertType::PAIMON_TO_PARQUET_ID)); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr new_field, + ProcessField(field, mode)); new_fields.push_back(new_field); } return arrow::schema(new_fields, schema->metadata()); @@ -75,7 +80,8 @@ arrow::Result> ParquetFieldIdConverter:: return metadata->Merge(*new_meta); }; - if (convert_type == ParquetFieldIdConverter::IdConvertType::PAIMON_TO_PARQUET_ID) { + if (convert_type == IdConvertType::PAIMON_TO_PARQUET_ID || + convert_type == IdConvertType::PAIMON_TO_PARQUET_WITH_ELEMENT_IDS) { // in write process return copy(DataField::FIELD_ID, PARQUET_FIELD_ID); } else if (convert_type == ParquetFieldIdConverter::IdConvertType::PARQUET_TO_PAIMON_ID) { @@ -86,10 +92,27 @@ arrow::Result> ParquetFieldIdConverter:: } arrow::Result> ParquetFieldIdConverter::ProcessField( - const std::shared_ptr& field, ParquetFieldIdConverter::IdConvertType convert_type) { + const std::shared_ptr& field, IdConvertType convert_type, + std::optional owner_id, int32_t depth) { ARROW_ASSIGN_OR_RAISE(std::shared_ptr updated_metadata, CopyId(field->metadata(), convert_type)); auto type = field->type(); + if (convert_type == IdConvertType::PAIMON_TO_PARQUET_WITH_ELEMENT_IDS && depth == 0 && + type->num_fields() && updated_metadata && updated_metadata->Contains(PARQUET_FIELD_ID)) { + ARROW_ASSIGN_OR_RAISE(auto id, updated_metadata->Get(PARQUET_FIELD_ID)); + owner_id = StringUtils::StringToValue(id); + } + auto repeated_child = [&](const std::shared_ptr& child, + bool map_key = false) -> arrow::Result> { + ARROW_ASSIGN_OR_RAISE(auto result, ProcessField(child, convert_type, owner_id, depth + 1)); + if (!owner_id || (result->metadata() && result->metadata()->Contains(PARQUET_FIELD_ID))) { + return result; + } + int32_t id = map_key ? SpecialFields::GetMapKeyFieldId(*owner_id, depth + 1) + : SpecialFields::GetArrayElementFieldId(*owner_id, depth + 1); + return result->WithMergedMetadata( + KeyValueMetadata::Make({PARQUET_FIELD_ID}, {std::to_string(id)})); + }; if (type->id() == arrow::Type::STRUCT) { auto struct_type = checked_pointer_cast(type); std::vector> new_fields; @@ -101,23 +124,20 @@ arrow::Result> ParquetFieldIdConverter::ProcessField( return field->WithType(new_type)->WithMergedMetadata(updated_metadata); } else if (type->id() == arrow::Type::LIST) { auto list_type = checked_pointer_cast(type); - ARROW_ASSIGN_OR_RAISE(auto new_value_field, - ProcessField(list_type->value_field(), convert_type)); + ARROW_ASSIGN_OR_RAISE(auto new_value_field, repeated_child(list_type->value_field())); auto new_type = arrow::list(new_value_field); return field->WithType(new_type)->WithMergedMetadata(updated_metadata); } else if (type->id() == arrow::Type::FIXED_SIZE_LIST) { auto vector_type = checked_pointer_cast(type); - ARROW_ASSIGN_OR_RAISE(auto new_value_field, - ProcessField(vector_type->value_field(), convert_type)); + ARROW_ASSIGN_OR_RAISE(auto new_value_field, repeated_child(vector_type->value_field())); auto new_type = arrow::fixed_size_list(new_value_field, vector_type->list_size()); return field->WithType(new_type)->WithMergedMetadata(updated_metadata); } else if (type->id() == arrow::Type::MAP) { auto map_type = checked_pointer_cast(type); - ARROW_ASSIGN_OR_RAISE(auto new_key_field, - ProcessField(map_type->key_field(), convert_type)); - ARROW_ASSIGN_OR_RAISE(auto new_item_field, - ProcessField(map_type->item_field(), convert_type)); - auto new_type = arrow::map(new_key_field->type(), new_item_field); + ARROW_ASSIGN_OR_RAISE(auto new_key_field, repeated_child(map_type->key_field(), true)); + ARROW_ASSIGN_OR_RAISE(auto new_item_field, repeated_child(map_type->item_field())); + auto new_type = std::make_shared(new_key_field, new_item_field, + map_type->keys_sorted()); return field->WithType(new_type)->WithMergedMetadata(updated_metadata); } diff --git a/src/paimon/format/parquet/parquet_field_id_converter.h b/src/paimon/format/parquet/parquet_field_id_converter.h index 4e2666869..a4ea0351e 100644 --- a/src/paimon/format/parquet/parquet_field_id_converter.h +++ b/src/paimon/format/parquet/parquet_field_id_converter.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include "arrow/api.h" @@ -45,10 +46,12 @@ class ParquetFieldIdConverter { enum class IdConvertType { PARQUET_TO_PAIMON_ID = 1, PAIMON_TO_PARQUET_ID = 2, + PAIMON_TO_PARQUET_WITH_ELEMENT_IDS, }; static arrow::Result> ProcessField( - const std::shared_ptr& field, IdConvertType convert_type); + const std::shared_ptr& field, IdConvertType convert_type, + std::optional owner_id = std::nullopt, int32_t depth = 0); static arrow::Result> CopyId( const std::shared_ptr& metadata, IdConvertType convert_type); diff --git a/src/paimon/format/parquet/parquet_field_id_converter_test.cpp b/src/paimon/format/parquet/parquet_field_id_converter_test.cpp index d2053f120..6df14ef70 100644 --- a/src/paimon/format/parquet/parquet_field_id_converter_test.cpp +++ b/src/paimon/format/parquet/parquet_field_id_converter_test.cpp @@ -26,6 +26,7 @@ #include "arrow/api.h" #include "gtest/gtest.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/core/schema/table_schema.h" #include "paimon/status.h" @@ -161,6 +162,34 @@ TEST_F(ParquetFieldIdConverterTest, TestSimple) { ASSERT_EQ(expected_field_infos, old_field_infos); } +TEST_F(ParquetFieldIdConverterTest, GeospatialContainerFieldIds) { + const std::string json = R"({ + "id":10,"name":"mapping","type":{"type":"MAP","key":"GEOMETRY NOT NULL", + "value":{"type":"ARRAY","element":{"type":"ROW","fields":[ + {"id":20,"name":"locations","type":{"type":"ARRAY","element":"GEOGRAPHY"}} + ]}}} + })"; + ASSERT_OK_AND_ASSIGN(auto data_field, DataField::FromJsonString(json)); + auto schema = DataField::ConvertDataFieldsToArrowSchema({data_field}); + ASSERT_OK_AND_ASSIGN(auto converted, + ParquetFieldIdConverter::AddParquetIdsFromPaimonIds(schema)); + std::vector actual; + PrintFieldMetadata(converted, ParquetFieldIdConverter::IdConvertType::PAIMON_TO_PARQUET_ID, + &actual); + const std::vector expected = { + {"mapping", arrow::Type::MAP, "10"}, {"key", arrow::Type::BINARY, "536860670"}, + {"value", arrow::Type::LIST, "536881152"}, {"item", arrow::Type::STRUCT, "536881153"}, + {"locations", arrow::Type::LIST, "20"}, {"item", arrow::Type::BINARY, "536891392"}}; + ASSERT_EQ(actual, expected); + ASSERT_TRUE(GeospatialType::Equals(*schema->field(0), *converted->field(0))); + ASSERT_OK_AND_ASSIGN(auto repeated, + ParquetFieldIdConverter::AddParquetIdsFromPaimonIds(converted)); + ASSERT_TRUE(converted->Equals(*repeated, true)); + ASSERT_OK_AND_ASSIGN(auto restored, + ParquetFieldIdConverter::GetPaimonIdsFromParquetIds(converted)); + ASSERT_TRUE(GeospatialType::Equals(*schema->field(0), *restored->field(0))); +} + TEST_F(ParquetFieldIdConverterTest, TestNestedType) { arrow::FieldVector fields = { arrow::field("f0", diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index 50f3b9735..71131b349 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -43,6 +43,7 @@ #include "arrow/util/thread_pool.h" #include "fmt/format.h" #include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -874,10 +875,16 @@ Result<::parquet::ArrowReaderProperties> ParquetFileBatchReader::CreateArrowRead // Nested column index computation -Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptr& read_type, - const std::shared_ptr& file_type, +Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptr& read_field, + const std::shared_ptr& file_field, int32_t* leaf_index, std::vector* indices) { + const auto& read_type = read_field->type(); + const auto& file_type = file_field->type(); + if (file_type->id() != read_type->id() || GeospatialType::IsGeospatial(*file_field) || + GeospatialType::IsGeospatial(*read_field)) { + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateEvolution(*file_field, *read_field)); + } if (file_type->id() == arrow::Type::STRUCT) { for (const auto& file_child : file_type->fields()) { std::shared_ptr read_child = nullptr; @@ -888,8 +895,8 @@ Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptrtype(), file_child->type(), - leaf_index, indices)); + PAIMON_RETURN_NOT_OK( + CollectLeafIndices(read_child, file_child, leaf_index, indices)); } else { SkipLeafIndices(file_child->type(), leaf_index); } @@ -904,7 +911,7 @@ Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptr(*read_type); const auto& file_list = static_cast(*file_type); - PAIMON_RETURN_NOT_OK(CollectLeafIndices(read_list.value_type(), file_list.value_type(), + PAIMON_RETURN_NOT_OK(CollectLeafIndices(read_list.value_field(), file_list.value_field(), leaf_index, indices)); } else if (file_type->id() == arrow::Type::FIXED_SIZE_LIST) { if (!HasSameNestedProjectionShape(read_type, file_type)) { @@ -914,8 +921,8 @@ Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptr(*read_type); const auto& file_vector = checked_cast(*file_type); - PAIMON_RETURN_NOT_OK(CollectLeafIndices(read_vector.value_type(), file_vector.value_type(), - leaf_index, indices)); + PAIMON_RETURN_NOT_OK(CollectLeafIndices(read_vector.value_field(), + file_vector.value_field(), leaf_index, indices)); } else if (file_type->id() == arrow::Type::MAP) { if (!HasSameNestedProjectionShape(read_type, file_type)) { return Status::Invalid(fmt::format( @@ -925,9 +932,9 @@ Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptr(*read_type); const auto& file_map = static_cast(*file_type); PAIMON_RETURN_NOT_OK( - CollectLeafIndices(read_map.key_type(), file_map.key_type(), leaf_index, indices)); + CollectLeafIndices(read_map.key_field(), file_map.key_field(), leaf_index, indices)); PAIMON_RETURN_NOT_OK( - CollectLeafIndices(read_map.item_type(), file_map.item_type(), leaf_index, indices)); + CollectLeafIndices(read_map.item_field(), file_map.item_field(), leaf_index, indices)); } else { // Leaf column — collect its index. indices->push_back((*leaf_index)++); @@ -974,8 +981,8 @@ Result> ParquetFileBatchReader::ComputeNestedColumnIndices( read_field->name())); } int32_t leaf_index = file_field_leaf_starts[file_field_idx]; - PAIMON_RETURN_NOT_OK(CollectLeafIndices( - read_field->type(), file_fields[file_field_idx]->type(), &leaf_index, &indices)); + PAIMON_RETURN_NOT_OK( + CollectLeafIndices(read_field, file_fields[file_field_idx], &leaf_index, &indices)); } return indices; } diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.h b/src/paimon/format/parquet/parquet_file_batch_reader.h index 565dec9b2..f9774b0f6 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.h +++ b/src/paimon/format/parquet/parquet_file_batch_reader.h @@ -199,12 +199,9 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { } } - /// Recursively collect leaf column indices for the sub-fields in read_type - /// that match file_type by paimon field ID. Unmatched sub-fields in file_type - /// have their leaf indices skipped. Partial projection inside LIST, MAP, or - /// FIXED_SIZE_LIST is not supported and will return Invalid. - static Status CollectLeafIndices(const std::shared_ptr& read_type, - const std::shared_ptr& file_type, + /// Advance the file's leaf index past this field and collect the projected leaves. + static Status CollectLeafIndices(const std::shared_ptr& read_field, + const std::shared_ptr& file_field, int32_t* leaf_index, std::vector* indices); /// Skip over all leaf column indices of the given file_type without collecting. @@ -212,7 +209,7 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { int32_t* leaf_index); /// Compute leaf column indices by recursively matching read_schema against - /// file_schema using paimon field IDs. STRUCT supports sub-field projection + /// file_schema using field names. STRUCT supports sub-field projection /// (unmatched sub-fields are skipped). LIST, MAP, and FIXED_SIZE_LIST require /// matching nested shapes. static Result> ComputeNestedColumnIndices( diff --git a/src/paimon/format/parquet/parquet_read_type_adapter.cpp b/src/paimon/format/parquet/parquet_read_type_adapter.cpp index 436b6c6bc..6b6007b4e 100644 --- a/src/paimon/format/parquet/parquet_read_type_adapter.cpp +++ b/src/paimon/format/parquet/parquet_read_type_adapter.cpp @@ -25,6 +25,7 @@ #include "arrow/type.h" #include "fmt/format.h" #include "paimon/common/data/blob_utils.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" @@ -63,9 +64,9 @@ Result> ParquetReadTypeAdapter::NormalizeFileTy NormalizeFileType(src_map_type->key_type())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr item_type, NormalizeFileType(src_map_type->item_type())); - return std::make_shared( - src_map_type->key_field()->WithType(key_type), - src_map_type->item_field()->WithType(item_type)); + return std::make_shared(src_map_type->key_field()->WithType(key_type), + src_map_type->item_field()->WithType(item_type), + src_map_type->keys_sorted()); } case arrow::Type::type::LIST: { auto* src_list_type = checked_cast(src_data_type.get()); @@ -94,6 +95,9 @@ Result ParquetReadTypeAdapter::NeedsArrayConversion( Result ParquetReadTypeAdapter::NeedsArrayConversionImpl( const std::shared_ptr& src_field, const std::shared_ptr& target_field) { + if (GeospatialType::IsGeospatial(*src_field) || GeospatialType::IsGeospatial(*target_field)) { + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateEvolution(*src_field, *target_field)); + } if (IsCompatibleBlob(src_field, target_field)) { return false; } @@ -126,29 +130,25 @@ Result ParquetReadTypeAdapter::NeedsArrayConversionImpl( fmt::format("source type {} and target type {} number of fields mismatch", src_data_type->ToString(), target_data_type->ToString())); } + bool needs_conversion = false; for (int32_t i = 0; i < src_struct_type->num_fields(); ++i) { - PAIMON_ASSIGN_OR_RAISE(bool need_cast, + PAIMON_ASSIGN_OR_RAISE(bool child_needs_conversion, NeedsArrayConversionImpl(src_struct_type->field(i), target_struct_type->field(i))); - if (need_cast) { - return true; - } + needs_conversion |= child_needs_conversion; } - return false; + return needs_conversion; } case arrow::Type::type::MAP: { auto* src_map_type = checked_cast(src_data_type.get()); auto* target_map_type = checked_cast(target_data_type.get()); PAIMON_ASSIGN_OR_RAISE( - bool need_cast, + bool key_needs_conversion, NeedsArrayConversionImpl(src_map_type->key_field(), target_map_type->key_field())); - if (need_cast) { - return true; - } - PAIMON_ASSIGN_OR_RAISE(need_cast, + PAIMON_ASSIGN_OR_RAISE(bool value_needs_conversion, NeedsArrayConversionImpl(src_map_type->item_field(), target_map_type->item_field())); - return need_cast; + return key_needs_conversion || value_needs_conversion; } case arrow::Type::type::LIST: { auto* src_list_type = checked_cast(src_data_type.get()); @@ -172,7 +172,8 @@ Result> ParquetReadTypeAdapter::AdaptArray( const std::shared_ptr& target_data_type, const std::shared_ptr& arrow_pool) { PAIMON_ASSIGN_OR_RAISE(bool need_cast, NeedsArrayConversion(array->type(), target_data_type)); - if (!need_cast) { + if (!need_cast && (!GeospatialType::ContainsGeospatial(target_data_type->fields()) || + array->type()->Equals(*target_data_type, /*check_metadata=*/true))) { return array; } @@ -195,25 +196,21 @@ Result> ParquetReadTypeAdapter::AdaptArrayImpl( arrow::Type::type type = array->type()->id(); switch (type) { case arrow::Type::type::STRUCT: { - auto* struct_array = checked_cast(array.get()); - arrow::ArrayVector target_sub_arrays; - std::vector target_names; - target_sub_arrays.reserve(struct_array->num_fields()); - target_names.reserve(struct_array->num_fields()); - for (int32_t i = 0; i < struct_array->num_fields(); i++) { - const auto& field = struct_array->field(i); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr sub_array, - AdaptArrayImpl(field, target_data_type->field(i)->type(), arrow_pool)); - target_sub_arrays.push_back(sub_array); - target_names.push_back(target_data_type->field(i)->name()); + // The parent's offset already applies to its children. Adapt unsliced child data + // so a sliced batch keeps its original offsets and validity bitmap. + auto data = array->data()->Copy(); + arrow::FieldVector target_fields; + target_fields.reserve(target_data_type->num_fields()); + for (int32_t i = 0; i < target_data_type->num_fields(); ++i) { + const auto& field = target_data_type->field(i); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr sub_array, + AdaptArrayImpl(arrow::MakeArray(data->child_data[i]), + field->type(), arrow_pool)); + data->child_data[i] = sub_array->data(); + target_fields.push_back(field->WithType(sub_array->type())); } - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - std::shared_ptr new_array, - arrow::StructArray::Make(target_sub_arrays, target_names, - struct_array->null_bitmap(), struct_array->null_count(), - struct_array->offset())); - return new_array; + data->type = arrow::struct_(std::move(target_fields)); + return arrow::MakeArray(data); } case arrow::Type::type::MAP: { auto* map_array = checked_cast(array.get()); @@ -225,9 +222,11 @@ Result> ParquetReadTypeAdapter::AdaptArrayImpl( std::shared_ptr item_array, AdaptArrayImpl(map_array->items(), map_type->item_type(), arrow_pool)); return std::make_shared( - arrow::map(key_array->type(), item_array->type()), map_array->length(), - map_array->value_offsets(), key_array, item_array, map_array->null_bitmap(), - map_array->null_count(), map_array->offset()); + std::make_shared( + map_type->key_field()->WithType(key_array->type()), + map_type->item_field()->WithType(item_array->type()), map_type->keys_sorted()), + map_array->length(), map_array->value_offsets(), key_array, item_array, + map_array->null_bitmap(), map_array->null_count(), map_array->offset()); } case arrow::Type::type::LIST: { auto* list_array = checked_cast(array.get()); @@ -236,9 +235,9 @@ Result> ParquetReadTypeAdapter::AdaptArrayImpl( std::shared_ptr value_array, AdaptArrayImpl(list_array->values(), list_type->value_type(), arrow_pool)); return std::make_shared( - arrow::list(value_array->type()), list_array->length(), list_array->value_offsets(), - value_array, list_array->null_bitmap(), list_array->null_count(), - list_array->offset()); + arrow::list(list_type->value_field()->WithType(value_array->type())), + list_array->length(), list_array->value_offsets(), value_array, + list_array->null_bitmap(), list_array->null_count(), list_array->offset()); } case arrow::Type::type::TIMESTAMP: { auto* ts_array = checked_cast(array.get()); diff --git a/src/paimon/format/parquet/parquet_read_type_adapter_test.cpp b/src/paimon/format/parquet/parquet_read_type_adapter_test.cpp index 403e7ea90..4e80c6f2d 100644 --- a/src/paimon/format/parquet/parquet_read_type_adapter_test.cpp +++ b/src/paimon/format/parquet/parquet_read_type_adapter_test.cpp @@ -24,7 +24,9 @@ #include "arrow/ipc/api.h" #include "gtest/gtest.h" #include "paimon/common/data/blob_utils.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/memory/memory_pool.h" #include "paimon/testing/utils/testharness.h" @@ -171,6 +173,9 @@ TEST(ParquetReadTypeAdapterTest, TestAdaptArray) { ])") .ValueOrDie()); ASSERT_TRUE(result_array->Equals(expected_array)) << result_array->ToString(); + ASSERT_OK_AND_ASSIGN(auto slice, ParquetReadTypeAdapter::AdaptArray( + array->Slice(1), arrow::struct_(target_fields), pool)); + ASSERT_TRUE(slice->Equals(expected_array->Slice(1))); } TEST(ParquetReadTypeAdapterTest, TestAdaptArrayValidatesResultType) { @@ -186,6 +191,70 @@ TEST(ParquetReadTypeAdapterTest, TestAdaptArrayValidatesResultType) { "timestamp[ms]"); } +TEST(ParquetReadTypeAdapterTest, TimestampConversionCannotRelabelGeospatialSibling) { + ASSERT_OK_AND_ASSIGN(auto geometry, GeospatialType::GeometryField("g")); + ASSERT_OK_AND_ASSIGN(auto geography, GeospatialType::GeographyField("g")); + auto millis = arrow::timestamp(arrow::TimeUnit::MILLI); + auto seconds = arrow::timestamp(arrow::TimeUnit::SECOND); + auto reject = [](const std::shared_ptr& source, + const std::shared_ptr& target) { + auto array = arrow::MakeArrayOfNull(source, 1); + ASSERT_TRUE(array.ok()) << array.status().ToString(); + ASSERT_NOK_WITH_MSG( + ParquetReadTypeAdapter::AdaptArray(*array, target, GetArrowPool(GetDefaultPool())), + "Incompatible geospatial type change"); + }; + reject(arrow::struct_({arrow::field("ts", millis), geometry}), + arrow::struct_({arrow::field("ts", seconds), geography})); + reject(arrow::map(millis, geometry), arrow::map(seconds, geography)); +} + +TEST(ParquetReadTypeAdapterTest, GeospatialMetadataSurvivesSlicedNestedArrays) { + ASSERT_OK_AND_ASSIGN(auto source_field, GeospatialType::GeometryField("g", "epsg:3857")); + ASSERT_OK_AND_ASSIGN(auto target_field, GeospatialType::GeometryField("g", "EPSG:3857")); + auto nested_type = [](const std::shared_ptr& field) { + return arrow::struct_({field, arrow::field("row", arrow::struct_({field})), + arrow::field("array", arrow::list(field)), + arrow::field("map", arrow::map(arrow::int32(), field))}); + }; + auto source_type = nested_type(source_field); + auto target_type = nested_type(target_field); + const char* json = R"([ + ["a", ["b"], ["c"], [[1, "d"]]], + null, + [null, null, null, null], + ["e", ["f"], ["g", null], [[2, "h"]]] + ])"; + auto make_array = + [&](const std::shared_ptr& type) -> Result> { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, + arrow::ipc::internal::json::ArrayFromJSON(type, json)); + auto row = checked_pointer_cast(array); + auto map = checked_pointer_cast(row->field(3)); + auto children = row->fields(); + children[3] = std::make_shared( + type->field(3)->type(), map->length(), map->value_offsets(), map->keys(), map->items(), + map->null_bitmap(), map->null_count()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr result, + arrow::StructArray::Make(children, type->fields(), row->null_bitmap(), + row->null_count())); + return result; + }; + ASSERT_OK_AND_ASSIGN(auto source, make_array(source_type)); + ASSERT_OK_AND_ASSIGN(auto expected, make_array(target_type)); + ASSERT_TRUE(source->type()->Equals(*source_type, true)); + ASSERT_TRUE(expected->type()->Equals(*target_type, true)); + for (int64_t offset : {0, 1}) { + ASSERT_OK_AND_ASSIGN(auto result, + ParquetReadTypeAdapter::AdaptArray(source->Slice(offset), target_type, + GetArrowPool(GetDefaultPool()))); + ASSERT_TRUE(result->ValidateFull().ok()); + ASSERT_TRUE(result->type()->Equals(*target_type, true)); + ASSERT_TRUE(result->Equals(expected->Slice(offset))) << result->ToString(); + } +} + TEST(ParquetReadTypeAdapterTest, TestNormalizeFileType) { auto timezone = DateTimeUtils::GetLocalTimezoneName(); arrow::FieldVector fields = { diff --git a/src/paimon/format/parquet/parquet_schema_util.cpp b/src/paimon/format/parquet/parquet_schema_util.cpp index 472b81fec..713d6b6d8 100644 --- a/src/paimon/format/parquet/parquet_schema_util.cpp +++ b/src/paimon/format/parquet/parquet_schema_util.cpp @@ -127,6 +127,8 @@ Result> FromByteArray(const ::parquet::LogicalType& l case ::parquet::LogicalType::Type::ENUM: case ::parquet::LogicalType::Type::JSON: case ::parquet::LogicalType::Type::BSON: + case ::parquet::LogicalType::Type::GEOMETRY: + case ::parquet::LogicalType::Type::GEOGRAPHY: return ::arrow::binary(); default: return Status::NotImplemented("Unhandled logical logical_type ", diff --git a/src/paimon/format/parquet/parquet_stats_extractor.cpp b/src/paimon/format/parquet/parquet_stats_extractor.cpp index dc9b92b57..a6c100d83 100644 --- a/src/paimon/format/parquet/parquet_stats_extractor.cpp +++ b/src/paimon/format/parquet/parquet_stats_extractor.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include "arrow/memory_pool.h" #include "arrow/type.h" @@ -288,6 +289,8 @@ ParquetStatsExtractor::ExtractFromMetadata(const ::parquet::FileMetaData& file_m result_stats.reserve(field_count); std::unordered_map> merged_stats; + std::vector> geospatial_null_counts( + file_metadata.schema()->num_columns(), 0); for (int32_t row_group_idx = 0; row_group_idx < file_metadata.num_row_groups(); ++row_group_idx) { @@ -295,6 +298,20 @@ ParquetStatsExtractor::ExtractFromMetadata(const ::parquet::FileMetaData& file_m file_metadata.RowGroup(row_group_idx); for (int32_t col_idx = 0; col_idx < row_group->num_columns(); ++col_idx) { auto column_chunk = row_group->ColumnChunk(col_idx); + const auto& logical_type = file_metadata.schema()->Column(col_idx)->logical_type(); + if (logical_type->is_geometry() || logical_type->is_geography()) { + auto& null_count = geospatial_null_counts[col_idx]; + if (null_count) { + auto stats = column_chunk->statistics(); + if (stats && stats->HasNullCount()) { + *null_count += stats->null_count(); + } else { + // A partial null count could incorrectly prune the entire file. + null_count.reset(); + } + } + continue; + } if (!column_chunk->is_stats_set()) { continue; } @@ -322,6 +339,13 @@ ParquetStatsExtractor::ExtractFromMetadata(const ::parquet::FileMetaData& file_m result_stats.push_back(ColumnStats::CreateNestedColumnStats(nested_type, std::nullopt)); } else { auto primitive_node = checked_pointer_cast<::parquet::schema::PrimitiveNode>(node); + const auto& logical_type = primitive_node->logical_type(); + if (logical_type->is_geometry() || logical_type->is_geography()) { + int32_t column_idx = file_metadata.schema()->GetColumnIndex(*primitive_node); + result_stats.push_back(ColumnStats::CreateStringColumnStats( + std::nullopt, std::nullopt, geospatial_null_counts[column_idx])); + continue; + } auto iter = merged_stats.find(node->name()); const std::shared_ptr<::parquet::Statistics>& parquet_stats = iter == merged_stats.end() ? nullptr : iter->second; diff --git a/src/paimon/format/parquet/predicate_converter.cpp b/src/paimon/format/parquet/predicate_converter.cpp index be933cfae..d9fe07161 100644 --- a/src/paimon/format/parquet/predicate_converter.cpp +++ b/src/paimon/format/parquet/predicate_converter.cpp @@ -187,6 +187,11 @@ Result PredicateConverter::ConvertLeaf( const auto& literals = leaf_predicate->Literals(); const auto& function = leaf_predicate->GetFunction(); auto function_type = function.GetType(); + if ((leaf_predicate->GetFieldType() == FieldType::GEOMETRY || + leaf_predicate->GetFieldType() == FieldType::GEOGRAPHY) && + function_type != Function::Type::IS_NULL && function_type != Function::Type::IS_NOT_NULL) { + return AlwaysTrue(); + } if (FloatingPointPredicateUtils::IsType(leaf_predicate->GetFieldType()) && FloatingPointPredicateUtils::IsComparison(function_type)) { return ConvertFloatingPointLeaf(leaf_predicate); diff --git a/src/paimon/format/parquet/predicate_converter_test.cpp b/src/paimon/format/parquet/predicate_converter_test.cpp index a2bfa7098..93f8c5cd9 100644 --- a/src/paimon/format/parquet/predicate_converter_test.cpp +++ b/src/paimon/format/parquet/predicate_converter_test.cpp @@ -45,18 +45,13 @@ Literal BigIntLiteral(int64_t value) { TEST(PredicateConverterTest, TestSimple) { // "struct"; - { - auto predicate = - PredicateBuilder::IsNull(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT); + for (FieldType type : {FieldType::BIGINT, FieldType::GEOMETRY, FieldType::GEOGRAPHY}) { + auto predicate = PredicateBuilder::IsNull(0, "f0", type); ASSERT_OK_AND_ASSIGN(auto expression, PredicateConverter::Convert( predicate, /*predicate_node_count_limit=*/100)); ASSERT_EQ("is_null(f0, {nan_is_null=false})", expression.ToString()); - } - { - auto predicate = - PredicateBuilder::IsNotNull(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT); - ASSERT_OK_AND_ASSIGN(auto expression, PredicateConverter::Convert( - predicate, /*predicate_node_count_limit=*/100)); + predicate = PredicateBuilder::IsNotNull(0, "f0", type); + ASSERT_OK_AND_ASSIGN(expression, PredicateConverter::Convert(predicate, 100)); ASSERT_EQ("invert(is_null(f0, {nan_is_null=false}))", expression.ToString()); } { @@ -201,6 +196,15 @@ TEST(PredicateConverterTest, TestSimple) { predicate, /*predicate_node_count_limit=*/100)); ASSERT_EQ("true", expression.ToString()); } + for (FieldType type : {FieldType::GEOMETRY, FieldType::GEOGRAPHY}) { + auto wkb = Literal(FieldType::BINARY, "wkb", 3); + for (const auto& predicate : {PredicateBuilder::Equal(0, "f0", type, wkb), + PredicateBuilder::LessThan(0, "f0", type, wkb), + PredicateBuilder::In(0, "f0", type, {wkb})}) { + ASSERT_OK_AND_ASSIGN(auto expression, PredicateConverter::Convert(predicate, 100)); + ASSERT_EQ("true", expression.ToString()); + } + } { auto predicate = PredicateBuilder::LessOrEqual( /*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, Literal(FieldType::BIGINT)); diff --git a/src/paimon/format/vortex/vortex_file_batch_reader.cpp b/src/paimon/format/vortex/vortex_file_batch_reader.cpp index f930a6cde..f70866d87 100644 --- a/src/paimon/format/vortex/vortex_file_batch_reader.cpp +++ b/src/paimon/format/vortex/vortex_file_batch_reader.cpp @@ -31,6 +31,7 @@ #include "fmt/format.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/predicate/predicate_filter.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -508,6 +509,7 @@ Status VortexFileBatchReader::SetReadSchema( // Leave the active scan and its row mapping unchanged if validation or import fails. PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr imported_read_schema, arrow::ImportSchema(read_schema)); + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateFileFormat(*imported_read_schema, "vortex")); PAIMON_RETURN_NOT_OK( ArrowSchemaValidator::ValidateNoRedundantFields(imported_read_schema->fields())); PAIMON_RETURN_NOT_OK(ValidateReadFieldReferences( diff --git a/src/paimon/format/vortex/vortex_file_format.cpp b/src/paimon/format/vortex/vortex_file_format.cpp index 39b670db6..1e6ff488c 100644 --- a/src/paimon/format/vortex/vortex_file_format.cpp +++ b/src/paimon/format/vortex/vortex_file_format.cpp @@ -20,6 +20,7 @@ #include "paimon/format/vortex/vortex_file_format.h" #include "arrow/c/bridge.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/options_utils.h" #include "paimon/format/vortex/vortex_defs.h" @@ -55,6 +56,7 @@ Result> VortexFileFormat::CreateWriterBuilder( } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr typed_schema, arrow::ImportSchema(schema)); + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateFileFormat(*typed_schema, "vortex")); return std::make_unique(typed_schema, options_); } @@ -65,6 +67,7 @@ Result> VortexFileFormat::CreateStatsExtra } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr typed_schema, arrow::ImportSchema(schema)); + PAIMON_RETURN_NOT_OK(GeospatialType::ValidateFileFormat(*typed_schema, "vortex")); return std::make_unique(typed_schema); } diff --git a/src/paimon/testing/utils/read_result_collector.h b/src/paimon/testing/utils/read_result_collector.h index 89f1c04a7..fa2d66a20 100644 --- a/src/paimon/testing/utils/read_result_collector.h +++ b/src/paimon/testing/utils/read_result_collector.h @@ -99,7 +99,7 @@ class ReadResultCollector { // them. Tests use this overload to verify that returned batches own every // allocator resource needed by their release callbacks. static Result> CollectResult( - std::unique_ptr batch_reader) { + std::unique_ptr batch_reader, bool preserve_schema = false) { std::vector batches; // ReadBatch owns the C structs, but deleting those structs does not invoke their release // callbacks. Explicitly release any buffered batches when a later read or import fails. @@ -127,7 +127,7 @@ class ReadResultCollector { ImportReadBatch(std::move(batch))); arrays.push_back(std::move(array)); } - return BuildChunkedArray(arrays); + return BuildChunkedArray(arrays, preserve_schema); } static Result> CollectResultOneBatch( @@ -245,10 +245,15 @@ class ReadResultCollector { } static Result> BuildChunkedArray( - const arrow::ArrayVector& arrays) { + const arrow::ArrayVector& arrays, bool preserve_schema = false) { if (arrays.empty()) { return std::shared_ptr(); } + if (preserve_schema) { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr chunked_array, + arrow::ChunkedArray::Make(arrays)); + return chunked_array; + } // Accumulate all batches and convert dictionary arrays together to expose overlapping // dictionaries from multiple stripes. arrow::ArrayVector converted_arrays; diff --git a/test/inte/geospatial/GeospatialInterop.java b/test/inte/geospatial/GeospatialInterop.java new file mode 100644 index 000000000..569fc80cb --- /dev/null +++ b/test/inte/geospatial/GeospatialInterop.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.paimon.data.DataGetters; +import org.apache.paimon.data.GenericArray; +import org.apache.paimon.data.GenericMap; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalMap; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.format.FileFormat; +import org.apache.paimon.format.FormatReaderContext; +import org.apache.paimon.format.FormatWriter; +import org.apache.paimon.format.parquet.ParquetSchemaConverter; +import org.apache.paimon.format.parquet.ParquetUtil; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.PositionOutputStream; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.options.Options; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.EdgeAlgorithm; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.CloseableIterator; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class GeospatialInterop { + private static final byte[] COLLECTION = {1, 7, 0, 0, 0, 0, 0, 0, 0}; + private static final byte[] POINT = point(1, 2); + private static final byte[] EMPTY_POINT = point(Double.NaN, Double.NaN); + private static final String[] EDGES = {"spherical", "vincenty", "thomas", "andoyer", "karney"}; + + private static byte[] point(double x, double y) { + return ByteBuffer.allocate(21).order(ByteOrder.LITTLE_ENDIAN) + .put((byte) 1).putInt(1).putDouble(x).putDouble(y).array(); + } + + private static RowType schema() { + DataType geometry = DataTypes.GEOMETRY("EPSG:3857"); + List fields = new ArrayList<>(); + fields.add(DataTypes.FIELD(0, "geom", geometry)); + for (String edge : EDGES) { + fields.add(DataTypes.FIELD(fields.size(), edge, + DataTypes.GEOGRAPHY("EPSG:4326", EdgeAlgorithm.fromName(edge)))); + } + fields.add(DataTypes.FIELD(6, "nested", DataTypes.ROW( + DataTypes.FIELD(20, "g", geometry), DataTypes.FIELD(21, "h", fields.get(5).type())))); + fields.add(DataTypes.FIELD(7, "arr", DataTypes.ARRAY(geometry))); + fields.add(DataTypes.FIELD(8, "mapping", DataTypes.MAP(geometry, fields.get(5).type()))); + fields.add(DataTypes.FIELD(9, "bytes", DataTypes.BYTES())); + return new RowType(fields); + } + + private static byte[] value(int row) { + return row == 0 ? COLLECTION : row == 1 ? null : POINT; + } + + private static void write(FileFormat format, LocalFileIO io, Path path, RowType schema) + throws Exception { + try (PositionOutputStream out = io.newOutputStream(path, false); + FormatWriter writer = format.createWriterFactory(schema).create(out, "uncompressed")) { + for (int i = 0; i < 3; i++) { + byte[] wkb = value(i); + GenericRow row = new GenericRow(schema.getFieldCount()); + for (int col = 0; col < 6; col++) { + row.setField(col, wkb); + } + row.setField(6, GenericRow.of(wkb, wkb)); + row.setField(7, new GenericArray(new Object[] {wkb})); + row.setField(8, GenericMap.fromBinaryKeyMap(Collections.singletonMap(EMPTY_POINT, wkb))); + row.setField(9, wkb); + writer.addElement(row); + } + } + } + + private static void require(boolean condition, String message) { + if (!condition) { + throw new AssertionError(message); + } + } + + private static void checkWkb(DataGetters data, int column, byte[] expected) { + require(data.isNullAt(column) == (expected == null), "Unexpected null at column " + column); + if (expected != null) { + require(Arrays.equals(expected, data.getBinary(column)), "WKB changed at column " + column); + } + } + + private static void check(FileFormat format, LocalFileIO io, Path path, RowType schema) + throws Exception { + long size = io.getFileSize(path); + try (var parquet = ParquetUtil.getParquetReader(io, path, size, new Options())) { + var footer = parquet.getFooter(); + RowType inferred = ParquetSchemaConverter.convertToPaimonRowType( + footer.getFileMetaData().getSchema()); + require(schema.equals(inferred), "Unexpected native Parquet schema: " + inferred); + var columns = footer.getBlocks().get(0).getColumns(); + for (int i = 0; i < columns.size(); i++) { + var stats = columns.get(i).getStatistics(); + require(stats.isNumNullsSet() && stats.getNumNulls() == (i == 9 ? 0 : 1), + "Null count for " + columns.get(i).getPath()); + require(stats.hasNonNullValue() == (i == 11), "Min/max for " + columns.get(i).getPath()); + } + } + FormatReaderContext context = new FormatReaderContext(io, path, size, null, null); + try (CloseableIterator rows = format + .createReaderFactory(schema, schema, Collections.emptyList()) + .createReader(context).toCloseableIterator()) { + for (int i = 0; i < 3; i++) { + require(rows.hasNext(), "Missing row " + i); + InternalRow row = rows.next(); + byte[] expected = value(i); + for (int col = 0; col < 6; col++) { + checkWkb(row, col, expected); + } + InternalRow nested = row.getRow(6, 2); + checkWkb(nested, 0, expected); + checkWkb(nested, 1, expected); + InternalArray array = row.getArray(7); + require(array.size() == 1, "Array size"); + checkWkb(array, 0, expected); + InternalMap map = row.getMap(8); + require(map.size() == 1, "Map size"); + checkWkb(map.keyArray(), 0, EMPTY_POINT); + checkWkb(map.valueArray(), 0, expected); + checkWkb(row, 9, expected); + } + require(!rows.hasNext(), "Unexpected extra row"); + } + } + + public static void main(String[] args) throws Exception { + FileFormat format = FileFormat.fromIdentifier("parquet", new Options()); + LocalFileIO io = LocalFileIO.create(); + Path path = new Path(Paths.get(args[1]).toUri()); + RowType schema = schema(); + if (args[0].equals("write")) { + write(format, io, path, schema); + } else { + check(format, io, path, schema); + } + } +} diff --git a/test/inte/geospatial/pom.xml b/test/inte/geospatial/pom.xml new file mode 100644 index 000000000..2941c4196 --- /dev/null +++ b/test/inte/geospatial/pom.xml @@ -0,0 +1,42 @@ + + + 4.0.0 + org.apache.paimon + paimon-cpp-geospatial-interop + 1.0-SNAPSHOT + + 2.2-SNAPSHOT + + + + org.apache.paimon + paimon-bundle + ${paimon.version} + + + org.apache.hadoophadoop-common3.3.6 + + + org.apache.hadoophadoop-mapreduce-client-core3.3.6 + + + diff --git a/test/inte/geospatial/run.sh b/test/inte/geospatial/run.sh new file mode 100755 index 000000000..53eca2a92 --- /dev/null +++ b/test/inte/geospatial/run.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail +script_dir="$(cd "$(dirname "$0")" && pwd)" +repo_dir="$(cd "$script_dir/../../.." && pwd)" +test_binary="${1:-$repo_dir/build/debug/paimon-parquet-format-test}" +interop_dir="$(mktemp -d)" +trap 'rm -rf "$interop_dir"' EXIT +mvn -q -f "$script_dir/pom.xml" dependency:build-classpath \ + -Dpaimon.version="${PAIMON_JAVA_VERSION:-2.2-SNAPSHOT}" -Dmdep.outputFile="$interop_dir/classpath" +classpath="$(cat "$interop_dir/classpath")" +javac -cp "$classpath" -d "$interop_dir" "$script_dir/GeospatialInterop.java" +java -cp "$interop_dir:$classpath" GeospatialInterop write "$interop_dir/java.parquet" +PAIMON_GEOSPATIAL_INTEROP_DIR="$interop_dir" "$test_binary" --gtest_filter=GeospatialParquetTest.JavaInteroperability +java -cp "$interop_dir:$classpath" GeospatialInterop check "$interop_dir/cpp.parquet" diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index c211f4425..65b560ada 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -39,6 +39,7 @@ #include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/reader/reader_utils.h" #include "paimon/common/types/data_type.h" +#include "paimon/common/types/geospatial_type.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/path_util.h" @@ -376,6 +377,72 @@ TEST_P(WriteAndReadInteTest, TestAppendSimple) { ASSERT_TRUE(success); } +TEST_P(WriteAndReadInteTest, GeospatialProjectionAndNullPredicates) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet") { + GTEST_SKIP(); + } + ASSERT_OK_AND_ASSIGN(auto geom, GeospatialType::GeometryField("shape", "EPSG:3857")); + ASSERT_OK_AND_ASSIGN(auto geog, + GeospatialType::GeographyField("position", "EPSG:4326", "karney")); + auto nested = arrow::field("nested", arrow::struct_({geog})); + auto fields = arrow::FieldVector{arrow::field("id", arrow::int32()), geom, nested}; + std::map options = {{Options::FILE_FORMAT, file_format}, + {Options::FILE_SYSTEM, file_system}, + {Options::BUCKET, "-1"}}; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, arrow::schema(fields), {}, {}, options, false)); + const std::string rows = R"([ + [0, "\u0001\u0007\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + ["\u0001\u0007\u0000\u0000\u0000\u0000\u0000\u0000\u0000"]], + [1, null, [null]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), rows, {}, 0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), 0, std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), std::nullopt, false)); + const auto path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + for (bool is_null : {false, true}) { + auto predicate = is_null ? PredicateBuilder::IsNull(1, "shape", FieldType::GEOMETRY) + : PredicateBuilder::IsNotNull(1, "shape", FieldType::GEOMETRY); + ReadContextBuilder builder(path); + builder.SetOptions(options) + .SetReadFieldNames({"nested", "shape", "id"}) + .SetPredicate(predicate) + .EnablePredicateFilter(true); + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(splits)); + ASSERT_OK_AND_ASSIGN(auto result, + ReadResultCollector::CollectResult(std::move(reader), true)); + ASSERT_EQ(result->length(), 1); + ASSERT_EQ(result->num_chunks(), 1); + auto row = checked_pointer_cast(result->chunk(0)); + ASSERT_EQ(row->num_fields(), 4); + ASSERT_TRUE(GeospatialType::Equals(*row->type()->field(1), *nested)); + ASSERT_TRUE(GeospatialType::Equals(*row->type()->field(2), *geom)); + ASSERT_EQ(checked_pointer_cast(row->field(3))->Value(0), + is_null ? 1 : 0); + auto position = checked_pointer_cast(row->field(1))->field(0); + for (const auto& value : {row->field(2), position}) { + ASSERT_EQ(value->IsNull(0), is_null); + if (!is_null) { + ASSERT_EQ(checked_pointer_cast(value)->GetView(0), + std::string("\x01\x07\0\0\0\0\0\0\0", 9)); + } + } + } + options[Options::FILE_FORMAT_PER_LEVEL] = "0:parquet,1:orc"; + ReadContextBuilder builder(path); + builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(context)), "GEOMETRY/GEOGRAPHY"); +} + TEST_P(WriteAndReadInteTest, TestAppendReadWithNestedPredicateAcrossBatches) { auto [file_format, file_system] = GetParam(); arrow::FieldVector fields = {