HIVE-29353:Add basic support for encoding GeoHash & decoding it to Geometry #6813
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
| package org.apache.hadoop.hive.ql.udf.esri; | ||
|
|
||
| import ch.hsr.geohash.BoundingBox; | ||
| import ch.hsr.geohash.GeoHash; | ||
| import org.locationtech.jts.geom.Coordinate; | ||
| import org.locationtech.jts.geom.Polygon; | ||
|
|
||
| public final class GeoHashUtils { | ||
|
|
||
| public static final int MIN_CHARACTER_PRECISION = 1; | ||
|
|
||
| /** | ||
| * Maximum geohash length in base-32 characters for {@link GeoHash#geoHashStringWithCharacterPrecision}. | ||
| */ | ||
| public static final int MAX_CHARACTER_PRECISION = 12; | ||
|
|
||
| public static final int DEFAULT_CHARACTER_PRECISION = 12; | ||
|
|
||
| private GeoHashUtils() { | ||
| } | ||
|
|
||
| public static String geohashForPoint(double longitude, double latitude, int characterPrecision) { | ||
| return GeoHash.geoHashStringWithCharacterPrecision(latitude, longitude, characterPrecision); | ||
| } | ||
|
|
||
| /** | ||
| * Returns a rectangular polygon for the geohash cell (closed ring, lon/lat coordinates). | ||
| * | ||
| * @param geohash base-32 geohash string (non-empty) | ||
| * @param characterPrecision number of leading characters to use (1–12, at most {@code geohash} | ||
| * length) | ||
| */ | ||
| public static Polygon geohashCellPolygon(String geohash, int characterPrecision) { | ||
|
Contributor
There was a problem hiding this comment. I wonder why this is the only public method in this class that got javadoc. I would suggest adding documentation to the other public methods as well.
Contributor
Author
There was a problem hiding this comment. for other methods their names are suffieciently self explainatory, this one required some explaination regarding type of polygon returned and geohash string kind needed so added javadoc for it only. I think we can avoid javadoc for others wdyt? |
||
| if (geohash == null || geohash.isEmpty()) { | ||
| return null; | ||
| } | ||
| if (characterPrecision < MIN_CHARACTER_PRECISION || | ||
| characterPrecision > MAX_CHARACTER_PRECISION || | ||
| characterPrecision > geohash.length()) { | ||
| return null; | ||
| } | ||
| String hashPrefix = geohash.substring(0, characterPrecision); | ||
|
Contributor
There was a problem hiding this comment. I'm trying to understand what happens here: we receive a geohash and a precision. And we only use the first part of the hash.
Contributor
Author
There was a problem hiding this comment. this is postgis compliant behaviour |
||
| BoundingBox box = GeoHash.fromGeohashString(hashPrefix).getBoundingBox(); | ||
| double west = box.getWestLongitude(); | ||
| double east = box.getEastLongitude(); | ||
| double south = box.getSouthLatitude(); | ||
| double north = box.getNorthLatitude(); | ||
| // Closed ring (west,south) -> (west,north) -> (east,north) -> (east,south) -> close. | ||
| Coordinate[] ring = new Coordinate[] { | ||
| new Coordinate(west, south), | ||
| new Coordinate(west, north), | ||
| new Coordinate(east, north), | ||
| new Coordinate(east, south), | ||
| new Coordinate(west, south) | ||
| }; | ||
| return GeometryUtils.GEOMETRY_FACTORY.createPolygon(ring); | ||
| } | ||
|
|
||
| public static int resolveEncodePrecision(Integer precisionArg) { | ||
| int precision = precisionArg == null ? DEFAULT_CHARACTER_PRECISION : precisionArg; | ||
| if (precision < MIN_CHARACTER_PRECISION || precision > MAX_CHARACTER_PRECISION) { | ||
| return -1; | ||
| } | ||
| return precision; | ||
| } | ||
|
|
||
| public static int resolveDecodePrecision(Integer precisionArg, int geohashLength) { | ||
| int precision = precisionArg == null ? geohashLength : precisionArg; | ||
| if (precision < MIN_CHARACTER_PRECISION || | ||
| precision > geohashLength || | ||
| geohashLength > MAX_CHARACTER_PRECISION) { | ||
| return -1; | ||
| } | ||
| return precision; | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,83 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
| package org.apache.hadoop.hive.ql.udf.esri; | ||
|
|
||
| import org.apache.hadoop.hive.ql.exec.Description; | ||
| import org.apache.hadoop.io.BytesWritable; | ||
| import org.apache.hadoop.io.IntWritable; | ||
| import org.apache.hadoop.io.Text; | ||
| import org.locationtech.jts.geom.Geometry; | ||
| import org.locationtech.jts.geom.Point; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| @Description( | ||
| name = "ST_GeoHash", | ||
| value = """ | ||
| _FUNC_(point) - geohash for a point geometry | ||
| _FUNC_(point, precision) - geohash at given character precision | ||
| """, | ||
| extended = """ | ||
| SELECT _FUNC_(ST_Point(-126.965375, 43.234528), 12); -- 9pttyydekk4t | ||
| """) | ||
| public class ST_GeoHash extends ST_Geometry { | ||
|
Check warning on line 39 in ql/src/java/org/apache/hadoop/hive/ql/udf/esri/ST_GeoHash.java
|
||
|
|
||
| static final Logger LOG = LoggerFactory.getLogger(ST_GeoHash.class.getName()); | ||
|
|
||
| public Text evaluate(BytesWritable geomref) { | ||
| return evaluate(geomref, null); | ||
| } | ||
|
|
||
| public Text evaluate(BytesWritable geomref, IntWritable precisionArg) { | ||
|
Contributor
There was a problem hiding this comment. nit: I think
Contributor
Author
There was a problem hiding this comment. the same variable name is used in almost all the other geospatial udfs so used same for consistency. |
||
| if (geomref == null || geomref.getLength() == 0) { | ||
|
Contributor
There was a problem hiding this comment. Tiny suggestion about readability: commons-lang3 has utility methods that helps a lot around strings, like
Contributor
There was a problem hiding this comment. Question about performance vs readability: if (geomref == null || geomref.getLength() < 5) {
return OGCType.UNKNOWN;
}This is basically almost the exact same check as the check here. Does it worth to do both checks or one of them is enough? |
||
| LogUtils.Log_ArgumentsNull(LOG); | ||
| return null; | ||
| } | ||
|
|
||
| GeometryUtils.OGCType type = GeometryUtils.getType(geomref); | ||
| if (type != GeometryUtils.OGCType.ST_POINT) { | ||
| LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_POINT, type); | ||
| return null; | ||
| } | ||
|
|
||
| Geometry geom = GeometryUtils.geometryFromEsriShape(geomref); | ||
| if (geom == null) { | ||
| return null; | ||
| } | ||
| Point point = (Point) geom; | ||
| return geohashText(point.getX(), point.getY(), precisionArg); | ||
|
Contributor
There was a problem hiding this comment.
|
||
| } | ||
|
|
||
| private Text geohashText(double longitude, double latitude, IntWritable precisionArg) { | ||
| int precision = | ||
| GeoHashUtils.resolveEncodePrecision(precisionArg == null ? null : precisionArg.get()); | ||
| if (precision < 0) { | ||
|
Contributor
There was a problem hiding this comment. nit about readability: What if adding an extra contstant to |
||
| LogUtils.Log_InvalidPrecision(LOG, GeoHashUtils.MIN_CHARACTER_PRECISION, | ||
| GeoHashUtils.MAX_CHARACTER_PRECISION); | ||
| return null; | ||
| } | ||
| try { | ||
| String hash = GeoHashUtils.geohashForPoint(longitude, latitude, precision); | ||
| return new Text(hash); | ||
| } catch (Exception e) { | ||
| LogUtils.Log_InternalError(LOG, "ST_GeoHash: " + e); | ||
| return null; | ||
| } | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
| package org.apache.hadoop.hive.ql.udf.esri; | ||
|
|
||
| import org.apache.hadoop.hive.ql.exec.Description; | ||
| import org.apache.hadoop.io.BytesWritable; | ||
| import org.apache.hadoop.io.IntWritable; | ||
| import org.apache.hadoop.io.Text; | ||
| import org.locationtech.jts.geom.Polygon; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| @Description( | ||
| name = "ST_GeomFromGeoHash", | ||
| value = """ | ||
| _FUNC_(geohash) - polygon for the geohash cell | ||
| _FUNC_(geohash, precision) - polygon using the first precision characters | ||
| """, | ||
| extended = """ | ||
| SELECT ST_AsText(_FUNC_('9ptty', 5)); | ||
| """) | ||
| public class ST_GeomFromGeoHash extends ST_Geometry { | ||
|
Check warning on line 38 in ql/src/java/org/apache/hadoop/hive/ql/udf/esri/ST_GeomFromGeoHash.java
|
||
|
|
||
| static final Logger LOG = LoggerFactory.getLogger(ST_GeomFromGeoHash.class.getName()); | ||
|
|
||
| public BytesWritable evaluate(Text geohashText) { | ||
| return evaluate(geohashText, null); | ||
| } | ||
|
|
||
| public BytesWritable evaluate(Text geohashText, IntWritable precisionArg) { | ||
| String geohash = geohashText != null ? geohashText.toString().trim() : null; | ||
| if (geohash == null || geohash.isEmpty()) { | ||
| LogUtils.Log_ArgumentsNull(LOG); | ||
| return null; | ||
| } | ||
|
|
||
| int characterPrecision = GeoHashUtils.resolveDecodePrecision(precisionArg == null ? null : precisionArg.get(), | ||
| geohash.length()); | ||
| if (characterPrecision < 0) { | ||
| LogUtils.Log_InvalidPrecision(LOG, GeoHashUtils.MIN_CHARACTER_PRECISION, | ||
| Math.min(geohash.length(), GeoHashUtils.MAX_CHARACTER_PRECISION)); | ||
|
Contributor
There was a problem hiding this comment. If the geohash.length() is 10, there will be the log message: It is not true because the precision should be between 1 and 12.
Contributor
Author
There was a problem hiding this comment. when we are construct geometry from geoshash string, specifying precision beyond string length doesn't yield anything and will be wrong as per behaviour: #6813 (comment) |
||
| return null; | ||
| } | ||
|
|
||
| try { | ||
| Polygon polygon = GeoHashUtils.geohashCellPolygon(geohash, characterPrecision); | ||
| if (polygon == null) { | ||
| return null; | ||
| } | ||
| return GeometryUtils.geometryToEsriShapeBytesWritable(polygon); | ||
| } catch (Exception e) { | ||
| LogUtils.Log_InternalError(LOG, "ST_GeomFromGeoHash: " + e); | ||
| return null; | ||
| } | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
compile is the default dependency scope: https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html