Skip to content

[feature](lance) push down common string and boolean predicates #67051

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Jay-ju wants to merge 2 commits into
base: branch-4.1
Choose a base branch
from
Open

[feature](lance) push down common string and boolean predicates #67051

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
597d3fd
[feature](lance) push down common string and boolean predicates
Jay-ju Aug 23, 2026
f613605
[fix](lance) keep unsafe string predicates residual
Jay-ju Aug 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions ...-core/src/main/java/org/apache/doris/datasource/lance/source/LancePredicateConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,17 @@
import org.apache.doris.analysis.DecimalLiteral;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.FloatLiteral;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.InPredicate;
import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.analysis.IsNullPredicate;
import org.apache.doris.analysis.LargeIntLiteral;
import org.apache.doris.analysis.LikePredicate;
import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.NullLiteral;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.thrift.TFunctionBinaryType;

import io.substrait.expression.Expression;
import io.substrait.expression.ExpressionCreator;
Expand Down Expand Up @@ -131,6 +134,15 @@ private Optional<Expression> convert(Expr expr) {
if (expr instanceof IsNullPredicate) {
return convertIsNull((IsNullPredicate) expr);
}
if (expr instanceof LikePredicate) {
return convertLike((LikePredicate) expr);
}
if (expr instanceof FunctionCallExpr) {
return convertStringFunction((FunctionCallExpr) expr);
}
if (expr instanceof SlotRef) {
return convertBooleanSlot((SlotRef) expr);
}
return Optional.empty();
}

Expand Down Expand Up @@ -257,6 +269,63 @@ private Optional<Expression> convertIsNull(IsNullPredicate predicate) {
return Optional.of(comparisonFunction(function, fieldReference(field)));
}

private Optional<Expression> convertLike(LikePredicate predicate) {
if (predicate.getOp() != LikePredicate.Operator.LIKE) {
return Optional.empty();
}
return convertStringPredicate("like:str_str", predicate.getChild(0), predicate.getChild(1), true);
}

private Optional<Expression> convertStringFunction(FunctionCallExpr function) {
if (function.getFnName() == null || function.getFn() == null
|| function.getFn().getBinaryType() != TFunctionBinaryType.BUILTIN
|| function.getChildren().size() != 2) {
return Optional.empty();
}
String functionName = function.getFnName().getFunction().toLowerCase(Locale.ROOT);

@github-actions github-actions Bot Aug 24, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Preserve the resolved function identity before dispatching by name. FunctionRegistry intentionally allows a UDF to shadow a built-in via prefer_udf_over_builtin, while a qualified call selects the UDF directly, and Nereids translates Java/Python UDFs to this same FunctionCallExpr with a non-BUILTIN catalog function. A two-argument UDF named starts_with, ends_with, or like can therefore reach this switch, be replaced by Lance's built-in, and be removed from the Doris residual in LanceScanNode; an empty residual can also enable limit pushdown. Please require a non-null resolved Doris BUILTIN before the switch (leaving unknown/UDF identities residual) and cover an analyzed same-named UDF.

@Jay-ju Jay-ju Aug 29, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in f613605. FunctionCallExpr pushdown now requires a resolved Doris BUILTIN function; unresolved functions and same-named UDFs remain as Doris residuals. Added a unit case with a resolved JAVA_UDF named starts_with.

switch (functionName) {
case "like":
return convertStringPredicate(
"like:str_str", function.getChild(0), function.getChild(1), true);
case "starts_with":
return convertStringPredicate(
"starts_with:str_str", function.getChild(0), function.getChild(1), false);
case "ends_with":
return convertStringPredicate(
"ends_with:str_str", function.getChild(0), function.getChild(1), false);
default:
return Optional.empty();
}
}

private Optional<Expression> convertStringPredicate(
String function, Expr input, Expr pattern, boolean rejectEscapedPattern) {
SlotRef slot = directSlot(input);
LiteralExpr literal = directLiteral(pattern);
ResolvedField field = slot == null ? null : findField(slot);
if (field == null || !isStringType(field.field.getType()) || !(literal instanceof StringLiteral)) {
return Optional.empty();
}
String patternValue = literal.getStringValue();
// Doris uses backslash as LIKE's default escape character, while the Substrait function
// has no escape argument. Keep escaped LIKE patterns in Doris rather than changing meaning.
if (patternValue.indexOf('\0') >= 0
|| (rejectEscapedPattern && patternValue.indexOf('\\') >= 0)) {
return Optional.empty();
}
return Optional.of(stringFunction(function, fieldReference(field),
ExpressionCreator.string(false, patternValue)));
}

private Optional<Expression> convertBooleanSlot(SlotRef slot) {
ResolvedField field = findField(slot);
if (field == null || !(field.field.getType() instanceof ArrowType.Bool)) {
return Optional.empty();
}
return Optional.of(comparisonFunction("equal:any_any",
fieldReference(field), ExpressionCreator.bool(false, true)));
}

// convert doris literal to Substrait literal with arrow type
private Optional<Expression> convertLiteral(ArrowType type, LiteralExpr literal) {
if (type instanceof ArrowType.Bool && literal instanceof BoolLiteral) {
Expand Down Expand Up @@ -450,6 +519,10 @@ private static boolean isPushdownType(ArrowType type) {
|| type instanceof ArrowType.LargeUtf8;
}

private static boolean isStringType(ArrowType type) {
return type instanceof ArrowType.Utf8 || type instanceof ArrowType.LargeUtf8;
}

// slotref with ordinal index with Substrait Type
private Expression fieldReference(ResolvedField field) {
return FieldReference.newRootStructReference(field.ordinal, toSubstraitType(field.field));
Expand Down Expand Up @@ -502,6 +575,10 @@ private static Expression booleanFunction(String key, List<Expression> arguments
return scalarFunction(DefaultExtensionCatalog.FUNCTIONS_BOOLEAN, key, arguments);
}

private static Expression stringFunction(String key, Expression... arguments) {
return scalarFunction(DefaultExtensionCatalog.FUNCTIONS_STRING, key, Arrays.asList(arguments));
}

private static Expression scalarFunction(String uri, String key, List<Expression> arguments) {
SimpleExtension.ScalarFunctionVariant declaration = EXTENSIONS.getScalarFunction(
SimpleExtension.FunctionAnchor.of(uri, key));
Expand Down
90 changes: 90 additions & 0 deletions fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LancePredicateConverterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,19 @@
import org.apache.doris.analysis.DecimalLiteral;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.FunctionName;
import org.apache.doris.analysis.InPredicate;
import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.analysis.IsNullPredicate;
import org.apache.doris.analysis.LargeIntLiteral;
import org.apache.doris.analysis.LikePredicate;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.catalog.ScalarFunction;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.datasource.lance.source.LancePredicateConverter;
import org.apache.doris.thrift.TFunctionBinaryType;

import io.substrait.proto.ExtendedExpression;
import org.apache.arrow.vector.types.DateUnit;
Expand Down Expand Up @@ -215,6 +219,85 @@ public void testComparisonInAndIsNull() {
Assertions.assertEquals(3, result.getPushedConjuncts().size());
}

@Test
public void testStringPredicates() {
Expr legacyLike = new LikePredicate(LikePredicate.Operator.LIKE,
new SlotRef(null, "label"), new StringLiteral("ready%"));
Expr like = stringFunction(
"like", new SlotRef(null, "label"), new StringLiteral("%ead_"));
Expr startsWith = stringFunction(
"starts_with", new SlotRef(null, "label"), new StringLiteral("ready"));
Expr endsWith = stringFunction(
"ends_with", new SlotRef(null, "large_label"), new StringLiteral("done"));

LancePredicateConverter.ConversionResult result =
converter.convert(Arrays.asList(legacyLike, like, startsWith, endsWith));

ExtendedExpression envelope = Assertions.assertDoesNotThrow(
() -> ExtendedExpression.parseFrom(result.getSubstraitFilter()));
String serialized = envelope.toString();
Assertions.assertTrue(serialized.contains("like:str_str"));
Assertions.assertTrue(serialized.contains("starts_with:str_str"));
Assertions.assertTrue(serialized.contains("ends_with:str_str"));
Assertions.assertEquals(4, result.getPushedConjuncts().size());
}

@Test
public void testUnsupportedStringPredicatesRemainResidual() {
Expr regexp = new LikePredicate(LikePredicate.Operator.REGEXP,
new SlotRef(null, "label"), new StringLiteral("ready.*"));
Expr escapedLike = new LikePredicate(LikePredicate.Operator.LIKE,
new SlotRef(null, "label"), new StringLiteral("ready\\%"));
Expr explicitEscape = stringFunction("like",
new SlotRef(null, "label"), new StringLiteral("ready!%"), new StringLiteral("!"));
Expr nonLiteralPattern = stringFunction(
"starts_with", new SlotRef(null, "label"), new SlotRef(null, "event-type"));
Expr nonStringInput = stringFunction(
"ends_with", new SlotRef(null, "row_id"), new StringLiteral("1"));

LancePredicateConverter.ConversionResult result = converter.convert(
Arrays.asList(regexp, escapedLike, explicitEscape, nonLiteralPattern, nonStringInput));

Assertions.assertEquals(0, result.getSubstraitFilter().length);
Assertions.assertTrue(result.getPushedConjuncts().isEmpty());
}

@Test
public void testResolvedUdfAndNulStringPredicatesRemainResidual() {
FunctionCallExpr udf = stringFunction(
"starts_with", new SlotRef(null, "label"), new StringLiteral("ready"));
udf.getFn().setBinaryType(TFunctionBinaryType.JAVA_UDF);
Expr legacyNulLike = new LikePredicate(LikePredicate.Operator.LIKE,
new SlotRef(null, "label"), new StringLiteral("m\0_"));
Expr functionNulLike = stringFunction(
"like", new SlotRef(null, "label"), new StringLiteral("m\0_"));

LancePredicateConverter.ConversionResult result =
converter.convert(Arrays.asList(udf, legacyNulLike, functionNulLike));

Assertions.assertEquals(0, result.getSubstraitFilter().length);
Assertions.assertTrue(result.getPushedConjuncts().isEmpty());
}

@Test
public void testDirectBooleanPredicates() {
LancePredicateConverter boolConverter = new LancePredicateConverter(new Schema(
Collections.singletonList(Field.nullable("active", ArrowType.Bool.INSTANCE))));
Expr active = new SlotRef(null, "active");
Expr notActive = new CompoundPredicate(
CompoundPredicate.Operator.NOT, new SlotRef(null, "active"), null);

LancePredicateConverter.ConversionResult result =
boolConverter.convert(Arrays.asList(active, notActive));

ExtendedExpression envelope = Assertions.assertDoesNotThrow(
() -> ExtendedExpression.parseFrom(result.getSubstraitFilter()));
String serialized = envelope.toString();
Assertions.assertTrue(serialized.contains("equal:any_any"));
Assertions.assertTrue(serialized.contains("not:bool"));
Assertions.assertEquals(2, result.getPushedConjuncts().size());
}

@Test
public void testNullableNullSafeEqualityPreservesTwoValuedSemantics() {
Expr nullableNullSafeEqual = new BinaryPredicate(BinaryPredicate.Operator.EQ_FOR_NULL,
Expand Down Expand Up @@ -473,6 +556,13 @@ private void assertUnsignedIntegerLiteral(
Assertions.assertEquals(1, result.getPushedConjuncts().size());
}

private FunctionCallExpr stringFunction(String name, Expr... arguments) {
FunctionCallExpr function = new FunctionCallExpr(name, Arrays.asList(arguments));
function.setFn(new ScalarFunction(new FunctionName(name),
Collections.nCopies(arguments.length, Type.VARCHAR), Type.BOOLEAN, false, true));
return function;
}

private void assertNullSafeEqualityComposition(Expr predicate) {
LancePredicateConverter.ConversionResult result =
converter.convert(Collections.singletonList(predicate));
Expand Down
55 changes: 55 additions & 0 deletions regression-test/data/external_table_p0/lance/test_lance_scalar_predicate_pushdown.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,61 @@
9
10

-- !select_bool_direct --
5
6
7
9
10

-- !select_bool_direct_not --
2
3
4
8

-- !select_utf8_starts_with --
7
8

-- !select_utf8_ends_with --
4
6

-- !select_utf8_like_prefix --
2
4
10

-- !select_utf8_like_contains --
3
7
10

-- !select_utf8_like_single_wildcard --
7
8

-- !select_utf8_not_like --
2
3
4
5
6
9
10

-- !select_utf8_like_explicit_escape --

-- !select_utf8_like_nul_residual --
2
4
10

-- !select_utf8_regexp_residual --
7
8

-- !select_float32_eq --
7
8
Expand Down
68 changes: 67 additions & 1 deletion regression-test/suites/external_table_p0/lance/test_lance_scalar_predicate_pushdown.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ suite("test_lance_scalar_predicate_pushdown", "p0,external") {
*
* | Lance / Arrow type | Doris type | Operators exercised |
* |---|---|---|
* | bool | boolean | =, !=, <>, <=>, IN, NOT IN, IS NULL, IS NOT NULL, OR, NOT |
* | bool | boolean | Direct predicate, =, !=, <>, <=>, IN, NOT IN, IS NULL, IS NOT NULL, OR, NOT |
* | float32 | float | All operators below |
* | float64 | double | All operators below |
* | decimal128 | decimal(18,2) | All operators below |
Expand Down Expand Up @@ -84,6 +84,19 @@ suite("test_lance_scalar_predicate_pushdown", "p0,external") {
}
}

Closure verifyResidual = { String query, String expression ->
explain {
sql(query)
notContains "lancePushdownPredicate="
check { explainString ->
String residual = explainString.readLines()
.find { line -> line.trim().startsWith("predicates:") }
return residual != null
&& residual.toLowerCase().contains(expression.toLowerCase())
}
}
}

Closure verifyOrderedScalarPushdown = { String tableName, String typeName, String columnName, Map values ->
String eqQuery = """ SELECT row_id FROM ${tableName} WHERE ${columnName} = ${values.equal} ORDER BY row_id; """
verifyFullyPushedDown(eqQuery, columnName)
Expand Down Expand Up @@ -202,10 +215,63 @@ suite("test_lance_scalar_predicate_pushdown", "p0,external") {
String boolReversedQuery = """ SELECT row_id FROM predicate_pushdown WHERE true = bool_value ORDER BY row_id; """
verifyFullyPushedDown(boolReversedQuery, "bool_value")
quickTest("select_bool_reversed", boolReversedQuery)

String boolDirectQuery = """ SELECT row_id FROM predicate_pushdown WHERE bool_value ORDER BY row_id; """
verifyFullyPushedDown(boolDirectQuery, "bool_value")
quickTest("select_bool_direct", boolDirectQuery)

String boolDirectNotQuery = """ SELECT row_id FROM predicate_pushdown WHERE NOT bool_value ORDER BY row_id; """
verifyFullyPushedDown(boolDirectNotQuery, "bool_value")
quickTest("select_bool_direct_not", boolDirectNotQuery)
}

verifyBooleanPushdown()

String startsWithQuery =
""" SELECT row_id FROM predicate_pushdown WHERE starts_with(utf8_value, 'ten') ORDER BY row_id; """
verifyFullyPushedDown(startsWithQuery, "utf8_value")
quickTest("select_utf8_starts_with", startsWithQuery)

String endsWithQuery =
""" SELECT row_id FROM predicate_pushdown WHERE ends_with(utf8_value, 'one') ORDER BY row_id; """
verifyFullyPushedDown(endsWithQuery, "utf8_value")
quickTest("select_utf8_ends_with", endsWithQuery)

String likePrefixQuery =
""" SELECT row_id FROM predicate_pushdown WHERE utf8_value LIKE 'm%' ORDER BY row_id; """
verifyFullyPushedDown(likePrefixQuery, "utf8_value")
quickTest("select_utf8_like_prefix", likePrefixQuery)

String likeContainsQuery =
""" SELECT row_id FROM predicate_pushdown WHERE utf8_value LIKE '%a%' ORDER BY row_id; """
verifyFullyPushedDown(likeContainsQuery, "utf8_value")
quickTest("select_utf8_like_contains", likeContainsQuery)

String likeSingleWildcardQuery =
""" SELECT row_id FROM predicate_pushdown WHERE utf8_value LIKE 'ten-_' ORDER BY row_id; """
verifyFullyPushedDown(likeSingleWildcardQuery, "utf8_value")
quickTest("select_utf8_like_single_wildcard", likeSingleWildcardQuery)

String notLikeQuery =
""" SELECT row_id FROM predicate_pushdown WHERE utf8_value NOT LIKE 'ten-%' ORDER BY row_id; """
verifyFullyPushedDown(notLikeQuery, "utf8_value")
quickTest("select_utf8_not_like", notLikeQuery)

String explicitEscapeQuery =
""" SELECT row_id FROM predicate_pushdown WHERE utf8_value LIKE 'ten!_%' ESCAPE '!' ORDER BY row_id; """
verifyResidual(explicitEscapeQuery, "like")
quickTest("select_utf8_like_explicit_escape", explicitEscapeQuery)

String nulLikeQuery =
""" SELECT row_id FROM predicate_pushdown WHERE utf8_value LIKE 'm\\0_' ORDER BY row_id; """
verifyResidual(nulLikeQuery, "like")
quickTest("select_utf8_like_nul_residual", nulLikeQuery)

String regexpQuery =
""" SELECT row_id FROM predicate_pushdown WHERE utf8_value REGEXP '^ten-' ORDER BY row_id; """
verifyResidual(regexpQuery, "regexp")
quickTest("select_utf8_regexp_residual", regexpQuery)

verifyOrderedScalarPushdown("predicate_pushdown", "float32", "float32_value", [
equal: "10",
threshold: "0",
Expand Down
Loading