Skip to content

[SPARK-59355][SQL] Simplify LIKE patterns containing escaped wildcards #58644

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
david-mollitor-db wants to merge 1 commit into
base: master
Choose a base branch
from
+187 −67
Open

[SPARK-59355][SQL] Simplify LIKE patterns containing escaped wildcards #58644

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
154 changes: 110 additions & 44 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -804,60 +804,126 @@ object SupportedBinaryExpr {
* pattern.
*/
object LikeSimplification extends Rule[LogicalPlan] with PredicateHelper {
// if guards below protect from escapes on trailing %.
// Cases like "something\%" are not optimized, but this does not affect correctness.
// Consecutive wildcard characters are equivalent to a single wildcard character.
// These regexes classify a pattern that has no escape character (`%`/`_` are the only
// wildcards). Patterns that do contain the escape character are decoded by `decodeLikePattern`
// first. Consecutive wildcard characters are equivalent to a single wildcard character.
private val startsWith = "([^_%]+)%+".r
private val endsWith = "%+([^_%]+)".r
private val startsAndEndsWith = "([^_%]+)%+([^_%]+)".r
private val contains = "%+([^_%]+)%+".r
private val equalTo = "([^_%]*)".r

private def startsWithExpr(input: Expression, prefix: String): Expression =
StartsWith(input, Literal.create(prefix, input.dataType))

private def endsWithExpr(input: Expression, postfix: String): Expression =
EndsWith(input, Literal.create(postfix, input.dataType))

private def containsExpr(input: Expression, infix: String): Expression =
Contains(input, Literal.create(infix, input.dataType))

private def equalToExpr(input: Expression, str: String): Expression =
EqualTo(input, Literal.create(str, input.dataType))

// 'a%a' is basically 'a%' && '%a', but a length guard is required to stop 'a' matching 'a%a'.
// When the collation matches raw bytes (supportsBinaryEquality), StartsWith/EndsWith pin the
// literal bytes of the prefix and suffix, so a byte-length guard (OctetLength, O(1) via the
// stored numBytes) accepts exactly the same inputs as the code-point guard and is cheaper.
// Otherwise the anchors are collation-aware (LIKE reaches this only for UTF8_LCASE) and can
// match a code point whose UTF-8 length differs from the pattern's -- a single multibyte code
// point could then satisfy both anchors and clear the byte guard -- so the code-point (Length)
// guard must be kept for correctness.
private def startsAndEndsWithExpr(
input: Expression, prefix: String, postfix: String): Expression = {
val lengthGuard = input.dataType match {
case st: StringType if st.supportsBinaryEquality =>
GreaterThanOrEqual(OctetLength(input),
Literal.create(UTF8String.fromString(prefix).numBytes
+ UTF8String.fromString(postfix).numBytes))
case _ =>
GreaterThanOrEqual(Length(input),
Literal.create(prefix.codePointCount(0, prefix.length)
+ postfix.codePointCount(0, postfix.length)))
}
And(lengthGuard, And(startsWithExpr(input, prefix), endsWithExpr(input, postfix)))
}

// A decoded LIKE pattern token: a maximal run of literal characters, or a `%` wildcard
// (consecutive `%` collapse into one).
private sealed trait LikeToken
private case class LikeLiteral(str: String) extends LikeToken
private case object LikeWildcard extends LikeToken

// Decodes a `pattern` that contains the escape character into literal/wildcard tokens, or None
// if it cannot be simplified: an invalid escape (the escape char not followed by `%`, `_`, or
// the escape char) or a trailing escape char -- both make LIKE throw at runtime, so the rule
// must leave them alone -- or an unescaped `_` wildcard, which this rule does not handle.
private def decodeLikePattern(pattern: String, escapeChar: Char): Option[Seq[LikeToken]] = {
val tokens = ArrayBuffer.empty[LikeToken]
val literal = new StringBuilder
def flushLiteral(): Unit = if (literal.length > 0) {
tokens += LikeLiteral(literal.toString)
literal.setLength(0)
}
var i = 0
while (i < pattern.length) {
val c = pattern.charAt(i)
if (c == escapeChar) {
if (i + 1 >= pattern.length) {
return None // trailing escape character
}
val next = pattern.charAt(i + 1)
if (next == '%' || next == '_' || next == escapeChar) {
literal.append(next)
i += 2
} else {
return None // invalid escape sequence
}
} else if (c == '%') {
flushLiteral()
if (tokens.isEmpty || tokens.last != LikeWildcard) {
tokens += LikeWildcard
}
i += 1
} else if (c == '_') {
return None // unescaped single-character wildcard, not handled here
} else {
literal.append(c)
i += 1
}
}
flushLiteral()
Some(tokens.toSeq)
}

private def simplifyLike(
input: Expression, pattern: String, escapeChar: Char = '\\'): Option[Expression] = {
if (pattern.contains(escapeChar)) {
// There are three different situations when pattern containing escapeChar:
// 1. pattern contains invalid escape sequence, e.g. 'm\aca'
// 2. pattern contains escaped wildcard character, e.g. 'ma\%ca'
// 3. pattern contains escaped escape character, e.g. 'ma\\ca'
// Although there are patterns can be optimized if we handle the escape first, we just
// skip this rule if pattern contains any escapeChar for simplicity.
None
} else {
if (!pattern.contains(escapeChar)) {
// Fast path: no escape character, so `%`/`_` are the only wildcards.
pattern match {
case startsWith(prefix) =>
Some(StartsWith(input, Literal.create(prefix, input.dataType)))
case endsWith(postfix) =>
Some(EndsWith(input, Literal.create(postfix, input.dataType)))
// 'a%a' pattern is basically same with 'a%' && '%a'.
// However, the additional length condition is required to prevent 'a' match 'a%a'.
case startsWith(prefix) => Some(startsWithExpr(input, prefix))
case endsWith(postfix) => Some(endsWithExpr(input, postfix))
case startsAndEndsWith(prefix, postfix) =>
// The length guard only rejects inputs too short to hold both the prefix and the
// suffix. When the collation matches raw bytes (supportsBinaryEquality),
// StartsWith/EndsWith pin the literal bytes of the prefix and suffix, so a
// byte-length guard (OctetLength, O(1) via the stored numBytes) accepts exactly the
// same inputs as the code-point guard and is cheaper. Otherwise the anchors are
// collation-aware (LIKE reaches this only for UTF8_LCASE) and can match a code point
// whose UTF-8 length differs from the pattern's -- a single multibyte code point
// could then satisfy both anchors and clear the byte guard -- so the code-point
// (Length) guard must be kept for correctness.
val lengthGuard = input.dataType match {
case st: StringType if st.supportsBinaryEquality =>
GreaterThanOrEqual(OctetLength(input),
Literal.create(UTF8String.fromString(prefix).numBytes
+ UTF8String.fromString(postfix).numBytes))
case _ =>
GreaterThanOrEqual(Length(input),
Literal.create(prefix.codePointCount(0, prefix.length)
+ postfix.codePointCount(0, postfix.length)))
}
Some(And(lengthGuard,
And(StartsWith(input, Literal.create(prefix, input.dataType)),
EndsWith(input, Literal.create(postfix, input.dataType)))))
case contains(infix) =>
Some(Contains(input, Literal.create(infix, input.dataType)))
case equalTo(str) =>
Some(EqualTo(input, Literal.create(str, input.dataType)))
Some(startsAndEndsWithExpr(input, prefix, postfix))
case contains(infix) => Some(containsExpr(input, infix))
case equalTo(str) => Some(equalToExpr(input, str))
case _ => None
}
} else if (escapeChar == '%' || escapeChar == '_') {
// Pathological: the escape character is itself a wildcard character, so `%`/`_` no longer
// act as wildcards and the shape detection does not apply. Skip, as before.
None
} else {
// The pattern uses escape sequences: decode them, then classify the decoded shape. The
// decoded literals are exact, so the resulting predicate accepts the same rows as the LIKE.
decodeLikePattern(pattern, escapeChar).flatMap {
case Seq(LikeLiteral(str)) => Some(equalToExpr(input, str))
case Seq(LikeLiteral(prefix), LikeWildcard) => Some(startsWithExpr(input, prefix))
case Seq(LikeWildcard, LikeLiteral(postfix)) => Some(endsWithExpr(input, postfix))
case Seq(LikeLiteral(prefix), LikeWildcard, LikeLiteral(postfix)) =>
Some(startsAndEndsWithExpr(input, prefix, postfix))
case Seq(LikeWildcard, LikeLiteral(infix), LikeWildcard) =>
Some(containsExpr(input, infix))
case _ => None
}
}
Expand Down
100 changes: 77 additions & 23 deletions ...lyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LikeSimplificationSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ class LikeSimplificationSuite extends PlanTest {

val optimized = Optimize.execute(originalQuery.analyze)
val correctAnswer = testRelation
.where(StartsWith($"a", "abc") || ($"a" like "abc\\%"))
.where(StartsWith($"a", "abc") || ($"a" === "abc%"))
.analyze

comparePlans(optimized, correctAnswer)
Expand All @@ -70,7 +70,7 @@ class LikeSimplificationSuite extends PlanTest {

val optimized = Optimize.execute(originalQuery.analyze)
val correctAnswer = testRelation
.where(($"a" like "abc\\%def") ||
.where(($"a" === "abc%def") ||
(OctetLength($"a") >= 6 && (StartsWith($"a", "abc") && EndsWith($"a", "def"))))
.analyze

Expand All @@ -84,7 +84,7 @@ class LikeSimplificationSuite extends PlanTest {

val optimized = Optimize.execute(originalQuery.analyze)
val correctAnswer = testRelation
.where(Contains($"a", "mn") || ($"a" like "%mn\\%"))
.where(Contains($"a", "mn") || EndsWith($"a", "mn%"))
.analyze

comparePlans(optimized, correctAnswer)
Expand All @@ -110,16 +110,63 @@ class LikeSimplificationSuite extends PlanTest {
}

test("test like escape syntax") {
// '#%' with escape '#' is a literal '%', so these decode to exact-match EqualTo.
val originalQuery1 = testRelation.where($"a".like("abc#%", '#'))
val optimized1 = Optimize.execute(originalQuery1.analyze)
comparePlans(optimized1, originalQuery1.analyze)
comparePlans(optimized1, testRelation.where($"a" === "abc%").analyze)

val originalQuery2 = testRelation.where($"a".like("abc#%abc", '#'))
val optimized2 = Optimize.execute(originalQuery2.analyze)
comparePlans(optimized2, originalQuery2.analyze)
comparePlans(optimized2, testRelation.where($"a" === "abc%abc").analyze)
}

test("SPARK-33677: LikeSimplification should be skipped if pattern contains any escapeChar") {
test("simplify LIKE patterns with escaped wildcards") {
// Escaped wildcard/escape characters decode to literals, so escaped patterns simplify too.
// EqualTo (no unescaped wildcard):
comparePlans(
Optimize.execute(testRelation.where($"a" like "ma\\%ca").analyze),
testRelation.where($"a" === "ma%ca").analyze)
comparePlans(
Optimize.execute(testRelation.where($"a" like "ma\\\\ca").analyze),
testRelation.where($"a" === "ma\\ca").analyze)
comparePlans(
Optimize.execute(testRelation.where($"a" like "a\\_b").analyze),
testRelation.where($"a" === "a_b").analyze)
// StartsWith / EndsWith / Contains with an escaped literal in the fixed part:
comparePlans(
Optimize.execute(testRelation.where($"a" like "abc\\%def%").analyze),
testRelation.where(StartsWith($"a", "abc%def")).analyze)
comparePlans(
Optimize.execute(testRelation.where($"a" like "%xyz\\%").analyze),
testRelation.where(EndsWith($"a", "xyz%")).analyze)
comparePlans(
Optimize.execute(testRelation.where($"a" like "%a\\%b%").analyze),
testRelation.where(Contains($"a", "a%b")).analyze)
// startsAndEndsWith with escaped literals on both sides ("a%b" and "c_d", 3 bytes each):
comparePlans(
Optimize.execute(testRelation.where($"a" like "a\\%b%c\\_d").analyze),
testRelation.where(OctetLength($"a") >= 6 &&
(StartsWith($"a", "a%b") && EndsWith($"a", "c_d"))).analyze)
}

test("do not simplify LIKE with invalid or pathological escapes") {
// Invalid escape (escape char not followed by %, _, or itself) -> kept (LIKE errors at eval).
val invalid = testRelation.where($"a" like "m\\aca").analyze
comparePlans(Optimize.execute(invalid), invalid)
// Trailing escape -> kept.
val trailing = testRelation.where($"a" like "abc\\").analyze
comparePlans(Optimize.execute(trailing), trailing)
// Escape char is itself a wildcard character -> kept.
val escPercent = testRelation.where($"a".like("a%%b", '%')).analyze
comparePlans(Optimize.execute(escPercent), escPercent)
val escUnderscore = testRelation.where($"a".like("a__b", '_')).analyze
comparePlans(Optimize.execute(escUnderscore), escUnderscore)
// Unescaped '_' single-char wildcard -> not handled by this rule.
val underscore = testRelation.where($"a" like "a_b").analyze
comparePlans(Optimize.execute(underscore), underscore)
}

test("SPARK-33677: LikeSimplification skips invalid/pathological escapes, simplifies valid") {
val originalQuery1 =
testRelation
.where(($"a" like "abc%") || ($"a" like "\\abc%"))
Expand Down Expand Up @@ -157,12 +204,14 @@ class LikeSimplificationSuite extends PlanTest {
.analyze
comparePlans(optimized4, correctAnswer4)

// 'abbc' with escape 'b': the 'b' escapes the next 'b' (a valid escaped-escape), so this is
// the exact literal "abc" and is now simplified rather than skipped.
val originalQuery5 =
testRelation
.where(($"a" like "abc") || ($"a" like ("abbc", 'b')))
val optimized5 = Optimize.execute(originalQuery5.analyze)
val correctAnswer5 = testRelation
.where(($"a" === "abc") || ($"a" like ("abbc", 'b')))
.where(($"a" === "abc") || ($"a" === "abc"))
.analyze
comparePlans(optimized5, correctAnswer5)
}
Expand Down Expand Up @@ -223,10 +272,10 @@ class LikeSimplificationSuite extends PlanTest {

val optimized = Optimize.execute(originalQuery.analyze)
val correctAnswer = testRelation
.where((((((StartsWith($"a", "abc") && EndsWith($"a", "xyz")) &&
(OctetLength($"a") >= 6 && (StartsWith($"a", "abc") && EndsWith($"a", "def")))) &&
Contains($"a", "mn")) && ($"a" === "")) && ($"a" === "abc")) &&
($"a" likeAll("abc\\%", "abc\\%def", "%mn\\%")))
.where(StartsWith($"a", "abc") && ($"a" === "abc%") && EndsWith($"a", "xyz") &&
($"a" === "abc%def") &&
(OctetLength($"a") >= 6 && (StartsWith($"a", "abc") && EndsWith($"a", "def"))) &&
Contains($"a", "mn") && EndsWith($"a", "mn%") && ($"a" === "") && ($"a" === "abc"))
.analyze

comparePlans(optimized, correctAnswer)
Expand All @@ -240,10 +289,11 @@ class LikeSimplificationSuite extends PlanTest {

val optimized = Optimize.execute(originalQuery.analyze)
val correctAnswer = testRelation
.where((((((Not(StartsWith($"a", "abc")) && Not(EndsWith($"a", "xyz"))) &&
Not(OctetLength($"a") >= 6 && (StartsWith($"a", "abc") && EndsWith($"a", "def")))) &&
Not(Contains($"a", "mn"))) && Not($"a" === "")) && Not($"a" === "abc")) &&
($"a" notLikeAll("abc\\%", "abc\\%def", "%mn\\%")))
.where(Not(StartsWith($"a", "abc")) && Not($"a" === "abc%") && Not(EndsWith($"a", "xyz")) &&
Not($"a" === "abc%def") &&
Not(OctetLength($"a") >= 6 && (StartsWith($"a", "abc") && EndsWith($"a", "def"))) &&
Not(Contains($"a", "mn")) && Not(EndsWith($"a", "mn%")) && Not($"a" === "") &&
Not($"a" === "abc"))
.analyze

comparePlans(optimized, correctAnswer)
Expand All @@ -257,10 +307,12 @@ class LikeSimplificationSuite extends PlanTest {

val optimized = Optimize.execute(originalQuery.analyze)
val correctAnswer = testRelation
.where(((StartsWith($"a", "abc") || EndsWith($"a", "xyz")) ||
(OctetLength($"a") >= 6 && (StartsWith($"a", "abc") && EndsWith($"a", "def")) ||
Contains($"a", "mn")) || (($"a" === "") || ($"a" === "abc")) ||
($"a" likeAny("abc\\%", "abc\\%def", "%mn\\%"))))
.where(
(((StartsWith($"a", "abc") || ($"a" === "abc%")) ||
(EndsWith($"a", "xyz") || ($"a" === "abc%def"))) ||
(((OctetLength($"a") >= 6 && (StartsWith($"a", "abc") && EndsWith($"a", "def"))) ||
Contains($"a", "mn")) || (EndsWith($"a", "mn%") || ($"a" === "")))) ||
($"a" === "abc"))
.analyze

comparePlans(optimized, correctAnswer)
Expand All @@ -274,10 +326,12 @@ class LikeSimplificationSuite extends PlanTest {

val optimized = Optimize.execute(originalQuery.analyze)
val correctAnswer = testRelation
.where((((Not(StartsWith($"a", "abc")) || Not(EndsWith($"a", "xyz"))) ||
(Not(OctetLength($"a") >= 6 && (StartsWith($"a", "abc") && EndsWith($"a", "def"))) ||
Not(Contains($"a", "mn")))) || (Not($"a" === "") || Not($"a" === "abc"))) ||
($"a" notLikeAny("abc\\%", "abc\\%def", "%mn\\%")))
.where(
(((Not(StartsWith($"a", "abc")) || Not($"a" === "abc%")) ||
(Not(EndsWith($"a", "xyz")) || Not($"a" === "abc%def"))) ||
((Not(OctetLength($"a") >= 6 && (StartsWith($"a", "abc") && EndsWith($"a", "def"))) ||
Not(Contains($"a", "mn"))) || (Not(EndsWith($"a", "mn%")) || Not($"a" === "")))) ||
Not($"a" === "abc"))
.analyze

comparePlans(optimized, correctAnswer)
Expand Down