diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala index cdc0444d74cc1..a05a93e2dd8fd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala @@ -804,60 +804,126 @@ object SupportedBinaryExpr { * pattern. */ object LikeSimplification extends Rule[LogicalPlan] with PredicateHelper { - // if guards below protect from escapes on trailing %. - // Cases like "something\%" are not optimized, but this does not affect correctness. - // Consecutive wildcard characters are equivalent to a single wildcard character. + // These regexes classify a pattern that has no escape character (`%`/`_` are the only + // wildcards). Patterns that do contain the escape character are decoded by `decodeLikePattern` + // first. Consecutive wildcard characters are equivalent to a single wildcard character. private val startsWith = "([^_%]+)%+".r private val endsWith = "%+([^_%]+)".r private val startsAndEndsWith = "([^_%]+)%+([^_%]+)".r private val contains = "%+([^_%]+)%+".r private val equalTo = "([^_%]*)".r + private def startsWithExpr(input: Expression, prefix: String): Expression = + StartsWith(input, Literal.create(prefix, input.dataType)) + + private def endsWithExpr(input: Expression, postfix: String): Expression = + EndsWith(input, Literal.create(postfix, input.dataType)) + + private def containsExpr(input: Expression, infix: String): Expression = + Contains(input, Literal.create(infix, input.dataType)) + + private def equalToExpr(input: Expression, str: String): Expression = + EqualTo(input, Literal.create(str, input.dataType)) + + // 'a%a' is basically 'a%' && '%a', but a length guard is required to stop 'a' matching 'a%a'. + // When the collation matches raw bytes (supportsBinaryEquality), StartsWith/EndsWith pin the + // literal bytes of the prefix and suffix, so a byte-length guard (OctetLength, O(1) via the + // stored numBytes) accepts exactly the same inputs as the code-point guard and is cheaper. + // Otherwise the anchors are collation-aware (LIKE reaches this only for UTF8_LCASE) and can + // match a code point whose UTF-8 length differs from the pattern's -- a single multibyte code + // point could then satisfy both anchors and clear the byte guard -- so the code-point (Length) + // guard must be kept for correctness. + private def startsAndEndsWithExpr( + input: Expression, prefix: String, postfix: String): Expression = { + val lengthGuard = input.dataType match { + case st: StringType if st.supportsBinaryEquality => + GreaterThanOrEqual(OctetLength(input), + Literal.create(UTF8String.fromString(prefix).numBytes + + UTF8String.fromString(postfix).numBytes)) + case _ => + GreaterThanOrEqual(Length(input), + Literal.create(prefix.codePointCount(0, prefix.length) + + postfix.codePointCount(0, postfix.length))) + } + And(lengthGuard, And(startsWithExpr(input, prefix), endsWithExpr(input, postfix))) + } + + // A decoded LIKE pattern token: a maximal run of literal characters, or a `%` wildcard + // (consecutive `%` collapse into one). + private sealed trait LikeToken + private case class LikeLiteral(str: String) extends LikeToken + private case object LikeWildcard extends LikeToken + + // Decodes a `pattern` that contains the escape character into literal/wildcard tokens, or None + // if it cannot be simplified: an invalid escape (the escape char not followed by `%`, `_`, or + // the escape char) or a trailing escape char -- both make LIKE throw at runtime, so the rule + // must leave them alone -- or an unescaped `_` wildcard, which this rule does not handle. + private def decodeLikePattern(pattern: String, escapeChar: Char): Option[Seq[LikeToken]] = { + val tokens = ArrayBuffer.empty[LikeToken] + val literal = new StringBuilder + def flushLiteral(): Unit = if (literal.length > 0) { + tokens += LikeLiteral(literal.toString) + literal.setLength(0) + } + var i = 0 + while (i < pattern.length) { + val c = pattern.charAt(i) + if (c == escapeChar) { + if (i + 1 >= pattern.length) { + return None // trailing escape character + } + val next = pattern.charAt(i + 1) + if (next == '%' || next == '_' || next == escapeChar) { + literal.append(next) + i += 2 + } else { + return None // invalid escape sequence + } + } else if (c == '%') { + flushLiteral() + if (tokens.isEmpty || tokens.last != LikeWildcard) { + tokens += LikeWildcard + } + i += 1 + } else if (c == '_') { + return None // unescaped single-character wildcard, not handled here + } else { + literal.append(c) + i += 1 + } + } + flushLiteral() + Some(tokens.toSeq) + } + private def simplifyLike( input: Expression, pattern: String, escapeChar: Char = '\\'): Option[Expression] = { - if (pattern.contains(escapeChar)) { - // There are three different situations when pattern containing escapeChar: - // 1. pattern contains invalid escape sequence, e.g. 'm\aca' - // 2. pattern contains escaped wildcard character, e.g. 'ma\%ca' - // 3. pattern contains escaped escape character, e.g. 'ma\\ca' - // Although there are patterns can be optimized if we handle the escape first, we just - // skip this rule if pattern contains any escapeChar for simplicity. - None - } else { + if (!pattern.contains(escapeChar)) { + // Fast path: no escape character, so `%`/`_` are the only wildcards. pattern match { - case startsWith(prefix) => - Some(StartsWith(input, Literal.create(prefix, input.dataType))) - case endsWith(postfix) => - Some(EndsWith(input, Literal.create(postfix, input.dataType))) - // 'a%a' pattern is basically same with 'a%' && '%a'. - // However, the additional length condition is required to prevent 'a' match 'a%a'. + case startsWith(prefix) => Some(startsWithExpr(input, prefix)) + case endsWith(postfix) => Some(endsWithExpr(input, postfix)) case startsAndEndsWith(prefix, postfix) => - // The length guard only rejects inputs too short to hold both the prefix and the - // suffix. When the collation matches raw bytes (supportsBinaryEquality), - // StartsWith/EndsWith pin the literal bytes of the prefix and suffix, so a - // byte-length guard (OctetLength, O(1) via the stored numBytes) accepts exactly the - // same inputs as the code-point guard and is cheaper. Otherwise the anchors are - // collation-aware (LIKE reaches this only for UTF8_LCASE) and can match a code point - // whose UTF-8 length differs from the pattern's -- a single multibyte code point - // could then satisfy both anchors and clear the byte guard -- so the code-point - // (Length) guard must be kept for correctness. - val lengthGuard = input.dataType match { - case st: StringType if st.supportsBinaryEquality => - GreaterThanOrEqual(OctetLength(input), - Literal.create(UTF8String.fromString(prefix).numBytes - + UTF8String.fromString(postfix).numBytes)) - case _ => - GreaterThanOrEqual(Length(input), - Literal.create(prefix.codePointCount(0, prefix.length) - + postfix.codePointCount(0, postfix.length))) - } - Some(And(lengthGuard, - And(StartsWith(input, Literal.create(prefix, input.dataType)), - EndsWith(input, Literal.create(postfix, input.dataType))))) - case contains(infix) => - Some(Contains(input, Literal.create(infix, input.dataType))) - case equalTo(str) => - Some(EqualTo(input, Literal.create(str, input.dataType))) + Some(startsAndEndsWithExpr(input, prefix, postfix)) + case contains(infix) => Some(containsExpr(input, infix)) + case equalTo(str) => Some(equalToExpr(input, str)) + case _ => None + } + } else if (escapeChar == '%' || escapeChar == '_') { + // Pathological: the escape character is itself a wildcard character, so `%`/`_` no longer + // act as wildcards and the shape detection does not apply. Skip, as before. + None + } else { + // The pattern uses escape sequences: decode them, then classify the decoded shape. The + // decoded literals are exact, so the resulting predicate accepts the same rows as the LIKE. + decodeLikePattern(pattern, escapeChar).flatMap { + case Seq(LikeLiteral(str)) => Some(equalToExpr(input, str)) + case Seq(LikeLiteral(prefix), LikeWildcard) => Some(startsWithExpr(input, prefix)) + case Seq(LikeWildcard, LikeLiteral(postfix)) => Some(endsWithExpr(input, postfix)) + case Seq(LikeLiteral(prefix), LikeWildcard, LikeLiteral(postfix)) => + Some(startsAndEndsWithExpr(input, prefix, postfix)) + case Seq(LikeWildcard, LikeLiteral(infix), LikeWildcard) => + Some(containsExpr(input, infix)) case _ => None } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LikeSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LikeSimplificationSuite.scala index 8b142f0c53d75..e0eb41fcd0b3c 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LikeSimplificationSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LikeSimplificationSuite.scala @@ -44,7 +44,7 @@ class LikeSimplificationSuite extends PlanTest { val optimized = Optimize.execute(originalQuery.analyze) val correctAnswer = testRelation - .where(StartsWith($"a", "abc") || ($"a" like "abc\\%")) + .where(StartsWith($"a", "abc") || ($"a" === "abc%")) .analyze comparePlans(optimized, correctAnswer) @@ -70,7 +70,7 @@ class LikeSimplificationSuite extends PlanTest { val optimized = Optimize.execute(originalQuery.analyze) val correctAnswer = testRelation - .where(($"a" like "abc\\%def") || + .where(($"a" === "abc%def") || (OctetLength($"a") >= 6 && (StartsWith($"a", "abc") && EndsWith($"a", "def")))) .analyze @@ -84,7 +84,7 @@ class LikeSimplificationSuite extends PlanTest { val optimized = Optimize.execute(originalQuery.analyze) val correctAnswer = testRelation - .where(Contains($"a", "mn") || ($"a" like "%mn\\%")) + .where(Contains($"a", "mn") || EndsWith($"a", "mn%")) .analyze comparePlans(optimized, correctAnswer) @@ -110,16 +110,63 @@ class LikeSimplificationSuite extends PlanTest { } test("test like escape syntax") { + // '#%' with escape '#' is a literal '%', so these decode to exact-match EqualTo. val originalQuery1 = testRelation.where($"a".like("abc#%", '#')) val optimized1 = Optimize.execute(originalQuery1.analyze) - comparePlans(optimized1, originalQuery1.analyze) + comparePlans(optimized1, testRelation.where($"a" === "abc%").analyze) val originalQuery2 = testRelation.where($"a".like("abc#%abc", '#')) val optimized2 = Optimize.execute(originalQuery2.analyze) - comparePlans(optimized2, originalQuery2.analyze) + comparePlans(optimized2, testRelation.where($"a" === "abc%abc").analyze) } - test("SPARK-33677: LikeSimplification should be skipped if pattern contains any escapeChar") { + test("simplify LIKE patterns with escaped wildcards") { + // Escaped wildcard/escape characters decode to literals, so escaped patterns simplify too. + // EqualTo (no unescaped wildcard): + comparePlans( + Optimize.execute(testRelation.where($"a" like "ma\\%ca").analyze), + testRelation.where($"a" === "ma%ca").analyze) + comparePlans( + Optimize.execute(testRelation.where($"a" like "ma\\\\ca").analyze), + testRelation.where($"a" === "ma\\ca").analyze) + comparePlans( + Optimize.execute(testRelation.where($"a" like "a\\_b").analyze), + testRelation.where($"a" === "a_b").analyze) + // StartsWith / EndsWith / Contains with an escaped literal in the fixed part: + comparePlans( + Optimize.execute(testRelation.where($"a" like "abc\\%def%").analyze), + testRelation.where(StartsWith($"a", "abc%def")).analyze) + comparePlans( + Optimize.execute(testRelation.where($"a" like "%xyz\\%").analyze), + testRelation.where(EndsWith($"a", "xyz%")).analyze) + comparePlans( + Optimize.execute(testRelation.where($"a" like "%a\\%b%").analyze), + testRelation.where(Contains($"a", "a%b")).analyze) + // startsAndEndsWith with escaped literals on both sides ("a%b" and "c_d", 3 bytes each): + comparePlans( + Optimize.execute(testRelation.where($"a" like "a\\%b%c\\_d").analyze), + testRelation.where(OctetLength($"a") >= 6 && + (StartsWith($"a", "a%b") && EndsWith($"a", "c_d"))).analyze) + } + + test("do not simplify LIKE with invalid or pathological escapes") { + // Invalid escape (escape char not followed by %, _, or itself) -> kept (LIKE errors at eval). + val invalid = testRelation.where($"a" like "m\\aca").analyze + comparePlans(Optimize.execute(invalid), invalid) + // Trailing escape -> kept. + val trailing = testRelation.where($"a" like "abc\\").analyze + comparePlans(Optimize.execute(trailing), trailing) + // Escape char is itself a wildcard character -> kept. + val escPercent = testRelation.where($"a".like("a%%b", '%')).analyze + comparePlans(Optimize.execute(escPercent), escPercent) + val escUnderscore = testRelation.where($"a".like("a__b", '_')).analyze + comparePlans(Optimize.execute(escUnderscore), escUnderscore) + // Unescaped '_' single-char wildcard -> not handled by this rule. + val underscore = testRelation.where($"a" like "a_b").analyze + comparePlans(Optimize.execute(underscore), underscore) + } + + test("SPARK-33677: LikeSimplification skips invalid/pathological escapes, simplifies valid") { val originalQuery1 = testRelation .where(($"a" like "abc%") || ($"a" like "\\abc%")) @@ -157,12 +204,14 @@ class LikeSimplificationSuite extends PlanTest { .analyze comparePlans(optimized4, correctAnswer4) + // 'abbc' with escape 'b': the 'b' escapes the next 'b' (a valid escaped-escape), so this is + // the exact literal "abc" and is now simplified rather than skipped. val originalQuery5 = testRelation .where(($"a" like "abc") || ($"a" like ("abbc", 'b'))) val optimized5 = Optimize.execute(originalQuery5.analyze) val correctAnswer5 = testRelation - .where(($"a" === "abc") || ($"a" like ("abbc", 'b'))) + .where(($"a" === "abc") || ($"a" === "abc")) .analyze comparePlans(optimized5, correctAnswer5) } @@ -223,10 +272,10 @@ class LikeSimplificationSuite extends PlanTest { val optimized = Optimize.execute(originalQuery.analyze) val correctAnswer = testRelation - .where((((((StartsWith($"a", "abc") && EndsWith($"a", "xyz")) && - (OctetLength($"a") >= 6 && (StartsWith($"a", "abc") && EndsWith($"a", "def")))) && - Contains($"a", "mn")) && ($"a" === "")) && ($"a" === "abc")) && - ($"a" likeAll("abc\\%", "abc\\%def", "%mn\\%"))) + .where(StartsWith($"a", "abc") && ($"a" === "abc%") && EndsWith($"a", "xyz") && + ($"a" === "abc%def") && + (OctetLength($"a") >= 6 && (StartsWith($"a", "abc") && EndsWith($"a", "def"))) && + Contains($"a", "mn") && EndsWith($"a", "mn%") && ($"a" === "") && ($"a" === "abc")) .analyze comparePlans(optimized, correctAnswer) @@ -240,10 +289,11 @@ class LikeSimplificationSuite extends PlanTest { val optimized = Optimize.execute(originalQuery.analyze) val correctAnswer = testRelation - .where((((((Not(StartsWith($"a", "abc")) && Not(EndsWith($"a", "xyz"))) && - Not(OctetLength($"a") >= 6 && (StartsWith($"a", "abc") && EndsWith($"a", "def")))) && - Not(Contains($"a", "mn"))) && Not($"a" === "")) && Not($"a" === "abc")) && - ($"a" notLikeAll("abc\\%", "abc\\%def", "%mn\\%"))) + .where(Not(StartsWith($"a", "abc")) && Not($"a" === "abc%") && Not(EndsWith($"a", "xyz")) && + Not($"a" === "abc%def") && + Not(OctetLength($"a") >= 6 && (StartsWith($"a", "abc") && EndsWith($"a", "def"))) && + Not(Contains($"a", "mn")) && Not(EndsWith($"a", "mn%")) && Not($"a" === "") && + Not($"a" === "abc")) .analyze comparePlans(optimized, correctAnswer) @@ -257,10 +307,12 @@ class LikeSimplificationSuite extends PlanTest { val optimized = Optimize.execute(originalQuery.analyze) val correctAnswer = testRelation - .where(((StartsWith($"a", "abc") || EndsWith($"a", "xyz")) || - (OctetLength($"a") >= 6 && (StartsWith($"a", "abc") && EndsWith($"a", "def")) || - Contains($"a", "mn")) || (($"a" === "") || ($"a" === "abc")) || - ($"a" likeAny("abc\\%", "abc\\%def", "%mn\\%")))) + .where( + (((StartsWith($"a", "abc") || ($"a" === "abc%")) || + (EndsWith($"a", "xyz") || ($"a" === "abc%def"))) || + (((OctetLength($"a") >= 6 && (StartsWith($"a", "abc") && EndsWith($"a", "def"))) || + Contains($"a", "mn")) || (EndsWith($"a", "mn%") || ($"a" === "")))) || + ($"a" === "abc")) .analyze comparePlans(optimized, correctAnswer) @@ -274,10 +326,12 @@ class LikeSimplificationSuite extends PlanTest { val optimized = Optimize.execute(originalQuery.analyze) val correctAnswer = testRelation - .where((((Not(StartsWith($"a", "abc")) || Not(EndsWith($"a", "xyz"))) || - (Not(OctetLength($"a") >= 6 && (StartsWith($"a", "abc") && EndsWith($"a", "def"))) || - Not(Contains($"a", "mn")))) || (Not($"a" === "") || Not($"a" === "abc"))) || - ($"a" notLikeAny("abc\\%", "abc\\%def", "%mn\\%"))) + .where( + (((Not(StartsWith($"a", "abc")) || Not($"a" === "abc%")) || + (Not(EndsWith($"a", "xyz")) || Not($"a" === "abc%def"))) || + ((Not(OctetLength($"a") >= 6 && (StartsWith($"a", "abc") && EndsWith($"a", "def"))) || + Not(Contains($"a", "mn"))) || (Not(EndsWith($"a", "mn%")) || Not($"a" === "")))) || + Not($"a" === "abc")) .analyze comparePlans(optimized, correctAnswer)