From 7e9e58e9ed3563fc492e60bdac6ba13a644bf3bc Mon Sep 17 00:00:00 2001 From: Mark Andreev Date: Wed, 26 Aug 2026 22:09:14 +0100 Subject: [PATCH] [SPARK-58980][SQL] Support GROUPS window frames --- .../resources/error/error-conditions.json | 15 + docs/sql-ref-ansi-compliance.md | 1 + docs/sql-ref-syntax-qry-select-window.md | 28 +- .../spark/sql/catalyst/parser/SqlBaseLexer.g4 | 1 + .../sql/catalyst/parser/SqlBaseParser.g4 | 4 + .../expressions/windowExpressions.scala | 42 + .../sql/catalyst/parser/AstBuilder.scala | 10 +- .../parser/ExpressionParserSuite.scala | 73 ++ .../sql/catalyst/parser/PlanParserSuite.scala | 17 + .../sql/execution/window/BoundOrdering.scala | 121 ++- .../SegmentTreeWindowFunctionFrame.scala | 40 +- .../window/WindowEvaluatorFactoryBase.scala | 90 +- .../window/WindowFunctionFrame.scala | 8 + .../postgreSQL/window_part1.sql.out | 28 + .../postgreSQL/window_part3.sql.out | 304 +++++++ .../analyzer-results/window-groups.sql.out | 318 +++++++ .../inputs/postgreSQL/window_part1.sql | 7 +- .../inputs/postgreSQL/window_part3.sql | 153 ++-- .../sql-tests/inputs/window-groups.sql | 91 ++ .../results/keywords-enforced.sql.out | 1 + .../sql-tests/results/keywords.sql.out | 1 + .../results/nonansi/keywords.sql.out | 1 + .../results/postgreSQL/window_part1.sql.out | 26 + .../results/postgreSQL/window_part3.sql.out | 377 ++++++++ .../sql-tests/results/window-groups.sql.out | 262 ++++++ .../sql/DataFrameWindowFramesSuite.scala | 825 +++++++++++++++++- .../execution/window/BoundOrderingSuite.scala | 228 +++++ .../SegmentTreeWindowFunctionSuite.scala | 155 +++- .../SegmentTreeWindowMetricsSuite.scala | 54 ++ .../UnboundedFollowingSegmentTreeSuite.scala | 75 ++ .../spark/sql/hive/HiveSparkSubmitSuite.scala | 79 +- 31 files changed, 3298 insertions(+), 137 deletions(-) create mode 100644 sql/core/src/test/resources/sql-tests/analyzer-results/window-groups.sql.out create mode 100644 sql/core/src/test/resources/sql-tests/inputs/window-groups.sql create mode 100644 sql/core/src/test/resources/sql-tests/results/window-groups.sql.out create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/window/BoundOrderingSuite.scala diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index c642e0be953aa..47f3f4b1530e6 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -1953,6 +1953,21 @@ "Filter expression of type is not a boolean." ] }, + "GROUPS_FRAME_NEGATIVE_OFFSET" : { + "message" : [ + "The offset of a groups frame must be non-negative." + ] + }, + "GROUPS_FRAME_NULL_OFFSET" : { + "message" : [ + "The bound of a groups frame must not be null." + ] + }, + "GROUPS_FRAME_WITHOUT_ORDER" : { + "message" : [ + "A groups window frame cannot be used in an unordered window specification." + ] + }, "HASH_MAP_TYPE" : { "message" : [ "Input to the function cannot contain elements of the \"MAP\" type. In Spark, same maps may have different hashcode, thus hash expressions are prohibited on \"MAP\" elements. To restore previous behavior set \"spark.sql.legacy.allowHashOnMapType\" to \"true\"." diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md index 5ab05fd026f75..7b32a3d58a7e2 100644 --- a/docs/sql-ref-ansi-compliance.md +++ b/docs/sql-ref-ansi-compliance.md @@ -584,6 +584,7 @@ Below is a list of all the keywords in Spark SQL. |GRANT|reserved|non-reserved|reserved| |GROUP|reserved|non-reserved|reserved| |GROUPING|non-reserved|non-reserved|reserved| +|GROUPS|non-reserved|non-reserved|reserved| |HANDLER|non-reserved|non-reserved|non-reserved| |HAVING|reserved|non-reserved|reserved| |HISTORY|non-reserved|non-reserved|non-reserved| diff --git a/docs/sql-ref-syntax-qry-select-window.md b/docs/sql-ref-syntax-qry-select-window.md index 3615252895592..d5ceeed64c532 100644 --- a/docs/sql-ref-syntax-qry-select-window.md +++ b/docs/sql-ref-syntax-qry-select-window.md @@ -66,7 +66,7 @@ window_function [ nulls_option ] OVER **Syntax:** - `{ RANGE | ROWS } { frame_start | BETWEEN frame_start AND frame_end }` + `{ RANGE | ROWS | GROUPS } { frame_start | BETWEEN frame_start AND frame_end }` * `frame_start` and `frame_end` have the following syntax: @@ -78,6 +78,13 @@ window_function [ nulls_option ] OVER **Note:** If `frame_end` is omitted it defaults to `CURRENT ROW`. + **Note:** `GROUPS` offsets count peer groups: rows with equal values for all window + `ORDER BY` expressions within a partition. `GROUPS` requires `ORDER BY` and supports multiple + ordering expressions. `CURRENT ROW` starts at the first row of the current peer group when + used as a frame start, and ends at its last row when used as a frame end. + `0 PRECEDING` and `0 FOLLOWING` are equivalent to `CURRENT ROW`. Offsets must be constant, + non-null, non-negative integer expressions. + ### Examples ```sql @@ -217,6 +224,25 @@ SELECT id, v, | 7| v| v| v| y| x| v| | 8|NULL|NULL|NULL| y| x| v| +--+----+----+----+---------+-----------+----------+ + +CREATE TABLE batches (batch_id INT, amount INT); + +INSERT INTO batches VALUES (1, 10), (1, 15), (2, 20), (3, 25), (3, 30), (9, 40); + +SELECT batch_id, amount, + SUM(amount) OVER (ORDER BY batch_id GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) AS moving_sum + FROM batches + ORDER BY batch_id, amount; ++--------+------+----------+ +|batch_id|amount|moving_sum| ++--------+------+----------+ +| 1| 10| 25| +| 1| 15| 25| +| 2| 20| 45| +| 3| 25| 75| +| 3| 30| 75| +| 9| 40| 95| ++--------+------+----------+ ``` ### Related Statements diff --git a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 index dcccba9edc041..c055a3deb4423 100644 --- a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 +++ b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 @@ -300,6 +300,7 @@ GLOBAL: 'GLOBAL'; GRANT: 'GRANT'; GROUP: 'GROUP'; GROUPING: 'GROUPING'; +GROUPS: 'GROUPS'; HANDLER: 'HANDLER'; HAVING: 'HAVING'; BINARY_HEX: 'X'; diff --git a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 index 3003182471884..039b2fe6a1dcf 100644 --- a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 +++ b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 @@ -1866,8 +1866,10 @@ windowSpec windowFrame : frameType=RANGE start=frameBound | frameType=ROWS start=frameBound + | frameType=GROUPS start=frameBound | frameType=RANGE BETWEEN start=frameBound AND end=frameBound | frameType=ROWS BETWEEN start=frameBound AND end=frameBound + | frameType=GROUPS BETWEEN start=frameBound AND end=frameBound ; frameBound @@ -2257,6 +2259,7 @@ ansiNonReserved | GEOMETRY | GLOBAL | GROUPING + | GROUPS | HANDLER | HISTORY | HOUR @@ -2701,6 +2704,7 @@ nonReserved | GRANT | GROUP | GROUPING + | GROUPS | HANDLER | HAVING | HISTORY diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala index e6c52c08e3b1f..f7d060102eaac 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala @@ -91,6 +91,8 @@ case class WindowSpecDefinition( "valueBoundaryType" -> toSQLType(f.valueBoundary.head.dataType) ) ) + case f: SpecifiedWindowFrame if f.frameType == GroupFrame && orderSpec.isEmpty => + DataTypeMismatch(errorSubClass = "GROUPS_FRAME_WITHOUT_ORDER") case _ => TypeCheckSuccess } } @@ -159,6 +161,39 @@ case object RangeFrame extends FrameType { override def sql: String = "RANGE" } +/** + * A frame whose integral offsets count peer groups in the window ordering. + */ +case object GroupFrame extends FrameType { + override def inputType: AbstractDataType = IntegerType + override def sql: String = "GROUPS" +} + +/** + * Validates a SQL GROUPS offset before PRECEDING negates it. Keep the original magnitude through + * analysis so parameter binding cannot turn a negative offset into a valid signed boundary. + */ +case class GroupFrameOffset(child: Expression) + extends RuntimeReplaceable with UnaryLike[Expression] { + override def replacement: Expression = child + override def foldable: Boolean = child.foldable + override def sql: String = child.sql + // Frame descriptions should display the offset, not its analysis-only validation wrapper. + override def toString: String = child.toString + + override def checkInputDataTypes(): TypeCheckResult = { + if (child.foldable && child.dataType == IntegerType && + Option(child.eval()).exists(_.asInstanceOf[Int] < 0)) { + DataTypeMismatch(errorSubClass = "GROUPS_FRAME_NEGATIVE_OFFSET") + } else { + TypeCheckSuccess + } + } + + override protected def withNewChildInternal(newChild: Expression): GroupFrameOffset = + copy(child = newChild) +} + /** * The trait used to represent special boundaries used in a window frame. */ @@ -272,6 +307,8 @@ case class SpecifiedWindowFrame( private def boundarySql(expr: Expression): String = expr match { case e: SpecialFrameBoundary => e.sql case UnaryMinus(n, _) => n.sql + " PRECEDING" + case IntegerLiteral(offset) if frameType == GroupFrame && offset < 0 => + s"${-offset} PRECEDING" case e: Expression => e.sql + " FOLLOWING" } @@ -302,6 +339,11 @@ case class SpecifiedWindowFrame( "expectedType" -> toSQLType(frameType.inputType) ) ) + case e: Expression if frameType == GroupFrame && e.eval() == null => + DataTypeMismatch( + errorSubClass = "GROUPS_FRAME_NULL_OFFSET", + messageParameters = Map("location" -> location) + ) case _ => TypeCheckSuccess } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index d65a9c7a36442..f4b8653cb0fc3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -4384,11 +4384,12 @@ class AstBuilder extends DataTypeAstBuilder val partition = ctx.partition.asScala.map(expression) val order = ctx.sortItem.asScala.map(visitSortItem) - // RANGE/ROWS BETWEEN ... + // RANGE/ROWS/GROUPS BETWEEN ... val frameSpecOption = Option(ctx.windowFrame).map { frame => val frameType = frame.frameType.getType match { case SqlBaseParser.RANGE => RangeFrame case SqlBaseParser.ROWS => RowFrame + case SqlBaseParser.GROUPS => GroupFrame } SpecifiedWindowFrame( @@ -4412,7 +4413,12 @@ class AstBuilder extends DataTypeAstBuilder if (!(e.resolved && e.foldable || e.isInstanceOf[Parameter])) { throw QueryParsingErrors.invalidWindowFrameBoundError(ctx) } - e + if (ctx.getParent.asInstanceOf[WindowFrameContext].frameType.getType == + SqlBaseParser.GROUPS) { + GroupFrameOffset(e) + } else { + e + } } ctx.boundType.getType match { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala index d46c85d8ed093..c77d8f489c895 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala @@ -583,6 +583,79 @@ class ExpressionParserSuite extends AnalysisTest { stop = 59)) } + private def groupsBoundary(boundary: Expression): Expression = boundary match { + case s: SpecialFrameBoundary => s + case UnaryMinus(offset, _) => -GroupFrameOffset(offset) + case IntegerLiteral(offset) if offset < 0 => -GroupFrameOffset(Literal(-offset)) + case offset => GroupFrameOffset(offset) + } + + test("groups window function expressions") { + val func = $"foo".function(star()) + def windowed( + partitioning: Seq[Expression] = Seq.empty, + ordering: Seq[SortOrder] = Seq.empty, + frame: WindowFrame = UnspecifiedFrame): Expression = { + WindowExpression(func, WindowSpecDefinition(partitioning, ordering, frame)) + } + + val boundaries = Seq( + // Single-bound forms. + ("unbounded preceding", UnboundedPreceding, CurrentRow), + ("10 preceding", -Literal(10), CurrentRow), + ("3 + 1 preceding", -Add(Literal(3), Literal(1)), CurrentRow), + ("0 preceding", -Literal(0), CurrentRow), + ("current row", CurrentRow, CurrentRow), + ("0 following", Literal(0), CurrentRow), + ("3 + 1 following", Add(Literal(3), Literal(1)), CurrentRow), + ("10 following", Literal(10), CurrentRow), + ("unbounded following", UnboundedFollowing, CurrentRow), // Will fail during analysis + + // BETWEEN forms. + ("between unbounded preceding and 5 following", UnboundedPreceding, Literal(5)), + ("between unbounded preceding and current row", UnboundedPreceding, CurrentRow), + ("between 10 preceding and current row", -Literal(10), CurrentRow), + ("between 0 preceding and current row", -Literal(0), CurrentRow), + ("between current row and current row", CurrentRow, CurrentRow), + ("between current row and 0 following", CurrentRow, Literal(0)), + ("between current row and 5 following", CurrentRow, Literal(5)), + ("between current row and unbounded following", CurrentRow, UnboundedFollowing), + ("between 10 preceding and unbounded following", -Literal(10), UnboundedFollowing), + ("between 0 preceding and unbounded following", -Literal(0), UnboundedFollowing), + ("between 10 preceding and 5 following", -Literal(10), Literal(5)), + ("between unbounded preceding and unbounded following", + UnboundedPreceding, UnboundedFollowing) + ) + boundaries.foreach { + case (boundarySql, begin, end) => + val query = s"foo(*) over (partition by a order by b groups $boundarySql)" + val expr = windowed(Seq($"a"), Seq($"b".asc), + SpecifiedWindowFrame(GroupFrame, groupsBoundary(begin), groupsBoundary(end))) + assertEqual(query, expr) + } + } + + test("GROUPS window frame sql output re-parses to an equal tree") { + val func = $"foo".function(star()) + val frames = Seq( + SpecifiedWindowFrame(GroupFrame, UnboundedPreceding, CurrentRow), + SpecifiedWindowFrame(GroupFrame, CurrentRow, CurrentRow), + SpecifiedWindowFrame(GroupFrame, UnboundedPreceding, UnboundedFollowing), + SpecifiedWindowFrame(GroupFrame, CurrentRow, UnboundedFollowing), + SpecifiedWindowFrame(GroupFrame, Literal(-2), Literal(-1)), + SpecifiedWindowFrame(GroupFrame, Literal(1), Literal(2)), + SpecifiedWindowFrame(GroupFrame, Literal(0), CurrentRow) + ) + frames.foreach { frame => + val expr = WindowExpression(func, + WindowSpecDefinition(Seq($"a"), Seq($"b".asc), + frame.copy(lower = groupsBoundary(frame.lower), upper = groupsBoundary(frame.upper)))) + val sqlText = s"foo(*) over (partition by a order by b ${frame.sql})" + val reparsed = defaultParser.parseExpression(sqlText) + compareExpressions(reparsed, expr) + } + } + test("row constructor") { // Note that '(a)' will be interpreted as a nested expression. assertEqual("(a, b)", CreateStruct(Seq($"a", $"b"))) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala index a0fd03ff00c0a..b773934cf6b22 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala @@ -1222,6 +1222,23 @@ class PlanParserSuite extends AnalysisTest { parsePlan("select approx, asof, distance, exact, nearest, similarity from t") } + test("GROUPS keyword is non-reserved (usable as identifier)") { + def checkGroupsAsIdentifier(): Unit = { + parsePlan("select groups from t") + parsePlan("select a as groups from t") + parsePlan("select * from groups") + parsePlan("select foo(*) over (order by groups) from t") + parsePlan( + "select foo(*) over (order by groups groups between 1 preceding and current row) from t") + } + checkGroupsAsIdentifier() + withSQLConf( + SQLConf.ANSI_ENABLED.key -> "true", + SQLConf.ENFORCE_RESERVED_KEYWORDS.key -> "true") { + checkGroupsAsIdentifier() + } + } + test("sampled relations") { val sql = "select * from t" assertEqual(s"$sql tablesample(100 rows)", diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/BoundOrdering.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/BoundOrdering.scala index d6a801954c1ac..8ef19defd9fb7 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/BoundOrdering.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/BoundOrdering.scala @@ -20,12 +20,16 @@ package org.apache.spark.sql.execution.window import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.Projection - /** * Function for comparing boundary values. */ private[window] abstract class BoundOrdering { def compare(inputRow: InternalRow, inputIndex: Int, outputRow: InternalRow, outputIndex: Int): Int + + /** + * Prepares state for a partition before any calls to [[compare]]. + */ + def prepare(): Unit = {} } /** @@ -56,3 +60,118 @@ private[window] final case class RangeBoundOrdering( outputIndex: Int): Int = ordering.compare(current(inputRow), bound(outputRow)) } + +/** + * Counts peer groups -- maximal runs of rows equal under `ordering` -- along a position that + * advances monotonically through a partition. Only the current group's ordering key is retained, + * so state is O(1) rather than O(#groups). + */ +private[window] final class PeerGroupCursor( + ordering: Ordering[InternalRow], + projection: Projection) { + + /** Ordering key of the current peer group; null before the first row. */ + private[this] var groupKey: InternalRow = null + private[this] var position: Int = -1 + private[this] var group: Int = 0 + + /** Rewinds the cursor for a new partition. */ + def reset(): Unit = { + groupKey = null + position = -1 + group = 0 + } + + /** + * Returns the peer-group number of `index`, whose row is `row`. `index` must be the current + * position or the one immediately after it. + */ + def groupOf(row: InternalRow, index: Int): Int = { + if (index > position) { + assert(index == position + 1, + s"peer-group cursor cannot skip from position $position to $index") + // Copy only the ordering key: both the projection and spilled iterators reuse buffers. + val key = projection(row) + if (groupKey == null) { + groupKey = key.copy() + } else if (ordering.compare(groupKey, key) != 0) { + group += 1 + groupKey = key.copy() + } + position = index + } + group + } +} + +/** + * Compares the peer-group indices of input and output rows, adjusted by `offset` groups. + * Negative offsets represent `PRECEDING` and positive offsets represent `FOLLOWING`. + * + * Group numbers come from two [[PeerGroupCursor]]s riding positions the frame already visits: + * the input cursor follows this bound's frame edge, the output cursor the output row. Nothing + * proportional to the partition is materialized. + * + * Neither bound of a two-sided frame sees every output row on its own -- a frame consults a + * bound only while its edge can still move, so under `GROUPS BETWEEN 3 PRECEDING AND 2 + * PRECEDING` the lower bound goes unconsulted until the frame first becomes non-empty -- so + * the two share an output cursor and are built together by [[GroupBoundOrdering.paired]]. + * + * Sharing is safe only because no output index is skipped, which [[PeerGroupCursor]] asserts + * on. Edges advance monotonically and stall only at the end of the partition, so each output + * index either has some bound consulted -- carrying the shared cursor forward one step -- or + * finds both edges already at the end, after which no bound is consulted again. + */ +private[window] final class GroupBoundOrdering private ( + ordering: Ordering[InternalRow], + projection: Projection, + outputCursor: PeerGroupCursor, + offset: Int) + extends BoundOrdering { + + private[this] val inputCursor = new PeerGroupCursor(ordering, projection) + + override def prepare(): Unit = { + inputCursor.reset() + outputCursor.reset() + } + + override def compare( + inputRow: InternalRow, + inputIndex: Int, + outputRow: InternalRow, + outputIndex: Int): Int = { + val inputGroup = inputCursor.groupOf(inputRow, inputIndex) + val outputGroup = outputCursor.groupOf(outputRow, outputIndex) + // Any wrap in `outputGroup + offset` cancels, as in `RowBoundOrdering`: the frame only + // compares an edge against a bound that can reach it. + inputGroup - (outputGroup + offset) + } +} + +private[window] object GroupBoundOrdering { + + // Cursors may share a projection: calls are sequential, and each retained key is copied + // before another cursor can reuse the projection's result buffer. + + /** + * Creates the single bound of a one-sided frame, which owns its output cursor: being the only + * bound, it is consulted on every output row. + */ + def apply( + ordering: Ordering[InternalRow], + projection: Projection, + offset: Int): GroupBoundOrdering = + new GroupBoundOrdering(ordering, projection, new PeerGroupCursor(ordering, projection), offset) + + /** Creates both bounds of a two-sided frame, sharing an output cursor. */ + def paired( + ordering: Ordering[InternalRow], + projection: Projection, + lowerOffset: Int, + upperOffset: Int): (GroupBoundOrdering, GroupBoundOrdering) = { + val outputCursor = new PeerGroupCursor(ordering, projection) + (new GroupBoundOrdering(ordering, projection, outputCursor, lowerOffset), + new GroupBoundOrdering(ordering, projection, outputCursor, upperOffset)) + } +} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowFunctionFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowFunctionFrame.scala index 496126150f54b..35f140569045a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowFunctionFrame.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowFunctionFrame.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.execution.window import org.apache.spark.TaskContext import org.apache.spark.memory.TaskMemoryManager import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, FrameType, MutableProjection, RangeFrame, RowFrame, UnsafeRow} +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, FrameType, GroupFrame, MutableProjection, RangeFrame, RowFrame, UnsafeRow} import org.apache.spark.sql.catalyst.expressions.aggregate.DeclarativeAggregate import org.apache.spark.sql.execution.ExternalAppendOnlyUnsafeRowArray import org.apache.spark.sql.execution.metric.SQLMetric @@ -62,8 +62,8 @@ private[window] final class SegmentTreeWindowFunctionFrame( numSegmentTreeFallbackFrames: Option[SQLMetric] = None) extends WindowFunctionFrame with AutoCloseable { - require(frameType == RowFrame || frameType == RangeFrame, - s"SegmentTreeWindowFunctionFrame supports RowFrame or RangeFrame, got $frameType") + require(frameType == RowFrame || frameType == RangeFrame || frameType == GroupFrame, + s"SegmentTreeWindowFunctionFrame supports RowFrame, RangeFrame or GroupFrame, got $frameType") // True when this is a shrinking-frame (UnboundedFollowing) instance. // Shorthand to avoid repeated `ubound.isEmpty` reads in hot loops. @@ -87,18 +87,19 @@ private[window] final class SegmentTreeWindowFunctionFrame( private[this] var boundIter: Iterator[UnsafeRow] = _ private[this] var nextRow: UnsafeRow = _ - // ---- RangeFrame-only driver state ---- + // ---- Value/group-bound driver state ---- // Two cursors over `rowArray`; `lowerRow` / `upperRow` hold the buffered - // head of each cursor, pre-fetched in `prepare` so - // `RangeBoundOrdering.compare` is never called with a null row on round 0. + // head of each cursor, pre-fetched in `prepare` so the bound's `compare` + // is never called with a null row on round 0. // // Spill-safety invariant: when `rowArray` spills, its iterator reuses a // single `UnsafeRow` whose pointer is rebound on each `next()`. Tolerated - // here because the cursor is **read-before-advance**: each `writeRange` + // here because the cursor is **read-before-advance**: each `writeOrdered` // iteration reads `lowerRow` / `upperRow` for comparison before calling // `getNextOrNull(...)`. DO NOT cache a historical row into a separate // field without an explicit `.copy()`; the shared reusable UnsafeRow - // would silently mutate. + // would silently mutate. `PeerGroupCursor` retains a copy of the current + // peer group's projected ordering key. private[this] var lowerIter: Iterator[UnsafeRow] = _ private[this] var upperIter: Iterator[UnsafeRow] = _ private[this] var lowerRow: UnsafeRow = _ @@ -168,13 +169,15 @@ private[window] final class SegmentTreeWindowFunctionFrame( // Count only on the successful segtree path: if `tree.build` throws, // the counter is not bumped. numSegmentTreeFrames.foreach(_ += 1) + lbound.prepare() + ubound.foreach(_.prepare()) if (shrinking) { // Upper bound pinned to partition end; never moves. upperBound = tree.size frameType match { case RowFrame => // RowFrame lower-bound advance is pure index arithmetic; no iterator. - case RangeFrame => + case RangeFrame | GroupFrame => lowerIter = rows.generateIterator() lowerRow = WindowFunctionFrame.getNextOrNull(lowerIter) } @@ -183,13 +186,12 @@ private[window] final class SegmentTreeWindowFunctionFrame( case RowFrame => boundIter = rows.generateIterator() nextRow = WindowFunctionFrame.getNextOrNull(boundIter) - case RangeFrame => + case RangeFrame | GroupFrame => lowerIter = rows.generateIterator() upperIter = rows.generateIterator() - // Pre-seed cursor heads so `RangeBoundOrdering.compare` never - // dereferences null on round 0. Either may be null if `rows` is - // empty; the advance loops' `!= null` / `< upperBound` guards - // handle that. + // Pre-seed cursor heads so the bound's `compare` never dereferences + // null on round 0. Either may be null if `rows` is empty; the advance + // loops' `!= null` / `< upperBound` guards handle that. lowerRow = WindowFunctionFrame.getNextOrNull(lowerIter) upperRow = WindowFunctionFrame.getNextOrNull(upperIter) } @@ -217,11 +219,11 @@ private[window] final class SegmentTreeWindowFunctionFrame( } frameType match { case RowFrame => writeRow(index, current) - case RangeFrame => writeRange(index, current) + case RangeFrame | GroupFrame => writeOrdered(index, current) } } - // `writeRow`/`writeRange` maintain the `(lowerBound, upperBound)` monotone + // `writeRow`/`writeOrdered` maintain the `(lowerBound, upperBound)` monotone // cursor invariant for both sliding and shrinking frame shapes: // // - Sliding (`ubound.isDefined`, mirrors `SlidingWindowFunctionFrame.write`): @@ -230,7 +232,7 @@ private[window] final class SegmentTreeWindowFunctionFrame( // loop advances `lowerBound`. Any future fix to Sliding's boundary // semantics must be mirrored here; equivalence is guarded by // `SegmentTreeWindowFunctionSuite` flag-on/off tests - // (`checkRangeEquivalence`, `feature flag off ...`, fallback tests) + // (`checkSqlEquivalence`, `feature flag off ...`, fallback tests) // against the Sliding baseline. // // - Shrinking (`ubound.isEmpty`, upper is `tree.size`): drop-only. The admit @@ -272,13 +274,13 @@ private[window] final class SegmentTreeWindowFunctionFrame( } } - private def writeRange(index: Int, current: InternalRow): Unit = { + private def writeOrdered(index: Int, current: InternalRow): Unit = { var boundsChanged = index == 0 if (!shrinking) { val ub = ubound.get // admit loop (upper edge). `RangeBoundOrdering.compare` ignores its index - // arguments; we pass `upperBound` for API symmetry with RowBoundOrdering. + // arguments; `GroupBoundOrdering` uses them to place the row in a peer group. while (upperRow != null && ub.compare(upperRow, upperBound, current, index) <= 0) { upperBound += 1 diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowEvaluatorFactoryBase.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowEvaluatorFactoryBase.scala index 40cba3d5ceb4c..4eaab779930a6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowEvaluatorFactoryBase.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowEvaluatorFactoryBase.scala @@ -22,7 +22,7 @@ import scala.collection.mutable.ArrayBuffer import org.apache.spark.{SparkException, TaskContext} import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.{Add, AggregateWindowFunction, Ascending, Attribute, BoundReference, CurrentRow, DateAdd, DateAddYMInterval, DecimalAddNoOverflowCheck, Descending, Expression, ExtractANSIIntervalDays, FrameLessOffsetWindowFunction, FrameType, IdentityProjection, IntegerLiteral, MutableProjection, NamedExpression, OffsetWindowFunction, PythonFuncExpression, RangeFrame, RowFrame, RowOrdering, SortOrder, SpecifiedWindowFrame, TimestampAddInterval, TimestampAddYMInterval, UnaryMinus, UnboundedFollowing, UnboundedPreceding, UnsafeProjection, WindowExpression} +import org.apache.spark.sql.catalyst.expressions.{Add, AggregateWindowFunction, Ascending, Attribute, BoundReference, CurrentRow, DateAdd, DateAddYMInterval, DecimalAddNoOverflowCheck, Descending, Expression, ExtractANSIIntervalDays, FrameLessOffsetWindowFunction, FrameType, GroupFrame, IdentityProjection, IntegerLiteral, MutableProjection, NamedExpression, OffsetWindowFunction, PythonFuncExpression, RangeFrame, RowFrame, RowOrdering, SortOrder, SpecifiedWindowFrame, TimestampAddInterval, TimestampAddYMInterval, UnaryMinus, UnboundedFollowing, UnboundedPreceding, UnsafeProjection, WindowExpression} import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, DeclarativeAggregate} import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.internal.SQLConf @@ -71,6 +71,9 @@ trait WindowEvaluatorFactoryBase { * * This method uses Code Generation. It can only be used on the executor side. * + * GROUPS bounds go through [[createSoleBoundOrdering]] or [[createBoundOrderingPair]], which + * decide whether the bound owns or shares its output cursor. + * * @param frame to evaluate. This can either be a Row or Range frame. * @param bound with respect to the row. * @param timeZone the session local timezone for time related calculations. @@ -133,6 +136,63 @@ trait WindowEvaluatorFactoryBase { case (RangeFrame, _) => throw SparkException.internalError("Non-Zero range offsets are not supported for windows " + "with multiple order expressions.") + + case (GroupFrame, _) => + // Unreachable; guards a future caller against giving a paired bound its own cursor. + throw SparkException.internalError("GROUPS bound orderings must be created by " + + "createSoleBoundOrdering or createBoundOrderingPair") + } + } + + /** The peer-group offset of an analyzed GROUPS bound; CURRENT ROW is offset zero. */ + private def groupsOffset(bound: Expression): Int = bound match { + case CurrentRow => 0 + case IntegerLiteral(offset) => offset + case _ => throw SparkException.internalError(s"Unhandled GROUPS window frame bound: $bound") + } + + /** Creates the projected ordering keys and an ordering over their positions. */ + private def createGroupOrdering(): (Ordering[InternalRow], UnsafeProjection) = { + val projection = UnsafeProjection.create(orderSpec.map(_.child), childOutput) + val ordering = orderSpec.zipWithIndex.map { case (sort, ordinal) => + SortOrder(BoundReference(ordinal, sort.child.dataType, sort.child.nullable), + sort.direction, sort.nullOrdering, Seq.empty) + } + (RowOrdering.create(ordering, Nil), projection) + } + + /** + * Creates the single bound ordering of a one-sided frame, whose other edge is unbounded. A + * GROUPS bound owns its output cursor here because, being the only bound, it is consulted on + * every output row; two-sided frames must use [[createBoundOrderingPair]]. + */ + private def createSoleBoundOrdering( + frameType: FrameType, bound: Expression, timeZone: String): BoundOrdering = { + frameType match { + case GroupFrame => + val (ordering, projection) = createGroupOrdering() + GroupBoundOrdering(ordering, projection, groupsOffset(bound)) + case _ => createBoundOrdering(frameType, bound, timeZone) + } + } + + /** + * Creates both bound orderings of a two-sided frame. GROUPS bounds are built as a pair + * because [[GroupBoundOrdering]] requires them to share an output cursor. + */ + private def createBoundOrderingPair( + frameType: FrameType, + lower: Expression, + upper: Expression, + timeZone: String): (BoundOrdering, BoundOrdering) = { + frameType match { + case GroupFrame => + val (ordering, projection) = createGroupOrdering() + GroupBoundOrdering.paired(ordering, projection, groupsOffset(lower), groupsOffset(upper)) + + case _ => + (createBoundOrdering(frameType, lower, timeZone), + createBoundOrdering(frameType, upper, timeZone)) } } @@ -276,7 +336,7 @@ trait WindowEvaluatorFactoryBase { new UnboundedPrecedingWindowFunctionFrame( target, processor, - createBoundOrdering(frameType, upper, timeZone)) + createSoleBoundOrdering(frameType, upper, timeZone)) } // Shrinking Frame. @@ -305,7 +365,7 @@ trait WindowEvaluatorFactoryBase { "an active TaskContext") } val tmm = tc.taskMemoryManager() - val lb = createBoundOrdering(frameType, lower, timeZone) + val lb = createSoleBoundOrdering(frameType, lower, timeZone) new SegmentTreeWindowFunctionFrame( target, processor, @@ -328,7 +388,7 @@ trait WindowEvaluatorFactoryBase { new UnboundedFollowingWindowFunctionFrame( target, processor, - createBoundOrdering(frameType, lower, timeZone)) + createSoleBoundOrdering(frameType, lower, timeZone)) } } @@ -349,8 +409,7 @@ trait WindowEvaluatorFactoryBase { "an active TaskContext") } val tmm = tc.taskMemoryManager() - val lb = createBoundOrdering(frameType, lower, timeZone) - val ub = createBoundOrdering(frameType, upper, timeZone) + val (lb, ub) = createBoundOrderingPair(frameType, lower, upper, timeZone) new SegmentTreeWindowFunctionFrame( target, processor, @@ -370,11 +429,8 @@ trait WindowEvaluatorFactoryBase { } } else { target: InternalRow => { - new SlidingWindowFunctionFrame( - target, - processor, - createBoundOrdering(frameType, lower, timeZone), - createBoundOrdering(frameType, upper, timeZone)) + val (lb, ub) = createBoundOrderingPair(frameType, lower, upper, timeZone) + new SlidingWindowFunctionFrame(target, processor, lb, ub) } } @@ -412,10 +468,11 @@ trait WindowEvaluatorFactoryBase { // RANGE accepted only for single-column order specs. Multi-column RANGE // with non-zero offset is already rejected by `createBoundOrdering`, so // gating here on `orderSpec.size == 1` matches the Sliding-path invariant. + // + // GROUPS supports peer equality over multiple order expressions. val frameTypeOk = frameType match { - case RowFrame => true + case RowFrame | GroupFrame => true case RangeFrame => orderSpec.size == 1 - case _ => false } conf.windowSegmentTreeEnabled && frameTypeOk && @@ -432,9 +489,10 @@ trait WindowEvaluatorFactoryBase { // RANGE the frame width is data-dependent (defined by order-key distance, // not row count), so no static width inference is possible; fall back to // a default budget and rely on the runtime LRU + TMM spiller. - assert(frameType == RowFrame || frameType == RangeFrame, - s"estimateMaxCachedBlocks expects RowFrame or RangeFrame, got $frameType") - if (frameType == RangeFrame) { + assert(frameType == RowFrame || frameType == RangeFrame || frameType == GroupFrame, + s"estimateMaxCachedBlocks expects RowFrame, RangeFrame or GroupFrame, got $frameType") + // GROUPS frame width is also data-dependent. + if (frameType == RangeFrame || frameType == GroupFrame) { return Some(8) } val w: Option[Int] = (lower, upper) match { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala index 644603e4710f2..787788efa54ad 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala @@ -460,6 +460,8 @@ final class SlidingWindowFunctionFrame( lowerBound = 0 upperBound = 0 buffer.clear() + lbound.prepare() + ubound.prepare() } /** Write the frame columns for the current row to the given target row. */ @@ -477,6 +479,10 @@ final class SlidingWindowFunctionFrame( // Add all rows to the buffer for which the input row value is equal to or less than // the output row upper bound. while (nextRow != null && ubound.compare(nextRow, upperBound, current, index) <= 0) { + // `nextRow` sits at `upperBound` but is passed as the row at `lowerBound`. The two + // coincide whenever this branch is taken: an admitted row is only below the lower bound + // when the buffer is empty, and then `lowerBound == upperBound`. This matters for + // `GroupBoundOrdering`, which unlike the other bounds reads both the row and the index. if (lbound.compare(nextRow, lowerBound, current, index) < 0) { lowerBound += 1 } else { @@ -597,6 +603,7 @@ final class UnboundedPrecedingWindowFunctionFrame( if (processor != null) { processor.initialize(input.length) } + ubound.prepare() } /** Write the frame columns for the current row to the given target row. */ @@ -660,6 +667,7 @@ final class UnboundedFollowingWindowFunctionFrame( override def prepare(rows: ExternalAppendOnlyUnsafeRowArray): Unit = { input = rows inputIndex = 0 + lbound.prepare() } /** Write the frame columns for the current row to the given target row. */ diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/window_part1.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/window_part1.sql.out index 1734e1aabede2..9d798a07f0978 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/window_part1.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/window_part1.sql.out @@ -546,6 +546,34 @@ Project [id#xL, sum_rows#xL] +- Range (1, 11, step=1) +-- !query +CREATE OR REPLACE TEMP VIEW v_window AS +SELECT i.id, sum(i.id) over (order by i.id groups between 1 preceding and 1 following) as sum_rows FROM range(1, 11) i +-- !query analysis +CreateViewCommand `v_window`, SELECT i.id, sum(i.id) over (order by i.id groups between 1 preceding and 1 following) as sum_rows FROM range(1, 11) i, false, true, LocalTempView, UNSUPPORTED, true + +- Project [id#xL, sum_rows#xL] + +- Project [id#xL, sum_rows#xL, sum_rows#xL] + +- Window [sum(id#xL) windowspecdefinition(id#xL ASC NULLS FIRST, specifiedwindowframe(GroupFrame, -1, 1)) AS sum_rows#xL], [id#xL ASC NULLS FIRST] + +- Project [id#xL] + +- SubqueryAlias i + +- Range (1, 11, step=1) + + +-- !query +SELECT * FROM v_window +-- !query analysis +Project [id#xL, sum_rows#xL] ++- SubqueryAlias v_window + +- View (`v_window`, [id#xL, sum_rows#xL]) + +- Project [cast(id#xL as bigint) AS id#xL, cast(sum_rows#xL as bigint) AS sum_rows#xL] + +- Project [id#xL, sum_rows#xL] + +- Project [id#xL, sum_rows#xL, sum_rows#xL] + +- Window [sum(id#xL) windowspecdefinition(id#xL ASC NULLS FIRST, specifiedwindowframe(GroupFrame, -1, 1)) AS sum_rows#xL], [id#xL ASC NULLS FIRST] + +- Project [id#xL] + +- SubqueryAlias i + +- Range (1, 11, step=1) + + -- !query DROP VIEW v_window -- !query analysis diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/window_part3.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/window_part3.sql.out index ab9b354ea7bda..99cda91279fb7 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/window_part3.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/window_part3.sql.out @@ -80,6 +80,197 @@ org.apache.spark.sql.AnalysisException } +-- !query +SELECT sum(unique1) over (order by four groups between unbounded preceding and current row), +unique1, four +FROM tenk1 WHERE unique1 < 10 +-- !query analysis +Project [sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)#xL, unique1#x, four#x] ++- Project [unique1#x, four#x, sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)#xL, sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)#xL] + +- Window [sum(unique1#x) windowspecdefinition(four#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, unboundedpreceding$(), currentrow$())) AS sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)#xL], [four#x ASC NULLS FIRST] + +- Project [unique1#x, four#x] + +- Filter (unique1#x < 10) + +- SubqueryAlias spark_catalog.default.tenk1 + +- Relation spark_catalog.default.tenk1[unique1#x,unique2#x,two#x,four#x,ten#x,twenty#x,hundred#x,thousand#x,twothousand#x,fivethous#x,tenthous#x,odd#x,even#x,stringu1#x,stringu2#x,string4#x] parquet + + +-- !query +SELECT sum(unique1) over (order by four groups between unbounded preceding and unbounded following), +unique1, four +FROM tenk1 WHERE unique1 < 10 +-- !query analysis +Project [sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)#xL, unique1#x, four#x] ++- Project [unique1#x, four#x, sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)#xL, sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)#xL] + +- Window [sum(unique1#x) windowspecdefinition(four#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, unboundedpreceding$(), unboundedfollowing$())) AS sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)#xL], [four#x ASC NULLS FIRST] + +- Project [unique1#x, four#x] + +- Filter (unique1#x < 10) + +- SubqueryAlias spark_catalog.default.tenk1 + +- Relation spark_catalog.default.tenk1[unique1#x,unique2#x,two#x,four#x,ten#x,twenty#x,hundred#x,thousand#x,twothousand#x,fivethous#x,tenthous#x,odd#x,even#x,stringu1#x,stringu2#x,string4#x] parquet + + +-- !query +SELECT sum(unique1) over (order by four groups between current row and unbounded following), +unique1, four +FROM tenk1 WHERE unique1 < 10 +-- !query analysis +Project [sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)#xL, unique1#x, four#x] ++- Project [unique1#x, four#x, sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)#xL, sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)#xL] + +- Window [sum(unique1#x) windowspecdefinition(four#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, currentrow$(), unboundedfollowing$())) AS sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)#xL], [four#x ASC NULLS FIRST] + +- Project [unique1#x, four#x] + +- Filter (unique1#x < 10) + +- SubqueryAlias spark_catalog.default.tenk1 + +- Relation spark_catalog.default.tenk1[unique1#x,unique2#x,two#x,four#x,ten#x,twenty#x,hundred#x,thousand#x,twothousand#x,fivethous#x,tenthous#x,odd#x,even#x,stringu1#x,stringu2#x,string4#x] parquet + + +-- !query +SELECT sum(unique1) over (order by four groups between 1 preceding and unbounded following), +unique1, four +FROM tenk1 WHERE unique1 < 10 +-- !query analysis +Project [sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING)#xL, unique1#x, four#x] ++- Project [unique1#x, four#x, sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING)#xL, sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING)#xL] + +- Window [sum(unique1#x) windowspecdefinition(four#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, -1, unboundedfollowing$())) AS sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING)#xL], [four#x ASC NULLS FIRST] + +- Project [unique1#x, four#x] + +- Filter (unique1#x < 10) + +- SubqueryAlias spark_catalog.default.tenk1 + +- Relation spark_catalog.default.tenk1[unique1#x,unique2#x,two#x,four#x,ten#x,twenty#x,hundred#x,thousand#x,twothousand#x,fivethous#x,tenthous#x,odd#x,even#x,stringu1#x,stringu2#x,string4#x] parquet + + +-- !query +SELECT sum(unique1) over (order by four groups between 1 following and unbounded following), +unique1, four +FROM tenk1 WHERE unique1 < 10 +-- !query analysis +Project [sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING)#xL, unique1#x, four#x] ++- Project [unique1#x, four#x, sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING)#xL, sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING)#xL] + +- Window [sum(unique1#x) windowspecdefinition(four#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, 1, unboundedfollowing$())) AS sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING)#xL], [four#x ASC NULLS FIRST] + +- Project [unique1#x, four#x] + +- Filter (unique1#x < 10) + +- SubqueryAlias spark_catalog.default.tenk1 + +- Relation spark_catalog.default.tenk1[unique1#x,unique2#x,two#x,four#x,ten#x,twenty#x,hundred#x,thousand#x,twothousand#x,fivethous#x,tenthous#x,odd#x,even#x,stringu1#x,stringu2#x,string4#x] parquet + + +-- !query +SELECT sum(unique1) over (order by four groups between unbounded preceding and 2 following), +unique1, four +FROM tenk1 WHERE unique1 < 10 +-- !query analysis +Project [sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN UNBOUNDED PRECEDING AND 2 FOLLOWING)#xL, unique1#x, four#x] ++- Project [unique1#x, four#x, sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN UNBOUNDED PRECEDING AND 2 FOLLOWING)#xL, sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN UNBOUNDED PRECEDING AND 2 FOLLOWING)#xL] + +- Window [sum(unique1#x) windowspecdefinition(four#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, unboundedpreceding$(), 2)) AS sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN UNBOUNDED PRECEDING AND 2 FOLLOWING)#xL], [four#x ASC NULLS FIRST] + +- Project [unique1#x, four#x] + +- Filter (unique1#x < 10) + +- SubqueryAlias spark_catalog.default.tenk1 + +- Relation spark_catalog.default.tenk1[unique1#x,unique2#x,two#x,four#x,ten#x,twenty#x,hundred#x,thousand#x,twothousand#x,fivethous#x,tenthous#x,odd#x,even#x,stringu1#x,stringu2#x,string4#x] parquet + + +-- !query +SELECT sum(unique1) over (order by four groups between 2 preceding and 1 preceding), +unique1, four +FROM tenk1 WHERE unique1 < 10 +-- !query analysis +Project [sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 2 PRECEDING AND 1 PRECEDING)#xL, unique1#x, four#x] ++- Project [unique1#x, four#x, sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 2 PRECEDING AND 1 PRECEDING)#xL, sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 2 PRECEDING AND 1 PRECEDING)#xL] + +- Window [sum(unique1#x) windowspecdefinition(four#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, -2, -1)) AS sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 2 PRECEDING AND 1 PRECEDING)#xL], [four#x ASC NULLS FIRST] + +- Project [unique1#x, four#x] + +- Filter (unique1#x < 10) + +- SubqueryAlias spark_catalog.default.tenk1 + +- Relation spark_catalog.default.tenk1[unique1#x,unique2#x,two#x,four#x,ten#x,twenty#x,hundred#x,thousand#x,twothousand#x,fivethous#x,tenthous#x,odd#x,even#x,stringu1#x,stringu2#x,string4#x] parquet + + +-- !query +SELECT sum(unique1) over (order by four groups between 2 preceding and 1 following), +unique1, four +FROM tenk1 WHERE unique1 < 10 +-- !query analysis +Project [sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 2 PRECEDING AND 1 FOLLOWING)#xL, unique1#x, four#x] ++- Project [unique1#x, four#x, sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 2 PRECEDING AND 1 FOLLOWING)#xL, sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 2 PRECEDING AND 1 FOLLOWING)#xL] + +- Window [sum(unique1#x) windowspecdefinition(four#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, -2, 1)) AS sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 2 PRECEDING AND 1 FOLLOWING)#xL], [four#x ASC NULLS FIRST] + +- Project [unique1#x, four#x] + +- Filter (unique1#x < 10) + +- SubqueryAlias spark_catalog.default.tenk1 + +- Relation spark_catalog.default.tenk1[unique1#x,unique2#x,two#x,four#x,ten#x,twenty#x,hundred#x,thousand#x,twothousand#x,fivethous#x,tenthous#x,odd#x,even#x,stringu1#x,stringu2#x,string4#x] parquet + + +-- !query +SELECT sum(unique1) over (order by four groups between 0 preceding and 0 following), +unique1, four +FROM tenk1 WHERE unique1 < 10 +-- !query analysis +Project [sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 0 PRECEDING AND 0 FOLLOWING)#xL, unique1#x, four#x] ++- Project [unique1#x, four#x, sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 0 PRECEDING AND 0 FOLLOWING)#xL, sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 0 PRECEDING AND 0 FOLLOWING)#xL] + +- Window [sum(unique1#x) windowspecdefinition(four#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, -0, 0)) AS sum(unique1) OVER (ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 0 PRECEDING AND 0 FOLLOWING)#xL], [four#x ASC NULLS FIRST] + +- Project [unique1#x, four#x] + +- Filter (unique1#x < 10) + +- SubqueryAlias spark_catalog.default.tenk1 + +- Relation spark_catalog.default.tenk1[unique1#x,unique2#x,two#x,four#x,ten#x,twenty#x,hundred#x,thousand#x,twothousand#x,fivethous#x,tenthous#x,odd#x,even#x,stringu1#x,stringu2#x,string4#x] parquet + + +-- !query +SELECT sum(unique1) over (partition by ten + order by four groups between 0 preceding and 0 following),unique1, four, ten +FROM tenk1 WHERE unique1 < 10 +-- !query analysis +Project [sum(unique1) OVER (PARTITION BY ten ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 0 PRECEDING AND 0 FOLLOWING)#xL, unique1#x, four#x, ten#x] ++- Project [unique1#x, four#x, ten#x, sum(unique1) OVER (PARTITION BY ten ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 0 PRECEDING AND 0 FOLLOWING)#xL, sum(unique1) OVER (PARTITION BY ten ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 0 PRECEDING AND 0 FOLLOWING)#xL] + +- Window [sum(unique1#x) windowspecdefinition(ten#x, four#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, -0, 0)) AS sum(unique1) OVER (PARTITION BY ten ORDER BY four ASC NULLS FIRST GROUPS BETWEEN 0 PRECEDING AND 0 FOLLOWING)#xL], [ten#x], [four#x ASC NULLS FIRST] + +- Project [unique1#x, four#x, ten#x] + +- Filter (unique1#x < 10) + +- SubqueryAlias spark_catalog.default.tenk1 + +- Relation spark_catalog.default.tenk1[unique1#x,unique2#x,two#x,four#x,ten#x,twenty#x,hundred#x,thousand#x,twothousand#x,fivethous#x,tenthous#x,odd#x,even#x,stringu1#x,stringu2#x,string4#x] parquet + + +-- !query +select first_value(enroll_date) over(order by enroll_date groups between 1 preceding and 1 following), +nth_value(enroll_date, 1) over(order by enroll_date groups between 1 preceding and 1 following), +salary, enroll_date from empsalary +-- !query analysis +Project [first_value(enroll_date) OVER (ORDER BY enroll_date ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#x, nth_value(enroll_date, 1) OVER (ORDER BY enroll_date ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#x, salary#x, enroll_date#x] ++- Project [salary#x, enroll_date#x, first_value(enroll_date) OVER (ORDER BY enroll_date ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#x, nth_value(enroll_date, 1) OVER (ORDER BY enroll_date ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#x, first_value(enroll_date) OVER (ORDER BY enroll_date ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#x, nth_value(enroll_date, 1) OVER (ORDER BY enroll_date ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#x] + +- Window [first_value(enroll_date#x, false) windowspecdefinition(enroll_date#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, -1, 1)) AS first_value(enroll_date) OVER (ORDER BY enroll_date ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#x, nth_value(enroll_date#x, 1, false) windowspecdefinition(enroll_date#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, -1, 1)) AS nth_value(enroll_date, 1) OVER (ORDER BY enroll_date ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#x], [enroll_date#x ASC NULLS FIRST] + +- Project [salary#x, enroll_date#x] + +- SubqueryAlias spark_catalog.default.empsalary + +- Relation spark_catalog.default.empsalary[depname#x,empno#x,salary#x,enroll_date#x] parquet + + +-- !query +select lead(salary) over(order by enroll_date groups between 1 preceding and 1 following) +from empsalary +-- !query analysis +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "_LEGACY_ERROR_TEMP_1035", + "messageParameters" : { + "prettyName" : "lead" + } +} + + +-- !query +select last(enroll_date) over(order by enroll_date groups between 1 preceding and 1 following), +salary, enroll_date from empsalary +-- !query analysis +Project [last(enroll_date) OVER (ORDER BY enroll_date ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#x, salary#x, enroll_date#x] ++- Project [salary#x, enroll_date#x, last(enroll_date) OVER (ORDER BY enroll_date ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#x, last(enroll_date) OVER (ORDER BY enroll_date ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#x] + +- Window [last(enroll_date#x, false) windowspecdefinition(enroll_date#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, -1, 1)) AS last(enroll_date) OVER (ORDER BY enroll_date ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#x], [enroll_date#x ASC NULLS FIRST] + +- Project [salary#x, enroll_date#x] + +- SubqueryAlias spark_catalog.default.empsalary + +- Relation spark_catalog.default.empsalary[depname#x,empno#x,salary#x,enroll_date#x] parquet + + +-- !query +select lag(salary) over(order by enroll_date groups between 1 preceding and 1 following) +from empsalary +-- !query analysis +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "_LEGACY_ERROR_TEMP_1035", + "messageParameters" : { + "prettyName" : "lag" + } +} + + -- !query WITH cte (x) AS ( SELECT * FROM range(1, 36, 2) @@ -124,6 +315,28 @@ WithCTE +- CTERelationRef xxxx, true, [x#xL], false, false, 18 +-- !query +WITH cte (x) AS ( + SELECT * FROM range(1, 36, 2) +) +SELECT x, (sum(x) over w) +FROM cte +WINDOW w AS (ORDER BY x groups between 1 preceding and 1 following) +-- !query analysis +WithCTE +:- CTERelationDef xxxx, false +: +- SubqueryAlias cte +: +- Project [id#xL AS x#xL] +: +- Project [id#xL] +: +- Range (1, 36, step=2) ++- Project [x#xL, sum(x) OVER (ORDER BY x ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#xL] + +- Project [x#xL, sum(x) OVER (ORDER BY x ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#xL, sum(x) OVER (ORDER BY x ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#xL] + +- Window [sum(x#xL) windowspecdefinition(x#xL ASC NULLS FIRST, specifiedwindowframe(GroupFrame, -1, 1)) AS sum(x) OVER (ORDER BY x ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#xL], [x#xL ASC NULLS FIRST] + +- Project [x#xL] + +- SubqueryAlias cte + +- CTERelationRef xxxx, true, [x#xL], false, false, 18 + + -- !query WITH cte (x) AS ( select 1 union all select 1 union all select 1 union all @@ -190,6 +403,39 @@ WithCTE +- CTERelationRef xxxx, true, [x#xL], false, false, 26 +-- !query +WITH cte (x) AS ( + select 1 union all select 1 union all select 1 union all + SELECT * FROM range(5, 50, 2) +) +SELECT x, (sum(x) over w) +FROM cte +WINDOW w AS (ORDER BY x groups between 1 preceding and 1 following) +-- !query analysis +WithCTE +:- CTERelationDef xxxx, false +: +- SubqueryAlias cte +: +- Project [1#xL AS x#xL] +: +- Union false, false +: :- Project [cast(1#x as bigint) AS 1#xL] +: : +- Union false, false +: : :- Union false, false +: : : :- Project [1 AS 1#x] +: : : : +- OneRowRelation +: : : +- Project [1 AS 1#x] +: : : +- OneRowRelation +: : +- Project [1 AS 1#x] +: : +- OneRowRelation +: +- Project [id#xL] +: +- Range (5, 50, step=2) ++- Project [x#xL, sum(x) OVER (ORDER BY x ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#xL] + +- Project [x#xL, sum(x) OVER (ORDER BY x ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#xL, sum(x) OVER (ORDER BY x ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#xL] + +- Window [sum(x#xL) windowspecdefinition(x#xL ASC NULLS FIRST, specifiedwindowframe(GroupFrame, -1, 1)) AS sum(x) OVER (ORDER BY x ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#xL], [x#xL ASC NULLS FIRST] + +- Project [x#xL] + +- SubqueryAlias cte + +- CTERelationRef xxxx, true, [x#xL], false, false, 26 + + -- !query SELECT count(*) OVER (PARTITION BY four) FROM (SELECT * FROM tenk1 UNION ALL SELECT * FROM tenk2)s LIMIT 0 -- !query analysis @@ -291,6 +537,64 @@ Project [f1#x, sum(f1) OVER (PARTITION BY f1, f2 ORDER BY f2 ASC NULLS FIRST RAN +- Relation spark_catalog.default.t1[f1#x,f2#x] parquet +-- !query +select f1, sum(f1) over (partition by f1, +groups between 1 preceding and 1 following) +from t1 where f1 = f2 +-- !query analysis +org.apache.spark.sql.catalyst.parser.ParseException +{ + "errorClass" : "PARSE_SYNTAX_ERROR", + "sqlState" : "42601", + "messageParameters" : { + "error" : "'preceding'", + "hint" : ": extra input 'preceding'" + } +} + + +-- !query +select f1, sum(f1) over (partition by f1 order by f2 +groups between 1 preceding and 1 following) +from t1 where f1 = f2 +-- !query analysis +Project [f1#x, sum(f1) OVER (PARTITION BY f1 ORDER BY f2 ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#xL] ++- Project [f1#x, f2#x, sum(f1) OVER (PARTITION BY f1 ORDER BY f2 ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#xL, sum(f1) OVER (PARTITION BY f1 ORDER BY f2 ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#xL] + +- Window [sum(f1#x) windowspecdefinition(f1#x, f2#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, -1, 1)) AS sum(f1) OVER (PARTITION BY f1 ORDER BY f2 ASC NULLS FIRST GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)#xL], [f1#x], [f2#x ASC NULLS FIRST] + +- Project [f1#x, f2#x] + +- Filter (f1#x = f2#x) + +- SubqueryAlias spark_catalog.default.t1 + +- Relation spark_catalog.default.t1[f1#x,f2#x] parquet + + +-- !query +select f1, sum(f1) over (partition by f1, f1 order by f2 +groups between 2 preceding and 1 preceding) +from t1 where f1 = f2 +-- !query analysis +Project [f1#x, sum(f1) OVER (PARTITION BY f1, f1 ORDER BY f2 ASC NULLS FIRST GROUPS BETWEEN 2 PRECEDING AND 1 PRECEDING)#xL] ++- Project [f1#x, f2#x, sum(f1) OVER (PARTITION BY f1, f1 ORDER BY f2 ASC NULLS FIRST GROUPS BETWEEN 2 PRECEDING AND 1 PRECEDING)#xL, sum(f1) OVER (PARTITION BY f1, f1 ORDER BY f2 ASC NULLS FIRST GROUPS BETWEEN 2 PRECEDING AND 1 PRECEDING)#xL] + +- Window [sum(f1#x) windowspecdefinition(f1#x, f1#x, f2#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, -2, -1)) AS sum(f1) OVER (PARTITION BY f1, f1 ORDER BY f2 ASC NULLS FIRST GROUPS BETWEEN 2 PRECEDING AND 1 PRECEDING)#xL], [f1#x, f1#x], [f2#x ASC NULLS FIRST] + +- Project [f1#x, f2#x] + +- Filter (f1#x = f2#x) + +- SubqueryAlias spark_catalog.default.t1 + +- Relation spark_catalog.default.t1[f1#x,f2#x] parquet + + +-- !query +select f1, sum(f1) over (partition by f1, f2 order by f2 +groups between 1 following and 2 following) +from t1 where f1 = f2 +-- !query analysis +Project [f1#x, sum(f1) OVER (PARTITION BY f1, f2 ORDER BY f2 ASC NULLS FIRST GROUPS BETWEEN 1 FOLLOWING AND 2 FOLLOWING)#xL] ++- Project [f1#x, f2#x, sum(f1) OVER (PARTITION BY f1, f2 ORDER BY f2 ASC NULLS FIRST GROUPS BETWEEN 1 FOLLOWING AND 2 FOLLOWING)#xL, sum(f1) OVER (PARTITION BY f1, f2 ORDER BY f2 ASC NULLS FIRST GROUPS BETWEEN 1 FOLLOWING AND 2 FOLLOWING)#xL] + +- Window [sum(f1#x) windowspecdefinition(f1#x, f2#x, f2#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, 1, 2)) AS sum(f1) OVER (PARTITION BY f1, f2 ORDER BY f2 ASC NULLS FIRST GROUPS BETWEEN 1 FOLLOWING AND 2 FOLLOWING)#xL], [f1#x, f2#x], [f2#x ASC NULLS FIRST] + +- Project [f1#x, f2#x] + +- Filter (f1#x = f2#x) + +- SubqueryAlias spark_catalog.default.t1 + +- Relation spark_catalog.default.t1[f1#x,f2#x] parquet + + -- !query SELECT rank() OVER (ORDER BY length('abc')) -- !query analysis diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/window-groups.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/window-groups.sql.out new file mode 100644 index 0000000000000..53362412eafd4 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/window-groups.sql.out @@ -0,0 +1,318 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +CREATE OR REPLACE TEMPORARY VIEW t AS SELECT * FROM VALUES +(1, 10), (1, 15), (2, 20), (3, 25), (3, 30), (9, 40) +AS t(batch_id, amount) +-- !query analysis +CreateViewCommand `t`, SELECT * FROM VALUES +(1, 10), (1, 15), (2, 20), (3, 25), (3, 30), (9, 40) +AS t(batch_id, amount), false, true, LocalTempView, UNSUPPORTED, true + +- Project [batch_id#x, amount#x] + +- SubqueryAlias t + +- LocalRelation [batch_id#x, amount#x] + + +-- !query +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id +GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) AS total FROM t ORDER BY batch_id, amount +-- !query analysis +Project [batch_id#x, total#xL] ++- Sort [batch_id#x ASC NULLS FIRST, amount#x ASC NULLS FIRST], true + +- Project [batch_id#x, total#xL, amount#x] + +- Project [batch_id#x, amount#x, total#xL, total#xL] + +- Window [sum(amount#x) windowspecdefinition(batch_id#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, -1, currentrow$())) AS total#xL], [batch_id#x ASC NULLS FIRST] + +- Project [batch_id#x, amount#x] + +- SubqueryAlias t + +- View (`t`, [batch_id#x, amount#x]) + +- Project [cast(batch_id#x as int) AS batch_id#x, cast(amount#x as int) AS amount#x] + +- Project [batch_id#x, amount#x] + +- SubqueryAlias t + +- LocalRelation [batch_id#x, amount#x] + + +-- !query +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id +GROUPS BETWEEN CURRENT ROW AND CURRENT ROW) AS total FROM t ORDER BY batch_id, amount +-- !query analysis +Project [batch_id#x, total#xL] ++- Sort [batch_id#x ASC NULLS FIRST, amount#x ASC NULLS FIRST], true + +- Project [batch_id#x, total#xL, amount#x] + +- Project [batch_id#x, amount#x, total#xL, total#xL] + +- Window [sum(amount#x) windowspecdefinition(batch_id#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, currentrow$(), currentrow$())) AS total#xL], [batch_id#x ASC NULLS FIRST] + +- Project [batch_id#x, amount#x] + +- SubqueryAlias t + +- View (`t`, [batch_id#x, amount#x]) + +- Project [cast(batch_id#x as int) AS batch_id#x, cast(amount#x as int) AS amount#x] + +- Project [batch_id#x, amount#x] + +- SubqueryAlias t + +- LocalRelation [batch_id#x, amount#x] + + +-- !query +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id +GROUPS BETWEEN CURRENT ROW AND 1 FOLLOWING) AS total FROM t ORDER BY batch_id, amount +-- !query analysis +Project [batch_id#x, total#xL] ++- Sort [batch_id#x ASC NULLS FIRST, amount#x ASC NULLS FIRST], true + +- Project [batch_id#x, total#xL, amount#x] + +- Project [batch_id#x, amount#x, total#xL, total#xL] + +- Window [sum(amount#x) windowspecdefinition(batch_id#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, currentrow$(), 1)) AS total#xL], [batch_id#x ASC NULLS FIRST] + +- Project [batch_id#x, amount#x] + +- SubqueryAlias t + +- View (`t`, [batch_id#x, amount#x]) + +- Project [cast(batch_id#x as int) AS batch_id#x, cast(amount#x as int) AS amount#x] + +- Project [batch_id#x, amount#x] + +- SubqueryAlias t + +- LocalRelation [batch_id#x, amount#x] + + +-- !query +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id +GROUPS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS total +FROM t ORDER BY batch_id, amount +-- !query analysis +Project [batch_id#x, total#xL] ++- Sort [batch_id#x ASC NULLS FIRST, amount#x ASC NULLS FIRST], true + +- Project [batch_id#x, total#xL, amount#x] + +- Project [batch_id#x, amount#x, total#xL, total#xL] + +- Window [sum(amount#x) windowspecdefinition(batch_id#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, unboundedpreceding$(), unboundedfollowing$())) AS total#xL], [batch_id#x ASC NULLS FIRST] + +- Project [batch_id#x, amount#x] + +- SubqueryAlias t + +- View (`t`, [batch_id#x, amount#x]) + +- Project [cast(batch_id#x as int) AS batch_id#x, cast(amount#x as int) AS amount#x] + +- Project [batch_id#x, amount#x] + +- SubqueryAlias t + +- LocalRelation [batch_id#x, amount#x] + + +-- !query +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id DESC +GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) AS total FROM t ORDER BY batch_id DESC, amount +-- !query analysis +Project [batch_id#x, total#xL] ++- Sort [batch_id#x DESC NULLS LAST, amount#x ASC NULLS FIRST], true + +- Project [batch_id#x, total#xL, amount#x] + +- Project [batch_id#x, amount#x, total#xL, total#xL] + +- Window [sum(amount#x) windowspecdefinition(batch_id#x DESC NULLS LAST, specifiedwindowframe(GroupFrame, -1, currentrow$())) AS total#xL], [batch_id#x DESC NULLS LAST] + +- Project [batch_id#x, amount#x] + +- SubqueryAlias t + +- View (`t`, [batch_id#x, amount#x]) + +- Project [cast(batch_id#x as int) AS batch_id#x, cast(amount#x as int) AS amount#x] + +- Project [batch_id#x, amount#x] + +- SubqueryAlias t + +- LocalRelation [batch_id#x, amount#x] + + +-- !query +CREATE OR REPLACE TEMPORARY VIEW t_multi AS SELECT * FROM VALUES +('2024-01-01', 1, 10), ('2024-01-01', 1, 20), ('2024-01-02', 2, 30), +('2024-01-03', 3, 45), ('2024-01-03', 3, 45) +AS t_multi(trade_date, batch_id, amount) +-- !query analysis +CreateViewCommand `t_multi`, SELECT * FROM VALUES +('2024-01-01', 1, 10), ('2024-01-01', 1, 20), ('2024-01-02', 2, 30), +('2024-01-03', 3, 45), ('2024-01-03', 3, 45) +AS t_multi(trade_date, batch_id, amount), false, true, LocalTempView, UNSUPPORTED, true + +- Project [trade_date#x, batch_id#x, amount#x] + +- SubqueryAlias t_multi + +- LocalRelation [trade_date#x, batch_id#x, amount#x] + + +-- !query +SELECT sum(amount) OVER (ORDER BY trade_date, batch_id +GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) AS total +FROM t_multi ORDER BY trade_date, batch_id +-- !query analysis +Project [total#xL] ++- Sort [trade_date#x ASC NULLS FIRST, batch_id#x ASC NULLS FIRST], true + +- Project [total#xL, trade_date#x, batch_id#x] + +- Project [amount#x, trade_date#x, batch_id#x, total#xL, total#xL] + +- Window [sum(amount#x) windowspecdefinition(trade_date#x ASC NULLS FIRST, batch_id#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, -1, currentrow$())) AS total#xL], [trade_date#x ASC NULLS FIRST, batch_id#x ASC NULLS FIRST] + +- Project [amount#x, trade_date#x, batch_id#x] + +- SubqueryAlias t_multi + +- View (`t_multi`, [trade_date#x, batch_id#x, amount#x]) + +- Project [cast(trade_date#x as string) AS trade_date#x, cast(batch_id#x as int) AS batch_id#x, cast(amount#x as int) AS amount#x] + +- Project [trade_date#x, batch_id#x, amount#x] + +- SubqueryAlias t_multi + +- LocalRelation [trade_date#x, batch_id#x, amount#x] + + +-- !query +CREATE OR REPLACE TEMPORARY VIEW t_nulls AS SELECT * FROM VALUES +(CAST(1 AS INT), 10), (1, 15), (2, 20), (CAST(NULL AS INT), 30) +AS t_nulls(batch_id, amount) +-- !query analysis +CreateViewCommand `t_nulls`, SELECT * FROM VALUES +(CAST(1 AS INT), 10), (1, 15), (2, 20), (CAST(NULL AS INT), 30) +AS t_nulls(batch_id, amount), false, true, LocalTempView, UNSUPPORTED, true + +- Project [batch_id#x, amount#x] + +- SubqueryAlias t_nulls + +- LocalRelation [batch_id#x, amount#x] + + +-- !query +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id NULLS FIRST +GROUPS BETWEEN CURRENT ROW AND CURRENT ROW) AS total +FROM t_nulls ORDER BY batch_id NULLS FIRST, amount +-- !query analysis +Project [batch_id#x, total#xL] ++- Sort [batch_id#x ASC NULLS FIRST, amount#x ASC NULLS FIRST], true + +- Project [batch_id#x, total#xL, amount#x] + +- Project [batch_id#x, amount#x, total#xL, total#xL] + +- Window [sum(amount#x) windowspecdefinition(batch_id#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, currentrow$(), currentrow$())) AS total#xL], [batch_id#x ASC NULLS FIRST] + +- Project [batch_id#x, amount#x] + +- SubqueryAlias t_nulls + +- View (`t_nulls`, [batch_id#x, amount#x]) + +- Project [cast(batch_id#x as int) AS batch_id#x, cast(amount#x as int) AS amount#x] + +- Project [batch_id#x, amount#x] + +- SubqueryAlias t_nulls + +- LocalRelation [batch_id#x, amount#x] + + +-- !query +SELECT batch_id, +sum(amount) OVER (ORDER BY batch_id + GROUPS BETWEEN CURRENT ROW AND CURRENT ROW) AS current_row_total, +sum(amount) OVER (ORDER BY batch_id + GROUPS BETWEEN 0 PRECEDING AND 0 FOLLOWING) AS zero_offset_total +FROM t ORDER BY batch_id, amount +-- !query analysis +Project [batch_id#x, current_row_total#xL, zero_offset_total#xL] ++- Sort [batch_id#x ASC NULLS FIRST, amount#x ASC NULLS FIRST], true + +- Project [batch_id#x, current_row_total#xL, zero_offset_total#xL, amount#x] + +- Project [batch_id#x, amount#x, current_row_total#xL, zero_offset_total#xL, current_row_total#xL, zero_offset_total#xL] + +- Window [sum(amount#x) windowspecdefinition(batch_id#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, currentrow$(), currentrow$())) AS current_row_total#xL, sum(amount#x) windowspecdefinition(batch_id#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, -0, 0)) AS zero_offset_total#xL], [batch_id#x ASC NULLS FIRST] + +- Project [batch_id#x, amount#x] + +- SubqueryAlias t + +- View (`t`, [batch_id#x, amount#x]) + +- Project [cast(batch_id#x as int) AS batch_id#x, cast(amount#x as int) AS amount#x] + +- Project [batch_id#x, amount#x] + +- SubqueryAlias t + +- LocalRelation [batch_id#x, amount#x] + + +-- !query +SELECT batch_id, count(amount) OVER( +GROUPS BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM t ORDER BY batch_id +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.GROUPS_FRAME_WITHOUT_ORDER", + "sqlState" : "42K09", + "messageParameters" : { + "sqlExpr" : "\"(GROUPS BETWEEN CURRENT ROW AND 1 FOLLOWING)\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 36, + "stopIndex" : 80, + "fragment" : "(\nGROUPS BETWEEN CURRENT ROW AND 1 FOLLOWING)" + } ] +} + + +-- !query +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id +GROUPS BETWEEN CAST(NULL AS INT) PRECEDING AND CURRENT ROW) AS total FROM t +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.GROUPS_FRAME_NULL_OFFSET", + "sqlState" : "42K09", + "messageParameters" : { + "location" : "lower", + "sqlExpr" : "\"GROUPS BETWEEN CAST(NULL AS INT) PRECEDING AND CURRENT ROW\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 35, + "stopIndex" : 112, + "fragment" : "(ORDER BY batch_id\nGROUPS BETWEEN CAST(NULL AS INT) PRECEDING AND CURRENT ROW)" + } ] +} + + +-- !query +SELECT 1 AS groups, 2 AS other ORDER BY groups +-- !query analysis +Sort [groups#x ASC NULLS FIRST], true ++- Project [1 AS groups#x, 2 AS other#x] + +- OneRowRelation + + +-- !query +CREATE OR REPLACE TEMPORARY VIEW t_ident AS SELECT * FROM VALUES +(1, 10), (1, 15), (2, 20), (3, 25), (3, 30), (9, 40) +AS t_ident(groups, amount) +-- !query analysis +CreateViewCommand `t_ident`, SELECT * FROM VALUES +(1, 10), (1, 15), (2, 20), (3, 25), (3, 30), (9, 40) +AS t_ident(groups, amount), false, true, LocalTempView, UNSUPPORTED, true + +- Project [groups#x, amount#x] + +- SubqueryAlias t_ident + +- LocalRelation [groups#x, amount#x] + + +-- !query +SELECT groups, amount FROM t_ident ORDER BY groups +-- !query analysis +Sort [groups#x ASC NULLS FIRST], true ++- Project [groups#x, amount#x] + +- SubqueryAlias t_ident + +- View (`t_ident`, [groups#x, amount#x]) + +- Project [cast(groups#x as int) AS groups#x, cast(amount#x as int) AS amount#x] + +- Project [groups#x, amount#x] + +- SubqueryAlias t_ident + +- LocalRelation [groups#x, amount#x] + + +-- !query +SELECT groups, sum(amount) OVER (ORDER BY groups +GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) AS total FROM t_ident ORDER BY groups, amount +-- !query analysis +Project [groups#x, total#xL] ++- Sort [groups#x ASC NULLS FIRST, amount#x ASC NULLS FIRST], true + +- Project [groups#x, total#xL, amount#x] + +- Project [groups#x, amount#x, total#xL, total#xL] + +- Window [sum(amount#x) windowspecdefinition(groups#x ASC NULLS FIRST, specifiedwindowframe(GroupFrame, -1, currentrow$())) AS total#xL], [groups#x ASC NULLS FIRST] + +- Project [groups#x, amount#x] + +- SubqueryAlias t_ident + +- View (`t_ident`, [groups#x, amount#x]) + +- Project [cast(groups#x as int) AS groups#x, cast(amount#x as int) AS amount#x] + +- Project [groups#x, amount#x] + +- SubqueryAlias t_ident + +- LocalRelation [groups#x, amount#x] + + +-- !query +CREATE OR REPLACE TEMPORARY VIEW t_regression AS SELECT * FROM VALUES +(1, 10, 10), (2, 20, 20), (3, 30, 10) +AS t_regression(key, row_amount, range_amount) +-- !query analysis +CreateViewCommand `t_regression`, SELECT * FROM VALUES +(1, 10, 10), (2, 20, 20), (3, 30, 10) +AS t_regression(key, row_amount, range_amount), false, true, LocalTempView, UNSUPPORTED, true + +- Project [key#x, row_amount#x, range_amount#x] + +- SubqueryAlias t_regression + +- LocalRelation [key#x, row_amount#x, range_amount#x] + + +-- !query +SELECT +sum(row_amount) OVER (ORDER BY key ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS rows_total, +sum(range_amount) OVER (ORDER BY key RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + AS range_total +FROM t_regression +-- !query analysis +Project [rows_total#xL, range_total#xL] ++- Project [row_amount#x, key#x, range_amount#x, rows_total#xL, range_total#xL, rows_total#xL, range_total#xL] + +- Window [sum(row_amount#x) windowspecdefinition(key#x ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rows_total#xL, sum(range_amount#x) windowspecdefinition(key#x ASC NULLS FIRST, specifiedwindowframe(RangeFrame, unboundedpreceding$(), currentrow$())) AS range_total#xL], [key#x ASC NULLS FIRST] + +- Project [row_amount#x, key#x, range_amount#x] + +- SubqueryAlias t_regression + +- View (`t_regression`, [key#x, row_amount#x, range_amount#x]) + +- Project [cast(key#x as int) AS key#x, cast(row_amount#x as int) AS row_amount#x, cast(range_amount#x as int) AS range_amount#x] + +- Project [key#x, row_amount#x, range_amount#x] + +- SubqueryAlias t_regression + +- LocalRelation [key#x, row_amount#x, range_amount#x] diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part1.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part1.sql index 3ebe9f91b2a08..22104ae95946f 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part1.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part1.sql @@ -345,10 +345,9 @@ SELECT * FROM v_window; -- exclude no others) as sum_rows FROM generate_series(1, 10) i; -- SELECT * FROM v_window; --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- CREATE OR REPLACE TEMP VIEW v_window AS --- SELECT i.id, sum(i.id) over (order by i.id groups between 1 preceding and 1 following) as sum_rows FROM range(1, 11) i; --- SELECT * FROM v_window; +CREATE OR REPLACE TEMP VIEW v_window AS +SELECT i.id, sum(i.id) over (order by i.id groups between 1 preceding and 1 following) as sum_rows FROM range(1, 11) i; +SELECT * FROM v_window; DROP VIEW v_window; -- [SPARK-29540] Thrift in some cases can't parse string to date diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part3.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part3.sql index 6f33a07631f7a..cbf5c28e15f8e 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part3.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part3.sql @@ -151,50 +151,41 @@ insert into datetimes values -- GROUPS tests --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- SELECT sum(unique1) over (order by four groups between unbounded preceding and current row), --- unique1, four --- FROM tenk1 WHERE unique1 < 10; +SELECT sum(unique1) over (order by four groups between unbounded preceding and current row), +unique1, four +FROM tenk1 WHERE unique1 < 10; --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- SELECT sum(unique1) over (order by four groups between unbounded preceding and unbounded following), --- unique1, four --- FROM tenk1 WHERE unique1 < 10; +SELECT sum(unique1) over (order by four groups between unbounded preceding and unbounded following), +unique1, four +FROM tenk1 WHERE unique1 < 10; --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- SELECT sum(unique1) over (order by four groups between current row and unbounded following), --- unique1, four --- FROM tenk1 WHERE unique1 < 10; +SELECT sum(unique1) over (order by four groups between current row and unbounded following), +unique1, four +FROM tenk1 WHERE unique1 < 10; --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- SELECT sum(unique1) over (order by four groups between 1 preceding and unbounded following), --- unique1, four --- FROM tenk1 WHERE unique1 < 10; +SELECT sum(unique1) over (order by four groups between 1 preceding and unbounded following), +unique1, four +FROM tenk1 WHERE unique1 < 10; --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- SELECT sum(unique1) over (order by four groups between 1 following and unbounded following), --- unique1, four --- FROM tenk1 WHERE unique1 < 10; +SELECT sum(unique1) over (order by four groups between 1 following and unbounded following), +unique1, four +FROM tenk1 WHERE unique1 < 10; --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- SELECT sum(unique1) over (order by four groups between unbounded preceding and 2 following), --- unique1, four --- FROM tenk1 WHERE unique1 < 10; +SELECT sum(unique1) over (order by four groups between unbounded preceding and 2 following), +unique1, four +FROM tenk1 WHERE unique1 < 10; --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- SELECT sum(unique1) over (order by four groups between 2 preceding and 1 preceding), --- unique1, four --- FROM tenk1 WHERE unique1 < 10; +SELECT sum(unique1) over (order by four groups between 2 preceding and 1 preceding), +unique1, four +FROM tenk1 WHERE unique1 < 10; --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- SELECT sum(unique1) over (order by four groups between 2 preceding and 1 following), --- unique1, four --- FROM tenk1 WHERE unique1 < 10; +SELECT sum(unique1) over (order by four groups between 2 preceding and 1 following), +unique1, four +FROM tenk1 WHERE unique1 < 10; --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- SELECT sum(unique1) over (order by four groups between 0 preceding and 0 following), --- unique1, four --- FROM tenk1 WHERE unique1 < 10; +SELECT sum(unique1) over (order by four groups between 0 preceding and 0 following), +unique1, four +FROM tenk1 WHERE unique1 < 10; -- [SPARK-28428] Spark `exclude` always expecting `()` -- SELECT sum(unique1) over (order by four groups between 2 preceding and 1 following @@ -211,41 +202,39 @@ insert into datetimes values -- exclude ties), unique1, four -- FROM tenk1 WHERE unique1 < 10; --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- SELECT sum(unique1) over (partition by ten --- order by four groups between 0 preceding and 0 following),unique1, four, ten --- FROM tenk1 WHERE unique1 < 10; +SELECT sum(unique1) over (partition by ten + order by four groups between 0 preceding and 0 following),unique1, four, ten +FROM tenk1 WHERE unique1 < 10; -- [SPARK-28428] Spark `exclude` always expecting `()` --- [SPARK-28648] Adds support to `groups` unit type in window clauses -- SELECT sum(unique1) over (partition by ten -- order by four groups between 0 preceding and 0 following exclude current row), unique1, four, ten -- FROM tenk1 WHERE unique1 < 10; -- [SPARK-28428] Spark `exclude` always expecting `()` --- [SPARK-28648] Adds support to `groups` unit type in window clauses -- SELECT sum(unique1) over (partition by ten -- order by four groups between 0 preceding and 0 following exclude group), unique1, four, ten -- FROM tenk1 WHERE unique1 < 10; -- [SPARK-28428] Spark `exclude` always expecting `()` --- [SPARK-28648] Adds support to `groups` unit type in window clauses -- SELECT sum(unique1) over (partition by ten -- order by four groups between 0 preceding and 0 following exclude ties), unique1, four, ten -- FROM tenk1 WHERE unique1 < 10; --- [SPARK-27951] ANSI SQL: NTH_VALUE function --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- select first_value(salary) over(order by enroll_date groups between 1 preceding and 1 following), --- lead(salary) over(order by enroll_date groups between 1 preceding and 1 following), --- nth_value(salary, 1) over(order by enroll_date groups between 1 preceding and 1 following), --- salary, enroll_date from empsalary; +-- Keep unsupported lead/lag frames separate so the value functions are also executed. +-- Use the order key as the value so the results do not depend on the order within a peer group. +select first_value(enroll_date) over(order by enroll_date groups between 1 preceding and 1 following), +nth_value(enroll_date, 1) over(order by enroll_date groups between 1 preceding and 1 following), +salary, enroll_date from empsalary; --- [SPARK-28508] Support for range frame+row frame in the same query --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- select last(salary) over(order by enroll_date groups between 1 preceding and 1 following), --- lag(salary) over(order by enroll_date groups between 1 preceding and 1 following), --- salary, enroll_date from empsalary; +select lead(salary) over(order by enroll_date groups between 1 preceding and 1 following) +from empsalary; + +select last(enroll_date) over(order by enroll_date groups between 1 preceding and 1 following), +salary, enroll_date from empsalary; + +select lag(salary) over(order by enroll_date groups between 1 preceding and 1 following) +from empsalary; -- [SPARK-27951] ANSI SQL: NTH_VALUE function -- select first_value(salary) over(order by enroll_date groups between 1 following and 3 following @@ -276,13 +265,12 @@ SELECT x, (sum(x) over w) FROM cte WINDOW w AS (ORDER BY x range between 1 preceding and 1 following); --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- WITH cte (x) AS ( --- SELECT * FROM range(1, 36, 2) --- ) --- SELECT x, (sum(x) over w) --- FROM cte --- WINDOW w AS (ORDER BY x groups between 1 preceding and 1 following); +WITH cte (x) AS ( + SELECT * FROM range(1, 36, 2) +) +SELECT x, (sum(x) over w) +FROM cte +WINDOW w AS (ORDER BY x groups between 1 preceding and 1 following); WITH cte (x) AS ( select 1 union all select 1 union all select 1 union all @@ -300,14 +288,13 @@ SELECT x, (sum(x) over w) FROM cte WINDOW w AS (ORDER BY x range between 1 preceding and 1 following); --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- WITH cte (x) AS ( --- select 1 union all select 1 union all select 1 union all --- SELECT * FROM range(5, 50, 2) --- ) --- SELECT x, (sum(x) over w) --- FROM cte --- WINDOW w AS (ORDER BY x groups between 1 preceding and 1 following); +WITH cte (x) AS ( + select 1 union all select 1 union all select 1 union all + SELECT * FROM range(5, 50, 2) +) +SELECT x, (sum(x) over w) +FROM cte +WINDOW w AS (ORDER BY x groups between 1 preceding and 1 following); -- with UNION SELECT count(*) OVER (PARTITION BY four) FROM (SELECT * FROM tenk1 UNION ALL SELECT * FROM tenk2)s LIMIT 0; @@ -338,10 +325,9 @@ select f1, sum(f1) over (partition by f1, f2 order by f2 range between 1 following and 2 following) from t1 where f1 = f2; --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- select f1, sum(f1) over (partition by f1, --- groups between 1 preceding and 1 following) --- from t1 where f1 = f2; +select f1, sum(f1) over (partition by f1, +groups between 1 preceding and 1 following) +from t1 where f1 = f2; -- Since EXPLAIN clause rely on host physical location, it is commented out -- explain @@ -349,20 +335,17 @@ from t1 where f1 = f2; -- range between 1 preceding and 1 following) -- from t1 where f1 = f2; --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- select f1, sum(f1) over (partition by f1 order by f2 --- groups between 1 preceding and 1 following) --- from t1 where f1 = f2; +select f1, sum(f1) over (partition by f1 order by f2 +groups between 1 preceding and 1 following) +from t1 where f1 = f2; --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- select f1, sum(f1) over (partition by f1, f1 order by f2 --- groups between 2 preceding and 1 preceding) --- from t1 where f1 = f2; +select f1, sum(f1) over (partition by f1, f1 order by f2 +groups between 2 preceding and 1 preceding) +from t1 where f1 = f2; --- [SPARK-28648] Adds support to `groups` unit type in window clauses --- select f1, sum(f1) over (partition by f1, f2 order by f2 --- groups between 1 following and 2 following) --- from t1 where f1 = f2; +select f1, sum(f1) over (partition by f1, f2 order by f2 +groups between 1 following and 2 following) +from t1 where f1 = f2; -- ordering by a non-integer constant is allowed SELECT rank() OVER (ORDER BY length('abc')); diff --git a/sql/core/src/test/resources/sql-tests/inputs/window-groups.sql b/sql/core/src/test/resources/sql-tests/inputs/window-groups.sql new file mode 100644 index 0000000000000..31f58535f7909 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/window-groups.sql @@ -0,0 +1,91 @@ +-- Tests for GROUPS window frames (SPARK-58980). +-- Canonical data reused throughout: batch_id has a tie (3, 25) / (3, 30) so peer-group +-- semantics are actually exercised, matching DataFrameWindowFramesSuite's test matrix. +CREATE OR REPLACE TEMPORARY VIEW t AS SELECT * FROM VALUES +(1, 10), (1, 15), (2, 20), (3, 25), (3, 30), (9, 40) +AS t(batch_id, amount); + +-- Case 1: `n PRECEDING` counts peer groups, not rows or key distance. +-- Expected: 25,25,45,75,75,95 +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id +GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) AS total FROM t ORDER BY batch_id, amount; + +-- Case 2: CURRENT ROW means the current row's entire peer group. +-- Expected: 25,25,20,55,55,40 +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id +GROUPS BETWEEN CURRENT ROW AND CURRENT ROW) AS total FROM t ORDER BY batch_id, amount; + +-- Case 3: `n FOLLOWING` advances one peer group. +-- Expected: 45,45,75,95,95,40 +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id +GROUPS BETWEEN CURRENT ROW AND 1 FOLLOWING) AS total FROM t ORDER BY batch_id, amount; + +-- Unbounded-both-ends GROUPS matches the equivalent RANGE query (no offset bounds). +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id +GROUPS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS total +FROM t ORDER BY batch_id, amount; + +-- Case 5: DESC ordering - offsets follow window order, not key order. +-- Expected: 40,95,95,75,45,45 +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id DESC +GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) AS total FROM t ORDER BY batch_id DESC, amount; + +-- Case 4: multi-column ORDER BY with an offset - GROUPS has no RANGE_FRAME_MULTI_ORDER +-- analogue, since multi-order support is a headline reason for the feature. +CREATE OR REPLACE TEMPORARY VIEW t_multi AS SELECT * FROM VALUES +('2024-01-01', 1, 10), ('2024-01-01', 1, 20), ('2024-01-02', 2, 30), +('2024-01-03', 3, 45), ('2024-01-03', 3, 45) +AS t_multi(trade_date, batch_id, amount); + +SELECT sum(amount) OVER (ORDER BY trade_date, batch_id +GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) AS total +FROM t_multi ORDER BY trade_date, batch_id; + +-- NULLs form one peer group: NULLS FIRST. +CREATE OR REPLACE TEMPORARY VIEW t_nulls AS SELECT * FROM VALUES +(CAST(1 AS INT), 10), (1, 15), (2, 20), (CAST(NULL AS INT), 30) +AS t_nulls(batch_id, amount); + +-- Case 6: NULLs form one peer group. +-- Expected: 25,25,20,30 +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id NULLS FIRST +GROUPS BETWEEN CURRENT ROW AND CURRENT ROW) AS total +FROM t_nulls ORDER BY batch_id NULLS FIRST, amount; + +-- 0 PRECEDING / 0 FOLLOWING are equivalent to CURRENT ROW. +SELECT batch_id, +sum(amount) OVER (ORDER BY batch_id + GROUPS BETWEEN CURRENT ROW AND CURRENT ROW) AS current_row_total, +sum(amount) OVER (ORDER BY batch_id + GROUPS BETWEEN 0 PRECEDING AND 0 FOLLOWING) AS zero_offset_total +FROM t ORDER BY batch_id, amount; + +-- Analysis error: GROUPS requires ORDER BY (test matrix case 7). +SELECT batch_id, count(amount) OVER( +GROUPS BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM t ORDER BY batch_id; + +-- Analysis error: null offsets are rejected (test matrix case 8). +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id +GROUPS BETWEEN CAST(NULL AS INT) PRECEDING AND CURRENT ROW) AS total FROM t; + +-- `groups` remains usable as an identifier (test matrix case 9). +SELECT 1 AS groups, 2 AS other ORDER BY groups; +CREATE OR REPLACE TEMPORARY VIEW t_ident AS SELECT * FROM VALUES +(1, 10), (1, 15), (2, 20), (3, 25), (3, 30), (9, 40) +AS t_ident(groups, amount); + +SELECT groups, amount FROM t_ident ORDER BY groups; +SELECT groups, sum(amount) OVER (ORDER BY groups +GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) AS total FROM t_ident ORDER BY groups, amount; + +-- Regression guard (test matrix case 10): existing ROWS and RANGE results are unchanged. +-- Expected: rows_total 10,30,60 ; range_total 10,30,40. +CREATE OR REPLACE TEMPORARY VIEW t_regression AS SELECT * FROM VALUES +(1, 10, 10), (2, 20, 20), (3, 30, 10) +AS t_regression(key, row_amount, range_amount); + +SELECT +sum(row_amount) OVER (ORDER BY key ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS rows_total, +sum(range_amount) OVER (ORDER BY key RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + AS range_total +FROM t_regression; diff --git a/sql/core/src/test/resources/sql-tests/results/keywords-enforced.sql.out b/sql/core/src/test/resources/sql-tests/results/keywords-enforced.sql.out index 84dbbaf041d2a..f0651ca1748b6 100644 --- a/sql/core/src/test/resources/sql-tests/results/keywords-enforced.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/keywords-enforced.sql.out @@ -177,6 +177,7 @@ GLOBAL false GRANT true GROUP true GROUPING false +GROUPS false HANDLER false HAVING true HISTORY false diff --git a/sql/core/src/test/resources/sql-tests/results/keywords.sql.out b/sql/core/src/test/resources/sql-tests/results/keywords.sql.out index 4b57e254ef964..cd19fa2f46753 100644 --- a/sql/core/src/test/resources/sql-tests/results/keywords.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/keywords.sql.out @@ -177,6 +177,7 @@ GLOBAL false GRANT false GROUP false GROUPING false +GROUPS false HANDLER false HAVING false HISTORY false diff --git a/sql/core/src/test/resources/sql-tests/results/nonansi/keywords.sql.out b/sql/core/src/test/resources/sql-tests/results/nonansi/keywords.sql.out index 4b57e254ef964..cd19fa2f46753 100644 --- a/sql/core/src/test/resources/sql-tests/results/nonansi/keywords.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/nonansi/keywords.sql.out @@ -177,6 +177,7 @@ GLOBAL false GRANT false GROUP false GROUPING false +GROUPS false HANDLER false HAVING false HISTORY false diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part1.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part1.sql.out index 7f16547b50f65..2b352e57a293c 100755 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part1.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part1.sql.out @@ -681,6 +681,32 @@ struct<> +-- !query +SELECT * FROM v_window +-- !query schema +struct +-- !query output +1 3 +10 19 +2 6 +3 9 +4 12 +5 15 +6 18 +7 21 +8 24 +9 27 + + +-- !query +CREATE OR REPLACE TEMP VIEW v_window AS +SELECT i.id, sum(i.id) over (order by i.id groups between 1 preceding and 1 following) as sum_rows FROM range(1, 11) i +-- !query schema +struct<> +-- !query output + + + -- !query SELECT * FROM v_window -- !query schema diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out index 95e8101801e57..584dcb803bd90 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out @@ -85,6 +85,263 @@ org.apache.spark.sql.AnalysisException } +-- !query +SELECT sum(unique1) over (order by four groups between unbounded preceding and current row), +unique1, four +FROM tenk1 WHERE unique1 < 10 +-- !query schema +struct +-- !query output +12 0 0 +12 4 0 +12 8 0 +27 1 1 +27 5 1 +27 9 1 +35 2 2 +35 6 2 +45 3 3 +45 7 3 + + +-- !query +SELECT sum(unique1) over (order by four groups between unbounded preceding and unbounded following), +unique1, four +FROM tenk1 WHERE unique1 < 10 +-- !query schema +struct +-- !query output +45 0 0 +45 1 1 +45 2 2 +45 3 3 +45 4 0 +45 5 1 +45 6 2 +45 7 3 +45 8 0 +45 9 1 + + +-- !query +SELECT sum(unique1) over (order by four groups between current row and unbounded following), +unique1, four +FROM tenk1 WHERE unique1 < 10 +-- !query schema +struct +-- !query output +10 3 3 +10 7 3 +18 2 2 +18 6 2 +33 1 1 +33 5 1 +33 9 1 +45 0 0 +45 4 0 +45 8 0 + + +-- !query +SELECT sum(unique1) over (order by four groups between 1 preceding and unbounded following), +unique1, four +FROM tenk1 WHERE unique1 < 10 +-- !query schema +struct +-- !query output +18 3 3 +18 7 3 +33 2 2 +33 6 2 +45 0 0 +45 1 1 +45 4 0 +45 5 1 +45 8 0 +45 9 1 + + +-- !query +SELECT sum(unique1) over (order by four groups between 1 following and unbounded following), +unique1, four +FROM tenk1 WHERE unique1 < 10 +-- !query schema +struct +-- !query output +10 2 2 +10 6 2 +18 1 1 +18 5 1 +18 9 1 +33 0 0 +33 4 0 +33 8 0 +NULL 3 3 +NULL 7 3 + + +-- !query +SELECT sum(unique1) over (order by four groups between unbounded preceding and 2 following), +unique1, four +FROM tenk1 WHERE unique1 < 10 +-- !query schema +struct +-- !query output +35 0 0 +35 4 0 +35 8 0 +45 1 1 +45 2 2 +45 3 3 +45 5 1 +45 6 2 +45 7 3 +45 9 1 + + +-- !query +SELECT sum(unique1) over (order by four groups between 2 preceding and 1 preceding), +unique1, four +FROM tenk1 WHERE unique1 < 10 +-- !query schema +struct +-- !query output +12 1 1 +12 5 1 +12 9 1 +23 3 3 +23 7 3 +27 2 2 +27 6 2 +NULL 0 0 +NULL 4 0 +NULL 8 0 + + +-- !query +SELECT sum(unique1) over (order by four groups between 2 preceding and 1 following), +unique1, four +FROM tenk1 WHERE unique1 < 10 +-- !query schema +struct +-- !query output +27 0 0 +27 4 0 +27 8 0 +33 3 3 +33 7 3 +35 1 1 +35 5 1 +35 9 1 +45 2 2 +45 6 2 + + +-- !query +SELECT sum(unique1) over (order by four groups between 0 preceding and 0 following), +unique1, four +FROM tenk1 WHERE unique1 < 10 +-- !query schema +struct +-- !query output +10 3 3 +10 7 3 +12 0 0 +12 4 0 +12 8 0 +15 1 1 +15 5 1 +15 9 1 +8 2 2 +8 6 2 + + +-- !query +SELECT sum(unique1) over (partition by ten + order by four groups between 0 preceding and 0 following),unique1, four, ten +FROM tenk1 WHERE unique1 < 10 +-- !query schema +struct +-- !query output +0 0 0 0 +1 1 1 1 +2 2 2 2 +3 3 3 3 +4 4 0 4 +5 5 1 5 +6 6 2 6 +7 7 3 7 +8 8 0 8 +9 9 1 9 + + +-- !query +select first_value(enroll_date) over(order by enroll_date groups between 1 preceding and 1 following), +nth_value(enroll_date, 1) over(order by enroll_date groups between 1 preceding and 1 following), +salary, enroll_date from empsalary +-- !query schema +struct +-- !query output +2006-10-01 2006-10-01 3900 2006-12-23 +2006-10-01 2006-10-01 5000 2006-10-01 +2006-10-01 2006-10-01 6000 2006-10-01 +2006-12-23 2006-12-23 4800 2007-08-01 +2006-12-23 2006-12-23 5200 2007-08-01 +2007-08-01 2007-08-01 4800 2007-08-08 +2007-08-08 2007-08-08 5200 2007-08-15 +2007-08-15 2007-08-15 3500 2007-12-10 +2007-12-10 2007-12-10 4200 2008-01-01 +2007-12-10 2007-12-10 4500 2008-01-01 + + +-- !query +select lead(salary) over(order by enroll_date groups between 1 preceding and 1 following) +from empsalary +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "_LEGACY_ERROR_TEMP_1035", + "messageParameters" : { + "prettyName" : "lead" + } +} + + +-- !query +select last(enroll_date) over(order by enroll_date groups between 1 preceding and 1 following), +salary, enroll_date from empsalary +-- !query schema +struct +-- !query output +2006-12-23 5000 2006-10-01 +2006-12-23 6000 2006-10-01 +2007-08-01 3900 2006-12-23 +2007-08-08 4800 2007-08-01 +2007-08-08 5200 2007-08-01 +2007-08-15 4800 2007-08-08 +2007-12-10 5200 2007-08-15 +2008-01-01 3500 2007-12-10 +2008-01-01 4200 2008-01-01 +2008-01-01 4500 2008-01-01 + + +-- !query +select lag(salary) over(order by enroll_date groups between 1 preceding and 1 following) +from empsalary +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "_LEGACY_ERROR_TEMP_1035", + "messageParameters" : { + "prettyName" : "lag" + } +} + + -- !query WITH cte (x) AS ( SELECT * FROM range(1, 36, 2) @@ -145,6 +402,36 @@ struct +-- !query output +1 4 +11 33 +13 39 +15 45 +17 51 +19 57 +21 63 +23 69 +25 75 +27 81 +29 87 +3 9 +31 93 +33 99 +35 68 +5 15 +7 21 +9 27 + + -- !query WITH cte (x) AS ( select 1 union all select 1 union all select 1 union all @@ -223,6 +510,45 @@ struct +-- !query output +1 8 +1 8 +1 8 +11 33 +13 39 +15 45 +17 51 +19 57 +21 63 +23 69 +25 75 +27 81 +29 87 +31 93 +33 99 +35 105 +37 111 +39 117 +41 123 +43 129 +45 135 +47 141 +49 96 +5 15 +7 21 +9 27 + + -- !query SELECT count(*) OVER (PARTITION BY four) FROM (SELECT * FROM tenk1 UNION ALL SELECT * FROM tenk2)s LIMIT 0 -- !query schema @@ -304,6 +630,57 @@ struct +-- !query output +org.apache.spark.sql.catalyst.parser.ParseException +{ + "errorClass" : "PARSE_SYNTAX_ERROR", + "sqlState" : "42601", + "messageParameters" : { + "error" : "'preceding'", + "hint" : ": extra input 'preceding'" + } +} + + +-- !query +select f1, sum(f1) over (partition by f1 order by f2 +groups between 1 preceding and 1 following) +from t1 where f1 = f2 +-- !query schema +struct +-- !query output +1 1 +2 2 + + +-- !query +select f1, sum(f1) over (partition by f1, f1 order by f2 +groups between 2 preceding and 1 preceding) +from t1 where f1 = f2 +-- !query schema +struct +-- !query output +1 NULL +2 NULL + + +-- !query +select f1, sum(f1) over (partition by f1, f2 order by f2 +groups between 1 following and 2 following) +from t1 where f1 = f2 +-- !query schema +struct +-- !query output +1 NULL +2 NULL + + -- !query SELECT rank() OVER (ORDER BY length('abc')) -- !query schema diff --git a/sql/core/src/test/resources/sql-tests/results/window-groups.sql.out b/sql/core/src/test/resources/sql-tests/results/window-groups.sql.out new file mode 100644 index 0000000000000..0974ba08c76c1 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/window-groups.sql.out @@ -0,0 +1,262 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +CREATE OR REPLACE TEMPORARY VIEW t AS SELECT * FROM VALUES +(1, 10), (1, 15), (2, 20), (3, 25), (3, 30), (9, 40) +AS t(batch_id, amount) +-- !query schema +struct<> +-- !query output + + + +-- !query +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id +GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) AS total FROM t ORDER BY batch_id, amount +-- !query schema +struct +-- !query output +1 25 +1 25 +2 45 +3 75 +3 75 +9 95 + + +-- !query +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id +GROUPS BETWEEN CURRENT ROW AND CURRENT ROW) AS total FROM t ORDER BY batch_id, amount +-- !query schema +struct +-- !query output +1 25 +1 25 +2 20 +3 55 +3 55 +9 40 + + +-- !query +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id +GROUPS BETWEEN CURRENT ROW AND 1 FOLLOWING) AS total FROM t ORDER BY batch_id, amount +-- !query schema +struct +-- !query output +1 45 +1 45 +2 75 +3 95 +3 95 +9 40 + + +-- !query +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id +GROUPS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS total +FROM t ORDER BY batch_id, amount +-- !query schema +struct +-- !query output +1 140 +1 140 +2 140 +3 140 +3 140 +9 140 + + +-- !query +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id DESC +GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) AS total FROM t ORDER BY batch_id DESC, amount +-- !query schema +struct +-- !query output +9 40 +3 95 +3 95 +2 75 +1 45 +1 45 + + +-- !query +CREATE OR REPLACE TEMPORARY VIEW t_multi AS SELECT * FROM VALUES +('2024-01-01', 1, 10), ('2024-01-01', 1, 20), ('2024-01-02', 2, 30), +('2024-01-03', 3, 45), ('2024-01-03', 3, 45) +AS t_multi(trade_date, batch_id, amount) +-- !query schema +struct<> +-- !query output + + + +-- !query +SELECT sum(amount) OVER (ORDER BY trade_date, batch_id +GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) AS total +FROM t_multi ORDER BY trade_date, batch_id +-- !query schema +struct +-- !query output +30 +30 +60 +120 +120 + + +-- !query +CREATE OR REPLACE TEMPORARY VIEW t_nulls AS SELECT * FROM VALUES +(CAST(1 AS INT), 10), (1, 15), (2, 20), (CAST(NULL AS INT), 30) +AS t_nulls(batch_id, amount) +-- !query schema +struct<> +-- !query output + + + +-- !query +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id NULLS FIRST +GROUPS BETWEEN CURRENT ROW AND CURRENT ROW) AS total +FROM t_nulls ORDER BY batch_id NULLS FIRST, amount +-- !query schema +struct +-- !query output +NULL 30 +1 25 +1 25 +2 20 + + +-- !query +SELECT batch_id, +sum(amount) OVER (ORDER BY batch_id + GROUPS BETWEEN CURRENT ROW AND CURRENT ROW) AS current_row_total, +sum(amount) OVER (ORDER BY batch_id + GROUPS BETWEEN 0 PRECEDING AND 0 FOLLOWING) AS zero_offset_total +FROM t ORDER BY batch_id, amount +-- !query schema +struct +-- !query output +1 25 25 +1 25 25 +2 20 20 +3 55 55 +3 55 55 +9 40 40 + + +-- !query +SELECT batch_id, count(amount) OVER( +GROUPS BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM t ORDER BY batch_id +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.GROUPS_FRAME_WITHOUT_ORDER", + "sqlState" : "42K09", + "messageParameters" : { + "sqlExpr" : "\"(GROUPS BETWEEN CURRENT ROW AND 1 FOLLOWING)\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 36, + "stopIndex" : 80, + "fragment" : "(\nGROUPS BETWEEN CURRENT ROW AND 1 FOLLOWING)" + } ] +} + + +-- !query +SELECT batch_id, sum(amount) OVER (ORDER BY batch_id +GROUPS BETWEEN CAST(NULL AS INT) PRECEDING AND CURRENT ROW) AS total FROM t +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.GROUPS_FRAME_NULL_OFFSET", + "sqlState" : "42K09", + "messageParameters" : { + "location" : "lower", + "sqlExpr" : "\"GROUPS BETWEEN CAST(NULL AS INT) PRECEDING AND CURRENT ROW\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 35, + "stopIndex" : 112, + "fragment" : "(ORDER BY batch_id\nGROUPS BETWEEN CAST(NULL AS INT) PRECEDING AND CURRENT ROW)" + } ] +} + + +-- !query +SELECT 1 AS groups, 2 AS other ORDER BY groups +-- !query schema +struct +-- !query output +1 2 + + +-- !query +CREATE OR REPLACE TEMPORARY VIEW t_ident AS SELECT * FROM VALUES +(1, 10), (1, 15), (2, 20), (3, 25), (3, 30), (9, 40) +AS t_ident(groups, amount) +-- !query schema +struct<> +-- !query output + + + +-- !query +SELECT groups, amount FROM t_ident ORDER BY groups +-- !query schema +struct +-- !query output +1 10 +1 15 +2 20 +3 25 +3 30 +9 40 + + +-- !query +SELECT groups, sum(amount) OVER (ORDER BY groups +GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) AS total FROM t_ident ORDER BY groups, amount +-- !query schema +struct +-- !query output +1 25 +1 25 +2 45 +3 75 +3 75 +9 95 + + +-- !query +CREATE OR REPLACE TEMPORARY VIEW t_regression AS SELECT * FROM VALUES +(1, 10, 10), (2, 20, 20), (3, 30, 10) +AS t_regression(key, row_amount, range_amount) +-- !query schema +struct<> +-- !query output + + + +-- !query +SELECT +sum(row_amount) OVER (ORDER BY key ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS rows_total, +sum(range_amount) OVER (ORDER BY key RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + AS range_total +FROM t_regression +-- !query schema +struct +-- !query output +10 10 +30 30 +60 40 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFramesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFramesSuite.scala index d818e70a29db0..c29abfe4738aa 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFramesSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFramesSuite.scala @@ -17,15 +17,22 @@ package org.apache.spark.sql -import org.apache.spark.sql.catalyst.expressions.{Literal, NonFoldableLiteral} +import java.util.Locale + +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.expressions.{GroupFrame, Literal, + NonFoldableLiteral, RangeFrame, SpecifiedWindowFrame, WindowExpression} import org.apache.spark.sql.catalyst.optimizer.EliminateWindowPartitions import org.apache.spark.sql.catalyst.plans.logical.{Window => WindowNode} import org.apache.spark.sql.classic.ExpressionColumnNode +import org.apache.spark.sql.execution.{ExtendedMode, SortExec} +import org.apache.spark.sql.execution.exchange.Exchange +import org.apache.spark.sql.execution.window.WindowExec import org.apache.spark.sql.expressions.Window import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession -import org.apache.spark.sql.types.CalendarIntervalType +import org.apache.spark.sql.types.{CalendarIntervalType, DayTimeIntervalType, IntegerType} /** * Window frame testing for DataFrame API. @@ -649,4 +656,818 @@ class DataFrameWindowFramesSuite extends SharedSparkSession { Row("a", 1, "x", "x"), Row("b", 0, null, null))) } + + test("GROUPS frame requires an ORDER BY") { + withTempView("t") { + Seq((1, 1), (2, 2)).toDF("key", "value").createOrReplaceTempView("t") + checkError( + exception = intercept[AnalysisException]( + spark.sql( + "select sum(value) over (partition by key groups between " + + "unbounded preceding and current row) from t").collect()), + condition = "DATATYPE_MISMATCH.GROUPS_FRAME_WITHOUT_ORDER", + parameters = Map( + "sqlExpr" -> + "\"(PARTITION BY key GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)\""), + queryContext = Array( + ExpectedContext( + fragment = "(partition by key groups between " + + "unbounded preceding and current row)", + start = 23, + stop = 91))) + } + } + + test("GROUPS frame rejects a non-integral offset (decimal)") { + withTempView("t") { + Seq((1, 1), (2, 2)).toDF("key", "value").createOrReplaceTempView("t") + checkError( + exception = intercept[AnalysisException]( + spark.sql( + "select sum(value) over (order by key groups between " + + "1.5 preceding and current row) from t").collect()), + condition = "DATATYPE_MISMATCH.SPECIFIED_WINDOW_FRAME_UNACCEPTED_TYPE", + parameters = Map( + "sqlExpr" -> "\"GROUPS BETWEEN 1.5 PRECEDING AND CURRENT ROW\"", + "location" -> "lower", + "exprType" -> "\"DECIMAL(2,1)\"", + "expectedType" -> "\"INT\""), + queryContext = Array( + ExpectedContext( + fragment = "(order by key groups between " + + "1.5 preceding and current row)", + start = 23, + stop = 81))) + } + } + + test("GROUPS frame rejects a non-integral offset (interval)") { + withTempView("t") { + Seq((1, 1), (2, 2)).toDF("key", "value").createOrReplaceTempView("t") + checkError( + exception = intercept[AnalysisException]( + spark.sql( + "select sum(value) over (order by key groups between " + + "interval 1 day preceding and current row) from t").collect()), + condition = "DATATYPE_MISMATCH.SPECIFIED_WINDOW_FRAME_UNACCEPTED_TYPE", + parameters = Map( + "sqlExpr" -> "\"GROUPS BETWEEN INTERVAL '1' DAY PRECEDING AND CURRENT ROW\"", + "location" -> "lower", + "exprType" -> "\"INTERVAL DAY\"", + "expectedType" -> "\"INT\""), + queryContext = Array( + ExpectedContext( + fragment = "(order by key groups between " + + "interval 1 day preceding and current row)", + start = 23, + stop = 92))) + } + } + + test("GROUPS frame rejects a null offset") { + withTempView("t") { + Seq((1, 1), (2, 2)).toDF("key", "value").createOrReplaceTempView("t") + checkError( + exception = intercept[AnalysisException]( + spark.sql( + "select sum(value) over (order by key groups between " + + "cast(null as int) preceding and current row) from t").collect()), + condition = "DATATYPE_MISMATCH.GROUPS_FRAME_NULL_OFFSET", + parameters = Map( + "sqlExpr" -> "\"GROUPS BETWEEN CAST(NULL AS INT) PRECEDING AND CURRENT ROW\"", + "location" -> "lower"), + queryContext = Array( + ExpectedContext( + fragment = "(order by key groups between " + + "cast(null as int) preceding and current row)", + start = 23, + stop = 95))) + } + } + + // Use VALUES because the single-pass resolver does not support CreateViewCommand. + test("GROUPS frame rejects a null offset with single-pass resolver") { + withSQLConf( + SQLConf.ANALYZER_DUAL_RUN_LEGACY_AND_SINGLE_PASS_RESOLVER.key -> "false", + SQLConf.ANALYZER_SINGLE_PASS_RESOLVER_ENABLED_TENTATIVELY.key -> "false", + SQLConf.ANALYZER_SINGLE_PASS_RESOLVER_ENABLED.key -> "true") { + checkError( + exception = intercept[AnalysisException]( + spark.sql( + "select sum(value) over (order by key groups between " + + "cast(null as int) following and unbounded following) " + + "from values (1, 1), (2, 2) as t(key, value)").collect()), + condition = "DATATYPE_MISMATCH.GROUPS_FRAME_NULL_OFFSET", + parameters = Map( + "sqlExpr" -> ("\"GROUPS BETWEEN CAST(NULL AS INT) FOLLOWING AND " + + "UNBOUNDED FOLLOWING\""), + "location" -> "lower"), + queryContext = Array( + ExpectedContext( + fragment = "(order by key groups between " + + "cast(null as int) following and unbounded following)", + start = 23, + stop = 103))) + } + } + + test("GROUPS rejects negative literal, expression and parameter offsets") { + for (singlePass <- Seq(false, true)) { + withSQLConf( + SQLConf.ANALYZER_DUAL_RUN_LEGACY_AND_SINGLE_PASS_RESOLVER.key -> "false", + SQLConf.ANALYZER_SINGLE_PASS_RESOLVER_ENABLED_TENTATIVELY.key -> "false", + SQLConf.ANALYZER_SINGLE_PASS_RESOLVER_ENABLED.key -> singlePass.toString) { + for (offset <- Seq("-1", "-2147483648", "1 - 2", ":offset"); + boundary <- Seq( + s"$offset preceding and current row", + s"$offset following and unbounded following", + s"unbounded preceding and $offset preceding", + s"current row and $offset following")) { + val error = intercept[AnalysisException] { + spark.sql( + s"select sum(v) over (order by k groups between $boundary) " + + "from values (1, 10), (2, 20) as t(k, v)", + Map("offset" -> -1)).collect() + } + assert(error.getCondition == "DATATYPE_MISMATCH.GROUPS_FRAME_NEGATIVE_OFFSET") + } + } + } + } + + test("GROUPS accepts zero and positive parameter offsets") { + for (offset <- Seq(0, 1, Int.MaxValue)) { + val query = + "select sum(v) over (order by k groups between :offset preceding and " + + "current row) from values (1, 10), (1, 20), (2, 30) as t(k, v)" + checkAnswer(spark.sql(query, Map("offset" -> offset)), + Seq(Row(30L), Row(30L), Row(if (offset == 0) 30L else 60L))) + } + } + + // Returns the Sort, Exchange, and WindowExec counts in the executed plan. + private def planShape(df: DataFrame): (Int, Int, Int) = { + val plan = df.queryExecution.executedPlan + (plan.collect { case s: SortExec => s }.size, + plan.collect { case e: Exchange => e }.size, + plan.collect { case w: WindowExec => w }.size) + } + + // No-offset GROUPS frames have the same results and plan shape as RANGE frames. + private def checkGroupsFrameMatchesRange(frameBoundary: String): Unit = { + withTempView("t") { + // Include ties to exercise peer-group semantics. + Seq((1, 10), (1, 15), (2, 20), (3, 25), (3, 30), (9, 40)) + .toDF("batch_id", "amount").createOrReplaceTempView("t") + val groupsDf = spark.sql( + s"select sum(amount) over (partition by batch_id % 2 order by batch_id " + + s"groups $frameBoundary) as total from t") + val rangeDf = spark.sql( + s"select sum(amount) over (partition by batch_id % 2 order by batch_id " + + s"range $frameBoundary) as total from t") + checkAnswer(groupsDf, rangeDf) + + // Disable AQE to inspect operators directly. Build fresh DataFrames because executedPlan + // is cached and the frames above were created with AQE enabled. + withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { + val groupsShape = planShape(spark.sql( + s"select sum(amount) over (partition by batch_id % 2 order by batch_id " + + s"groups $frameBoundary) as total from t")) + val rangeShape = planShape(spark.sql( + s"select sum(amount) over (partition by batch_id % 2 order by batch_id " + + s"range $frameBoundary) as total from t")) + assert(groupsShape._1 > 0 && groupsShape._2 > 0 && groupsShape._3 > 0, + s"expected non-zero Sort/Exchange/WindowExec counts, got $groupsShape") + assert(groupsShape == rangeShape, + s"GROUPS plan shape differs from RANGE plan shape for '$frameBoundary'") + } + + // Physical RANGE execution must not change the logical frame representation. + val frameSqls = groupsDf.queryExecution.analyzed.collect { case w: WindowNode => + w.windowExpressions.flatMap(_.collect { case e: WindowExpression => + e.windowSpec.frameSpecification.sql + }) + }.flatten + assert(frameSqls.nonEmpty, "expected at least one window frame in the analyzed plan") + frameSqls.foreach { sql => + assert(sql.contains("GROUPS"), s"expected GROUPS in frame sql: $sql") + } + + val explainText = groupsDf.queryExecution.explainString(ExtendedMode) + assert(explainText.contains("GroupFrame"), + s"expected GroupFrame in explain output:\n$explainText") + assert(!explainText.contains("RangeFrame"), + s"did not expect RangeFrame in explain output:\n$explainText") + } + } + + test("GROUPS between unbounded preceding and unbounded following " + + "matches RANGE") { + checkGroupsFrameMatchesRange("between unbounded preceding and unbounded following") + } + + test("GROUPS between unbounded preceding and current row matches RANGE") { + checkGroupsFrameMatchesRange("between unbounded preceding and current row") + } + + test("GROUPS between current row and unbounded following matches RANGE") { + checkGroupsFrameMatchesRange("between current row and unbounded following") + } + + // Case 2: CURRENT ROW means the current row's entire peer group. + test("GROUPS between current row and current row (test matrix case 2)") { + withTempView("t") { + Seq((1, 10), (1, 15), (2, 20), (3, 25), (3, 30), (9, 40)) + .toDF("batch_id", "amount").createOrReplaceTempView("t") + checkAnswer( + spark.sql( + "select sum(amount) over (order by batch_id " + + "groups between current row and current row) as total from t"), + Seq(Row(25), Row(25), Row(20), Row(55), Row(55), Row(40))) + } + } + + // Case 6: NULLs form one peer group. + test("GROUPS with NULL peer group (test matrix case 6)") { + withTempView("t") { + Seq((Some(1), 10), (Some(1), 15), (Some(2), 20), (None, 30)) + .toDF("batch_id", "amount").createOrReplaceTempView("t") + checkAnswer( + spark.sql( + "select sum(amount) over (order by batch_id nulls first " + + "groups between current row and current row) as total from t"), + Seq(Row(25), Row(25), Row(20), Row(30))) + } + } + + // Exercise an offset GroupBoundOrdering with multiple NULL keys in one peer group. + test("GROUPS offset frame with NULLS FIRST peer group") { + withTempView("t") { + Seq( + (None, 1), (None, 2), (Some(1), 10), (Some(1), 15), (Some(2), 20), + (Some(3), 25), (Some(3), 30), (Some(9), 40)) + .toDF("batch_id", "amount").createOrReplaceTempView("t") + val groupsDf = spark.sql( + "select batch_id, sum(amount) over (order by batch_id nulls first " + + "groups between 1 preceding and current row) as total from t") + // Peer groups are {null, null}, {1, 1}, {2}, {3, 3}, {9}. + checkAnswer( + groupsDf, + Seq( + Row(null, 3), Row(null, 3), Row(1, 28), Row(1, 28), Row(2, 45), + Row(3, 75), Row(3, 75), Row(9, 95))) + // RANGE over DENSE_RANK provides an equivalent peer-group oracle. + checkAnswer( + groupsDf, + spark.sql( + """ + |select batch_id, total from ( + | select batch_id, amount, sum(amount) over ( + | order by dense_rank() over (order by batch_id nulls first) + | range between 1 preceding and current row) as total + | from t + |) + |""".stripMargin)) + } + } + + test("GROUPS offset frame with NULLS LAST peer group") { + withTempView("t") { + Seq( + (Some(1), 10), (Some(1), 15), (Some(2), 20), (Some(3), 25), (Some(3), 30), + (Some(9), 40), (None, 1), (None, 2)) + .toDF("batch_id", "amount").createOrReplaceTempView("t") + val groupsDf = spark.sql( + "select batch_id, sum(amount) over (order by batch_id nulls last " + + "groups between 1 preceding and current row) as total from t") + // Peer groups are {1, 1}, {2}, {3, 3}, {9}, {null, null}. + checkAnswer( + groupsDf, + Seq( + Row(1, 25), Row(1, 25), Row(2, 45), Row(3, 75), + Row(3, 75), Row(9, 95), Row(null, 43), Row(null, 43))) + checkAnswer( + groupsDf, + spark.sql( + """ + |select batch_id, total from ( + | select batch_id, amount, sum(amount) over ( + | order by dense_rank() over (order by batch_id nulls last) + | range between 1 preceding and current row) as total + | from t + |) + |""".stripMargin)) + } + } + + // Case 1: `n PRECEDING` counts peer groups, not rows or key distance. + test("GROUPS between 1 preceding and current row (test matrix case 1)") { + withTempView("t") { + Seq((1, 10), (1, 15), (2, 20), (3, 25), (3, 30), (9, 40)) + .toDF("batch_id", "amount").createOrReplaceTempView("t") + // Include batch_id to verify each total is assigned to the correct rows. + checkAnswer( + spark.sql( + "select batch_id, sum(amount) over (order by batch_id " + + "groups between 1 preceding and current row) as total from t"), + Seq( + Row(1, 25), Row(1, 25), Row(2, 45), Row(3, 75), Row(3, 75), Row(9, 95))) + } + } + + // GROUPS offsets remain integral and are not coerced to the ORDER BY type. + private def specifiedFrame(df: DataFrame): SpecifiedWindowFrame = { + val frames = df.queryExecution.analyzed.collect { case w: WindowNode => + w.windowExpressions.flatMap(_.collect { + case e: WindowExpression => e.windowSpec.frameSpecification + }) + }.flatten + assert(frames.size === 1, s"expected exactly one window frame, got $frames") + frames.head.asInstanceOf[SpecifiedWindowFrame] + } + + test("GROUPS offset over a DATE order key is not coerced to an interval") { + withTempView("t") { + Seq(("2020-01-01", 10), ("2020-01-01", 15), ("2020-01-02", 20), + ("2020-01-03", 25), ("2020-01-03", 30), ("2020-01-09", 40)) + .toDF("d", "amount") + .selectExpr("CAST(d AS DATE) AS d", "amount") + .createOrReplaceTempView("t") + + val groupsDf = spark.sql( + "select amount, sum(amount) over (order by d " + + "groups between 1 preceding and current row) as total from t") + val groupsFrame = specifiedFrame(groupsDf) + assert(groupsFrame.frameType === GroupFrame, + s"expected GroupFrame, got ${groupsFrame.frameType}") + assert(groupsFrame.lower.dataType == IntegerType, + s"GROUPS offset must stay an integer, not be cast to the DATE order-key type " + + s"or an interval, got ${groupsFrame.lower} : ${groupsFrame.lower.dataType}") + val analyzedText = groupsDf.queryExecution.analyzed.toString + assert(!analyzedText.toLowerCase(Locale.ROOT).contains("interval"), + s"GROUPS offset must not be cast to an interval:\n$analyzedText") + val optimizedText = groupsDf.queryExecution.optimizedPlan.toString + assert(optimizedText.contains("GroupFrame"), + s"expected GroupFrame to survive optimization:\n$optimizedText") + assert(!optimizedText.toLowerCase(Locale.ROOT).contains("interval"), + s"GROUPS offset must not be cast to an interval after optimization:\n$optimizedText") + checkAnswer(groupsDf.select("total"), + Seq(Row(25), Row(25), Row(45), Row(75), Row(75), Row(95))) + + // RANGE over a DATE key uses an interval boundary. + val rangeDf = spark.sql( + "select amount, sum(amount) over (order by d " + + "range between interval 1 day preceding and current row) as total from t") + val rangeFrame = specifiedFrame(rangeDf) + assert(rangeFrame.frameType === RangeFrame, + s"expected RangeFrame, got ${rangeFrame.frameType}") + assert(rangeFrame.lower.dataType.isInstanceOf[DayTimeIntervalType], + s"RANGE offset over a DATE order key should be an interval, " + + s"got ${rangeFrame.lower} : ${rangeFrame.lower.dataType}") + } + } + + // With one row per peer group, offset GROUPS and ROWS plans are equivalent. + test("GROUPS offset frame plan shape has no extra Sort/Exchange/WindowExec") { + withTempView("t") { + Seq((1, 10), (2, 15), (3, 20), (4, 25), (5, 30)) + .toDF("batch_id", "amount").createOrReplaceTempView("t") + withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { + val groupsShape = planShape(spark.sql( + "select sum(amount) over (order by batch_id " + + "groups between 1 preceding and current row) as total from t")) + val rowsShape = planShape(spark.sql( + "select sum(amount) over (order by batch_id " + + "rows between 1 preceding and current row) as total from t")) + assert(groupsShape._1 == 1 && groupsShape._3 == 1, + s"expected exactly one Sort and one WindowExec, got $groupsShape") + assert(groupsShape == rowsShape, + s"GROUPS offset plan shape differs from the equivalent ROWS plan shape: " + + s"groups=$groupsShape rows=$rowsShape") + } + } + } + + // GROUPS uses the generic aggregate path for offset window functions. + test("first_value/nth_value over a GROUPS frame use peer-group semantics") { + withTempView("t") { + Seq((1, 10), (1, 15), (2, 20), (3, 25), (3, 30), (9, 40)) + .toDF("batch_id", "amount").createOrReplaceTempView("t") + checkAnswer( + spark.sql( + "select batch_id, " + + "first_value(amount) over (order by batch_id " + + " groups between 1 preceding and 1 following) as first_amt, " + + "nth_value(amount, 2) over (order by batch_id " + + " groups between 1 preceding and 1 following) as second_amt " + + "from t"), + Seq( + Row(1, 10, 15), Row(1, 10, 15), Row(2, 10, 15), + Row(3, 20, 25), Row(3, 20, 25), Row(9, 25, 30))) + } + } + + // Case 3: `n FOLLOWING` advances one peer group. + test("GROUPS between current row and 1 following (test matrix case 3)") { + withTempView("t") { + Seq((1, 10), (1, 15), (2, 20), (3, 25), (3, 30), (9, 40)) + .toDF("batch_id", "amount").createOrReplaceTempView("t") + // Include batch_id to verify each total is assigned to the correct rows. + checkAnswer( + spark.sql( + "select batch_id, sum(amount) over (order by batch_id " + + "groups between current row and 1 following) as total from t"), + Seq( + Row(1, 45), Row(1, 45), Row(2, 75), Row(3, 95), Row(3, 95), Row(9, 40))) + } + } + + // Case 4: GROUPS offsets support multi-column ordering. + test("GROUPS with multi-column ORDER BY and an offset (test matrix case 4)") { + withTempView("t") { + Seq( + ("2024-01-01", 1, 10), + ("2024-01-01", 1, 20), + ("2024-01-02", 2, 30), + ("2024-01-03", 3, 45), + ("2024-01-03", 3, 45)) + .toDF("trade_date", "batch_id", "amount").createOrReplaceTempView("t") + checkAnswer( + spark.sql( + "select sum(amount) over (order by trade_date, batch_id " + + "groups between 1 preceding and current row) as total from t"), + // Peer groups are {10, 20}, {30}, and {45, 45}. + Seq(Row(30), Row(30), Row(60), Row(120), Row(120))) + } + } + + // Case 5: offsets count in window-ordering direction, not key-value direction. + test("GROUPS with DESC order counts in window order (test matrix case 5)") { + withTempView("t") { + Seq((1, 10), (1, 15), (2, 20), (3, 25), (3, 30), (9, 40)) + .toDF("batch_id", "amount").createOrReplaceTempView("t") + // Include batch_id to verify totals follow descending window order. + checkAnswer( + spark.sql( + "select batch_id, sum(amount) over (order by batch_id desc " + + "groups between 1 preceding and current row) as total from t"), + Seq( + Row(9, 40), Row(3, 95), Row(3, 95), Row(2, 75), Row(1, 45), Row(1, 45))) + } + } + + // 0 PRECEDING / 0 FOLLOWING must equal CURRENT ROW. + test("GROUPS 0 PRECEDING / 0 FOLLOWING equal CURRENT ROW") { + withTempView("t") { + Seq((1, 10), (1, 15), (2, 20), (3, 25), (3, 30), (9, 40)) + .toDF("batch_id", "amount").createOrReplaceTempView("t") + val current = spark.sql( + "select sum(amount) over (order by batch_id " + + "groups between current row and current row) as total from t") + val zeroPreceding = spark.sql( + "select sum(amount) over (order by batch_id " + + "groups between 0 preceding and current row) as total from t") + val zeroFollowing = spark.sql( + "select sum(amount) over (order by batch_id " + + "groups between current row and 0 following) as total from t") + val bothZero = spark.sql( + "select sum(amount) over (order by batch_id " + + "groups between 0 preceding and 0 following) as total from t") + checkAnswer(zeroPreceding, current) + checkAnswer(zeroFollowing, current) + checkAnswer(bothZero, current) + } + } + + // RANGE over DENSE_RANK is equivalent to GROUPS over the original ordering. + private def denseRankRangeOracle(lower: String, upper: String): DataFrame = { + spark.sql( + s""" + |select total from ( + | select amount, sum(amount) over ( + | order by dense_rank() over (order by batch_id) + | range between $lower and $upper) as total + | from t + |) + |""".stripMargin) + } + + private def checkGroupsAgainstDenseRankOracle(lower: String, upper: String): Unit = { + val groupsDf = spark.sql( + s"select sum(amount) over (order by batch_id groups between $lower and $upper) " + + "as total from t") + checkAnswer(groupsDf, denseRankRangeOracle(lower, upper)) + } + + test("GROUPS boundary matrix matches the DENSE_RANK + RANGE oracle") { + withTempView("t") { + Seq((1, 10), (1, 15), (2, 20), (3, 25), (3, 30), (9, 40)) + .toDF("batch_id", "amount").createOrReplaceTempView("t") + val lowers = Seq("unbounded preceding", "1 preceding", "current row", "1 following") + val uppers = Seq("1 preceding", "current row", "1 following", "unbounded following") + for (lower <- lowers; upper <- uppers) { + // Only this combination is rejected statically; other reversed bounds yield empty frames. + val invalid = (lower, upper) match { + case ("1 following", "1 preceding") => true + case _ => false + } + if (!invalid) { + withClue(s"lower='$lower' upper='$upper'") { + checkGroupsAgainstDenseRankOracle(lower, upper) + } + } else { + withClue(s"lower='$lower' upper='$upper'") { + checkError( + exception = intercept[AnalysisException] { + spark.sql( + "select sum(amount) over (order by batch_id groups between " + + "1 following and 1 preceding) as total from t").collect() + }, + condition = "DATATYPE_MISMATCH.SPECIFIED_WINDOW_FRAME_WRONG_COMPARISON", + parameters = Map( + "sqlExpr" -> "\"GROUPS BETWEEN 1 FOLLOWING AND 1 PRECEDING\"", + "comparison" -> "less than or equal"), + queryContext = Array( + ExpectedContext( + fragment = "(order by batch_id groups between " + + "1 following and 1 preceding)", + start = 24, + stop = 85))) + } + } + } + } + } + + // Peer-shape edges. + test("GROUPS peer-shape edge - one group for the whole partition") { + withTempView("t") { + Seq((1, 10), (1, 15), (1, 20)).toDF("batch_id", "amount").createOrReplaceTempView("t") + checkAnswer( + spark.sql( + "select sum(amount) over (order by batch_id " + + "groups between 1 preceding and 1 following) as total from t"), + Seq(Row(45), Row(45), Row(45))) + } + } + + // One group per row: GROUPS must then equal ROWS. + test("GROUPS peer-shape edge - one group per row equals ROWS") { + withTempView("t") { + Seq((1, 10), (2, 15), (3, 20), (4, 25), (5, 30)) + .toDF("batch_id", "amount").createOrReplaceTempView("t") + val groupsDf = spark.sql( + "select sum(amount) over (order by batch_id " + + "groups between 1 preceding and 1 following) as total from t") + val rowsDf = spark.sql( + "select sum(amount) over (order by batch_id " + + "rows between 1 preceding and 1 following) as total from t") + checkAnswer(groupsDf, rowsDf) + } + } + + test("GROUPS peer-shape edge - a single row") { + withTempView("t") { + Seq((1, 10)).toDF("batch_id", "amount").createOrReplaceTempView("t") + checkAnswer( + spark.sql( + "select sum(amount) over (order by batch_id " + + "groups between 1 preceding and 1 following) as total from t"), + Seq(Row(10))) + } + } + + test("GROUPS peer-shape edge - an empty partition") { + withTempView("t") { + Seq.empty[(Int, Int)].toDF("batch_id", "amount").createOrReplaceTempView("t") + checkAnswer( + spark.sql( + "select sum(amount) over (order by batch_id " + + "groups between 1 preceding and 1 following) as total from t"), + Seq.empty[Row]) + } + } + + // A frame entirely outside the partition is an empty frame, not an error: aggregates return + // their empty-input value (SUM -> NULL, COUNT -> 0). + test("GROUPS frame entirely outside the partition returns empty-input value") { + withTempView("t") { + Seq((1, 10), (2, 20), (3, 30)).toDF("batch_id", "amount").createOrReplaceTempView("t") + checkAnswer( + spark.sql( + "select sum(amount) over (order by batch_id " + + "groups between 5 following and 6 following) as sum_total, " + + "count(amount) over (order by batch_id " + + "groups between 5 following and 6 following) as count_total from t"), + Seq(Row(null, 0), Row(null, 0), Row(null, 0))) + checkAnswer( + spark.sql( + "select sum(amount) over (order by batch_id " + + "groups between 6 preceding and 5 preceding) as sum_total, " + + "count(amount) over (order by batch_id " + + "groups between 6 preceding and 5 preceding) as count_total from t"), + Seq(Row(null, 0), Row(null, 0), Row(null, 0))) + } + } + + // Compare random inputs, including NULL keys, with the DENSE_RANK plus RANGE oracle. + test("GROUPS randomised differential oracle vs DENSE_RANK + RANGE") { + val rand = new scala.util.Random(58980L) + withTempView("t") { + val rows = (1 to 200).map { _ => + val batchId = if (rand.nextInt(10) == 0) None else Some(rand.nextInt(5)) + (batchId, rand.nextInt(30)) + } + rows.toDF("batch_id", "amount").createOrReplaceTempView("t") + val boundaries = Seq( + ("unbounded preceding", "current row"), + ("2 preceding", "current row"), + ("current row", "2 following"), + ("1 preceding", "1 following"), + ("current row", "unbounded following")) + boundaries.foreach { case (lower, upper) => + withClue(s"lower='$lower' upper='$upper'") { + checkGroupsAgainstDenseRankOracle(lower, upper) + } + } + } + } + + // Frames, and the peer-group cursors in their bound orderings, are reused across a task's + // partitions. Every other GROUPS test runs on one partition, or on partitions of identical + // group shape, so none would notice group structure carried over from the previous one. + test("GROUPS over partitions with differing peer-group structure") { + withTempView("t") { + // pk=0: every row its own peer group. pk=1: one peer group covering the partition. + // pk=2: mixed group sizes. pk=3: a single row. + val rows = + (0 until 12).map(i => (0, i, i + 1)) ++ + (0 until 12).map(i => (1, 7, i + 1)) ++ + Seq((2, 0, 1), (2, 0, 2), (2, 0, 3), (2, 1, 4), (2, 2, 5), (2, 2, 6), (2, 3, 7)) ++ + Seq((3, 0, 1)) + rows.toDF("pk", "batch_id", "amount").createOrReplaceTempView("t") + val boundaries = Seq( + ("unbounded preceding", "current row"), + ("2 preceding", "current row"), + ("current row", "2 following"), + ("1 preceding", "1 following"), + ("2 preceding", "1 preceding"), + ("current row", "unbounded following")) + // One shuffle partition puts all four window partitions through one frame instance. + // `minPartitionRows = 8` also splits them across execution paths: the 12-row partitions + // take the segment tree, the 7- and 1-row ones its fallback frame, which shares its + // bound orderings. + val configs = Seq( + Seq(SQLConf.SHUFFLE_PARTITIONS.key -> "1"), + Seq(SQLConf.SHUFFLE_PARTITIONS.key -> "1", + SQLConf.WINDOW_SEGMENT_TREE_ENABLED.key -> "true", + SQLConf.WINDOW_SEGMENT_TREE_MIN_PARTITION_ROWS.key -> "8")) + for (config <- configs; (lower, upper) <- boundaries) { + withSQLConf(config: _*) { + withClue(s"config=$config lower='$lower' upper='$upper'") { + checkAnswer( + spark.sql( + s"select pk, sum(amount) over (partition by pk order by batch_id " + + s"groups between $lower and $upper) as total from t"), + spark.sql( + s""" + |select pk, total from ( + | select pk, sum(amount) over ( + | partition by pk + | order by dense_rank() over (partition by pk order by batch_id) + | range between $lower and $upper) as total + | from t + |) + |""".stripMargin)) + } + } + } + } + } + + // Force multiple spills and compare with in-memory execution. + test("GROUPS results are unaffected by partition spilling") { + withTempView("t") { + val rows = (1 to 500).map(i => (i % 40, i)) + rows.toDF("batch_id", "amount").createOrReplaceTempView("t") + val query = + "select sum(amount) over (order by batch_id " + + "groups between 2 preceding and 2 following) as total from t" + val noSpill = spark.sql(query).collect() + val spilled = withSQLConf( + SQLConf.WINDOW_EXEC_BUFFER_SPILL_THRESHOLD.key -> "25", + SQLConf.WINDOW_EXEC_BUFFER_IN_MEMORY_THRESHOLD.key -> "1") { + spark.sql(query).collect() + } + assert(spilled.sameElements(noSpill), + s"spilled results differ from non-spilling results:\n" + + s"no-spill: ${noSpill.mkString(",")}\nspilled: ${spilled.mkString(",")}") + } + } + + // Verify that GROUPS and RANGE use the same peer equality for special order-key values. + test("GROUPS peer groups for float special values match RANGE CURRENT ROW") { + withTempView("t") { + Seq(0.0, -0.0, Double.NaN, Double.NaN, 1.0) + .zipWithIndex.map { case (v, i) => (v, i) } + .toDF("value", "amount").createOrReplaceTempView("t") + // Signed zeros are peers, as are NaN values. + checkAnswer( + spark.sql( + "select sum(amount) over (order by value " + + "groups between current row and current row) as total from t"), + spark.sql( + "select sum(amount) over (order by value " + + "range between current row and current row) as total from t")) + // Check offset behavior against the DENSE_RANK plus RANGE oracle. + checkAnswer( + spark.sql( + "select sum(amount) over (order by value " + + "groups between 1 preceding and current row) as total from t"), + spark.sql( + """ + |select total from ( + | select amount, sum(amount) over ( + | order by dense_rank() over (order by value) + | range between 1 preceding and current row) as total + | from t + |) + |""".stripMargin)) + } + } + + test("GROUPS peer groups for collated strings match RANGE CURRENT ROW") { + withTempView("t") { + Seq(("abc", 1), ("ABC", 2), ("abd", 3)) + .toDF("value", "amount").createOrReplaceTempView("t") + spark.sql( + "select value collate UTF8_LCASE as value, amount from t").createOrReplaceTempView("tc") + checkAnswer( + spark.sql( + "select sum(amount) over (order by value " + + "groups between current row and current row) as total from tc"), + spark.sql( + "select sum(amount) over (order by value " + + "range between current row and current row) as total from tc")) + checkAnswer( + spark.sql( + "select sum(amount) over (order by value " + + "groups between 1 preceding and current row) as total from tc"), + spark.sql( + """ + |select total from ( + | select amount, sum(amount) over ( + | order by dense_rank() over (order by value) + | range between 1 preceding and current row) as total + | from tc + |) + |""".stripMargin)) + } + } + + test("CREATE VIEW over a no-offset GROUPS query round-trips") { + // Persistent views cannot reference temporary views. + withTable("t") { + withView("v") { + Seq((1, 10), (1, 15), (2, 20), (3, 25), (3, 30), (9, 40)) + .toDF("batch_id", "amount").write.saveAsTable("t") + spark.sql( + "create view v as select batch_id, sum(amount) over (order by batch_id " + + "groups between current row and current row) as total from t") + val storedText = + spark.sessionState.catalog.getTempViewOrPermanentTableMetadata( + TableIdentifier("v")).viewText.get + // View text preserves the keyword's original case. + assert(storedText.toUpperCase(Locale.ROOT).contains("GROUPS"), storedText) + // Re-create the view to verify that its stored text parses. + withView("v2") { + spark.sql(s"create view v2 as $storedText") + checkAnswer(spark.table("v2"), spark.table("v")) + } + checkAnswer( + spark.table("v").orderBy("batch_id", "total"), + Seq( + Row(1, 25), Row(1, 25), Row(2, 20), Row(3, 55), Row(3, 55), Row(9, 40))) + } + } + } + + test("no regression in existing ROWS and RANGE frame results") { + withTempView("t") { + Seq((1, 10, 10), (2, 20, 20), (3, 30, 10)) + .toDF("key", "row_amount", "range_amount") + .createOrReplaceTempView("t") + checkAnswer( + spark.sql( + """ + |select + | sum(row_amount) over ( + | order by key rows between unbounded preceding and current row) as rows_total, + | sum(range_amount) over ( + | order by key range between unbounded preceding and current row) as range_total + |from t + |""".stripMargin), + Row(10, 10) :: Row(30, 30) :: Row(60, 40) :: Nil) + } + } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/window/BoundOrderingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/window/BoundOrderingSuite.scala new file mode 100644 index 0000000000000..fd84478b0ab4c --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/window/BoundOrderingSuite.scala @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.window + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.{BoundReference, IdentityProjection, UnsafeProjection, UnsafeRow} +import org.apache.spark.sql.types.IntegerType + +/** + * Tests for the peer-group machinery behind GROUPS frames: [[PeerGroupCursor]], which counts + * peer groups incrementally along a monotonically advancing row position, and + * [[GroupBoundOrdering]], which turns two such cursors into a frame bound. + */ +class BoundOrderingSuite extends SparkFunSuite { + + private val ordering: Ordering[InternalRow] = Ordering.by[InternalRow, Int](_.getInt(0)) + + private def row(value: Int): UnsafeRow = { + val r = new UnsafeRow(1) + r.pointTo(new Array[Byte](64), 16) + r.setInt(0, value) + r + } + + /** + * Walks `values` through a cursor one position at a time and returns the group number + * reported at each position. + */ + private def walk(values: Seq[Int]): Seq[Int] = { + val cursor = new PeerGroupCursor(ordering, IdentityProjection) + values.zipWithIndex.map { case (v, i) => cursor.groupOf(row(v), i) } + } + + test("empty partition never advances the cursor") { + assert(walk(Seq.empty) === Seq.empty) + } + + test("single row is a single group") { + assert(walk(Seq(5)) === Seq(0)) + } + + test("all rows in one group") { + assert(walk(Seq.fill(50)(7)) === Seq.fill(50)(0)) + } + + test("every row is its own group") { + assert(walk(0 until 50) === (0 until 50)) + } + + test("mixed group sizes") { + // 80 singleton groups (values 0 until 80) followed by 3 groups of 20 rows each. + val singletons = 0 until 80 + val repeated = (1000 until 1003).flatMap(v => Seq.fill(20)(v)) + val expected = (0 until 80) ++ Seq.fill(20)(80) ++ Seq.fill(20)(81) ++ Seq.fill(20)(82) + assert(walk(singletons ++ repeated) === expected) + } + + test("re-asking for the current position does not advance the cursor") { + val cursor = new PeerGroupCursor(ordering, IdentityProjection) + assert(cursor.groupOf(row(1), 0) === 0) + // The row is ignored while the position is unchanged: frames pass whichever row they + // have on hand when an edge has not moved. + assert(cursor.groupOf(row(99), 0) === 0) + assert(cursor.groupOf(row(99), 0) === 0) + assert(cursor.groupOf(row(1), 1) === 0) + assert(cursor.groupOf(row(2), 2) === 1) + assert(cursor.groupOf(row(7), 2) === 1) + } + + test("skipping a position fails loudly instead of miscounting groups") { + val cursor = new PeerGroupCursor(ordering, IdentityProjection) + assert(cursor.groupOf(row(1), 0) === 0) + val e = intercept[AssertionError] { + cursor.groupOf(row(3), 2) + } + assert(e.getMessage.contains("cannot skip from position 0 to 2")) + } + + test("reset rewinds the cursor for a new partition") { + val cursor = new PeerGroupCursor(ordering, IdentityProjection) + assert(cursor.groupOf(row(1), 0) === 0) + assert(cursor.groupOf(row(2), 1) === 1) + cursor.reset() + // Without the rewind, position 0 would be a skip backwards and the group count would carry + // over from the previous partition. + assert(cursor.groupOf(row(2), 0) === 0) + assert(cursor.groupOf(row(2), 1) === 0) + assert(cursor.groupOf(row(9), 2) === 1) + } + + test("cursor retains a copy of the group representative") { + // Iterators over a spilled partition hand out one mutable row buffer over and over. + val cursor = new PeerGroupCursor(ordering, IdentityProjection) + val buffer = row(0) + val groups = Seq(1, 1, 2, 2, 2, 3).zipWithIndex.map { case (v, i) => + buffer.setInt(0, v) + cursor.groupOf(buffer, i) + } + assert(groups === Seq(0, 0, 1, 1, 1, 2)) + } + + test("peer cursors retain only projected keys when input and projection buffers are reused") { + val projection = UnsafeProjection.create(Seq(BoundReference(1, IntegerType, nullable = false))) + val keyOrdering = new Ordering[InternalRow] { + override def compare(left: InternalRow, right: InternalRow): Int = { + Seq(left, right).foreach { key => + assert(key.numFields == 1) + assert(key.asInstanceOf[UnsafeRow].getSizeInBytes == 16) + } + ordering.compare(left, right) + } + } + val first = new PeerGroupCursor(keyOrdering, projection) + val second = new PeerGroupCursor(keyOrdering, projection) + val input = InternalRow(new Array[Byte](64 * 1024), 1, 10) + assert(first.groupOf(input, 0) == 0) + input.setInt(1, 9) + assert(second.groupOf(input, 0) == 0) + input.setInt(1, 1) + input.setInt(2, 20) + assert(first.groupOf(input, 1) == 0) + input.setInt(1, 2) + assert(first.groupOf(input, 2) == 1) + input.setInt(1, 9) + assert(second.groupOf(input, 1) == 0) + } + + /** + * Drives a bound the way a frame does: `edge` gives the input position (this bound's frame + * edge) for each output position, and both positions are backed by `values`. + */ + private def compareAlong( + bound: GroupBoundOrdering, + values: Seq[Int], + edge: Seq[Int]): Seq[Int] = { + edge.zipWithIndex.map { case (inputIndex, outputIndex) => + bound.compare(row(values(inputIndex)), inputIndex, row(values(outputIndex)), outputIndex) + } + } + + test("single bound compares peer-group distance against its offset") { + // Peer groups: {0,1} -> 0, {2,3} -> 1, {4} -> 2. + val values = Seq(1, 1, 2, 2, 3) + // Edge parked on the output row itself: distance is always 0, so the comparison is the + // negated offset. + val currentRowEdge = values.indices + assert(compareAlong( + GroupBoundOrdering(ordering, IdentityProjection, 0), values, currentRowEdge) === + Seq(0, 0, 0, 0, 0)) + assert(compareAlong( + GroupBoundOrdering(ordering, IdentityProjection, -1), values, currentRowEdge) === + Seq(1, 1, 1, 1, 1)) + assert(compareAlong( + GroupBoundOrdering(ordering, IdentityProjection, 2), values, currentRowEdge) === + Seq(-2, -2, -2, -2, -2)) + } + + test("single bound tracks an edge that advances independently of the output row") { + val values = Seq(1, 1, 2, 2, 3) + // Edge trailing one row behind the output row: groups 0, 0, 0, 1, 1 against output groups + // 0, 0, 1, 1, 2, with a 1 PRECEDING offset. + val edge = Seq(0, 0, 1, 2, 3) + assert(compareAlong(GroupBoundOrdering(ordering, IdentityProjection, -1), values, edge) === + Seq(1, 1, 0, 1, 0)) + } + + test("prepare rewinds both cursors between partitions") { + val bound = GroupBoundOrdering(ordering, IdentityProjection, 0) + val partition1 = Seq(1, 1, 2) + assert(compareAlong(bound, partition1, partition1.indices) === Seq(0, 0, 0)) + bound.prepare() + val partition2 = Seq(5, 6, 7, 8) + assert(compareAlong(bound, partition2, partition2.indices) === Seq(0, 0, 0, 0)) + } + + test("paired bounds share the output cursor so neither has to see every output row") { + // GROUPS BETWEEN 3 PRECEDING AND 2 PRECEDING over five singleton groups. The upper bound is + // consulted for every output row; the lower bound is not consulted until the frame first + // becomes non-empty, and must still see the correct output group when it is. + val values = Seq(10, 20, 30, 40, 50) + val (lower, upper) = GroupBoundOrdering.paired( + ordering, IdentityProjection, lowerOffset = -3, upperOffset = -2) + + def upperAt(index: Int, edge: Int): Int = + upper.compare(row(values(edge)), edge, row(values(index)), index) + def lowerAt(index: Int, edge: Int): Int = + lower.compare(row(values(edge)), edge, row(values(index)), index) + + // Output rows 0 and 1: the upper edge cannot leave position 0; the lower bound is not asked. + assert(upperAt(0, 0) === 2) + assert(upperAt(1, 0) === 1) + // Output row 2: the lower bound is consulted for the first time, against output group 2, + // which only the shared cursor knows. + assert(upperAt(2, 0) === 0) + assert(upperAt(2, 1) === 1) + assert(lowerAt(2, 0) === 1) + // Output row 3: frame is groups 0..1. + assert(upperAt(3, 1) === 0) + assert(upperAt(3, 2) === 1) + assert(lowerAt(3, 0) === 0) + } + + test("prepare on either half of a pair rewinds the shared output cursor") { + val values = Seq(1, 2, 3) + val (lower, upper) = GroupBoundOrdering.paired( + ordering, IdentityProjection, lowerOffset = 0, upperOffset = 0) + assert(compareAlong(upper, values, values.indices) === Seq(0, 0, 0)) + // Only the upper half is re-prepared; the reset must still take effect for both. + upper.prepare() + assert(compareAlong(lower, values, values.indices) === Seq(0, 0, 0)) + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowFunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowFunctionSuite.scala index 5bf74c351c08e..445af362da867 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowFunctionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowFunctionSuite.scala @@ -441,7 +441,7 @@ class SegmentTreeWindowFunctionSuite extends SharedSparkSession { // MIN/MAX non-invertible, guaranteeing seg-tree path is exercised. /** Run `sql` twice (flag off / on) and checkAnswer equality. */ - private def checkRangeEquivalence(df: DataFrame, query: String): Unit = { + private def checkSqlEquivalence(df: DataFrame, query: String): Unit = { df.createOrReplaceTempView("t") try { val baseline = withSQLConf(disableSegTree.toSeq: _*) { @@ -467,7 +467,7 @@ class SegmentTreeWindowFunctionSuite extends SharedSparkSession { "WHEN 0 THEN 1 WHEN 1 THEN 3 WHEN 2 THEN 4 WHEN 3 THEN 4 " + "WHEN 4 THEN 7 WHEN 5 THEN 10 ELSE 15 END + (CAST(id AS INT) / 7) * 20 AS INT) AS k", "CAST((id * 31) % 97 AS INT) AS v") - checkRangeEquivalence(df, + checkSqlEquivalence(df, """SELECT id, pk, | MIN(v) OVER (PARTITION BY pk ORDER BY k | RANGE BETWEEN 2 PRECEDING AND 2 FOLLOWING) AS mn, @@ -486,7 +486,7 @@ class SegmentTreeWindowFunctionSuite extends SharedSparkSession { "(CASE CAST(id AS INT) % 3 WHEN 0 THEN 1 WHEN 1 THEN 3 ELSE 4 END), 0) " + "AS TIMESTAMP) AS ts", "CAST((id * 17) % 53 AS INT) AS v") - checkRangeEquivalence(df, + checkSqlEquivalence(df, """SELECT id, pk, | MAX(v) OVER (PARTITION BY pk ORDER BY ts | RANGE BETWEEN INTERVAL '1' HOUR PRECEDING @@ -503,7 +503,7 @@ class SegmentTreeWindowFunctionSuite extends SharedSparkSession { (i, i % 2, k, (i * 13) % 41) } val df = rows.toDF("id", "pk", "k", "v") - checkRangeEquivalence(df, + checkSqlEquivalence(df, """SELECT id, pk, k, | MIN(v) OVER (PARTITION BY pk ORDER BY k | RANGE BETWEEN 0 PRECEDING AND 0 FOLLOWING) AS mn, @@ -520,7 +520,7 @@ class SegmentTreeWindowFunctionSuite extends SharedSparkSession { "(CAST(id AS INT) / 5) AS pk", "CAST((id * 7) % 23 AS INT) AS k", "CAST((id * 19) % 101 AS INT) AS v") - checkRangeEquivalence(df, + checkSqlEquivalence(df, """SELECT id, pk, | MIN(v) OVER (PARTITION BY pk ORDER BY k | RANGE BETWEEN 100 PRECEDING AND 100 FOLLOWING) AS mn, @@ -543,7 +543,7 @@ class SegmentTreeWindowFunctionSuite extends SharedSparkSession { (i, i % 2, kOpt, (i * 11) % 37) } val df = rows.toDF("id", "pk", "k", "v") - checkRangeEquivalence(df, + checkSqlEquivalence(df, """SELECT id, pk, | MIN(v) OVER (PARTITION BY pk ORDER BY k ASC NULLS FIRST | RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS mn_nf, @@ -556,6 +556,149 @@ class SegmentTreeWindowFunctionSuite extends SharedSparkSession { |FROM t""".stripMargin) } + // ---- GROUPS frames ---- + + /** Two partitions with two rows per peer group. */ + private def groupsTiedDF: DataFrame = { + val rows = (0 until 60).map(i => (i, i % 2, i / 4, (i * 13) % 41)) + rows.toDF("id", "pk", "k", "v") + } + + test("-- GROUPS offset bounds with multi-row peer groups (MIN/MAX/SUM/COUNT/AVG)") { + // A two-group offset spans six rows, distinguishing group and row offsets. + checkSqlEquivalence(groupsTiedDF, + """SELECT id, pk, + | MIN(v) OVER w AS mn, MAX(v) OVER w AS mx, SUM(v) OVER w AS sm, + | COUNT(v) OVER w AS ct, AVG(v) OVER w AS av + |FROM t + |WINDOW w AS (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN 2 PRECEDING AND 2 FOLLOWING)""".stripMargin) + } + + test("-- GROUPS with CURRENT ROW on either edge (mixed bound kinds)") { + // Cover every combination of row-reading and index-only bounds. + checkSqlEquivalence(groupsTiedDF, + """SELECT id, pk, + | MIN(v) OVER (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN 2 PRECEDING AND CURRENT ROW) AS a, + | MAX(v) OVER (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN CURRENT ROW AND 2 FOLLOWING) AS b, + | SUM(v) OVER (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN CURRENT ROW AND CURRENT ROW) AS c + |FROM t""".stripMargin) + } + + test("-- GROUPS with an entirely one-sided frame (0 PRECEDING / 0 FOLLOWING)") { + // Zero offsets use index-only bounds but must equal CURRENT ROW. + checkSqlEquivalence(groupsTiedDF, + """SELECT id, pk, + | SUM(v) OVER (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN 0 PRECEDING AND 0 FOLLOWING) AS c, + | SUM(v) OVER (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN CURRENT ROW AND CURRENT ROW) AS d + |FROM t""".stripMargin) + } + + test("-- GROUPS with multi-column ORDER BY (no single-column restriction)") { + // Unlike RANGE offsets, GROUPS offsets support multiple order expressions. + val rows = (0 until 48).map(i => (i, i % 2, i / 8, (i / 2) % 4, (i * 7) % 29)) + val df = rows.toDF("id", "pk", "k1", "k2", "v") + checkSqlEquivalence(df, + """SELECT id, pk, + | MIN(v) OVER (PARTITION BY pk ORDER BY k1, k2 + | GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS mn, + | MAX(v) OVER (PARTITION BY pk ORDER BY k1, k2 + | GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS mx + |FROM t""".stripMargin) + } + + test("-- GROUPS with NULL order key and DESC / NULLS FIRST / NULLS LAST") { + // Group indices follow the configured sort order, including NULL placement. + val rows = (0 until 42).map { i => + val kOpt: Option[Int] = (i % 7) match { + case 0 | 3 => None + case 1 | 2 => Some(1) + case 4 => Some(2) + case _ => Some(3) + } + (i, i % 2, kOpt, (i * 11) % 37) + } + val df = rows.toDF("id", "pk", "k", "v") + checkSqlEquivalence(df, + """SELECT id, pk, + | MIN(v) OVER (PARTITION BY pk ORDER BY k ASC NULLS FIRST + | GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS a, + | MAX(v) OVER (PARTITION BY pk ORDER BY k ASC NULLS LAST + | GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS b, + | SUM(v) OVER (PARTITION BY pk ORDER BY k DESC NULLS FIRST + | GROUPS BETWEEN 2 PRECEDING AND CURRENT ROW) AS c, + | COUNT(v) OVER (PARTITION BY pk ORDER BY k DESC NULLS LAST + | GROUPS BETWEEN CURRENT ROW AND 2 FOLLOWING) AS d + |FROM t""".stripMargin) + } + + test("-- GROUPS frame wider than the partition (admit/drop loops saturate)") { + // Bounds saturate at the partition edges. + checkSqlEquivalence(groupsTiedDF, + """SELECT id, pk, + | MIN(v) OVER (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN 1000 PRECEDING AND 1000 FOLLOWING) AS wide, + | MAX(v) OVER (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN 1000 PRECEDING AND CURRENT ROW) AS lo, + | SUM(v) OVER (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN CURRENT ROW AND 1000 FOLLOWING) AS hi + |FROM t""".stripMargin) + } + + test("-- GROUPS frame spanning segment-tree block boundaries") { + // Use a frame wider than the minimum block size to exercise block merging. + withSQLConf(SQLConf.WINDOW_SEGMENT_TREE_BLOCK_SIZE.key -> segTreeBlock) { + val rows = (0 until 120).map(i => (i, i % 2, i / 2, (i * 17) % 53)) + val df = rows.toDF("id", "pk", "k", "v") + checkSqlEquivalence(df, + """SELECT id, pk, + | MIN(v) OVER (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN 20 PRECEDING AND 20 FOLLOWING) AS mn, + | SUM(v) OVER (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN 20 PRECEDING AND 20 FOLLOWING) AS sm + |FROM t""".stripMargin) + } + } + + /** Compares two queries with segment-tree evaluation enabled. */ + private def checkSameUnderSegTree(df: DataFrame, left: String, right: String): Unit = { + df.createOrReplaceTempView("t") + try { + withSQLConf(enableSegTree.toSeq: _*) { + val l = spark.sql(left).collect().sortBy(_.toString) + val r = spark.sql(right).collect().sortBy(_.toString) + assert(l.toSeq === r.toSeq, + s"queries disagree under the segment tree.\nLeft: ${l.toSeq}\nRight: ${r.toSeq}") + } + } finally { + spark.catalog.dropTempView("t") + } + } + + test("-- GROUPS over all-distinct order keys degenerates to ROWS") { + // With one row per peer group, GROUPS is equivalent to ROWS. + val df = (0 until 60).map(i => (i, i % 2, i, (i * 23) % 47)).toDF("id", "pk", "k", "v") + checkSameUnderSegTree(df, + """SELECT id, MIN(v) OVER (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN 3 PRECEDING AND 2 FOLLOWING) AS r FROM t""".stripMargin, + """SELECT id, MIN(v) OVER (PARTITION BY pk ORDER BY k + | ROWS BETWEEN 3 PRECEDING AND 2 FOLLOWING) AS r FROM t""".stripMargin) + } + + test("-- GROUPS over a single peer group covers the whole partition") { + // With one peer group, any offset selects the entire partition. + val df = (0 until 60).map(i => (i, i % 2, 1, (i * 23) % 47)).toDF("id", "pk", "k", "v") + checkSameUnderSegTree(df, + """SELECT id, SUM(v) OVER (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS r FROM t""".stripMargin, + """SELECT id, SUM(v) OVER (PARTITION BY pk) AS r FROM t""".stripMargin) + } + // Decimal overflow / BinaryType MIN/MAX across block merge; UDAF fallback. // Trap: blockSize=16 is SQLConf minimum; frame > blockSize ensures the // seg-tree merge path is actually crossed. diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowMetricsSuite.scala index d22c1cdc90da2..2109fe8844d32 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowMetricsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/window/SegmentTreeWindowMetricsSuite.scala @@ -178,6 +178,60 @@ class SegmentTreeWindowMetricsSuite } } + private def groupsDF(frame: String): org.apache.spark.sql.DataFrame = spark.sql( + s"select id, min(v) over (partition by pk order by id $frame) as mn from " + + "(select id, (id % 3) AS pk, CAST(id AS INT) AS v from range(0, 120))") + + // Use ROWS as a control for the expected frame count. + test("GROUPS frame takes the segment-tree path") { + withSQLConf( + SQLConf.WINDOW_SEGMENT_TREE_ENABLED.key -> "true", + SQLConf.WINDOW_SEGMENT_TREE_MIN_PARTITION_ROWS.key -> "1", + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { + val rowsDf = baseDF.select($"id", min($"v").over(winSpec).as("mn")) + val rowsMetrics = windowMetricValues(rowsDf) + assert(rowsMetrics("number of segment-tree frames prepared") === 3L, + s"expected the ROWS control query to take the segment-tree path, got $rowsMetrics") + + // Cover moving and shrinking factory branches. + Seq( + "groups between 3 preceding and 3 following", + "groups between current row and 3 following", + "groups between 3 preceding and current row", + "groups between 3 preceding and unbounded following").foreach { frame => + val m = windowMetricValues(groupsDF(frame)) + assert(m("number of segment-tree frames prepared") === 3L, + s"GROUPS frame '$frame' must take the segment-tree path, got $m") + assert(m("number of segment-tree fallback frames prepared") === 0L, + s"GROUPS frame '$frame' must not fall back, got $m") + } + } + } + + test("GROUPS frame honours the min-partition-rows fallback") { + withSQLConf( + SQLConf.WINDOW_SEGMENT_TREE_ENABLED.key -> "true", + // Force every partition below the segment-tree threshold. + SQLConf.WINDOW_SEGMENT_TREE_MIN_PARTITION_ROWS.key -> "1000", + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { + val m = windowMetricValues(groupsDF("groups between 3 preceding and 3 following")) + assert(m("number of segment-tree fallback frames prepared") === 3L, + s"expected 3 GROUPS fallback frames (one per partition under threshold), got $m") + assert(m("number of segment-tree frames prepared") === 0L, + s"segtree counter must be 0 when all GROUPS partitions fall back, got $m") + } + } + + test("feature flag off leaves GROUPS counters at zero") { + withSQLConf( + SQLConf.WINDOW_SEGMENT_TREE_ENABLED.key -> "false", + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { + val m = windowMetricValues(groupsDF("groups between 3 preceding and 3 following")) + assert(m("number of segment-tree frames prepared") === 0L, s"got $m") + assert(m("number of segment-tree fallback frames prepared") === 0L, s"got $m") + } + } + test("T4 (G4) mixed segtree + fallback, non-aliasing order") { withSQLConf( SQLConf.WINDOW_SEGMENT_TREE_ENABLED.key -> "true", diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/window/UnboundedFollowingSegmentTreeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/window/UnboundedFollowingSegmentTreeSuite.scala index 67316ba7a0489..b303d17cebc4d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/window/UnboundedFollowingSegmentTreeSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/window/UnboundedFollowingSegmentTreeSuite.scala @@ -476,6 +476,81 @@ class UnboundedFollowingSegmentTreeSuite extends SharedSparkSession { |FROM t""".stripMargin) } + // ---- GROUPS shrinking frames ---- + + /** Two partitions with two rows per peer group. */ + private def groupsTiedDF: DataFrame = + (0 until 60).map(i => (i, i % 2, i / 4, (i * 13) % 41)).toDF("id", "pk", "k", "v") + + test("GROUPS BETWEEN n PRECEDING AND UNBOUNDED FOLLOWING") { + checkSqlEquivalence(groupsTiedDF, + """SELECT id, pk, + | MIN(v) OVER w AS mn, MAX(v) OVER w AS mx, SUM(v) OVER w AS sm, AVG(v) OVER w AS av + |FROM t + |WINDOW w AS (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN 2 PRECEDING AND UNBOUNDED FOLLOWING)""".stripMargin) + } + + test("GROUPS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING (row-reading lower bound)") { + // CURRENT ROW starts at the first row of the peer group. + checkSqlEquivalence(groupsTiedDF, + """SELECT id, pk, + | MIN(v) OVER (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) AS mn, + | COUNT(v) OVER (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) AS ct + |FROM t""".stripMargin) + } + + test("GROUPS BETWEEN n FOLLOWING AND UNBOUNDED FOLLOWING (lower bound is positive)") { + // A positive lower offset can produce an empty frame near the partition end. + checkSqlEquivalence(groupsTiedDF, + """SELECT id, pk, + | SUM(v) OVER (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN 3 FOLLOWING AND UNBOUNDED FOLLOWING) AS sm, + | COUNT(v) OVER (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN 3 FOLLOWING AND UNBOUNDED FOLLOWING) AS ct + |FROM t""".stripMargin) + } + + test("GROUPS shrinking frame with NULL order key and DESC") { + val rows = (0 until 42).map { i => + val kOpt: Option[Int] = if (i % 7 == 0 || i % 7 == 3) None else Some(i % 7) + (i, i % 2, kOpt, (i * 11) % 37) + } + checkSqlEquivalence(rows.toDF("id", "pk", "k", "v"), + """SELECT id, pk, + | MIN(v) OVER (PARTITION BY pk ORDER BY k ASC NULLS FIRST + | GROUPS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) AS a, + | MAX(v) OVER (PARTITION BY pk ORDER BY k DESC NULLS LAST + | GROUPS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) AS b + |FROM t""".stripMargin) + } + + test("GROUPS shrinking partition below minPartitionRows falls back to legacy frame") { + val df = groupsTiedDF + val query = + """SELECT id, pk, + | MIN(v) OVER (PARTITION BY pk ORDER BY k + | GROUPS BETWEEN 2 PRECEDING AND UNBOUNDED FOLLOWING) AS mn + |FROM t""".stripMargin + df.createOrReplaceTempView("t") + try { + val baseline = withSQLConf(disableSegTree.toSeq: _*) { + spark.sql(query).collect().sortBy(_.toString) + } + // Force every partition onto the fallback path. + withSQLConf( + SQLConf.WINDOW_SEGMENT_TREE_ENABLED.key -> "true", + SQLConf.WINDOW_SEGMENT_TREE_MIN_PARTITION_ROWS.key -> "1024") { + val actual = spark.sql(query).collect().sortBy(_.toString) + assert(actual.toSeq === baseline.toSeq) + } + } finally { + spark.catalog.dropTempView("t") + } + } + // ============================================================ // Feature-flag off: legacy frame is used // ============================================================ diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala index d028be42e6e52..2b4836b8e274e 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala @@ -39,7 +39,7 @@ import org.apache.spark.sql.catalyst.catalog._ import org.apache.spark.sql.execution.command.DDLUtils import org.apache.spark.sql.expressions.Window import org.apache.spark.sql.hive.test.{HiveTestJars, TestHiveContext} -import org.apache.spark.sql.internal.SQLConf.{LEGACY_TIME_PARSER_POLICY, SHUFFLE_PARTITIONS} +import org.apache.spark.sql.internal.SQLConf.{ADAPTIVE_EXECUTION_ENABLED, LEGACY_TIME_PARSER_POLICY, SHUFFLE_PARTITIONS, WINDOW_SEGMENT_TREE_ENABLED, WINDOW_SEGMENT_TREE_MIN_PARTITION_ROWS} import org.apache.spark.sql.internal.StaticSQLConf.WAREHOUSE_PATH import org.apache.spark.sql.types.{DecimalType, StructType} import org.apache.spark.tags.{ExtendedHiveTest, SlowHiveTest} @@ -221,6 +221,20 @@ class HiveSparkSubmitSuite runSparkSubmit(args) } + test("SPARK-58980 GROUPS window frames in cluster mode") { + val unusedJar = TestUtils.createJarWithClasses(Seq.empty) + val args = Seq( + "--class", SPARK_58980.getClass.getName.stripSuffix("$"), + "--name", "GroupsWindowClusterTest", + "--master", "local-cluster[2,1,512]", + "--conf", s"${EXECUTOR_MEMORY.key}=512m", + "--conf", "spark.ui.enabled=false", + "--conf", "spark.master.rest.enabled=false", + "--driver-java-options", "-Dderby.system.durability=test", + unusedJar.toString) + runSparkSubmit(args) + } + test("set spark.sql.warehouse.dir") { val unusedJar = TestUtils.createJarWithClasses(Seq.empty) val args = Seq( @@ -823,6 +837,69 @@ object SPARK_14244 extends QueryTest { } } +object SPARK_58980 extends QueryTest { + protected var spark: SparkSession = _ + + def main(args: Array[String]): Unit = { + TestUtils.configTestLog4j2("INFO") + + val sparkContext = new SparkContext( + new SparkConf() + .set(UI_ENABLED, false) + .set(SHUFFLE_PARTITIONS.key, "8") + .set(ADAPTIVE_EXECUTION_ENABLED.key, "false")) + + val hiveContext = new TestHiveContext(sparkContext) + spark = hiveContext.sparkSession + + try { + // Each account's six rows originate in different input partitions. ORDER BY has four + // peer groups, including ties and a gap, so ROWS and RANGE cannot produce these results. + spark.range(0, 64 * 6, 1, 8) + .selectExpr("id % 64 AS account", "CAST(id DIV 64 AS INT) AS item") + .createOrReplaceTempView("input") + spark.sql( + """ + |SELECT account, item, + | CASE item WHEN 0 THEN 1 WHEN 1 THEN 1 WHEN 2 THEN 2 + | WHEN 3 THEN 3 WHEN 4 THEN 3 ELSE 9 END AS batch, + | (CASE item WHEN 0 THEN 10 WHEN 1 THEN 15 WHEN 2 THEN 20 + | WHEN 3 THEN 25 WHEN 4 THEN 30 ELSE 40 END) * (account + 1) AS amount + |FROM input + |""".stripMargin).createOrReplaceTempView("batches") + + val totals = Seq(Row(25L, null), Row(25L, null), Row(45L, 25L), + Row(75L, 45L), Row(75L, 45L), Row(95L, 75L)) + val expected = (0 until 64).flatMap { account => + totals.zipWithIndex.map { case (total, item) => + val factor = account + 1L + Row(account.toLong, item, total.getLong(0) * factor, + if (total.isNullAt(1)) null else total.getLong(1) * factor) + } + } + for (segmentTree <- Seq(false, true)) { + withSQLConf( + WINDOW_SEGMENT_TREE_ENABLED.key -> segmentTree.toString, + WINDOW_SEGMENT_TREE_MIN_PARTITION_ROWS.key -> "1") { + checkAnswer( + spark.sql( + """ + |SELECT account, item, + | sum(amount) OVER (PARTITION BY account ORDER BY batch + | GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) AS trailing, + | sum(amount) OVER (PARTITION BY account ORDER BY batch + | GROUPS BETWEEN 2 PRECEDING AND 1 PRECEDING) AS preceding + |FROM batches + |""".stripMargin), + expected) + } + } + } finally { + sparkContext.stop() + } + } +} + object SPARK_18360 { def main(args: Array[String]): Unit = { val spark = SparkSession.builder()