diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Compiler.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Compiler.kt index 6df4eaf590..ae2950342a 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Compiler.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Compiler.kt @@ -4,8 +4,12 @@ import org.partiql.eval.PartiQLEngine import org.partiql.eval.internal.operator.Operator import org.partiql.eval.internal.operator.rel.RelAggregate import org.partiql.eval.internal.operator.rel.RelDistinct +import org.partiql.eval.internal.operator.rel.RelExceptAll +import org.partiql.eval.internal.operator.rel.RelExceptDistinct import org.partiql.eval.internal.operator.rel.RelExclude import org.partiql.eval.internal.operator.rel.RelFilter +import org.partiql.eval.internal.operator.rel.RelIntersectAll +import org.partiql.eval.internal.operator.rel.RelIntersectDistinct import org.partiql.eval.internal.operator.rel.RelJoinInner import org.partiql.eval.internal.operator.rel.RelJoinLeft import org.partiql.eval.internal.operator.rel.RelJoinOuterFull @@ -18,6 +22,8 @@ import org.partiql.eval.internal.operator.rel.RelScanIndexed import org.partiql.eval.internal.operator.rel.RelScanIndexedPermissive import org.partiql.eval.internal.operator.rel.RelScanPermissive import org.partiql.eval.internal.operator.rel.RelSort +import org.partiql.eval.internal.operator.rel.RelUnionAll +import org.partiql.eval.internal.operator.rel.RelUnionDistinct import org.partiql.eval.internal.operator.rel.RelUnpivot import org.partiql.eval.internal.operator.rex.ExprCallDynamic import org.partiql.eval.internal.operator.rex.ExprCallStatic @@ -308,6 +314,33 @@ internal class Compiler( } } + override fun visitRelOpSetExcept(node: Rel.Op.Set.Except, ctx: StaticType?): Operator { + val lhs = visitRel(node.lhs, ctx) + val rhs = visitRel(node.rhs, ctx) + return when (node.quantifier) { + Rel.Op.Set.Quantifier.ALL -> RelExceptAll(lhs, rhs) + Rel.Op.Set.Quantifier.DISTINCT -> RelExceptDistinct(lhs, rhs) + } + } + + override fun visitRelOpSetIntersect(node: Rel.Op.Set.Intersect, ctx: StaticType?): Operator { + val lhs = visitRel(node.lhs, ctx) + val rhs = visitRel(node.rhs, ctx) + return when (node.quantifier) { + Rel.Op.Set.Quantifier.ALL -> RelIntersectAll(lhs, rhs) + Rel.Op.Set.Quantifier.DISTINCT -> RelIntersectDistinct(lhs, rhs) + } + } + + override fun visitRelOpSetUnion(node: Rel.Op.Set.Union, ctx: StaticType?): Operator { + val lhs = visitRel(node.lhs, ctx) + val rhs = visitRel(node.rhs, ctx) + return when (node.quantifier) { + Rel.Op.Set.Quantifier.ALL -> RelUnionAll(lhs, rhs) + Rel.Op.Set.Quantifier.DISTINCT -> RelUnionDistinct(lhs, rhs) + } + } + override fun visitRelOpLimit(node: Rel.Op.Limit, ctx: StaticType?): Operator { val input = visitRel(node.input, ctx) val limit = visitRex(node.limit, ctx) diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/helpers/IteratorChain.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/helpers/IteratorChain.kt new file mode 100644 index 0000000000..55a2c832e8 --- /dev/null +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/helpers/IteratorChain.kt @@ -0,0 +1,27 @@ +package org.partiql.eval.internal.helpers + +internal class IteratorChain( + iterators: Array> +) : IteratorPeeking() { + + private var iterator: Iterator> = when (iterators.isEmpty()) { + true -> listOf(emptyList().iterator()).iterator() + false -> iterators.iterator() + } + private var current: Iterator = iterator.next() + + override fun peek(): T? { + return when (current.hasNext()) { + true -> current.next() + false -> { + while (iterator.hasNext()) { + current = iterator.next() + if (current.hasNext()) { + return current.next() + } + } + return null + } + } + } +} diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/helpers/IteratorPeeking.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/helpers/IteratorPeeking.kt new file mode 100644 index 0000000000..55f6c8fd2b --- /dev/null +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/helpers/IteratorPeeking.kt @@ -0,0 +1,36 @@ +package org.partiql.eval.internal.helpers + +/** + * For [Iterator]s that MUST materialize data in order to execute [hasNext], this abstract class caches the + * result of [peek] to implement both [hasNext] and [next]. + * + * With this implementation, invoking hasNext() multiple times will not iterate unnecessarily. Invoking next() without + * invoking hasNext() is allowed -- however, it is highly recommended to avoid doing so. + */ +internal abstract class IteratorPeeking : Iterator { + + internal var next: T? = null + + /** + * @return NULL when there is not another [T] to be produced. Returns a [T] when able to. + * + * @see IteratorPeeking + */ + abstract fun peek(): T? + + override fun hasNext(): Boolean { + if (next != null) { + return true + } + this.next = peek() + return this.next != null + } + + override fun next(): T { + val next = next + ?: peek() + ?: error("There were no more elements, however, next() was called. Please use hasNext() beforehand.") + this.next = null + return next + } +} diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelDistinct.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelDistinct.kt index b4d041ccf7..15dcc1ba5a 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelDistinct.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelDistinct.kt @@ -10,9 +10,8 @@ internal class RelDistinct( private val seen = mutableSetOf() - override fun open(env: Environment) { + override fun openPeeking(env: Environment) { input.open(env) - super.open(env) } override fun peek(): Record? { @@ -25,9 +24,8 @@ internal class RelDistinct( return null } - override fun close() { + override fun closePeeking() { seen.clear() input.close() - super.close() } } diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelExceptAll.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelExceptAll.kt new file mode 100644 index 0000000000..f62a433b32 --- /dev/null +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelExceptAll.kt @@ -0,0 +1,53 @@ +package org.partiql.eval.internal.operator.rel + +import org.partiql.eval.internal.Environment +import org.partiql.eval.internal.Record +import org.partiql.eval.internal.operator.Operator + +internal class RelExceptAll( + private val lhs: Operator.Relation, + private val rhs: Operator.Relation, +) : RelPeeking() { + + private val seen: MutableMap = mutableMapOf() + private var init: Boolean = false + + override fun openPeeking(env: Environment) { + lhs.open(env) + rhs.open(env) + init = false + seen.clear() + } + + override fun peek(): Record? { + if (!init) { + seed() + } + for (row in lhs) { + val remaining = seen[row] ?: 0 + if (remaining > 0) { + seen[row] = remaining - 1 + continue + } + return row + } + return null + } + + override fun closePeeking() { + lhs.close() + rhs.close() + seen.clear() + } + + /** + * Read the entire right-hand-side into our search structure. + */ + private fun seed() { + init = true + for (row in rhs) { + val n = seen[row] ?: 0 + seen[row] = n + 1 + } + } +} diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelExcept.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelExceptDistinct.kt similarity index 89% rename from partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelExcept.kt rename to partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelExceptDistinct.kt index 93ac2eb581..9874aabaea 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelExcept.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelExceptDistinct.kt @@ -10,7 +10,7 @@ import org.partiql.eval.internal.operator.Operator * @property lhs * @property rhs */ -internal class RelExcept( +internal class RelExceptDistinct( private val lhs: Operator.Relation, private val rhs: Operator.Relation, ) : RelPeeking() { @@ -18,12 +18,11 @@ internal class RelExcept( private var seen: MutableSet = mutableSetOf() private var init: Boolean = false - override fun open(env: Environment) { + override fun openPeeking(env: Environment) { lhs.open(env) rhs.open(env) init = false seen = mutableSetOf() - super.open(env) } override fun peek(): Record? { @@ -38,11 +37,10 @@ internal class RelExcept( return null } - override fun close() { + override fun closePeeking() { lhs.close() rhs.close() seen.clear() - super.close() } /** diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelFilter.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelFilter.kt index 5c8b38949d..2081f14d58 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelFilter.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelFilter.kt @@ -13,10 +13,9 @@ internal class RelFilter( private lateinit var env: Environment - override fun open(env: Environment) { + override fun openPeeking(env: Environment) { this.env = env input.open(env) - super.open(env) } override fun peek(): Record? { @@ -28,9 +27,8 @@ internal class RelFilter( return null } - override fun close() { + override fun closePeeking() { input.close() - super.close() } @OptIn(PartiQLValueExperimental::class) diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelIntersectAll.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelIntersectAll.kt new file mode 100644 index 0000000000..33f2ba869a --- /dev/null +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelIntersectAll.kt @@ -0,0 +1,54 @@ +package org.partiql.eval.internal.operator.rel + +import org.partiql.eval.internal.Environment +import org.partiql.eval.internal.Record +import org.partiql.eval.internal.operator.Operator + +internal class RelIntersectAll( + private val lhs: Operator.Relation, + private val rhs: Operator.Relation, +) : RelPeeking() { + + private val seen: MutableMap = mutableMapOf() + private var init: Boolean = false + + override fun openPeeking(env: Environment) { + lhs.open(env) + rhs.open(env) + init = false + seen.clear() + } + + override fun peek(): Record? { + if (!init) { + seed() + } + for (row in rhs) { + seen.computeIfPresent(row) { _, y -> + when (y) { + 0 -> null + else -> y - 1 + } + }?.let { return row } + } + return null + } + + override fun closePeeking() { + lhs.close() + rhs.close() + seen.clear() + } + + /** + * Read the entire left-hand-side into our search structure. + */ + private fun seed() { + init = true + for (row in lhs) { + seen.computeIfPresent(row) { _, y -> + y + 1 + } ?: seen.put(row, 1) + } + } +} diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelIntersect.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelIntersectDistinct.kt similarity index 71% rename from partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelIntersect.kt rename to partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelIntersectDistinct.kt index 67c83b9e9a..2700924104 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelIntersect.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelIntersectDistinct.kt @@ -4,20 +4,19 @@ import org.partiql.eval.internal.Environment import org.partiql.eval.internal.Record import org.partiql.eval.internal.operator.Operator -internal class RelIntersect( +internal class RelIntersectDistinct( private val lhs: Operator.Relation, private val rhs: Operator.Relation, ) : RelPeeking() { - private var seen: MutableSet = mutableSetOf() + private val seen: MutableSet = mutableSetOf() private var init: Boolean = false - override fun open(env: Environment) { + override fun openPeeking(env: Environment) { lhs.open(env) rhs.open(env) init = false - seen = mutableSetOf() - super.open(env) + seen.clear() } override fun peek(): Record? { @@ -25,18 +24,17 @@ internal class RelIntersect( seed() } for (row in rhs) { - if (seen.contains(row)) { + if (seen.remove(row)) { return row } } return null } - override fun close() { + override fun closePeeking() { lhs.close() rhs.close() seen.clear() - super.close() } /** @@ -44,8 +42,7 @@ internal class RelIntersect( */ private fun seed() { init = true - while (true) { - val row = lhs.next() ?: break + for (row in lhs) { seen.add(row) } } diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinNestedLoop.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinNestedLoop.kt index ed761c10d2..cb39e48188 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinNestedLoop.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelJoinNestedLoop.kt @@ -19,7 +19,7 @@ internal abstract class RelJoinNestedLoop : RelPeeking() { private var lhsRecord: Record? = null private lateinit var env: Environment - override fun open(env: Environment) { + override fun openPeeking(env: Environment) { this.env = env lhs.open(env) if (lhs.hasNext().not()) { @@ -27,7 +27,6 @@ internal abstract class RelJoinNestedLoop : RelPeeking() { } lhsRecord = lhs.next() rhs.open(env.push(lhsRecord!!)) - super.open(env) } abstract fun join(condition: Boolean, lhs: Record, rhs: Record): Record? @@ -69,10 +68,9 @@ internal abstract class RelJoinNestedLoop : RelPeeking() { return toReturn } - override fun close() { + override fun closePeeking() { lhs.close() rhs.close() - super.close() } @OptIn(PartiQLValueExperimental::class) diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelPeeking.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelPeeking.kt index b8fd37685b..6206b783bf 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelPeeking.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelPeeking.kt @@ -2,44 +2,38 @@ package org.partiql.eval.internal.operator.rel import org.partiql.eval.internal.Environment import org.partiql.eval.internal.Record +import org.partiql.eval.internal.helpers.IteratorPeeking import org.partiql.eval.internal.operator.Operator /** * For [Operator.Relation]'s that MUST materialize data in order to execute [hasNext], this abstract class caches the * result of [peek] to implement both [hasNext] and [next]. */ -internal abstract class RelPeeking : Operator.Relation { +internal abstract class RelPeeking : Operator.Relation, IteratorPeeking() { - private var _next: Record? = null + /** + * This shall have the same functionality as [open]. Implementers of [RelPeeking] shall not override [open]. + */ + abstract fun openPeeking(env: Environment) /** - * @return Null when there is not another record to be produced. Returns a [Record] when able to. - * - * @see RelPeeking + * This shall have the same functionality as [close]. Implementers of [RelPeeking] shall not override [close]. */ - abstract fun peek(): Record? + abstract fun closePeeking() + /** + * Implementers shall not override this method. + */ override fun open(env: Environment) { - _next = null - } - - override fun hasNext(): Boolean { - if (_next != null) { - return true - } - this._next = peek() - return this._next != null - } - - override fun next(): Record { - val next = _next - ?: peek() - ?: error("There was not a record to be produced, however, next() was called. Please use hasNext() beforehand.") - this._next = null - return next + next = null + openPeeking(env) } + /** + * Implementers shall not override this method. + */ override fun close() { - _next = null + next = null + closePeeking() } } diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelUnion.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelUnionAll.kt similarity index 96% rename from partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelUnion.kt rename to partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelUnionAll.kt index 58d66709a3..663abadd23 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelUnion.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelUnionAll.kt @@ -4,7 +4,7 @@ import org.partiql.eval.internal.Environment import org.partiql.eval.internal.Record import org.partiql.eval.internal.operator.Operator -internal class RelUnion( +internal class RelUnionAll( private val lhs: Operator.Relation, private val rhs: Operator.Relation, ) : Operator.Relation { diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelUnionDistinct.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelUnionDistinct.kt new file mode 100644 index 0000000000..bafa6cb28e --- /dev/null +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelUnionDistinct.kt @@ -0,0 +1,38 @@ +package org.partiql.eval.internal.operator.rel + +import org.partiql.eval.internal.Environment +import org.partiql.eval.internal.Record +import org.partiql.eval.internal.helpers.IteratorChain +import org.partiql.eval.internal.operator.Operator + +internal class RelUnionDistinct( + private val lhs: Operator.Relation, + private val rhs: Operator.Relation, +) : RelPeeking() { + + private val seen: MutableSet = mutableSetOf() + private lateinit var input: Iterator + + override fun openPeeking(env: Environment) { + lhs.open(env) + rhs.open(env) + seen.clear() + input = IteratorChain(arrayOf(lhs, rhs)) + } + + override fun peek(): Record? { + for (record in input) { + if (!seen.contains(record)) { + seen.add(record) + return record + } + } + return null + } + + override fun closePeeking() { + lhs.close() + rhs.close() + seen.clear() + } +} diff --git a/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt b/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt index 6de65d6937..c294c4b255 100644 --- a/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt +++ b/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt @@ -70,46 +70,6 @@ class PartiQLEngineDefaultTest { @Execution(ExecutionMode.CONCURRENT) fun globalsTests(tc: SuccessTestCase) = tc.assert() - @Test - fun singleTest() { - val tc = SuccessTestCase( - input = """ - SELECT o.name AS orderName, - (SELECT c.name FROM customers c WHERE c.id=o.custId) AS customerName - FROM orders o - """.trimIndent(), - expected = bagValue( - structValue( - "orderName" to stringValue("foo") - ), - structValue( - "orderName" to stringValue("bar"), - "customerName" to stringValue("Helen") - ), - ), - globals = listOf( - SuccessTestCase.Global( - name = "customers", - value = """ - [{id:1, name: "Mary"}, - {id:2, name: "Helen"}, - {id:1, name: "John"} - ] - """ - ), - SuccessTestCase.Global( - name = "orders", - value = """ - [{custId:1, name: "foo"}, - {custId:2, name: "bar"} - ] - """ - ), - ) - ) - tc.assert() - } - companion object { @JvmStatic diff --git a/partiql-plan/src/main/resources/partiql_plan.ion b/partiql-plan/src/main/resources/partiql_plan.ion index 0c272060b8..cf723fce0b 100644 --- a/partiql-plan/src/main/resources/partiql_plan.ion +++ b/partiql-plan/src/main/resources/partiql_plan.ion @@ -261,20 +261,29 @@ rel::{ ], }, - union::{ - lhs: rel, - rhs: rel, - }, + set::[ + union::{ + quantifier: quantifier, + lhs: rel, + rhs: rel, + }, - intersect::{ - lhs: rel, - rhs: rel, - }, + intersect::{ + quantifier: quantifier, + lhs: rel, + rhs: rel, + }, - except::{ - lhs: rel, - rhs: rel, - }, + except::{ + quantifier: quantifier, + lhs: rel, + rhs: rel, + }, + + _::[ + quantifier::[ ALL, DISTINCT ], + ] + ], limit::{ input: rel, diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Nodes.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Nodes.kt index dc09b1f705..370ad08e8f 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Nodes.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/ir/Nodes.kt @@ -20,7 +20,6 @@ import org.partiql.planner.internal.ir.builder.RelOpAggregateCallResolvedBuilder import org.partiql.planner.internal.ir.builder.RelOpAggregateCallUnresolvedBuilder import org.partiql.planner.internal.ir.builder.RelOpDistinctBuilder import org.partiql.planner.internal.ir.builder.RelOpErrBuilder -import org.partiql.planner.internal.ir.builder.RelOpExceptBuilder import org.partiql.planner.internal.ir.builder.RelOpExcludeBuilder import org.partiql.planner.internal.ir.builder.RelOpExcludePathBuilder import org.partiql.planner.internal.ir.builder.RelOpExcludeStepBuilder @@ -30,16 +29,17 @@ import org.partiql.planner.internal.ir.builder.RelOpExcludeTypeStructKeyBuilder import org.partiql.planner.internal.ir.builder.RelOpExcludeTypeStructSymbolBuilder import org.partiql.planner.internal.ir.builder.RelOpExcludeTypeStructWildcardBuilder import org.partiql.planner.internal.ir.builder.RelOpFilterBuilder -import org.partiql.planner.internal.ir.builder.RelOpIntersectBuilder import org.partiql.planner.internal.ir.builder.RelOpJoinBuilder import org.partiql.planner.internal.ir.builder.RelOpLimitBuilder import org.partiql.planner.internal.ir.builder.RelOpOffsetBuilder import org.partiql.planner.internal.ir.builder.RelOpProjectBuilder import org.partiql.planner.internal.ir.builder.RelOpScanBuilder import org.partiql.planner.internal.ir.builder.RelOpScanIndexedBuilder +import org.partiql.planner.internal.ir.builder.RelOpSetExceptBuilder +import org.partiql.planner.internal.ir.builder.RelOpSetIntersectBuilder +import org.partiql.planner.internal.ir.builder.RelOpSetUnionBuilder import org.partiql.planner.internal.ir.builder.RelOpSortBuilder import org.partiql.planner.internal.ir.builder.RelOpSortSpecBuilder -import org.partiql.planner.internal.ir.builder.RelOpUnionBuilder import org.partiql.planner.internal.ir.builder.RelOpUnpivotBuilder import org.partiql.planner.internal.ir.builder.RelTypeBuilder import org.partiql.planner.internal.ir.builder.RexBuilder @@ -876,9 +876,7 @@ internal data class Rel( is Distinct -> visitor.visitRelOpDistinct(this, ctx) is Filter -> visitor.visitRelOpFilter(this, ctx) is Sort -> visitor.visitRelOpSort(this, ctx) - is Union -> visitor.visitRelOpUnion(this, ctx) - is Intersect -> visitor.visitRelOpIntersect(this, ctx) - is Except -> visitor.visitRelOpExcept(this, ctx) + is Set -> visitor.visitRelOpSet(this, ctx) is Limit -> visitor.visitRelOpLimit(this, ctx) is Offset -> visitor.visitRelOpOffset(this, ctx) is Project -> visitor.visitRelOpProject(this, ctx) @@ -1020,64 +1018,82 @@ internal data class Rel( internal fun builder(): RelOpSortBuilder = RelOpSortBuilder() } } + internal sealed class Set : Op() { - internal data class Union( - @JvmField internal val lhs: Rel, - @JvmField internal val rhs: Rel, - ) : Op() { - public override val children: List by lazy { - val kids = mutableListOf() - kids.add(lhs) - kids.add(rhs) - kids.filterNotNull() + public override fun accept(visitor: PlanVisitor, ctx: C): R = when (this) { + is Union -> visitor.visitRelOpSetUnion(this, ctx) + is Intersect -> visitor.visitRelOpSetIntersect(this, ctx) + is Except -> visitor.visitRelOpSetExcept(this, ctx) } - public override fun accept(visitor: PlanVisitor, ctx: C): R = - visitor.visitRelOpUnion(this, ctx) + internal data class Union( + @JvmField internal val quantifier: Quantifier, + @JvmField internal val lhs: Rel, + @JvmField internal val rhs: Rel, + @JvmField internal val isOuter: Boolean, + ) : Set() { + public override val children: List by lazy { + val kids = mutableListOf() + kids.add(lhs) + kids.add(rhs) + kids.filterNotNull() + } - internal companion object { - @JvmStatic - internal fun builder(): RelOpUnionBuilder = RelOpUnionBuilder() - } - } + public override fun accept(visitor: PlanVisitor, ctx: C): R = + visitor.visitRelOpSetUnion(this, ctx) - internal data class Intersect( - @JvmField internal val lhs: Rel, - @JvmField internal val rhs: Rel, - ) : Op() { - public override val children: List by lazy { - val kids = mutableListOf() - kids.add(lhs) - kids.add(rhs) - kids.filterNotNull() + internal companion object { + @JvmStatic + internal fun builder(): RelOpSetUnionBuilder = RelOpSetUnionBuilder() + } } - public override fun accept(visitor: PlanVisitor, ctx: C): R = - visitor.visitRelOpIntersect(this, ctx) + internal data class Intersect( + @JvmField internal val quantifier: Quantifier, + @JvmField internal val lhs: Rel, + @JvmField internal val rhs: Rel, + @JvmField internal val isOuter: Boolean, + ) : Set() { + public override val children: List by lazy { + val kids = mutableListOf() + kids.add(lhs) + kids.add(rhs) + kids.filterNotNull() + } - internal companion object { - @JvmStatic - internal fun builder(): RelOpIntersectBuilder = RelOpIntersectBuilder() - } - } + public override fun accept(visitor: PlanVisitor, ctx: C): R = + visitor.visitRelOpSetIntersect(this, ctx) - internal data class Except( - @JvmField internal val lhs: Rel, - @JvmField internal val rhs: Rel, - ) : Op() { - public override val children: List by lazy { - val kids = mutableListOf() - kids.add(lhs) - kids.add(rhs) - kids.filterNotNull() + internal companion object { + @JvmStatic + internal fun builder(): RelOpSetIntersectBuilder = RelOpSetIntersectBuilder() + } } - public override fun accept(visitor: PlanVisitor, ctx: C): R = - visitor.visitRelOpExcept(this, ctx) + internal data class Except( + @JvmField internal val quantifier: Quantifier, + @JvmField internal val lhs: Rel, + @JvmField internal val rhs: Rel, + @JvmField internal val isOuter: Boolean, + ) : Set() { + public override val children: List by lazy { + val kids = mutableListOf() + kids.add(lhs) + kids.add(rhs) + kids.filterNotNull() + } - internal companion object { - @JvmStatic - internal fun builder(): RelOpExceptBuilder = RelOpExceptBuilder() + public override fun accept(visitor: PlanVisitor, ctx: C): R = + visitor.visitRelOpSetExcept(this, ctx) + + internal companion object { + @JvmStatic + internal fun builder(): RelOpSetExceptBuilder = RelOpSetExceptBuilder() + } + } + + internal enum class Quantifier { + ALL, DISTINCT } } diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/PlanTransform.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/PlanTransform.kt index 7fdcc25c29..0d0c7de206 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/PlanTransform.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/PlanTransform.kt @@ -328,21 +328,29 @@ internal class PlanTransform( } ) - override fun visitRelOpUnion(node: Rel.Op.Union, ctx: Unit) = org.partiql.plan.Rel.Op.Union( + override fun visitRelOpSetExcept(node: Rel.Op.Set.Except, ctx: Unit) = org.partiql.plan.Rel.Op.Set.Except( lhs = visitRel(node.lhs, ctx), rhs = visitRel(node.rhs, ctx), + quantifier = visitRelOpSetQuantifier(node.quantifier) ) - override fun visitRelOpIntersect(node: Rel.Op.Intersect, ctx: Unit) = org.partiql.plan.Rel.Op.Intersect( + override fun visitRelOpSetIntersect(node: Rel.Op.Set.Intersect, ctx: Unit) = org.partiql.plan.Rel.Op.Set.Intersect( lhs = visitRel(node.lhs, ctx), rhs = visitRel(node.rhs, ctx), + quantifier = visitRelOpSetQuantifier(node.quantifier) ) - override fun visitRelOpExcept(node: Rel.Op.Except, ctx: Unit) = org.partiql.plan.Rel.Op.Except( + override fun visitRelOpSetUnion(node: Rel.Op.Set.Union, ctx: Unit) = org.partiql.plan.Rel.Op.Set.Union( lhs = visitRel(node.lhs, ctx), rhs = visitRel(node.rhs, ctx), + quantifier = visitRelOpSetQuantifier(node.quantifier) ) + private fun visitRelOpSetQuantifier(node: Rel.Op.Set.Quantifier) = when (node) { + Rel.Op.Set.Quantifier.ALL -> org.partiql.plan.Rel.Op.Set.Quantifier.ALL + Rel.Op.Set.Quantifier.DISTINCT -> org.partiql.plan.Rel.Op.Set.Quantifier.DISTINCT + } + override fun visitRelOpLimit(node: Rel.Op.Limit, ctx: Unit) = org.partiql.plan.Rel.Op.Limit( input = visitRel(node.input, ctx), limit = visitRex(node.limit, ctx), diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/RelConverter.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/RelConverter.kt index 5abdb2bbde..21259c3bed 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/RelConverter.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/RelConverter.kt @@ -42,7 +42,6 @@ import org.partiql.planner.internal.ir.relOpAggregate import org.partiql.planner.internal.ir.relOpAggregateCallUnresolved import org.partiql.planner.internal.ir.relOpDistinct import org.partiql.planner.internal.ir.relOpErr -import org.partiql.planner.internal.ir.relOpExcept import org.partiql.planner.internal.ir.relOpExclude import org.partiql.planner.internal.ir.relOpExcludePath import org.partiql.planner.internal.ir.relOpExcludeStep @@ -52,7 +51,6 @@ import org.partiql.planner.internal.ir.relOpExcludeTypeStructKey import org.partiql.planner.internal.ir.relOpExcludeTypeStructSymbol import org.partiql.planner.internal.ir.relOpExcludeTypeStructWildcard import org.partiql.planner.internal.ir.relOpFilter -import org.partiql.planner.internal.ir.relOpIntersect import org.partiql.planner.internal.ir.relOpJoin import org.partiql.planner.internal.ir.relOpLimit import org.partiql.planner.internal.ir.relOpOffset @@ -61,7 +59,6 @@ import org.partiql.planner.internal.ir.relOpScan import org.partiql.planner.internal.ir.relOpScanIndexed import org.partiql.planner.internal.ir.relOpSort import org.partiql.planner.internal.ir.relOpSortSpec -import org.partiql.planner.internal.ir.relOpUnion import org.partiql.planner.internal.ir.relOpUnpivot import org.partiql.planner.internal.ir.relType import org.partiql.planner.internal.ir.rex @@ -435,9 +432,6 @@ internal object RelConverter { /** * Append SQL set operator if present - * - * TODO combine/compare schemas - * TODO set quantifier */ private fun convertSetOp(input: Rel, setOp: Expr.SFW.SetOp?): Rel { if (setOp == null) { @@ -446,10 +440,14 @@ internal object RelConverter { val type = input.type.copy(props = emptySet()) val lhs = input val rhs = visitExprSFW(setOp.operand, nil) + val quantifier = when (setOp.type.setq) { + SetQuantifier.ALL -> Rel.Op.Set.Quantifier.ALL + null, SetQuantifier.DISTINCT -> Rel.Op.Set.Quantifier.DISTINCT + } val op = when (setOp.type.type) { - SetOp.Type.UNION -> relOpUnion(lhs, rhs) - SetOp.Type.INTERSECT -> relOpIntersect(lhs, rhs) - SetOp.Type.EXCEPT -> relOpExcept(lhs, rhs) + SetOp.Type.UNION -> Rel.Op.Set.Union(quantifier, lhs, rhs, false) + SetOp.Type.EXCEPT -> Rel.Op.Set.Except(quantifier, lhs, rhs, false) + SetOp.Type.INTERSECT -> Rel.Op.Set.Intersect(quantifier, lhs, rhs, false) } return rel(type, op) } diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/RexConverter.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/RexConverter.kt index 11db9199b2..e4ab533d53 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/RexConverter.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/RexConverter.kt @@ -20,6 +20,8 @@ import org.partiql.ast.AstNode import org.partiql.ast.DatetimeField import org.partiql.ast.Expr import org.partiql.ast.Select +import org.partiql.ast.SetOp +import org.partiql.ast.SetQuantifier import org.partiql.ast.Type import org.partiql.ast.visitor.AstBaseVisitor import org.partiql.planner.internal.Env @@ -821,6 +823,41 @@ internal object RexConverter { override fun visitExprSFW(node: Expr.SFW, context: Env): Rex = RelConverter.apply(node, context) + override fun visitExprBagOp(node: Expr.BagOp, ctx: Env): Rex { + val lhs = Rel( + type = Rel.Type(listOf(Rel.Binding("_0", StaticType.ANY)), props = emptySet()), + op = Rel.Op.Scan(visitExpr(node.lhs, ctx)) + ) + val rhs = Rel( + type = Rel.Type(listOf(Rel.Binding("_1", StaticType.ANY)), props = emptySet()), + op = Rel.Op.Scan(visitExpr(node.rhs, ctx)) + ) + val quantifier = when (node.type.setq) { + SetQuantifier.ALL -> Rel.Op.Set.Quantifier.ALL + null, SetQuantifier.DISTINCT -> Rel.Op.Set.Quantifier.DISTINCT + } + val isOuter = node.outer == true + val op = when (node.type.type) { + SetOp.Type.UNION -> Rel.Op.Set.Union(quantifier, lhs, rhs, isOuter) + SetOp.Type.EXCEPT -> Rel.Op.Set.Except(quantifier, lhs, rhs, isOuter) + SetOp.Type.INTERSECT -> Rel.Op.Set.Intersect(quantifier, lhs, rhs, isOuter) + } + val rel = Rel( + type = Rel.Type(listOf(Rel.Binding("_0", StaticType.ANY)), props = emptySet()), + op = op + ) + return Rex( + type = StaticType.ANY, + op = Rex.Op.Select( + constructor = Rex( + StaticType.ANY, + Rex.Op.Var.Unresolved(Identifier.Symbol("_0", Identifier.CaseSensitivity.SENSITIVE), Rex.Op.Var.Scope.LOCAL) + ), + rel = rel + ) + ) + } + // Helpers private fun negate(call: Rex.Op.Call): Rex.Op.Call { diff --git a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/PlanTyper.kt b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/PlanTyper.kt index 0c1018972f..5bab5333fe 100644 --- a/partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/PlanTyper.kt +++ b/partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/PlanTyper.kt @@ -91,6 +91,7 @@ import org.partiql.value.PartiQLValueExperimental import org.partiql.value.TextValue import org.partiql.value.boolValue import org.partiql.value.stringValue +import kotlin.math.max /** * Rewrites an untyped algebraic translation of the query to be both typed and have resolved variables. @@ -218,16 +219,95 @@ internal class PlanTyper(private val env: Env) { return rel(type, op) } - override fun visitRelOpUnion(node: Rel.Op.Union, ctx: Rel.Type?): Rel { - TODO("Type RelOp Union") + override fun visitRelOpSetExcept(node: Rel.Op.Set.Except, ctx: Rel.Type?): Rel { + val lhs = visitRel(node.lhs, node.lhs.type) + val rhs = visitRel(node.rhs, node.rhs.type) + // Check for Compatibility + if (!setOpSchemaSizesMatch(lhs, rhs)) { + return createRelErrForSetOpMismatchSizes() + } + if (!node.isOuter && !setOpSchemaTypesMatch(lhs, rhs)) { + return createRelErrForSetOpMismatchTypes() + } + // Compute Schema + val type = Rel.Type(lhs.type.schema, props = emptySet()) + return Rel(type, node.copy(lhs = lhs, rhs = rhs)) + } + + override fun visitRelOpSetIntersect(node: Rel.Op.Set.Intersect, ctx: Rel.Type?): Rel { + val lhs = visitRel(node.lhs, node.lhs.type) + val rhs = visitRel(node.rhs, node.rhs.type) + // Check for Compatibility + if (!setOpSchemaSizesMatch(lhs, rhs)) { + return createRelErrForSetOpMismatchSizes() + } + if (!node.isOuter && !setOpSchemaTypesMatch(lhs, rhs)) { + return createRelErrForSetOpMismatchTypes() + } + // Compute Schema + val type = Rel.Type(lhs.type.schema, props = emptySet()) + return Rel(type, node.copy(lhs = lhs, rhs = rhs)) + } + + override fun visitRelOpSetUnion(node: Rel.Op.Set.Union, ctx: Rel.Type?): Rel { + val lhs = visitRel(node.lhs, node.lhs.type) + val rhs = visitRel(node.rhs, node.rhs.type) + // Check for Compatibility + if (!setOpSchemaSizesMatch(lhs, rhs)) { + return createRelErrForSetOpMismatchSizes() + } + if (!node.isOuter && !setOpSchemaTypesMatch(lhs, rhs)) { + return createRelErrForSetOpMismatchTypes() + } + // Compute Schema + val size = max(lhs.type.schema.size, rhs.type.schema.size) + val schema = List(size) { + val lhsBinding = lhs.type.schema.getOrNull(it) ?: Rel.Binding("_$it", MISSING) + val rhsBinding = rhs.type.schema.getOrNull(it) ?: Rel.Binding("_$it", MISSING) + val bindingName = when (lhsBinding.name == rhsBinding.name) { + true -> lhsBinding.name + false -> "_$it" + } + Rel.Binding(bindingName, unionOf(lhsBinding.type, rhsBinding.type)) + } + val type = Rel.Type(schema, props = emptySet()) + return Rel(type, node.copy(lhs = lhs, rhs = rhs)) + } + + /** + * @return whether each type of the [lhs] is equal to its counterpart on the [rhs] + * @param lhs should be typed already + * @param rhs should be typed already + */ + private fun setOpSchemaTypesMatch(lhs: Rel, rhs: Rel): Boolean { + // TODO: [RFC-0007](https://github.com/partiql/partiql-lang/blob/main/RFCs/0007-rfc-bag-operators.md) + // states that the types must be "comparable". The below code ONLY makes sure that types need to be + // the same. In the future, we need to add support for checking comparable types. + for (i in 0..lhs.type.schema.lastIndex) { + val lhsBindingType = lhs.type.schema[i].type + val rhsBindingType = rhs.type.schema[i].type + if (lhsBindingType != rhsBindingType) { + return false + } + } + return true + } + + /** + * @return whether the [lhs] and [rhs] schemas are of equal size + * @param lhs should be typed already + * @param rhs should be typed already + */ + private fun setOpSchemaSizesMatch(lhs: Rel, rhs: Rel): Boolean { + return lhs.type.schema.size == rhs.type.schema.size } - override fun visitRelOpIntersect(node: Rel.Op.Intersect, ctx: Rel.Type?): Rel { - TODO("Type RelOp Intersect") + private fun createRelErrForSetOpMismatchSizes(): Rel { + return Rel(Rel.Type(emptyList(), emptySet()), Rel.Op.Err("LHS and RHS of SET OP do not have the same number of bindings.")) } - override fun visitRelOpExcept(node: Rel.Op.Except, ctx: Rel.Type?): Rel { - TODO("Type RelOp Except") + private fun createRelErrForSetOpMismatchTypes(): Rel { + return Rel(Rel.Type(emptyList(), emptySet()), Rel.Op.Err("LHS and RHS of SET OP do not have the same type.")) } override fun visitRelOpLimit(node: Rel.Op.Limit, ctx: Rel.Type?): Rel { diff --git a/partiql-planner/src/main/resources/partiql_plan_internal.ion b/partiql-planner/src/main/resources/partiql_plan_internal.ion index a8f5479817..590134f693 100644 --- a/partiql-planner/src/main/resources/partiql_plan_internal.ion +++ b/partiql-planner/src/main/resources/partiql_plan_internal.ion @@ -296,20 +296,38 @@ rel::{ ], }, - union::{ - lhs: rel, - rhs: rel, - }, - intersect::{ - lhs: rel, - rhs: rel, - }, + // In each variant, is_outer is an internal-only field. It is specifically used to aid in typing the plan and throwing potential errors. + // For example, if a user were to write: `<< { 'a': 1 } >>` UNION << { 'b': 'hello' } >>, then this would FAIL + // due to [RFC-0007](https://github.com/partiql/partiql-lang/blob/main/RFCs/0007-rfc-bag-operators.md). However, + // if a user were to use OUTER UNION, then it would work. Under the hood at execution, the operator is the same -- + // however, at planning time, with static type analysis, we can fail queries prior to their execution. + set::[ + union::{ + quantifier: quantifier, + lhs: rel, + rhs: rel, + is_outer: bool + }, - except::{ - lhs: rel, - rhs: rel, - }, + intersect::{ + quantifier: quantifier, + lhs: rel, + rhs: rel, + is_outer: bool + }, + + except::{ + quantifier: quantifier, + lhs: rel, + rhs: rel, + is_outer: bool + }, + + _::[ + quantifier::[ ALL, DISTINCT ], + ] + ], limit::{ input: rel,