feat: add by:name to append transform

prqlc/prqlc/src/ir/pl/extra.rs

@@ -90,7 +90,10 @@ pub enum TransformKind {
         range: Range,
         pipeline: Box<Expr>,
     },
-    Append(Box<Expr>),
+    Append {
+        by: AppendBy,
+        bottom: Box<Expr>,
+    },
     Loop(Box<Expr>),
 }
 
@@ -115,6 +118,12 @@ pub enum JoinSide {
     Full,
 }
 
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, JsonSchema)]
+pub enum AppendBy {
+    Name,
+    Position,
+}
+
 impl Expr {
     pub fn new(kind: impl Into<ExprKind>) -> Self {
         Expr {

prqlc/prqlc/src/ir/pl/fold.rs

@@ -256,7 +256,10 @@ pub fn fold_transform_kind<T: ?Sized + PlFold>(
             with: Box::new(fold.fold_expr(*with)?),
             filter: Box::new(fold.fold_expr(*filter)?),
         },
-        Append(bottom) => Append(Box::new(fold.fold_expr(*bottom)?)),
+        Append { by, bottom } => Append {
+            by,
+            bottom: Box::new(fold.fold_expr(*bottom)?),
+        },
         Group { by, pipeline } => Group {
             by: Box::new(fold.fold_expr(*by)?),
             pipeline: Box::new(fold.fold_expr(*pipeline)?),

prqlc/prqlc/src/ir/rq/fold.rs

@@ -185,7 +185,10 @@ pub fn fold_transform<T: ?Sized + RqFold>(
             with: fold.fold_table_ref(with)?,
             filter: fold.fold_expr(filter)?,
         },
-        Append(bottom) => Append(fold.fold_table_ref(bottom)?),
+        Append { by, bottom } => Append {
+            by,
+            bottom: fold.fold_table_ref(bottom)?,
+        },
         Loop(transforms) => Loop(fold_transforms(fold, transforms)?),
     };
     Ok(transform)

prqlc/prqlc/src/ir/rq/transform.rs

@@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};
 use super::*;
 use crate::ir::generic::ColumnSort;
 use crate::ir::generic::WindowFrame;
-use crate::ir::pl::JoinSide;
+use crate::ir::pl::{AppendBy, JoinSide};
 
 /// Transformation of a table.
 #[derive(
@@ -27,7 +27,10 @@ pub enum Transform {
         with: TableRef,
         filter: Expr,
     },
-    Append(TableRef),
+    Append {
+        by: AppendBy,
+        bottom: TableRef,
+    },
     Loop(Vec<Transform>),
 }
 

prqlc/prqlc/src/semantic/lowering.rs

@@ -612,11 +612,11 @@ impl Lowerer {
                 };
                 self.pipeline.push(transform);
             }
-            pl::TransformKind::Append(bottom) => {
+            pl::TransformKind::Append { by, bottom } => {
                 let mut bottom = self.lower_table_ref(*bottom)?;
                 bottom.prefer_cte = false;
 
-                self.pipeline.push(Transform::Append(bottom));
+                self.pipeline.push(Transform::Append { by, bottom });
             }
             pl::TransformKind::Loop(pipeline) => {
                 let relation = self.lower_relation(*pipeline)?;

prqlc/prqlc/src/semantic/reporting.rs

@@ -257,7 +257,7 @@ impl PlFold for FrameCollector {
                         pl::TransformKind::Derive { assigns: ref e }
                         | pl::TransformKind::Select { assigns: ref e }
                         | pl::TransformKind::Filter { filter: ref e }
-                        | pl::TransformKind::Append(ref e)
+                        | pl::TransformKind::Append { bottom: ref e, .. }
                         | pl::TransformKind::Loop(ref e)
                         | pl::TransformKind::Group {
                             pipeline: ref e, ..

prqlc/prqlc/src/semantic/resolver/flatten.rs

@@ -151,9 +151,11 @@ impl PlFold for Flattener {
                         // in scope for downstream transforms in the outer pipeline. Per the PRQL
                         // spec a join retains the left (input) side's order, so snapshot the
                         // input's sort and restore it after folding the kind.
-                        let input_sort =
-                            matches!(kind, TransformKind::Join { .. } | TransformKind::Append(_))
-                                .then(|| self.sort.clone());
+                        let input_sort = matches!(
+                            kind,
+                            TransformKind::Join { .. } | TransformKind::Append { .. }
+                        )
+                        .then(|| self.sort.clone());
 
                         let kind = fold_transform_kind(self, kind)?;
 
@@ -177,8 +179,10 @@ impl PlFold for Flattener {
                 //   derive {`album_name` = `name`}
                 //   select {`artist_id`, `album_name`}
                 // ) (this.id == that.artist_id)
-                let sort = if matches!(kind, TransformKind::Join { .. } | TransformKind::Append(_))
-                {
+                let sort = if matches!(
+                    kind,
+                    TransformKind::Join { .. } | TransformKind::Append { .. }
+                ) {
                     vec![]
                 } else {
                     self.sort.clone()

prqlc/prqlc/src/semantic/resolver/mod.rs

@@ -44,7 +44,7 @@ impl Resolver<'_> {
 pub(super) mod test {
     use insta::assert_yaml_snapshot;
 
-    use crate::ir::pl::{Expr, Lineage, PlFold};
+    use crate::ir::pl::{AppendBy, Expr, Lineage, PlFold};
     use crate::{Errors, Result};
 
     pub fn erase_ids(expr: Expr) -> Expr {
@@ -463,7 +463,10 @@ pub(super) mod test {
         bottom_expr.lineage = Some(bottom_lineage);
 
         let transform_call = TransformCall {
-            kind: Box::new(TransformKind::Append(Box::new(bottom_expr))),
+            kind: Box::new(TransformKind::Append {
+                by: AppendBy::Position,
+                bottom: Box::new(bottom_expr),
+            }),
             input: Box::new(top_expr),
             partition: None,
             frame: crate::ir::pl::WindowFrame::default(),

prqlc/prqlc/src/semantic/resolver/transforms.rs

@@ -254,9 +254,35 @@ impl Resolver<'_> {
                 (transform_kind, tbl)
             }
             "append" => {
-                let [bottom, top] = unpack::<2>(func.args);
+                let [by, bottom, top] = unpack::<3>(func.args);
 
-                (TransformKind::Append(Box::new(bottom)), top)
+                let by = {
+                    let span = by.span;
+                    let ident = by
+                        .clone()
+                        .try_cast(ExprKind::into_ident, Some("by"), "ident")?;
+
+                    match ident.to_string().as_str() {
+                        "position" => AppendBy::Position,
+                        "name" => AppendBy::Name,
+                        _ => {
+                            return Err(Error::new(Reason::Expected {
+                                who: Some("`by`".to_string()),
+                                expected: "position or name".to_string(),
+                                found: ident.to_string(),
+                            })
+                            .with_span(span))
+                        }
+                    }
+                };
+
+                (
+                    TransformKind::Append {
+                        by,
+                        bottom: Box::new(bottom),
+                    },
+                    top,
+                )
             }
             "loop" => {
                 let [pipeline, tbl] = unpack::<2>(func.args);
@@ -600,7 +626,7 @@ impl Resolver<'_> {
                 let pipeline = pipeline.kind.into_function().unwrap().unwrap();
                 pipeline.return_ty.map(|x| *x)
             }
-            TransformKind::Append(bottom) => {
+            TransformKind::Append { bottom, .. } => {
                 let top = transform_call.input.ty.clone().unwrap();
                 let bottom = bottom.ty.clone().unwrap();
 
@@ -757,7 +783,7 @@ impl TransformCall {
                 let right = lineage_or_default(with)?;
                 join(left, right)
             }
-            Append(bottom) => {
+            Append { bottom, .. } => {
                 let top = lineage_or_default(&self.input)?;
                 let bottom = lineage_or_default(bottom)?;
                 append(top, bottom)?

prqlc/prqlc/src/semantic/std.prql

@@ -118,7 +118,12 @@ let window = func
   tbl <relation>
   -> <relation> internal window
 
-let append = `default_db.bottom`<relation> top<relation> -> <relation> internal append
+let append = func
+  `noresolve.by`:position
+  `default_db.bottom`<relation>
+  top<relation>
+  -> <relation> internal append
+
 let intersect = `default_db.bottom`<relation> top<relation> -> <relation> (
   t = top
   join (b = bottom) (tuple_reduce std.and (tuple_map _eq (tuple_zip t.* b.*)))

prqlc/prqlc/src/sql/dialect.rs

@@ -216,6 +216,11 @@ pub(super) trait DialectHandler: Any + Debug {
         true
     }
 
+    /// Support for UNION [..] BY NAME.
+    fn union_by_name(&self) -> bool {
+        false
+    }
+
     /// Support or EXCEPT ALL.
     /// When not supported, fallback to anti join.
     fn except_all(&self) -> bool {
@@ -243,6 +248,7 @@ pub(super) trait DialectHandler: Any + Debug {
         true
     }
 
+    /// Support for SELECT DISTINCT ON(partition)
     fn supports_distinct_on(&self) -> bool {
         false
     }
@@ -634,6 +640,11 @@ impl DialectHandler for BigQueryDialect {
         true
     }
 
+    fn union_by_name(&self) -> bool {
+        // https://docs.cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#union
+        true
+    }
+
     fn prefers_subquery_parentheses_shorthand(&self) -> bool {
         true
     }
@@ -684,6 +695,11 @@ impl DialectHandler for SnowflakeDialect {
         false
     }
 
+    fn union_by_name(&self) -> bool {
+        // https://docs.snowflake.com/en/sql-reference/operators-query
+        true
+    }
+
     fn interval_quoting_style(&self, _dtf: &DateTimeField) -> IntervalQuotingStyle {
         // Snowflake requires quotes around value and unit together
         // https://docs.snowflake.com/en/sql-reference/data-types-datetime#interval-constants
@@ -709,6 +725,11 @@ impl DialectHandler for DuckDbDialect {
     }
 
     fn supports_distinct_on(&self) -> bool {
+        // https://duckdb.org/docs/current/sql/query_syntax/select#distinct-on-clause
+        true
+    }
+
+    fn union_by_name(&self) -> bool {
         true
     }
 

prqlc/prqlc/src/sql/gen_query.rs

@@ -15,7 +15,7 @@ use super::operators::translate_operator;
 use super::pq::ast::{Cte, CteKind, RelationExpr, RelationExprKind, SqlRelation, SqlTransform};
 use super::{Context, Dialect};
 use crate::debug;
-use crate::ir::pl::{JoinSide, Literal};
+use crate::ir::pl::{AppendBy, JoinSide, Literal};
 use crate::ir::rq::{CId, Expr, ExprKind, RelationLiteral, RelationalQuery};
 use crate::utils::{BreakUp, Pluck};
 use crate::{Error, Result, WithErrorInfo};
@@ -292,10 +292,15 @@ fn translate_set_ops_pipeline(
             _ => unreachable!(),
         };
 
-        let (distinct, bottom) = match transform {
-            Union { distinct, bottom }
-            | Except { distinct, bottom }
-            | Intersect { distinct, bottom } => (distinct, bottom),
+        let (distinct, bottom, by) = match transform {
+            Union {
+                distinct,
+                bottom,
+                by,
+            } => (distinct, bottom, by),
+            Except { distinct, bottom } | Intersect { distinct, bottom } => {
+                (distinct, bottom, AppendBy::Position)
+            }
             _ => unreachable!(),
         };
 
@@ -330,14 +335,13 @@ fn translate_set_ops_pipeline(
         top = default_query(SetExpr::SetOperation {
             left,
             right,
-            set_quantifier: if distinct {
-                if context.dialect.set_ops_distinct() {
-                    sql_ast::SetQuantifier::Distinct
-                } else {
-                    sql_ast::SetQuantifier::None
-                }
-            } else {
-                sql_ast::SetQuantifier::All
+            set_quantifier: match (distinct, context.dialect.set_ops_distinct(), by) {
+                (false, _, AppendBy::Position) => sql_ast::SetQuantifier::All,
+                (false, _, AppendBy::Name) => sql_ast::SetQuantifier::AllByName,
+                (true, true, AppendBy::Position) => sql_ast::SetQuantifier::Distinct,
+                (true, true, AppendBy::Name) => sql_ast::SetQuantifier::DistinctByName,
+                (true, false, AppendBy::Position) => sql_ast::SetQuantifier::None,
+                (true, false, AppendBy::Name) => sql_ast::SetQuantifier::ByName,
             },
             op,
         });

prqlc/prqlc/src/sql/pq/ast.rs

@@ -10,7 +10,7 @@ use serde::Serialize;
 
 use super::context::RIId;
 use crate::ir::generic::ColumnSort;
-use crate::ir::pl::JoinSide;
+use crate::ir::pl::{AppendBy, JoinSide};
 use crate::ir::rq::{self, fold_column_sorts, RelationLiteral, RqFold};
 use crate::Result;
 
@@ -111,6 +111,7 @@ pub enum SqlTransform<Rel = RIId, Super = rq::Transform> {
     Union {
         bottom: Rel,
         distinct: bool,
+        by: AppendBy,
     },
 }
 
@@ -176,9 +177,14 @@ pub fn fold_sql_transform<
 
         SqlTransform::Distinct => SqlTransform::Distinct,
         SqlTransform::DistinctOn(ids) => SqlTransform::DistinctOn(fold.fold_cids(ids)?),
-        SqlTransform::Union { bottom, distinct } => SqlTransform::Union {
+        SqlTransform::Union {
+            bottom,
+            distinct,
+            by,
+        } => SqlTransform::Union {
             bottom: fold.fold_rel(bottom)?,
             distinct,
+            by,
         },
         SqlTransform::Except { bottom, distinct } => SqlTransform::Except {
             bottom: fold.fold_rel(bottom)?,

prqlc/prqlc/src/sql/pq/gen_query.rs

@@ -161,7 +161,7 @@ impl PqMapper<RIId, pq::RelationExpr, rq::Transform, ()> for TransformCompiler<'
                             rq::Transform::Sort(v) => pq::SqlTransform::Sort(v),
                             rq::Transform::Take(v) => pq::SqlTransform::Take(v),
                             rq::Transform::Compute(_)
-                            | rq::Transform::Append(_)
+                            | rq::Transform::Append { .. }
                             | rq::Transform::Loop(_) => {
                                 // these are not used from here on
                                 return Ok(None);

prqlc/prqlc/src/sql/pq/preprocess.rs

@@ -7,7 +7,7 @@ use super::anchor::{infer_complexity, CidCollector, Complexity};
 use super::ast::*;
 
 use crate::ir::generic::{ColumnSort, SortDirection, WindowFrame, WindowKind};
-use crate::ir::pl::{JoinSide, Literal};
+use crate::ir::pl::{AppendBy, JoinSide, Literal};
 use crate::ir::rq::{
     self, maybe_binop, new_binop, CId, Compute, Expr, ExprKind, RqFold, Transform, Window,
 };
@@ -284,7 +284,7 @@ pub(in crate::sql) fn union(
     let mut res = Vec::with_capacity(pipeline.len());
     let mut pipeline = pipeline.into_iter().peekable();
     while let Some(t) = pipeline.next() {
-        let Super(Append(bottom)) = t else {
+        let Super(Append { by, bottom }) = t else {
             res.push(t);
             continue;
         };
@@ -297,7 +297,18 @@ pub(in crate::sql) fn union(
             false
         };
 
-        res.push(SqlTransform::Union { bottom, distinct });
+        if by == AppendBy::Name && !ctx.dialect.union_by_name() {
+            return Err(Error::new_simple(format!(
+                "The dialect {:?} does not support UNION BY NAME",
+                ctx.dialect
+            )));
+        }
+
+        res.push(SqlTransform::Union {
+            bottom,
+            distinct,
+            by,
+        });
     }
     Ok(res)
 }