feat[layout]: list-of-struct nested projection

.github/workflows/ci-fork.yml

@@ -0,0 +1,73 @@
+name: CI (fork)
+
+on:
+  pull_request: { }
+
+permissions:
+  contents: read
+
+env:
+  CARGO_TERM_COLOR: auto
+
+jobs:
+  rust-fmt:
+    name: Rust (fmt)
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/setup-rust
+        with:
+          toolchain: nightly
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Rust format
+        run: cargo +nightly fmt --all --check
+
+  rust-clippy:
+    name: Rust (clippy)
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/setup-rust
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Rust clippy (workspace)
+        run: >
+          cargo clippy --locked --workspace --all-features --all-targets
+          --exclude vortex-bench
+          --exclude vortex-python
+          --exclude vortex-duckdb
+          --exclude vortex-fuzz
+          --exclude duckdb-bench
+          --exclude lance-bench
+          --exclude datafusion-bench
+          --exclude random-access-bench
+          --exclude compress-bench
+          -- -D warnings
+
+  rust-tests:
+    name: Rust (tests)
+    runs-on: ubuntu-latest
+    timeout-minutes: 240
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/setup-rust
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: taiki-e/install-action@v2
+        with:
+          tool: nextest
+      - name: Cargo nextest
+        run: >
+          cargo nextest run --locked --workspace --all-features --no-fail-fast
+          --exclude vortex-cuda
+          --exclude vortex-bench
+          --exclude vortex-python
+          --exclude vortex-duckdb
+          --exclude vortex-fuzz
+          --exclude duckdb-bench
+          --exclude lance-bench
+          --exclude datafusion-bench
+          --exclude random-access-bench
+          --exclude compress-bench

.github/workflows/ci.yml

@@ -9,9 +9,6 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }}
 on:
-  push:
-    branches: [develop]
-  pull_request: { }
   workflow_dispatch: { }
 
 permissions:

vortex-array/src/expr/analysis/immediate_access.rs

@@ -24,11 +24,13 @@ pub fn annotate_scope_access(scope: &StructFields) -> impl AnnotationFn<Annotati
             "cannot analyse select, simplify the expression"
         );
 
-        if let Some(field_name) = expr.as_opt::<GetItem>() {
-            if expr.child(0).is::<Root>() {
-                return vec![field_name.clone()];
-            }
-        } else if expr.is::<Root>() {
+        if let Some(field_name) = expr.as_opt::<GetItem>()
+            && expr.child(0).is::<Root>()
+        {
+            return vec![field_name.clone()];
+        }
+
+        if expr.is::<Root>() {
             return scope.names().iter().cloned().collect();
         }
 

vortex-array/src/expr/exprs/get_item.rs

@@ -31,9 +31,11 @@ use crate::expr::Pack;
 use crate::expr::ReduceCtx;
 use crate::expr::ReduceNode;
 use crate::expr::ReduceNodeRef;
+use crate::expr::SimplifyCtx;
 use crate::expr::StatsCatalog;
 use crate::expr::VTable;
 use crate::expr::VTableExt;
+use crate::expr::exprs::get_item_list::get_item_list;
 use crate::expr::exprs::root::root;
 use crate::expr::lit;
 use crate::expr::stats::Stat;
@@ -95,15 +97,7 @@ impl VTable for GetItem {
                 vortex_err!("Couldn't find the {} field in the input scope", field_name)
             })?;
 
-        // Match here to avoid cloning the dtype if nullability doesn't need to change
-        if matches!(
-            (struct_dtype.nullability(), field_dtype.nullability()),
-            (Nullability::Nullable, Nullability::NonNullable)
-        ) {
-            return Ok(field_dtype.with_nullability(Nullability::Nullable));
-        }
-
-        Ok(field_dtype)
+        Ok(field_dtype.union_nullability(struct_dtype.nullability()))
     }
 
     fn execute(
@@ -125,6 +119,30 @@ impl VTable for GetItem {
         .execute(args.ctx)
     }
 
+    fn simplify(
+        &self,
+        field_name: &FieldName,
+        expr: &Expression,
+        ctx: &dyn SimplifyCtx,
+    ) -> VortexResult<Option<Expression>> {
+        let child = expr.child(0);
+        let child_dtype = ctx.return_dtype(child)?;
+
+        let element_dtype = match child_dtype {
+            DType::List(element_dtype, _) => Some(element_dtype),
+            DType::FixedSizeList(element_dtype, ..) => Some(element_dtype),
+            _ => None,
+        };
+
+        if let Some(element_dtype) = element_dtype
+            && element_dtype.as_struct_fields_opt().is_some()
+        {
+            Ok(Some(get_item_list(field_name.clone(), child.clone())))
+        } else {
+            Ok(None)
+        }
+    }
+
     fn reduce(
         &self,
         field_name: &FieldName,
@@ -245,19 +263,25 @@ pub fn get_item(field: impl Into<FieldName>, child: Expression) -> Expression {
 
 #[cfg(test)]
 mod tests {
+    use std::sync::Arc;
+
     use vortex_buffer::buffer;
     use vortex_dtype::DType;
     use vortex_dtype::FieldNames;
     use vortex_dtype::Nullability;
     use vortex_dtype::Nullability::NonNullable;
     use vortex_dtype::PType;
     use vortex_dtype::StructFields;
+    use vortex_scalar::Scalar;
 
     use crate::Array;
     use crate::IntoArray;
+    use crate::arrays::FixedSizeListArray;
+    use crate::arrays::ListArray;
     use crate::arrays::StructArray;
     use crate::expr::exprs::binary::checked_add;
     use crate::expr::exprs::get_item::get_item;
+    use crate::expr::exprs::get_item_list::get_item_list;
     use crate::expr::exprs::literal::lit;
     use crate::expr::exprs::pack::pack;
     use crate::expr::exprs::root::root;
@@ -307,6 +331,182 @@ mod tests {
         );
     }
 
+    #[test]
+    fn get_item_list_of_struct() {
+        let element_dtype = Arc::new(DType::Struct(
+            [
+                ("a", DType::Primitive(PType::I32, NonNullable)),
+                ("b", DType::Utf8(NonNullable)),
+            ]
+            .into_iter()
+            .collect(),
+            NonNullable,
+        ));
+
+        let row_count = 4;
+        let items = ListArray::from_iter_opt_slow::<u32, _, _>(
+            [
+                Some(vec![
+                    Scalar::struct_(
+                        (*element_dtype).clone(),
+                        vec![
+                            Scalar::primitive(1i32, NonNullable),
+                            Scalar::utf8("x", NonNullable),
+                        ],
+                    ),
+                    Scalar::struct_(
+                        (*element_dtype).clone(),
+                        vec![
+                            Scalar::primitive(2i32, NonNullable),
+                            Scalar::utf8("y", NonNullable),
+                        ],
+                    ),
+                ]),
+                Some(Vec::new()),
+                None,
+                Some(vec![Scalar::struct_(
+                    (*element_dtype).clone(),
+                    vec![
+                        Scalar::primitive(3i32, NonNullable),
+                        Scalar::utf8("z", NonNullable),
+                    ],
+                )]),
+            ],
+            element_dtype,
+        )
+        .unwrap();
+
+        let ids = buffer![0i32, 1, 2, 3].into_array();
+
+        let data = StructArray::new(
+            FieldNames::from(["id", "items"]),
+            vec![ids, items],
+            row_count,
+            Validity::NonNullable,
+        )
+        .into_array();
+
+        // Regression for nested field projection on list-of-struct: `items.a`.
+        let projection = get_item_list("a", get_item("items", root()));
+        let out = data.apply(&projection).expect("apply");
+
+        assert_eq!(
+            out.dtype(),
+            &DType::List(
+                Arc::new(DType::Primitive(PType::I32, NonNullable)),
+                Nullability::Nullable
+            )
+        );
+
+        assert_eq!(
+            out.scalar_at(0)
+                .unwrap()
+                .as_list()
+                .elements()
+                .unwrap()
+                .to_vec(),
+            vec![
+                Scalar::primitive(1i32, NonNullable),
+                Scalar::primitive(2i32, NonNullable),
+            ]
+        );
+        assert!(
+            out.scalar_at(1)
+                .unwrap()
+                .as_list()
+                .elements()
+                .unwrap()
+                .is_empty()
+        );
+        assert!(out.scalar_at(2).unwrap().is_null());
+        assert_eq!(
+            out.scalar_at(3)
+                .unwrap()
+                .as_list()
+                .elements()
+                .unwrap()
+                .to_vec(),
+            vec![Scalar::primitive(3i32, NonNullable)]
+        );
+    }
+
+    #[test]
+    fn get_item_fixed_size_list_of_struct() {
+        let n_lists: usize = 3;
+        let list_size: u32 = 2;
+        let n_elements = n_lists * list_size as usize;
+
+        let struct_elems = StructArray::try_new(
+            FieldNames::from(["a", "b"]),
+            vec![
+                buffer![1i32, 2, 3, 4, 5, 6].into_array(),
+                buffer![10i64, 20, 30, 40, 50, 60].into_array(),
+            ],
+            n_elements,
+            Validity::from_iter([true, false, true, true, false, true]),
+        )
+        .unwrap()
+        .into_array();
+
+        let items = FixedSizeListArray::try_new(
+            struct_elems,
+            list_size,
+            Validity::from_iter([true, false, true]),
+            n_lists,
+        )
+        .unwrap()
+        .into_array();
+
+        let ids = buffer![0i32, 1, 2].into_array();
+
+        let data = StructArray::new(
+            FieldNames::from(["id", "items"]),
+            vec![ids, items],
+            n_lists,
+            Validity::NonNullable,
+        )
+        .into_array();
+
+        // FixedSizeList-of-struct projection: `items.a`, including struct-level nulls inside the list.
+        let projection = get_item_list("a", get_item("items", root()));
+        let out = data.apply(&projection).expect("apply");
+
+        assert_eq!(
+            out.dtype(),
+            &DType::FixedSizeList(
+                Arc::new(DType::Primitive(PType::I32, Nullability::Nullable)),
+                list_size,
+                Nullability::Nullable
+            )
+        );
+
+        assert_eq!(
+            out.scalar_at(0)
+                .unwrap()
+                .as_list()
+                .elements()
+                .unwrap()
+                .to_vec(),
+            vec![
+                Scalar::primitive(1i32, Nullability::Nullable),
+                Scalar::null(DType::Primitive(PType::I32, Nullability::Nullable)),
+            ]
+        );
+        assert!(out.scalar_at(1).unwrap().is_null());
+        assert_eq!(
+            out.scalar_at(2)
+                .unwrap()
+                .as_list()
+                .elements()
+                .unwrap()
+                .to_vec(),
+            vec![
+                Scalar::null(DType::Primitive(PType::I32, Nullability::Nullable)),
+                Scalar::primitive(6i32, Nullability::Nullable),
+            ]
+        );
+    }
+
     #[test]
     fn test_pack_get_item_rule() {
         // Create: pack(a: lit(1), b: lit(2)).get_item("b")