diff --git a/CHANGELOG.md b/CHANGELOG.md index 67cfe79..bd3084f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased ### Added +- Added `ByteSize::logical_bytes` for logical byte size accounting. - Table::get_views_for_target_batch_size for easier Table -> wire batching. - Producer-driven record batch stream export for FFI. diff --git a/src/traits/byte_size.rs b/src/traits/byte_size.rs index 433e1f3..ef13246 100644 --- a/src/traits/byte_size.rs +++ b/src/traits/byte_size.rs @@ -22,6 +22,12 @@ //! - Simple calculation where possible (e.g., size_of::() * n * m) //! - Includes data buffers, null masks, and nested structures //! +//! The trait carries two metrics. `est_bytes` reports the memory footprint +//! of the allocations, so it counts buffer capacity. `logical_bytes` reports +//! the data itself, so it counts buffer length and excludes any capacity +//! slack left behind by buffer growth. Memory budgeting uses `est_bytes`, +//! while throughput accounting and wire-size planning use `logical_bytes`. +//! //! ## Usage //! ```rust //! use minarrow::{IntegerArray, ByteSize, MaskedArray}; @@ -29,6 +35,8 @@ //! let arr = IntegerArray::::from_slice(&[1, 2, 3, 4, 5]); //! let bytes = arr.est_bytes(); //! // Returns data buffer size: 5 * 8 = 40 bytes (plus small overhead) +//! let logical = arr.logical_bytes(); +//! // Returns the data payload: 5 * 8 = 40 bytes //! ``` use std::mem::size_of; @@ -53,6 +61,26 @@ pub trait ByteSize { /// - Stack size of the struct itself (only heap allocations) /// - Arc pointer overhead (counted once per allocation, not per reference) fn est_bytes(&self) -> usize; + + /// Returns the exact logical byte size of the data. + /// + /// The figure counts the bytes the values occupy in Arrow buffer terms. + /// Value buffers count `len * element_width`, offsets count + /// `(len + 1) * offset_width`, null masks count `ceil(len / 8)` + /// and dictionaries count their summed string contents. Capacity slack + /// left behind by buffer growth is excluded, which is the difference + /// from [`est_bytes`](ByteSize::est_bytes). That method reports the + /// memory footprint of the allocations, where this one reports the + /// data itself, so callers use it for throughput denominators and + /// wire-size planning where the figure must match the payload. + /// + /// Views report the window they cover rather than the backing array. + /// + /// ### Warning + /// The non-Arrow numerical container types (`Matrix`, `NdArray` and their chunked + /// and view forms, plus `XArray`) do not yet define logical byte + /// accounting currently and panic with `unimplemented!` when called. + fn logical_bytes(&self) -> usize; } // Base Buffer Type Implementations @@ -66,6 +94,12 @@ impl ByteSize for Vec64 { // Capacity in elements * size per element self.capacity() * size_of::() } + + #[inline] + fn logical_bytes(&self) -> usize { + // Populated elements * size per element + self.len() * size_of::() + } } /// ByteSize for Buffer - unified owned/shared buffer @@ -75,15 +109,29 @@ impl ByteSize for Buffer { // Capacity in elements * size per element self.capacity() * size_of::() } + + #[inline] + fn logical_bytes(&self) -> usize { + // A shared buffer reports its window length, so the figure covers + // the elements the buffer presents rather than the backing region. + self.len() * size_of::() + } } /// ByteSize for Bitmask - bit-packed bitmask impl ByteSize for Bitmask { #[inline] fn est_bytes(&self) -> usize { - // Bit-packed: (capacity + 7) / 8 bytes + // The capacity of the backing byte buffer, which is already + // byte-granular self.bits.est_bytes() } + + #[inline] + fn logical_bytes(&self) -> usize { + // Bit-packed is the owned bit count rounded up to whole bytes + (self.len() + 7) / 8 + } } // Concrete Array Type Implementations @@ -98,6 +146,11 @@ impl ByteSize for IntegerArray { let mask_bytes = self.null_mask.as_ref().map_or(0, |m| m.est_bytes()); data_bytes + mask_bytes } + + #[inline] + fn logical_bytes(&self) -> usize { + self.data.logical_bytes() + self.null_mask.as_ref().map_or(0, |m| m.logical_bytes()) + } } /// ByteSize for FloatArray @@ -108,6 +161,11 @@ impl ByteSize for FloatArray { let mask_bytes = self.null_mask.as_ref().map_or(0, |m| m.est_bytes()); data_bytes + mask_bytes } + + #[inline] + fn logical_bytes(&self) -> usize { + self.data.logical_bytes() + self.null_mask.as_ref().map_or(0, |m| m.logical_bytes()) + } } /// ByteSize for StringArray @@ -119,6 +177,14 @@ impl ByteSize for StringArray { let mask_bytes = self.null_mask.as_ref().map_or(0, |m| m.est_bytes()); data_bytes + offsets_bytes + mask_bytes } + + #[inline] + fn logical_bytes(&self) -> usize { + // The string payload plus its offsets buffer plus any null mask bytes + self.data.logical_bytes() + + self.offsets.logical_bytes() + + self.null_mask.as_ref().map_or(0, |m| m.logical_bytes()) + } } /// ByteSize for CategoricalArray @@ -126,11 +192,28 @@ impl ByteSize for CategoricalArray { #[inline] fn est_bytes(&self) -> usize { let data_bytes = self.data.est_bytes(); - // Approximate the dictionary container at - // `values_count * size_of::()` - let unique_values_bytes = self.unique_values().len() * std::mem::size_of::(); + // The dictionary allocates in two parts. The Vec64 stores the + // String structs at 24 bytes each, and every String owns its + // character buffer as a separate allocation, so both count. + // Under shared_dict the published prefix stands in for the + // struct count, as the sharing group owns the allocation. + #[cfg(not(feature = "shared_dict"))] + let struct_bytes = self.unique_values.capacity() * std::mem::size_of::(); + #[cfg(feature = "shared_dict")] + let struct_bytes = self.unique_values().len() * std::mem::size_of::(); + let character_bytes: usize = self.unique_values().iter().map(|s| s.capacity()).sum(); let mask_bytes = self.null_mask.as_ref().map_or(0, |m| m.est_bytes()); - data_bytes + unique_values_bytes + mask_bytes + data_bytes + struct_bytes + character_bytes + mask_bytes + } + + #[inline] + fn logical_bytes(&self) -> usize { + // The index buffer plus the dictionary contents at their summed + // string lengths. + let dict_bytes: usize = self.unique_values().iter().map(|s| s.len()).sum(); + self.data.logical_bytes() + + dict_bytes + + self.null_mask.as_ref().map_or(0, |m| m.logical_bytes()) } } @@ -142,6 +225,12 @@ impl ByteSize for BooleanArray { let mask_bytes = self.null_mask.as_ref().map_or(0, |m| m.est_bytes()); data_bytes + mask_bytes } + + #[inline] + fn logical_bytes(&self) -> usize { + // Bit-packed values plus any null mask bytes + self.data.logical_bytes() + self.null_mask.as_ref().map_or(0, |m| m.logical_bytes()) + } } /// ByteSize for DatetimeArray (when datetime feature is enabled) @@ -156,6 +245,11 @@ impl ByteSize for DatetimeArray { let mask_bytes = self.null_mask.as_ref().map_or(0, |m| m.est_bytes()); data_bytes + mask_bytes } + + #[inline] + fn logical_bytes(&self) -> usize { + self.data.logical_bytes() + self.null_mask.as_ref().map_or(0, |m| m.logical_bytes()) + } } // Mid-Level Enum Implementations @@ -183,6 +277,26 @@ impl ByteSize for NumericArray { NumericArray::Null => 0, } } + + fn logical_bytes(&self) -> usize { + match self { + #[cfg(feature = "extended_numeric_types")] + NumericArray::Int8(arr) => arr.logical_bytes(), + #[cfg(feature = "extended_numeric_types")] + NumericArray::Int16(arr) => arr.logical_bytes(), + NumericArray::Int32(arr) => arr.logical_bytes(), + NumericArray::Int64(arr) => arr.logical_bytes(), + #[cfg(feature = "extended_numeric_types")] + NumericArray::UInt8(arr) => arr.logical_bytes(), + #[cfg(feature = "extended_numeric_types")] + NumericArray::UInt16(arr) => arr.logical_bytes(), + NumericArray::UInt32(arr) => arr.logical_bytes(), + NumericArray::UInt64(arr) => arr.logical_bytes(), + NumericArray::Float32(arr) => arr.logical_bytes(), + NumericArray::Float64(arr) => arr.logical_bytes(), + NumericArray::Null => 0, + } + } } /// ByteSize for TextArray enum @@ -206,6 +320,26 @@ impl ByteSize for TextArray { TextArray::Null => 0, } } + + fn logical_bytes(&self) -> usize { + match self { + TextArray::String32(arr) => arr.logical_bytes(), + #[cfg(feature = "large_string")] + TextArray::String64(arr) => arr.logical_bytes(), + #[cfg(feature = "default_categorical_8")] + TextArray::Categorical8(arr) => arr.logical_bytes(), + #[cfg(feature = "extended_categorical")] + TextArray::Categorical16(arr) => arr.logical_bytes(), + #[cfg(any( + not(feature = "default_categorical_8"), + feature = "extended_categorical" + ))] + TextArray::Categorical32(arr) => arr.logical_bytes(), + #[cfg(feature = "extended_categorical")] + TextArray::Categorical64(arr) => arr.logical_bytes(), + TextArray::Null => 0, + } + } } #[cfg(feature = "datetime")] @@ -221,6 +355,14 @@ impl ByteSize for TemporalArray { TemporalArray::Null => 0, } } + + fn logical_bytes(&self) -> usize { + match self { + TemporalArray::Datetime32(arr) => arr.logical_bytes(), + TemporalArray::Datetime64(arr) => arr.logical_bytes(), + TemporalArray::Null => 0, + } + } } // Top-Level Array Enum Implementation @@ -239,6 +381,17 @@ impl ByteSize for Array { Array::Null => 0, } } + + fn logical_bytes(&self) -> usize { + match self { + Array::NumericArray(arr) => arr.logical_bytes(), + Array::TextArray(arr) => arr.logical_bytes(), + #[cfg(feature = "datetime")] + Array::TemporalArray(arr) => arr.logical_bytes(), + Array::BooleanArray(arr) => arr.logical_bytes(), + Array::Null => 0, + } + } } // High-Level Structure Implementations @@ -253,6 +406,19 @@ impl ByteSize for Field { // Name string allocation self.name.capacity() } + + #[inline] + fn logical_bytes(&self) -> usize { + // The field counts the name plus every metadata key and value. + // The dtype and nullable markers are fixed descriptors, so they + // contribute no variable bytes. + self.name.len() + + self + .metadata + .iter() + .map(|(k, v)| k.len() + v.len()) + .sum::() + } } /// ByteSize for FieldArray - field metadata + array data @@ -261,6 +427,11 @@ impl ByteSize for FieldArray { fn est_bytes(&self) -> usize { self.field.est_bytes() + self.array.est_bytes() } + + #[inline] + fn logical_bytes(&self) -> usize { + self.field.logical_bytes() + self.array.logical_bytes() + } } /// ByteSize for Table - sum of all column arrays @@ -268,6 +439,12 @@ impl ByteSize for Table { fn est_bytes(&self) -> usize { self.cols.iter().map(|col| col.est_bytes()).sum() } + + fn logical_bytes(&self) -> usize { + // The table name plus each column, where every column counts its + // field schema strings through FieldArray + self.name.len() + self.cols.iter().map(|col| col.logical_bytes()).sum::() + } } // View Type Implementations @@ -287,6 +464,135 @@ impl ByteSize for ArrayV { 0 } } + + /// The view reports the exact bytes of its window. + /// Fixed-width windows count `len * element_width`, string + /// windows read the payload span from the offsets buffer, bit-packed + /// windows round up to whole bytes and a categorical window counts the + /// dictionary contents in full because the indices reference the + /// complete dictionary. + fn logical_bytes(&self) -> usize { + let len = self.len(); + let offset = self.offset; + match &self.array { + Array::NumericArray(inner) => match inner { + #[cfg(feature = "extended_numeric_types")] + NumericArray::Int8(arr) => { + len * size_of::() + + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + #[cfg(feature = "extended_numeric_types")] + NumericArray::Int16(arr) => { + len * size_of::() + + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + NumericArray::Int32(arr) => { + len * size_of::() + + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + NumericArray::Int64(arr) => { + len * size_of::() + + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + #[cfg(feature = "extended_numeric_types")] + NumericArray::UInt8(arr) => { + len * size_of::() + + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + #[cfg(feature = "extended_numeric_types")] + NumericArray::UInt16(arr) => { + len * size_of::() + + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + NumericArray::UInt32(arr) => { + len * size_of::() + + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + NumericArray::UInt64(arr) => { + len * size_of::() + + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + NumericArray::Float32(arr) => { + len * size_of::() + + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + NumericArray::Float64(arr) => { + len * size_of::() + + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + NumericArray::Null => 0, + }, + Array::TextArray(inner) => match inner { + TextArray::String32(arr) => { + let payload = arr.offsets[offset + len] as usize + - arr.offsets[offset] as usize; + (len + 1) * size_of::() + + payload + + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + #[cfg(feature = "large_string")] + TextArray::String64(arr) => { + let payload = arr.offsets[offset + len] as usize + - arr.offsets[offset] as usize; + (len + 1) * size_of::() + + payload + + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + #[cfg(feature = "default_categorical_8")] + TextArray::Categorical8(arr) => { + let dict_bytes: usize = + arr.unique_values().iter().map(|s| s.len()).sum(); + len * size_of::() + + dict_bytes + + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + #[cfg(feature = "extended_categorical")] + TextArray::Categorical16(arr) => { + let dict_bytes: usize = + arr.unique_values().iter().map(|s| s.len()).sum(); + len * size_of::() + + dict_bytes + + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + #[cfg(any( + not(feature = "default_categorical_8"), + feature = "extended_categorical" + ))] + TextArray::Categorical32(arr) => { + let dict_bytes: usize = + arr.unique_values().iter().map(|s| s.len()).sum(); + len * size_of::() + + dict_bytes + + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + #[cfg(feature = "extended_categorical")] + TextArray::Categorical64(arr) => { + let dict_bytes: usize = + arr.unique_values().iter().map(|s| s.len()).sum(); + len * size_of::() + + dict_bytes + + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + TextArray::Null => 0, + }, + #[cfg(feature = "datetime")] + Array::TemporalArray(inner) => match inner { + TemporalArray::Datetime32(arr) => { + len * size_of::() + + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + TemporalArray::Datetime64(arr) => { + len * size_of::() + + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + TemporalArray::Null => 0, + }, + Array::BooleanArray(arr) => { + (len + 7) / 8 + arr.null_mask.as_ref().map_or(0, |_| (len + 7) / 8) + } + Array::Null => 0, + } + } } /// ByteSize for TableV - sum of column view estimates @@ -295,6 +601,18 @@ impl ByteSize for TableV { fn est_bytes(&self) -> usize { self.cols.iter().map(|col| col.est_bytes()).sum() } + + fn logical_bytes(&self) -> usize { + // The view counts its name plus the field schema strings and the + // window bytes of the active columns, so a column selection narrows + // the figure the same way it narrows every other access method + self.name.len() + + self + .active_col_indices() + .into_iter() + .map(|i| self.fields[i].logical_bytes() + self.cols[i].logical_bytes()) + .sum::() + } } #[cfg(all(feature = "chunked", feature = "views"))] @@ -306,6 +624,10 @@ impl ByteSize for SuperArrayV { fn est_bytes(&self) -> usize { self.slices.iter().map(|slice| slice.est_bytes()).sum() } + + fn logical_bytes(&self) -> usize { + self.slices.iter().map(|slice| slice.logical_bytes()).sum() + } } /// ByteSize for SuperTableV - sum of slice estimates @@ -314,6 +636,10 @@ impl ByteSize for SuperTableV { fn est_bytes(&self) -> usize { self.slices.iter().map(|slice| slice.est_bytes()).sum() } + + fn logical_bytes(&self) -> usize { + self.slices.iter().map(|slice| slice.logical_bytes()).sum() + } } /// ByteSize for Matrix (when matrix feature is enabled) @@ -326,6 +652,10 @@ impl ByteSize for Matrix { // Matrix contains data buffer for n_rows * n_cols elements self.data.est_bytes() } + + fn logical_bytes(&self) -> usize { + unimplemented!("Matrix has not yet implemented logical bytes.") + } } /// ByteSize for NdArray (when ndarray feature is enabled) @@ -338,6 +668,10 @@ impl ByteSize for NdArray { // Physical backing buffer, including any stride padding. self.data.est_bytes() } + + fn logical_bytes(&self) -> usize { + unimplemented!("NdArray has not yet implemented logical bytes.") + } } /// ByteSize for NdArrayV - proportional estimate from the backing array @@ -355,6 +689,10 @@ impl ByteSize for NdArrayV { 0 } } + + fn logical_bytes(&self) -> usize { + unimplemented!("NdArrayV has not yet implemented logical bytes.") + } } /// ByteSize for SuperNdArray - sum of batch estimates @@ -366,6 +704,10 @@ impl ByteSize for SuperNdArray { fn est_bytes(&self) -> usize { self.batches.iter().map(|batch| batch.est_bytes()).sum() } + + fn logical_bytes(&self) -> usize { + unimplemented!("SuperNdArray has not yet implemented logical bytes.") + } } /// ByteSize for SuperNdArrayV - sum of slice estimates @@ -377,6 +719,10 @@ impl ByteSize for SuperNdArrayV { fn est_bytes(&self) -> usize { self.slices.iter().map(|slice| slice.est_bytes()).sum() } + + fn logical_bytes(&self) -> usize { + unimplemented!("SuperNdArrayV has not yet implemented logical bytes.") + } } /// ByteSize for XArray - storage plus coordinate arrays @@ -402,6 +748,10 @@ impl ByteSize for XArray { .sum(); data_bytes + coord_bytes } + + fn logical_bytes(&self) -> usize { + unimplemented!("XArray has not yet implemented logical bytes.") + } } /// ByteSize for Cube (when cube feature is enabled) @@ -414,6 +764,10 @@ impl ByteSize for Cube { // Cube contains multiple tables self.tables.iter().map(|tbl| tbl.est_bytes()).sum() } + + fn logical_bytes(&self) -> usize { + self.tables.iter().map(|tbl| tbl.logical_bytes()).sum() + } } /// ByteSize for SuperArray (when chunked feature is enabled) @@ -426,6 +780,10 @@ impl ByteSize for SuperArray { // Sum of all chunk arrays self.chunks().iter().map(|chunk| chunk.est_bytes()).sum() } + + fn logical_bytes(&self) -> usize { + self.chunks().iter().map(|chunk| chunk.logical_bytes()).sum() + } } /// ByteSize for SuperTable (when chunked feature is enabled) @@ -438,6 +796,10 @@ impl ByteSize for SuperTable { // Sum of all batch tables self.batches.iter().map(|batch| batch.est_bytes()).sum() } + + fn logical_bytes(&self) -> usize { + self.batches.iter().map(|batch| batch.logical_bytes()).sum() + } } // Value Enum Implementation @@ -464,6 +826,40 @@ impl ByteSize for Scalar { _ => 0, // Other scalars are inline } } + + /// A scalar reports the width its value occupies as a single Arrow + /// element, so a boolean rounds up to the one bit-packed byte and a + /// string reports its populated length. + fn logical_bytes(&self) -> usize { + match self { + Scalar::Null => 0, + Scalar::Boolean(_) => 1, + #[cfg(feature = "extended_numeric_types")] + Scalar::Int8(_) => size_of::(), + #[cfg(feature = "extended_numeric_types")] + Scalar::Int16(_) => size_of::(), + Scalar::Int32(_) => size_of::(), + Scalar::Int64(_) => size_of::(), + #[cfg(feature = "extended_numeric_types")] + Scalar::UInt8(_) => size_of::(), + #[cfg(feature = "extended_numeric_types")] + Scalar::UInt16(_) => size_of::(), + Scalar::UInt32(_) => size_of::(), + Scalar::UInt64(_) => size_of::(), + Scalar::Float32(_) => size_of::(), + Scalar::Float64(_) => size_of::(), + Scalar::String32(s) => s.len(), + #[cfg(feature = "large_string")] + Scalar::String64(s) => s.len(), + #[cfg(feature = "datetime")] + Scalar::Datetime32(_) => size_of::(), + #[cfg(feature = "datetime")] + Scalar::Datetime64(_) => size_of::(), + // The variant carries no value + #[cfg(feature = "datetime")] + Scalar::Interval => 0, + } + } } /// ByteSize for Value enum - delegates to inner types @@ -538,4 +934,185 @@ impl ByteSize for Value { } } } + + /// The tabular variants delegate to their inner type. The numerical + /// container variants (`Matrix`, `NdArray` and their chunked and view + /// forms, plus `XArray`) and `Custom` do not define logical byte + /// accounting and panic with `unimplemented!` when called. + fn logical_bytes(&self) -> usize { + match self { + #[cfg(feature = "scalar_type")] + Value::Scalar(s) => s.logical_bytes(), + Value::Array(arr) => arr.logical_bytes(), + #[cfg(feature = "views")] + Value::ArrayView(av) => av.logical_bytes(), + Value::Table(tbl) => tbl.logical_bytes(), + #[cfg(feature = "views")] + Value::TableView(tv) => tv.logical_bytes(), + #[cfg(feature = "chunked")] + Value::SuperArray(sa) => sa.logical_bytes(), + #[cfg(all(feature = "chunked", feature = "views"))] + Value::SuperArrayView(sav) => sav.logical_bytes(), + #[cfg(feature = "chunked")] + Value::SuperTable(st) => st.logical_bytes(), + #[cfg(all(feature = "chunked", feature = "views"))] + Value::SuperTableView(stv) => stv.logical_bytes(), + Value::FieldArray(fa) => fa.logical_bytes(), + #[cfg(feature = "matrix")] + Value::Matrix(_) => { + unimplemented!("Matrix does not define logical byte accounting") + } + #[cfg(feature = "ndarray")] + Value::NdArray(_) => { + unimplemented!("NdArray does not define logical byte accounting") + } + #[cfg(all(feature = "ndarray", feature = "views"))] + Value::NdArrayView(_) => { + unimplemented!("NdArrayV does not define logical byte accounting") + } + #[cfg(all(feature = "ndarray", feature = "chunked"))] + Value::SuperNdArray(_) => { + unimplemented!("SuperNdArray does not define logical byte accounting") + } + #[cfg(all(feature = "ndarray", feature = "chunked", feature = "views"))] + Value::SuperNdArrayView(_) => { + unimplemented!("SuperNdArrayV does not define logical byte accounting") + } + #[cfg(feature = "xarray")] + Value::XArray(_) => { + unimplemented!("XArray does not define logical byte accounting") + } + #[cfg(feature = "cube")] + Value::Cube(c) => c.logical_bytes(), + Value::VecValue(vec) => vec.iter().map(|v| v.logical_bytes()).sum::(), + Value::BoxValue(boxed) => boxed.logical_bytes(), + Value::ArcValue(arc) => arc.logical_bytes(), + Value::Tuple2(tuple) => tuple.0.logical_bytes() + tuple.1.logical_bytes(), + Value::Tuple3(tuple) => { + tuple.0.logical_bytes() + tuple.1.logical_bytes() + tuple.2.logical_bytes() + } + Value::Tuple4(tuple) => { + tuple.0.logical_bytes() + + tuple.1.logical_bytes() + + tuple.2.logical_bytes() + + tuple.3.logical_bytes() + } + Value::Tuple5(tuple) => { + tuple.0.logical_bytes() + + tuple.1.logical_bytes() + + tuple.2.logical_bytes() + + tuple.3.logical_bytes() + + tuple.4.logical_bytes() + } + Value::Tuple6(tuple) => { + tuple.0.logical_bytes() + + tuple.1.logical_bytes() + + tuple.2.logical_bytes() + + tuple.3.logical_bytes() + + tuple.4.logical_bytes() + + tuple.5.logical_bytes() + } + Value::Custom(_) => { + unimplemented!("CustomValue does not define logical byte accounting") + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn integer_array_counts_data_and_mask() { + let mut arr = IntegerArray::::from_slice(&[1, 2, 3, 4, 5]); + assert_eq!(arr.logical_bytes(), 40); + arr.null_mask = Some(Bitmask::new_set_all(5, true)); + assert_eq!(arr.logical_bytes(), 41); + } + + #[test] + fn buffer_capacity_slack_is_excluded() { + let mut v: Vec64 = Vec64::with_capacity(100); + v.push(1); + v.push(2); + v.push(3); + assert_eq!(v.logical_bytes(), 12); + assert_eq!(v.est_bytes(), 400); + } + + #[test] + fn bitmask_rounds_bits_to_bytes() { + let mask = Bitmask::new_set_all(10, true); + assert_eq!(mask.logical_bytes(), 2); + } + + #[test] + fn string_array_excludes_capacity_slack() { + let strings: Vec64 = (0..1000).map(|i| format!("row_{}", i)).collect(); + let refs: Vec64<&str> = strings.iter().map(String::as_str).collect(); + let payload: usize = strings.iter().map(|s| s.len()).sum(); + let arr = StringArray::::from_vec64(refs, None); + assert_eq!(arr.logical_bytes(), payload + 1001 * size_of::()); + assert!(arr.est_bytes() >= arr.logical_bytes()); + } + + #[test] + fn categorical_array_counts_indices_and_dictionary_contents() { + let arr = CategoricalArray::::from_slices( + &[0, 1, 2, 1, 0, 2], + &["red".to_string(), "green".to_string(), "blue".to_string()], + ); + assert_eq!(arr.logical_bytes(), 6 * size_of::() + 12); + } + + #[test] + fn boolean_array_rounds_bits_to_bytes() { + let arr = BooleanArray::<()>::from_slice(&[true; 10]); + assert_eq!(arr.logical_bytes(), 2); + } + + #[cfg(feature = "views")] + #[test] + fn string_view_reports_exact_window_bytes() { + use crate::{ArrayV, vec64}; + let refs: Vec64<&str> = vec64!["a", "bb", "ccc", "dddd", "eeeee"]; + let arr = StringArray::::from_vec64(refs, None); + let view = ArrayV::new(crate::Array::from_string32(arr), 1, 3); + // Window covers "bb", "ccc" and "dddd", so the payload spans 9 + // bytes and the offsets window holds 4 entries + assert_eq!(view.logical_bytes(), 4 * size_of::() + 9); + } + + #[cfg(feature = "views")] + #[test] + fn full_width_table_view_matches_owned_table() { + use crate::{FieldArray, Table, arr_i32, arr_str32, vec64}; + let ids = vec64![1i32, 2, 3, 4, 5]; + let refs: Vec64<&str> = vec64!["a", "bb", "ccc", "dddd", "eeeee"]; + let table = Table::new( + "t".to_string(), + Some(vec![ + FieldArray::from_arr("ids", arr_i32!(ids)), + FieldArray::from_arr("labels", arr_str32!(refs)), + ]), + ); + assert_eq!(table.slice(0, 5).logical_bytes(), table.logical_bytes()); + // A mid-table window counts the table name "t", the field names + // "ids" and "labels", 3 ids, a 4-entry offsets window and the + // "bb" + "ccc" + "dddd" payload + assert_eq!( + table.slice(1, 3).logical_bytes(), + 1 + 3 + 6 + 3 * size_of::() + 4 * size_of::() + 9 + ); + } + + #[cfg(all(feature = "value_type", feature = "scalar_type"))] + #[test] + fn scalar_reports_element_width() { + assert_eq!(Scalar::Int64(7).logical_bytes(), 8); + assert_eq!(Scalar::String32("abc".into()).logical_bytes(), 3); + assert_eq!(Scalar::Boolean(true).logical_bytes(), 1); + assert_eq!(Scalar::Null.logical_bytes(), 0); + } }