diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index 871382d1ca7..f84ecb396fd 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -58,6 +58,7 @@ jobs: cache-dependency-path: vortex-web/package-lock.json - run: npm ci - run: npm run wasm + - run: npm run format:check - run: npm run lint - run: npm run typecheck - run: npm run build-storybook diff --git a/Cargo.lock b/Cargo.lock index f43fe0b8314..d76e4cdd3b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10937,8 +10937,14 @@ dependencies = [ name = "vortex-web-wasm" version = "0.1.0" dependencies = [ + "arrow-array 58.0.0", + "arrow-ipc 58.0.0", + "arrow-schema 58.0.0", "console_error_panic_hook", + "futures", "js-sys", + "serde", + "serde_json", "vortex", "wasm-bindgen", "wasm-bindgen-futures", diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000000..0cd0ca8e96d --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "vortex", + "lockfileVersion": 3, + "requires": true, + "packages": {} +} diff --git a/package.json b/package.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/package.json @@ -0,0 +1 @@ +{} diff --git a/vortex-web/.prettierrc.json b/vortex-web/.prettierrc.json new file mode 100644 index 00000000000..50bf5f7ae21 --- /dev/null +++ b/vortex-web/.prettierrc.json @@ -0,0 +1,7 @@ +{ + "singleQuote": true, + "trailingComma": "all", + "bracketSpacing": true, + "printWidth": 100, + "tabWidth": 2 +} diff --git a/vortex-web/.storybook/main.ts b/vortex-web/.storybook/main.ts index 93a5ba691cd..169c5bb6be7 100644 --- a/vortex-web/.storybook/main.ts +++ b/vortex-web/.storybook/main.ts @@ -4,13 +4,8 @@ import type { StorybookConfig } from '@storybook/react-vite'; const config: StorybookConfig = { - "stories": [ - "../src/**/*.mdx", - "../src/**/*.stories.@(js|jsx|mjs|ts|tsx)" - ], - "addons": [ - "@storybook/addon-docs", - ], - "framework": "@storybook/react-vite" + stories: ['../src/**/*.mdx', '../src/**/*.stories.@(js|jsx|mjs|ts|tsx)'], + addons: ['@storybook/addon-docs'], + framework: '@storybook/react-vite', }; -export default config; \ No newline at end of file +export default config; diff --git a/vortex-web/.storybook/preview.ts b/vortex-web/.storybook/preview.ts index 85daeef5177..e120ec5e169 100644 --- a/vortex-web/.storybook/preview.ts +++ b/vortex-web/.storybook/preview.ts @@ -1,18 +1,43 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -import type { Preview } from '@storybook/react-vite' -import '../src/index.css' +import type { Preview } from '@storybook/react-vite'; +import '../src/index.css'; const preview: Preview = { + globalTypes: { + theme: { + description: 'Color theme', + toolbar: { + title: 'Theme', + icon: 'paintbrush', + items: [ + { value: 'light', title: 'Light', icon: 'sun' }, + { value: 'dark', title: 'Dark', icon: 'moon' }, + ], + dynamicTitle: true, + }, + }, + }, + initialGlobals: { + theme: 'light', + }, + decorators: [ + (Story, context) => { + const theme = context.globals.theme || 'light'; + document.documentElement.classList.toggle('dark', theme === 'dark'); + document.documentElement.classList.toggle('light', theme === 'light'); + return Story(); + }, + ], parameters: { controls: { matchers: { - color: /(background|color)$/i, - date: /Date$/i, + color: /(background|color)$/i, + date: /Date$/i, }, }, }, }; -export default preview; \ No newline at end of file +export default preview; diff --git a/vortex-web/crate/Cargo.toml b/vortex-web/crate/Cargo.toml index 992d6341b5f..25b16583380 100644 --- a/vortex-web/crate/Cargo.toml +++ b/vortex-web/crate/Cargo.toml @@ -11,16 +11,27 @@ publish = false crate-type = ["cdylib", "rlib"] [dependencies] +arrow-array = { workspace = true } +arrow-ipc = { workspace = true } +arrow-schema = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } vortex = { path = "../../vortex", default-features = false, features = [ "files", ] } [target.'cfg(target_arch = "wasm32")'.dependencies] +futures = { workspace = true } wasm-bindgen = "0.2.104" wasm-bindgen-futures = { workspace = true } console_error_panic_hook = "0.1.7" js-sys = "0.3.81" -web-sys = { version = "0.3.81", features = ["console", "File", "FileReader"] } +web-sys = { version = "0.3.81", features = [ + "Blob", + "console", + "File", + "FileReader", +] } [lints] workspace = true diff --git a/vortex-web/crate/src/wasm.rs b/vortex-web/crate/src/wasm.rs index a4aecb0c823..bd4782079ff 100644 --- a/vortex-web/crate/src/wasm.rs +++ b/vortex-web/crate/src/wasm.rs @@ -1,13 +1,50 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::collections::VecDeque; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::task::Context; +use std::task::Poll; + +use arrow_array::RecordBatch; +use arrow_array::cast::AsArray; +use arrow_ipc::writer::StreamWriter; +use arrow_schema::DataType; +use arrow_schema::Field; +use arrow_schema::Schema; +use futures::FutureExt; +use futures::TryStreamExt; +use futures::future::BoxFuture; +use serde::Serialize; use vortex::VortexSessionDefault; -use vortex::buffer::ByteBuffer; +use vortex::array::ArrayRef; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; +use vortex::array::arrow::ArrowArrayExecutor; +use vortex::array::buffer::BufferHandle; +use vortex::array::dtype::DType; +use vortex::array::serde::ArrayParts; +use vortex::array::stream::ArrayStream; +use vortex::buffer::Alignment; +use vortex::buffer::ByteBufferMut; +use vortex::error::VortexResult; use vortex::file::OpenOptionsSessionExt; +use vortex::file::VERSION; +use vortex::file::VortexFile; +use vortex::io::CoalesceConfig; +use vortex::io::VortexReadAt; use vortex::io::runtime::wasm::WasmRuntime; use vortex::io::session::RuntimeSessionExt; +use vortex::layout::LayoutChildType; +use vortex::layout::LayoutRef; +use vortex::layout::layouts::flat::Flat; +use vortex::layout::scan::scan_builder::ScanBuilder; use vortex::session::VortexSession; +use vortex::session::registry::ReadContext; use wasm_bindgen::prelude::*; +use wasm_bindgen_futures::JsFuture; /// Initialize the WASM module (sets up panic hook for better error messages). #[wasm_bindgen(start)] @@ -15,30 +52,125 @@ pub fn init() { console_error_panic_hook::set_once(); } -/// Open a Vortex file from raw bytes and return a handle for exploration. +/// A `VortexReadAt` backed by a `web_sys::Blob`, enabling lazy range-based reads +/// via `Blob.slice()` + `arrayBuffer()`. +struct BlobReadAt { + blob: web_sys::Blob, + size: u64, +} + +// SAFETY: WASM is single-threaded — Blob is never accessed from multiple threads. +unsafe impl Send for BlobReadAt {} +unsafe impl Sync for BlobReadAt {} + +/// Wrapper to mark a `JsFuture` as `Send`. +/// +/// SAFETY: WASM is single-threaded, so `JsFuture` is never accessed from multiple threads. +struct SendFuture(JsFuture); + +unsafe impl Send for SendFuture {} + +impl Future for SendFuture { + type Output = Result; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + // SAFETY: We never move the inner JsFuture after pinning. + unsafe { Pin::new_unchecked(&mut self.0).poll(cx) } + } +} + +impl VortexReadAt for BlobReadAt { + fn concurrency(&self) -> usize { + 4 + } + + fn coalesce_config(&self) -> Option { + Some(CoalesceConfig::in_memory()) + } + + fn size(&self) -> BoxFuture<'static, VortexResult> { + let size = self.size; + async move { Ok(size) }.boxed() + } + + fn read_at( + &self, + offset: u64, + length: usize, + alignment: Alignment, + ) -> BoxFuture<'static, VortexResult> { + let start = offset as f64; + let end = (offset + length as u64) as f64; + let slice = self + .blob + .slice_with_f64_and_f64(start, end) + .expect("Blob.slice() failed"); + // SAFETY: WASM is single-threaded so the non-Send JsFuture is safe to wrap. + let future = SendFuture(JsFuture::from(slice.array_buffer())); + async move { + let array_buffer = future.await.expect("Blob.arrayBuffer() failed"); + let uint8 = js_sys::Uint8Array::new(&array_buffer); + let mut buffer = + ByteBufferMut::with_capacity_aligned(uint8.length() as usize, alignment); + buffer.extend_from_slice(&uint8.to_vec()); + Ok(BufferHandle::new_host(buffer.freeze())) + } + .boxed() + } +} + +/// Open a Vortex file from a `File` handle and return a handle for exploration. /// -/// Call this from JavaScript after reading a `.vortex` file via drag-and-drop. +/// The `File` (a `Blob`) is read lazily — only the footer is read at open time. #[wasm_bindgen] -pub fn open_vortex_file(data: &[u8]) -> Result { +pub async fn open_vortex_file(file: web_sys::File) -> Result { let session = VortexSession::default().with_handle(WasmRuntime::handle()); - let buffer = ByteBuffer::from(data.to_vec()); + let blob: &web_sys::Blob = file.as_ref(); + let file_size = blob.size() as usize; + let reader = Arc::new(BlobReadAt { + blob: blob.clone(), + size: file_size as u64, + }); let vxf = session .open_options() - .open_buffer(buffer) + .open(reader) + .await .map_err(|e| JsValue::from_str(&e.to_string()))?; - let row_count = vxf.row_count(); - let dtype = format!("{}", vxf.dtype()); + // Extract the array ReadContext from any FlatLayout in the tree. + let array_read_ctx = find_array_read_ctx(vxf.footer().layout()); - Ok(VortexFileHandle { row_count, dtype }) + Ok(VortexFileHandle { + vxf, + session, + file_size, + array_read_ctx, + }) +} + +/// Walk the layout tree to find the first FlatLayout and extract its ReadContext. +fn find_array_read_ctx(layout: &LayoutRef) -> Option { + if let Some(flat) = layout.as_opt::() { + return Some(flat.array_ctx().clone()); + } + if let Ok(children) = layout.children() { + for child in children { + if let Some(ctx) = find_array_read_ctx(&child) { + return Some(ctx); + } + } + } + None } /// A handle to an opened Vortex file, exposing metadata to JavaScript. #[wasm_bindgen] pub struct VortexFileHandle { - row_count: u64, - dtype: String, + vxf: VortexFile, + session: VortexSession, + file_size: usize, + array_read_ctx: Option, } #[wasm_bindgen] @@ -46,12 +178,675 @@ impl VortexFileHandle { /// The total number of rows in the file. #[wasm_bindgen(getter)] pub fn row_count(&self) -> u64 { - self.row_count + self.vxf.row_count() } /// The top-level DType of the file as a string. #[wasm_bindgen(getter)] pub fn dtype(&self) -> String { - self.dtype.clone() + format!("{}", self.vxf.dtype()) + } + + /// Returns the layout tree as a JSON string matching the TS `LayoutTreeNode` type. + pub fn layout_tree(&self) -> Result { + let root = self.vxf.footer().layout().clone(); + let tree = build_layout_tree(root, "root".to_string(), 0, &ChildKindJson::Root) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + + serde_json::to_string(&tree) + .map_err(|e| JsValue::from_str(&format!("JSON serialization failed: {e}"))) + } + + /// Returns the segment map as a JSON string matching the TS `SegmentMapEntry[]` type. + pub fn segment_map(&self) -> Result { + let footer = self.vxf.footer(); + let root_layout = footer.layout().clone(); + let segment_map = footer.segment_map(); + + // BFS to map each segment ID to its layout path and column name. + let mut segment_paths: Vec)>> = + vec![None; segment_map.len()]; + + let mut queue: VecDeque<(String, Option, LayoutRef)> = + VecDeque::from([(String::from("root"), None, root_layout)]); + + while let Some((path, column, layout)) = queue.pop_front() { + for segment in layout.segment_ids() { + let idx = *segment as usize; + if idx < segment_paths.len() { + segment_paths[idx] = Some((path.clone(), column.clone())); + } + } + + if let Ok(children) = layout.children() { + for (i, child_layout) in children.into_iter().enumerate() { + let child_type = layout.child_type(i); + let child_name = child_type.name(); + let child_path = format!("{path}.{child_name}"); + + // Track the column: use this field's name if it's a Field, otherwise + // inherit the parent's column. + let child_column = match &child_type { + LayoutChildType::Field(name) => Some(name.to_string()), + _ => column.clone(), + }; + + queue.push_back((child_path, child_column, child_layout)); + } + } + } + + let entries: Vec = segment_map + .iter() + .enumerate() + .map(|(i, spec)| { + let (layout_path, column) = segment_paths[i] + .clone() + .unwrap_or_else(|| (String::from(""), None)); + SegmentMapEntryJson { + index: i, + byte_offset: spec.offset, + byte_length: spec.length, + alignment: *spec.alignment, + column, + layout_path, + } + }) + .collect(); + + serde_json::to_string(&entries) + .map_err(|e| JsValue::from_str(&format!("JSON serialization failed: {e}"))) + } + + /// Returns file structure info as a JSON string matching the TS `FileStructureInfo` type. + pub fn file_structure(&self) -> Result { + let footer = self.vxf.footer(); + let segment_map = footer.segment_map(); + + let total_data_bytes: u64 = segment_map.iter().map(|s| s.length as u64).sum(); + + let total_metadata_bytes = + sum_metadata_bytes(footer.layout()).map_err(|e| JsValue::from_str(&e.to_string()))?; + + let info = FileStructureJson { + file_size: self.file_size as u64, + version: VERSION, + postscript_size: 64, + total_data_bytes, + total_metadata_bytes, + }; + + serde_json::to_string(&info) + .map_err(|e| JsValue::from_str(&format!("JSON serialization failed: {e}"))) + } + + /// Preview data from a specific layout node, returning Arrow IPC stream bytes. + /// + /// Navigates to the layout node identified by `node_id` (e.g. "root.customer_id.[0]"), + /// creates a layout reader, scans up to `row_limit` rows, and returns Arrow IPC bytes. + pub async fn preview_data( + &self, + node_id: &str, + row_limit: u32, + ) -> Result { + let layout = find_layout_by_id(self.vxf.footer().layout(), node_id) + .ok_or_else(|| JsValue::from_str(&format!("Layout node not found: {node_id}")))?; + + let segment_source = self.vxf.segment_source(); + let reader = layout + .new_reader(node_id.into(), segment_source, &self.session) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + + let stream = ScanBuilder::new(self.session.clone(), reader) + .with_limit(row_limit as u64) + .into_array_stream() + .map_err(|e| JsValue::from_str(&e.to_string()))?; + + let dtype = stream.dtype().clone(); + let chunks: Vec = stream + .try_collect() + .await + .map_err(|e| JsValue::from_str(&e.to_string()))?; + + let schema = + dtype_to_schema(&dtype, "value").map_err(|e| JsValue::from_str(&e.to_string()))?; + let arrow_schema = Arc::new(schema); + + let mut buf = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut buf, &arrow_schema) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + + for chunk in chunks { + let batch = array_to_record_batch(chunk, &dtype, &arrow_schema) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + writer + .write(&batch) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + } + + writer + .finish() + .map_err(|e| JsValue::from_str(&e.to_string()))?; + } + + Ok(js_sys::Uint8Array::from(buf.as_slice())) + } + + /// Fetch the array encoding tree for a flat layout node. + /// + /// Finds the layout by node ID, reads the segment, fully decodes the array + /// to extract dtype, child names, and buffer names from the encoding vtables. + pub async fn fetch_encoding_tree(&self, node_id: String) -> Result { + let ctx = self + .array_read_ctx + .as_ref() + .ok_or_else(|| JsValue::from_str("No array ReadContext available"))?; + + let layout = find_layout_by_id(self.vxf.footer().layout(), &node_id) + .ok_or_else(|| JsValue::from_str(&format!("Layout node not found: {node_id}")))?; + + let flat = layout + .as_opt::() + .ok_or_else(|| JsValue::from_str("Node is not a flat layout"))?; + + let segment_id = flat.segment_id(); + let dtype = layout.dtype().clone(); + let row_count = layout.row_count() as usize; + + let buf = self + .vxf + .segment_source() + .request(segment_id) + .await + .map_err(|e| JsValue::from_str(&e.to_string()))?; + + let parts = ArrayParts::try_from(buf).map_err(|e| JsValue::from_str(&e.to_string()))?; + + let array = parts + .decode(&dtype, row_count, ctx, &self.session) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + + let tree = build_array_encoding_tree_from_array(&array); + serde_json::to_string(&tree) + .map_err(|e| JsValue::from_str(&format!("JSON serialization failed: {e}"))) + } + + /// Fetch a buffer from a decoded array node. + /// + /// `layout_node_id` identifies the flat layout, `array_path` is a list of + /// child names to navigate within the decoded array tree (e.g. `["values", "encoded"]`), + /// and `buffer_index` selects which buffer of the target array node to return. + pub async fn fetch_array_buffer( + &self, + layout_node_id: String, + array_path: Vec, + buffer_index: usize, + ) -> Result { + use vortex::array::ArrayVisitor; + + let ctx = self + .array_read_ctx + .as_ref() + .ok_or_else(|| JsValue::from_str("No array ReadContext available"))?; + + let layout = + find_layout_by_id(self.vxf.footer().layout(), &layout_node_id).ok_or_else(|| { + JsValue::from_str(&format!("Layout node not found: {layout_node_id}")) + })?; + + let flat = layout + .as_opt::() + .ok_or_else(|| JsValue::from_str("Node is not a flat layout"))?; + + let segment_id = flat.segment_id(); + let dtype = layout.dtype().clone(); + let row_count = layout.row_count() as usize; + + let buf = self + .vxf + .segment_source() + .request(segment_id) + .await + .map_err(|e| JsValue::from_str(&e.to_string()))?; + + let parts = ArrayParts::try_from(buf).map_err(|e| JsValue::from_str(&e.to_string()))?; + + let root_array = parts + .decode(&dtype, row_count, ctx, &self.session) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + + // Navigate the array tree by child names. + let mut current = root_array; + for child_name in &array_path { + let named = current.named_children(); + let child = named + .into_iter() + .find(|(name, _)| name == child_name) + .map(|(_, arr)| arr) + .ok_or_else(|| { + JsValue::from_str(&format!("Array child not found: {child_name}")) + })?; + current = child; + } + + let handles = current.buffer_handles(); + let handle = handles.get(buffer_index).ok_or_else(|| { + JsValue::from_str(&format!( + "Buffer index {buffer_index} out of range ({})", + handles.len() + )) + })?; + + Ok(js_sys::Uint8Array::from(handle.as_host().as_slice())) + } + + /// Preview data from a specific array node within a flat layout. + /// + /// Decodes the array from the flat layout's segment, navigates to the + /// target array child by name path, and returns Arrow IPC bytes. + pub async fn preview_array_data( + &self, + layout_node_id: String, + array_path: Vec, + row_limit: u32, + ) -> Result { + use vortex::array::ArrayVisitor; + + let ctx = self + .array_read_ctx + .as_ref() + .ok_or_else(|| JsValue::from_str("No array ReadContext available"))?; + + let layout = + find_layout_by_id(self.vxf.footer().layout(), &layout_node_id).ok_or_else(|| { + JsValue::from_str(&format!("Layout node not found: {layout_node_id}")) + })?; + + let flat = layout + .as_opt::() + .ok_or_else(|| JsValue::from_str("Node is not a flat layout"))?; + + let segment_id = flat.segment_id(); + let dtype = layout.dtype().clone(); + let row_count = layout.row_count() as usize; + + let buf = self + .vxf + .segment_source() + .request(segment_id) + .await + .map_err(|e| JsValue::from_str(&e.to_string()))?; + + let parts = ArrayParts::try_from(buf).map_err(|e| JsValue::from_str(&e.to_string()))?; + + let root_array = parts + .decode(&dtype, row_count, ctx, &self.session) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + + // Navigate to the target array node. + let mut current = root_array; + for child_name in &array_path { + let named = current.named_children(); + let child = named + .into_iter() + .find(|(name, _)| name == child_name) + .map(|(_, arr)| arr) + .ok_or_else(|| { + JsValue::from_str(&format!("Array child not found: {child_name}")) + })?; + current = child; + } + + // Slice to row_limit. + let len = current.len().min(row_limit as usize); + if len < current.len() { + current = current + .slice(0..len) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + } + + // Convert to Arrow IPC. + let array_dtype = current.dtype().clone(); + let schema = dtype_to_schema(&array_dtype, "value") + .map_err(|e| JsValue::from_str(&e.to_string()))?; + let arrow_schema = Arc::new(schema); + + let batch = array_to_record_batch(current, &array_dtype, &arrow_schema) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + + let mut ipc_buf = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut ipc_buf, &arrow_schema) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + writer + .write(&batch) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + writer + .finish() + .map_err(|e| JsValue::from_str(&e.to_string()))?; + } + + Ok(js_sys::Uint8Array::from(ipc_buf.as_slice())) + } +} + +/// Recursively build the layout tree JSON structure. +fn build_layout_tree( + layout: LayoutRef, + id: String, + parent_row_offset: u64, + child_type: &ChildKindJson, +) -> VortexResult { + let row_count = layout.row_count(); + let children_result = layout.children(); + + let mut children_json = Vec::new(); + if let Ok(children) = children_result { + for (i, child_layout) in children.iter().enumerate() { + let ct = layout.child_type(i); + let child_name = ct.name(); + let child_id = format!("{id}.{child_name}"); + + let child_row_offset = match &ct { + LayoutChildType::Chunk((_, rel)) => parent_row_offset + rel, + LayoutChildType::Auxiliary(_) => 0, + LayoutChildType::Transparent(_) | LayoutChildType::Field(_) => parent_row_offset, + }; + + let child_kind = match &ct { + LayoutChildType::Transparent(name) => ChildKindJson::Transparent { + name: name.to_string(), + }, + LayoutChildType::Auxiliary(name) => ChildKindJson::Auxiliary { + name: name.to_string(), + }, + LayoutChildType::Chunk((idx, row_offset)) => ChildKindJson::Chunk { + chunk_index: *idx, + row_offset: parent_row_offset + row_offset, + }, + LayoutChildType::Field(name) => ChildKindJson::Field { + field_name: name.to_string(), + }, + }; + + children_json.push(build_layout_tree( + child_layout.clone(), + child_id, + child_row_offset, + &child_kind, + )?); + } + } + + // For flat layouts, extract the array encoding tree if available. + let array_encoding_tree = layout.as_opt::().and_then(|flat| { + let tree_buf = flat.array_tree()?; + let ctx = flat.array_ctx(); + let parts = ArrayParts::from_array_tree(tree_buf.as_ref().to_vec()).ok()?; + Some(build_array_encoding_tree(&parts, ctx)) + }); + + Ok(LayoutTreeNodeJson { + id, + encoding: layout.encoding().id().to_string(), + dtype: layout.dtype().to_string(), + row_count, + row_offset: parent_row_offset, + metadata_bytes: layout.metadata().len(), + segment_ids: layout.segment_ids().iter().map(|s| **s).collect(), + child_type: child_type.clone(), + children: children_json, + array_encoding_tree, + }) +} + +/// DFS to sum metadata bytes across all layout nodes. +fn sum_metadata_bytes(layout: &LayoutRef) -> VortexResult { + let mut total = 0u64; + for node in layout.depth_first_traversal() { + total += node?.metadata().len() as u64; + } + Ok(total) +} + +/// Recursively build the array encoding tree from `ArrayParts` (used for inline trees +/// where we don't have a fully decoded array). +fn build_array_encoding_tree(parts: &ArrayParts, ctx: &ReadContext) -> ArrayEncodingNodeJson { + let encoding = ctx + .resolve(parts.encoding_id()) + .map(|id| id.to_string()) + .unwrap_or_else(|| format!("unknown({})", parts.encoding_id())); + + let nchildren = parts.nchildren(); + let children: Vec = (0..nchildren) + .map(|i| build_array_encoding_tree(&parts.child(i), ctx)) + .collect(); + + ArrayEncodingNodeJson { + encoding, + dtype: String::new(), + metadata_bytes: parts.metadata().len(), + num_buffers: parts.nbuffers(), + buffer_lengths: parts.buffer_lengths(), + buffer_names: Vec::new(), + children, + child_names: (0..nchildren).map(|i| format!("child {i}")).collect(), } } + +/// Recursively build the array encoding tree from a fully decoded array, +/// extracting dtype, child names, and buffer names from the encoding vtables. +fn build_array_encoding_tree_from_array(array: &ArrayRef) -> ArrayEncodingNodeJson { + use vortex::array::ArrayVisitor; + + let encoding = array.encoding_id().to_string(); + let dtype = array.dtype().to_string(); + let buffer_names = array.buffer_names(); + let buffer_handles = array.buffer_handles(); + let buffer_lengths: Vec = buffer_handles.iter().map(|b| b.len()).collect(); + let metadata_bytes = array + .metadata() + .ok() + .flatten() + .map(|m| m.len()) + .unwrap_or(0); + + let named_children = array.named_children(); + let child_names: Vec = named_children + .iter() + .map(|(name, _)| name.clone()) + .collect(); + let children: Vec = named_children + .iter() + .map(|(_, child)| build_array_encoding_tree_from_array(child)) + .collect(); + + ArrayEncodingNodeJson { + encoding, + dtype, + metadata_bytes, + num_buffers: buffer_lengths.len(), + buffer_lengths, + buffer_names, + children, + child_names, + } +} + +/// Navigate the layout tree to find a node by its dot-separated ID path. +/// +/// IDs match the format: "root.field_name.chunked.[0]" where each segment +/// corresponds to a `LayoutChildType::name()`. +fn find_layout_by_id(root: &LayoutRef, node_id: &str) -> Option { + let segments: Vec<&str> = node_id.split('.').collect(); + if segments.is_empty() || segments[0] != "root" { + return None; + } + if segments.len() == 1 { + return Some(root.clone()); + } + + let mut current = root.clone(); + for seg in &segments[1..] { + let children = current.children().ok()?; + let mut found = false; + for (i, child) in children.into_iter().enumerate() { + let name = current.child_type(i).name(); + if name.as_ref() == *seg { + current = child; + found = true; + break; + } + } + if !found { + return None; + } + } + Some(current) +} + +/// Downgrade Arrow `*View` types to their non-view equivalents so the JS +/// `apache-arrow` library can decode them. +fn downgrade_arrow_type(dt: DataType) -> DataType { + match dt { + DataType::Utf8View => DataType::LargeUtf8, + DataType::BinaryView => DataType::LargeBinary, + DataType::Struct(fields) => DataType::Struct( + fields + .iter() + .map(|f| { + Arc::new(Field::new( + f.name(), + downgrade_arrow_type(f.data_type().clone()), + f.is_nullable(), + )) + }) + .collect(), + ), + DataType::List(f) => DataType::List(Arc::new(Field::new( + f.name(), + downgrade_arrow_type(f.data_type().clone()), + f.is_nullable(), + ))), + DataType::LargeList(f) => DataType::LargeList(Arc::new(Field::new( + f.name(), + downgrade_arrow_type(f.data_type().clone()), + f.is_nullable(), + ))), + other => other, + } +} + +/// Create an Arrow Schema from a Vortex DType, with view types downgraded. +fn dtype_to_schema(dtype: &DType, default_name: &str) -> VortexResult { + let schema = match dtype { + DType::Struct(..) => dtype.to_arrow_schema()?, + other => { + let arrow_dt = other.to_arrow_dtype()?; + let nullable = other.is_nullable(); + Schema::new(vec![Field::new(default_name, arrow_dt, nullable)]) + } + }; + // Downgrade view types in all fields. + Ok(Schema::new( + schema + .fields() + .iter() + .map(|f| { + Field::new( + f.name(), + downgrade_arrow_type(f.data_type().clone()), + f.is_nullable(), + ) + }) + .collect::>(), + )) +} + +/// Convert a Vortex ArrayRef into an Arrow RecordBatch using the given schema. +/// +/// Always uses `execute_arrow` with explicit types to ensure view types are avoided. +fn array_to_record_batch( + array: ArrayRef, + dtype: &DType, + schema: &Arc, +) -> VortexResult { + let data_type = match dtype { + DType::Struct(..) => DataType::Struct(schema.fields().clone()), + _ => schema.field(0).data_type().clone(), + }; + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let arrow = array.execute_arrow(Some(&data_type), &mut ctx)?; + match dtype { + DType::Struct(..) => Ok(RecordBatch::from(arrow.as_struct().clone())), + _ => Ok(RecordBatch::try_new(schema.clone(), vec![arrow])?), + } +} + +// --- JSON serialization types --- + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct LayoutTreeNodeJson { + id: String, + encoding: String, + dtype: String, + row_count: u64, + row_offset: u64, + metadata_bytes: usize, + segment_ids: Vec, + child_type: ChildKindJson, + children: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + array_encoding_tree: Option, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct ArrayEncodingNodeJson { + encoding: String, + dtype: String, + metadata_bytes: usize, + num_buffers: usize, + buffer_lengths: Vec, + buffer_names: Vec, + children: Vec, + child_names: Vec, +} + +#[derive(Serialize, Clone)] +#[serde(tag = "kind", rename_all = "camelCase")] +enum ChildKindJson { + #[serde(rename_all = "camelCase")] + Root, + #[serde(rename_all = "camelCase")] + Field { field_name: String }, + #[serde(rename_all = "camelCase")] + Chunk { chunk_index: usize, row_offset: u64 }, + #[serde(rename_all = "camelCase")] + Transparent { name: String }, + #[serde(rename_all = "camelCase")] + Auxiliary { name: String }, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct SegmentMapEntryJson { + index: usize, + byte_offset: u64, + byte_length: u32, + alignment: usize, + column: Option, + layout_path: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct FileStructureJson { + file_size: u64, + version: u16, + postscript_size: u64, + total_data_bytes: u64, + total_metadata_bytes: u64, +} diff --git a/vortex-web/eslint.config.ts b/vortex-web/eslint.config.ts index 72dde640163..ecc2fe28df0 100644 --- a/vortex-web/eslint.config.ts +++ b/vortex-web/eslint.config.ts @@ -5,10 +5,11 @@ import js from "@eslint/js"; import globals from "globals"; import reactHooks from "eslint-plugin-react-hooks"; import reactRefresh from "eslint-plugin-react-refresh"; +import prettier from "eslint-config-prettier"; import storybook from "eslint-plugin-storybook"; import tseslint from "typescript-eslint"; -export default tseslint.config({ ignores: ["dist"] }, { +export default tseslint.config({ ignores: ["dist", "storybook-static"] }, { extends: [js.configs.recommended, ...tseslint.configs.recommended], files: ["**/*.{ts,tsx}"], languageOptions: { @@ -26,4 +27,4 @@ export default tseslint.config({ ignores: ["dist"] }, { { allowConstantExport: true }, ], }, -}, storybook.configs["flat/recommended"]); +}, storybook.configs["flat/recommended"], prettier); diff --git a/vortex-web/index.html b/vortex-web/index.html index 77918e18716..6ed92bb0d33 100644 --- a/vortex-web/index.html +++ b/vortex-web/index.html @@ -8,6 +8,7 @@ Vortex Explorer + =12" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": ">=16.8", + "react-dom": ">=16.8" + } + }, + "node_modules/@tanstack/react-virtual": { + "version": "3.13.23", + "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.13.23.tgz", + "integrity": "sha512-XnMRnHQ23piOVj2bzJqHrRrLg4r+F86fuBcwteKfbIjJrtGxb4z7tIvPVAe4B+4UVwo9G4Giuz5fmapcrnZ0OQ==", + "license": "MIT", + "dependencies": { + "@tanstack/virtual-core": "3.13.23" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/@tanstack/table-core": { + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/@tanstack/table-core/-/table-core-8.21.3.tgz", + "integrity": "sha512-ldZXEhOBb8Is7xLs01fR3YEc3DERiz5silj8tnGkFZytt1abEvl/GhUmCE0PMLaMPTa3Jk4HbKmRlHmu+gCftg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, + "node_modules/@tanstack/virtual-core": { + "version": "3.13.23", + "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.13.23.tgz", + "integrity": "sha512-zSz2Z2HNyLjCplANTDyl3BcdQJc2k1+yyFoKhNRmCr7V7dY8o8q5m8uFTI1/Pg1kL+Hgrz6u3Xo6eFUB7l66cg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, "node_modules/@testing-library/dom": { "version": "10.4.1", "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.1.tgz", @@ -2014,6 +2090,25 @@ "assertion-error": "^2.0.1" } }, + "node_modules/@types/command-line-args": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.3.tgz", + "integrity": "sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw==", + "license": "MIT" + }, + "node_modules/@types/command-line-usage": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.4.tgz", + "integrity": "sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg==", + "license": "MIT" + }, + "node_modules/@types/d3-hierarchy": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/@types/d3-hierarchy/-/d3-hierarchy-3.1.7.tgz", + "integrity": "sha512-tJFtNoYBtRtkNysX1Xq4sxtjK8YgoWUNpIiUee0/jHGRwqvzYxkq0hGVbbOGSz+JgFxxRu4K8nb3YpG3CMARtg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/deep-eql": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", @@ -2049,6 +2144,15 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/node": { + "version": "24.12.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.0.tgz", + "integrity": "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, "node_modules/@types/react": { "version": "19.2.14", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", @@ -2517,7 +2621,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "license": "MIT", "dependencies": { "color-convert": "^2.0.1" @@ -2529,6 +2632,26 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/apache-arrow": { + "version": "21.1.0", + "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-21.1.0.tgz", + "integrity": "sha512-kQrYLxhC+NTVVZ4CCzGF6L/uPVOzJmD1T3XgbiUnP7oTeVFOFgEUu6IKNwCDkpFoBVqDKQivlX4RUFqqnWFlEA==", + "license": "Apache-2.0", + "dependencies": { + "@swc/helpers": "^0.5.11", + "@types/command-line-args": "^5.2.3", + "@types/command-line-usage": "^5.0.4", + "@types/node": "^24.0.3", + "command-line-args": "^6.0.1", + "command-line-usage": "^7.0.1", + "flatbuffers": "^25.1.24", + "json-bignum": "^0.0.3", + "tslib": "^2.6.2" + }, + "bin": { + "arrow2csv": "bin/arrow2csv.js" + } + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -2546,6 +2669,15 @@ "dequal": "^2.0.3" } }, + "node_modules/array-back": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.3.tgz", + "integrity": "sha512-SGDvmg6QTYiTxCBkYVmThcoa67uLl35pyzRHdpCGBOcqFy6BtwnphoFPk7LhJshD+Yk1Kt35WGWeZPTgwR4Fhw==", + "license": "MIT", + "engines": { + "node": ">=12.17" + } + }, "node_modules/assertion-error": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", @@ -2703,7 +2835,6 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.1.0", @@ -2716,6 +2847,21 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/chalk-template": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz", + "integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==", + "license": "MIT", + "dependencies": { + "chalk": "^4.1.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/chalk-template?sponsor=1" + } + }, "node_modules/check-error": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", @@ -2730,7 +2876,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "license": "MIT", "dependencies": { "color-name": "~1.1.4" @@ -2743,9 +2888,46 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, "license": "MIT" }, + "node_modules/command-line-args": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-6.0.2.tgz", + "integrity": "sha512-AIjYVxrV9X752LmPDLbVYv8aMCuHPSLZJXEo2qo/xJfv+NYhaZ4sMSF01rM+gHPaMgvPM0l5D/F+Qx+i2WfSmQ==", + "license": "MIT", + "dependencies": { + "array-back": "^6.2.3", + "find-replace": "^5.0.2", + "lodash.camelcase": "^4.3.0", + "typical": "^7.3.0" + }, + "engines": { + "node": ">=12.20" + }, + "peerDependencies": { + "@75lb/nature": "latest" + }, + "peerDependenciesMeta": { + "@75lb/nature": { + "optional": true + } + } + }, + "node_modules/command-line-usage": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.4.tgz", + "integrity": "sha512-85UdvzTNx/+s5CkSgBm/0hzP80RFHAa7PsfeADE5ezZF3uHz3/Tqj9gIKGT9PTtpycc3Ua64T0oVulGfKxzfqg==", + "license": "MIT", + "dependencies": { + "array-back": "^6.2.2", + "chalk-template": "^0.4.0", + "table-layout": "^4.1.1", + "typical": "^7.3.0" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -2789,6 +2971,15 @@ "dev": true, "license": "MIT" }, + "node_modules/d3-hierarchy": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz", + "integrity": "sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -3065,6 +3256,22 @@ } } }, + "node_modules/eslint-config-prettier": { + "version": "10.1.8", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-10.1.8.tgz", + "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", + "dev": true, + "license": "MIT", + "bin": { + "eslint-config-prettier": "bin/cli.js" + }, + "funding": { + "url": "https://opencollective.com/eslint-config-prettier" + }, + "peerDependencies": { + "eslint": ">=7.0.0" + } + }, "node_modules/eslint-plugin-react-hooks": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-5.2.0.tgz", @@ -3269,6 +3476,23 @@ "node": ">=16.0.0" } }, + "node_modules/find-replace": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-5.0.2.tgz", + "integrity": "sha512-Y45BAiE3mz2QsrN2fb5QEtO4qb44NcS7en/0y9PEVsg351HsLeVclP8QPMH79Le9sH3rs5RSwJu99W0WPZO43Q==", + "license": "MIT", + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@75lb/nature": "latest" + }, + "peerDependenciesMeta": { + "@75lb/nature": { + "optional": true + } + } + }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -3300,6 +3524,12 @@ "node": ">=16" } }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, "node_modules/flatted": { "version": "3.4.2", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", @@ -3436,7 +3666,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -3642,6 +3871,14 @@ "node": ">=6" } }, + "node_modules/json-bignum": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/json-bignum/-/json-bignum-0.0.3.tgz", + "integrity": "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -3977,6 +4214,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -4309,6 +4552,23 @@ "node": ">= 0.8.0" } }, + "node_modules/prettier": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.1.tgz", + "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==", + "dev": true, + "license": "MIT", + "peer": true, + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, "node_modules/pretty-format": { "version": "27.5.1", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz", @@ -4715,7 +4975,6 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, "license": "MIT", "dependencies": { "has-flag": "^4.0.0" @@ -4737,6 +4996,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/table-layout": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-4.1.1.tgz", + "integrity": "sha512-iK5/YhZxq5GO5z8wb0bY1317uDF3Zjpha0QFFLA8/trAoiLbQD0HUbMesEaxyzUgDxi2QlcbM8IvqOlEjgoXBA==", + "license": "MIT", + "dependencies": { + "array-back": "^6.2.2", + "wordwrapjs": "^5.1.0" + }, + "engines": { + "node": ">=12.17" + } + }, "node_modules/tailwindcss": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.2.2.tgz", @@ -4834,7 +5106,6 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, "license": "0BSD" }, "node_modules/type-check": { @@ -4889,6 +5160,21 @@ "typescript": ">=4.8.4 <6.0.0" } }, + "node_modules/typical": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.3.0.tgz", + "integrity": "sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw==", + "license": "MIT", + "engines": { + "node": ">=12.17" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "license": "MIT" + }, "node_modules/unplugin": { "version": "2.3.11", "resolved": "https://registry.npmjs.org/unplugin/-/unplugin-2.3.11.tgz", @@ -5065,6 +5351,15 @@ "node": ">=0.10.0" } }, + "node_modules/wordwrapjs": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-5.1.1.tgz", + "integrity": "sha512-0yweIbkINJodk27gX9LBGMzyQdBDan3s/dEAiwBOj+Mf0PPyWL6/rikalkv8EeD0E8jm4o5RXEOrFTP3NXbhJg==", + "license": "MIT", + "engines": { + "node": ">=12.17" + } + }, "node_modules/ws": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.0.tgz", diff --git a/vortex-web/package.json b/vortex-web/package.json index b4db8620f73..ae1ea81bd5a 100644 --- a/vortex-web/package.json +++ b/vortex-web/package.json @@ -10,6 +10,8 @@ "build": "npm run wasm:release && tsc -b && vite build", "check": "npm run wasm && npm run lint && npm run typecheck", "preview": "vite preview", + "format": "prettier --write 'src/**/*.{ts,tsx,css}' '.storybook/**/*.ts'", + "format:check": "prettier --check 'src/**/*.{ts,tsx,css}' '.storybook/**/*.ts'", "lint": "eslint .", "lint:fix": "eslint . --fix", "typecheck": "tsc -b --noEmit", @@ -17,6 +19,10 @@ "build-storybook": "storybook build" }, "dependencies": { + "@tanstack/react-table": "^8.21.3", + "@tanstack/react-virtual": "^3.13.23", + "apache-arrow": "^21.1.0", + "d3-hierarchy": "^3.1.2", "react": "^19.1.0", "react-dom": "^19.1.0" }, @@ -25,14 +31,17 @@ "@storybook/addon-docs": "^10.3.3", "@storybook/react-vite": "^10.3.3", "@tailwindcss/vite": "^4.1.4", + "@types/d3-hierarchy": "^3.1.7", "@types/react": "^19.1.2", "@types/react-dom": "^19.1.2", "@vitejs/plugin-react": "^4.4.1", "eslint": "^9.25.0", + "eslint-config-prettier": "^10.1.8", "eslint-plugin-react-hooks": "^5.2.0", "eslint-plugin-react-refresh": "^0.4.20", "eslint-plugin-storybook": "^10.3.3", "globals": "^16.0.0", + "prettier": "^3.8.1", "storybook": "^10.3.3", "tailwindcss": "^4.1.4", "typescript": "~5.7.0", diff --git a/vortex-web/src/App.tsx b/vortex-web/src/App.tsx index 63fb697ffb6..07b52b120fe 100644 --- a/vortex-web/src/App.tsx +++ b/vortex-web/src/App.tsx @@ -1,142 +1,193 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -import { useCallback, useEffect, useRef, useState, type DragEvent } from "react"; -import type { InitOutput } from "./wasm/pkg/vortex_web_wasm.d.ts"; - -interface FileInfo { - name: string; - rowCount: bigint; - dtype: string; -} +import { useCallback, useEffect, useMemo, useRef, useState, type DragEvent } from 'react'; +import type { VortexFileState, VortexFileContextValue } from './contexts/VortexFileContext'; +import { VortexFileProvider } from './contexts/VortexFileContext'; +import { SelectionProvider } from './contexts/SelectionContext'; +import type { LayoutTreeNode } from './components/swimlane/types'; +import { arrayTreeToLayoutChildren, findNodeById } from './components/swimlane/utils'; +import { FileDropScreen } from './components/explorer/FileDropScreen'; +import { FileHeader } from './components/explorer/FileHeader'; +import { MainArea } from './components/explorer/MainArea'; +import { StatusBar } from './components/explorer/StatusBar'; +import { VortexWorker } from './workers/VortexWorker'; function App() { - const [isDragging, setIsDragging] = useState(false); - const [fileInfo, setFileInfo] = useState(null); + const [fileState, setFileState] = useState(null); const [error, setError] = useState(null); const [loading, setLoading] = useState(false); - const wasmRef = useRef(null); + const [isDragging, setIsDragging] = useState(false); + const dragCounter = useRef(0); + const workerRef = useRef(null); useEffect(() => { - import("./wasm/pkg/vortex_web_wasm.js").then(async (wasm) => { - wasmRef.current = await wasm.default(); - }); + workerRef.current = new VortexWorker(); + return () => workerRef.current?.terminate(); }, []); - const openFile = useCallback( - async (file: File) => { - setError(null); - setLoading(true); - try { - const wasm = await import("./wasm/pkg/vortex_web_wasm.js"); - if (!wasmRef.current) { - wasmRef.current = await wasm.default(); - } - const bytes = new Uint8Array(await file.arrayBuffer()); - const handle = wasm.open_vortex_file(bytes); - setFileInfo({ - name: file.name, - rowCount: handle.row_count, - dtype: handle.dtype, - }); - handle.free(); - } catch (e) { - setError(e instanceof Error ? e.message : String(e)); - setFileInfo(null); - } finally { - setLoading(false); - } + const openFile = useCallback(async (file: File) => { + setError(null); + setLoading(true); + try { + const result = await workerRef.current!.openFile(file); + setFileState({ + fileName: file.name, + fileSize: file.size, + rowCount: result.rowCount, + version: result.fileStructure.version, + dtype: result.dtype, + layoutTree: result.layoutTree, + segments: result.segments, + fileStructure: result.fileStructure, + }); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + setFileState(null); + } finally { + setLoading(false); + } + }, []); + + const fetchEncodingTree = useCallback( + (nodeId: string) => workerRef.current!.fetchEncodingTree(nodeId), + [], + ); + + const previewData = useCallback( + (nodeId: string, rowLimit: number) => workerRef.current!.previewData(nodeId, rowLimit), + [], + ); + + /** Clone a tree, replacing the node at targetId with a modified version. */ + const cloneTreeWithUpdate = useCallback( + ( + root: LayoutTreeNode, + targetId: string, + update: (node: LayoutTreeNode) => LayoutTreeNode, + ): LayoutTreeNode => { + if (root.id === targetId) return update(root); + const newChildren = root.children.map((child) => + cloneTreeWithUpdate(child, targetId, update), + ); + if (newChildren === root.children) return root; + return { ...root, children: newChildren }; }, [], ); + const expandArrayTree = useCallback( + async (nodeId: string) => { + // Fetch the encoding tree (may be async). + const arrayTree = await workerRef.current!.fetchEncodingTree(nodeId); + if (!arrayTree) return; + + setFileState((prev) => { + if (!prev) return prev; + const node = findNodeById(prev.layoutTree, nodeId); + if (!node || node.encoding !== 'vortex.flat') return prev; + if (node.children.some((c) => c.isArrayNode)) return prev; + + const arrayChildren = arrayTreeToLayoutChildren(arrayTree, node); + const newTree = cloneTreeWithUpdate(prev.layoutTree, nodeId, (n) => ({ + ...n, + arrayEncodingTree: arrayTree, + children: [...n.children, ...arrayChildren], + })); + return { ...prev, layoutTree: newTree }; + }); + }, + [cloneTreeWithUpdate], + ); + + const fetchArrayBuffer = useCallback( + (layoutNodeId: string, arrayPath: string[], bufferIndex: number) => + workerRef.current!.fetchArrayBuffer(layoutNodeId, arrayPath, bufferIndex), + [], + ); + + const previewArrayData = useCallback( + (layoutNodeId: string, arrayPath: string[], rowLimit: number) => + workerRef.current!.previewArrayData(layoutNodeId, arrayPath, rowLimit), + [], + ); + + const fileContextValue = useMemo( + () => + fileState + ? { + ...fileState, + fetchEncodingTree, + previewData, + expandArrayTree, + fetchArrayBuffer, + previewArrayData, + } + : null, + [ + fileState, + fetchEncodingTree, + previewData, + expandArrayTree, + fetchArrayBuffer, + previewArrayData, + ], + ); + + const closeFile = useCallback(() => setFileState(null), []); + + const handleDragEnter = useCallback((e: DragEvent) => { + e.preventDefault(); + dragCounter.current++; + if (dragCounter.current === 1) setIsDragging(true); + }, []); + const handleDragOver = useCallback((e: DragEvent) => { e.preventDefault(); - setIsDragging(true); }, []); const handleDragLeave = useCallback((e: DragEvent) => { e.preventDefault(); - setIsDragging(false); + dragCounter.current--; + if (dragCounter.current === 0) setIsDragging(false); }, []); const handleDrop = useCallback( (e: DragEvent) => { e.preventDefault(); + dragCounter.current = 0; setIsDragging(false); const file = e.dataTransfer.files[0]; - if (file) { - openFile(file); - } + if (file) openFile(file); }, [openFile], ); - const handleClick = useCallback(() => { - const input = document.createElement("input"); - input.type = "file"; - input.accept = ".vortex,.vtx"; - input.onchange = () => { - const file = input.files?.[0]; - if (file) { - openFile(file); - } - }; - input.click(); - }, [openFile]); + if (!fileContextValue) { + return ; + } return ( -
-

- Vortex Explorer -

- -
- {loading ? ( -

Loading...

- ) : fileInfo ? ( -
-

{fileInfo.name}

-

- - {fileInfo.rowCount.toLocaleString()} - {" "} - rows -

- 
-              {fileInfo.dtype}
-
-
- ) : ( -
-

- Drop a{" "} - - .vortex - {" "} - file here -

-

- or click to browse -

-
- )} -
- - {error && ( -

- {error} -

- )} -
+ + +
+ + + + {isDragging && ( +
+

Drop to open file

+
+ )} +
+ + ); } diff --git a/vortex-web/src/components/DataTable.tsx b/vortex-web/src/components/DataTable.tsx new file mode 100644 index 00000000000..6f752e01821 --- /dev/null +++ b/vortex-web/src/components/DataTable.tsx @@ -0,0 +1,451 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useMemo, useRef, useState } from 'react'; +import { + useReactTable, + getCoreRowModel, + getSortedRowModel, + flexRender, + type ColumnDef, + type SortingState, +} from '@tanstack/react-table'; +import { useVirtualizer } from '@tanstack/react-virtual'; + +// --- Column statistics --- + +interface ColumnStats { + kind: 'numeric' | 'string' | 'boolean' | 'other'; + count: number; + nullCount: number; + min?: number; + max?: number; + mean?: number; + histogram?: number[]; + cardinality?: number; + trueCount?: number; + falseCount?: number; +} + +function computeStats(values: unknown[]): ColumnStats { + const count = values.length; + let nullCount = 0; + const nums: number[] = []; + let hasStrings = false; + let trueCount = 0; + let falseCount = 0; + let hasBools = false; + + for (const v of values) { + if (v == null) { + nullCount++; + } else if (typeof v === 'boolean') { + hasBools = true; + if (v) trueCount++; + else falseCount++; + } else if (typeof v === 'number' || typeof v === 'bigint') { + nums.push(Number(v)); + } else if (typeof v === 'string') { + hasStrings = true; + } + } + + if (hasBools) { + return { kind: 'boolean', count, nullCount, trueCount, falseCount }; + } + + if (nums.length > 0) { + let min = Infinity, + max = -Infinity, + sum = 0; + for (const n of nums) { + if (n < min) min = n; + if (n > max) max = n; + sum += n; + } + const bins = 20; + const histogram = new Array(bins).fill(0); + if (max > min) { + const range = max - min; + for (const n of nums) { + const idx = Math.min(bins - 1, Math.floor(((n - min) / range) * bins)); + histogram[idx]++; + } + } else { + histogram[0] = nums.length; + } + return { kind: 'numeric', count, nullCount, min, max, mean: sum / nums.length, histogram }; + } + + if (hasStrings) { + const uniq = new Set(); + for (const v of values) if (typeof v === 'string') uniq.add(v); + return { kind: 'string', count, nullCount, cardinality: uniq.size }; + } + + return { kind: 'other', count, nullCount }; +} + +// --- Sparkline --- + +function SparkHistogram({ histogram, height = 12 }: { histogram: number[]; height?: number }) { + const max = Math.max(...histogram); + if (max === 0) return null; + + // Check if all values fell in a single bin (constant column). + const nonZero = histogram.filter((v) => v > 0).length; + if (nonZero <= 1) { + // Render a flat line to indicate constant/single-value. + const barW = 2; + const gap = 0.5; + const w = histogram.length * (barW + gap) - gap; + return ( + + + + ); + } + + const barW = 2; + const gap = 0.5; + const w = histogram.length * (barW + gap) - gap; + return ( + + {histogram.map((v, i) => { + const barH = Math.max(0.5, (v / max) * height); + return ( + + ); + })} + + ); +} + +// --- Header summary (inline, compact) --- + +function HeaderSummary({ stats }: { stats: ColumnStats }) { + if (stats.kind === 'numeric') { + const isConst = stats.min === stats.max && stats.nullCount === 0; + if (isConst) { + return ( + + const + + ); + } + if (stats.histogram) { + return ; + } + } + if (stats.kind === 'boolean') { + const total = stats.trueCount! + stats.falseCount!; + if (total === 0) return null; + const allTrue = stats.falseCount === 0 && stats.nullCount === 0; + const allFalse = stats.trueCount === 0 && stats.nullCount === 0; + if (allTrue || allFalse) + return const; + const pct = Math.round((stats.trueCount! / total) * 100); + return ( +
+
+
+
+
+ {pct}% +
+ ); + } + if (stats.kind === 'string' && stats.cardinality != null) { + if (stats.cardinality === 1 && stats.nullCount === 0) { + return const; + } + return ( + + {stats.cardinality}v + + ); + } + return null; +} + +// --- Header tooltip (shown on hover) --- + +function HeaderTooltip({ stats, approximate }: { stats: ColumnStats; approximate: boolean }) { + const p = approximate ? '~' : ''; + const fmt = (n: number) => { + const s = + Math.abs(n) >= 1e6 || (Math.abs(n) < 0.01 && n !== 0) + ? n.toExponential(2) + : n.toLocaleString(undefined, { maximumFractionDigits: 2 }); + return p + s; + }; + + return ( +
+
+ {stats.count.toLocaleString()} rows{approximate ? ' (sampled)' : ''} +
+ {stats.nullCount > 0 && ( +
{stats.nullCount.toLocaleString()} nulls
+ )} + {stats.kind === 'numeric' && ( + <> +
min: {fmt(stats.min!)}
+
max: {fmt(stats.max!)}
+
mean: {fmt(stats.mean!)}
+ {stats.histogram && ( +
+ +
+ )} + + )} + {stats.kind === 'boolean' && ( + <> +
true: {stats.trueCount!.toLocaleString()}
+
false: {stats.falseCount!.toLocaleString()}
+ + )} + {stats.kind === 'string' &&
{stats.cardinality!.toLocaleString()} distinct values
} +
+ ); +} + +// --- Hoverable header with tooltip --- + +function ColumnHeader({ + name, + stats, + approximate, +}: { + name: string; + stats: ColumnStats; + approximate: boolean; +}) { + const [showTip, setShowTip] = useState(false); + const timeoutRef = useRef>(); + + const onEnter = () => { + clearTimeout(timeoutRef.current); + timeoutRef.current = setTimeout(() => setShowTip(true), 400); + }; + const onLeave = () => { + clearTimeout(timeoutRef.current); + setShowTip(false); + }; + + return ( +
+
+ {name} + +
+ {showTip && ( +
+ +
+ )} +
+ ); +} + +// --- Formatting --- + +function formatCell(value: unknown): string { + if (value == null) return ''; + if (typeof value === 'bigint') return value.toString(); + if (typeof value === 'number') { + if (Number.isInteger(value)) return value.toLocaleString(); + return value.toLocaleString(undefined, { maximumFractionDigits: 4 }); + } + return String(value); +} + +// --- Main component --- + +export type CellRenderer = (value: unknown, row: Record) => React.ReactNode; + +export interface DataTableProps { + columns: string[]; + rows: Record[]; + rowHeight?: number; + onRowClick?: (rowIndex: number) => void; + onRowHover?: (rowIndex: number | null) => void; + /** Custom cell renderers keyed by column name. */ + cellRenderers?: Record; + /** If true, stats are approximate (data was truncated by a row limit). */ + approximate?: boolean; +} + +export function DataTable({ + columns, + rows, + rowHeight = 24, + onRowClick, + onRowHover, + cellRenderers, + approximate = false, +}: DataTableProps) { + const [sorting, setSorting] = useState([]); + const parentRef = useRef(null); + + const columnStats = useMemo(() => { + const stats: Record = {}; + for (const col of columns) { + stats[col] = computeStats(rows.map((r) => r[col])); + } + return stats; + }, [columns, rows]); + + const columnDefs = useMemo>[]>( + () => [ + { + id: '__row_num', + header: '#', + size: 50, + enableSorting: false, + cell: (info) => ( + {info.row.index} + ), + }, + ...columns.map( + (col): ColumnDef> => ({ + accessorKey: col, + header: () => ( + + ), + cell: (info) => { + const renderer = cellRenderers?.[col]; + if (renderer) { + return renderer(info.getValue(), info.row.original); + } + const val = info.getValue(); + if (val == null) { + return null; + } + return formatCell(val); + }, + sortingFn: 'auto', + }), + ), + ], + [columns, columnStats], + ); + + const table = useReactTable({ + data: rows, + columns: columnDefs, + state: { sorting }, + onSortingChange: setSorting, + getCoreRowModel: getCoreRowModel(), + getSortedRowModel: getSortedRowModel(), + }); + + const { rows: tableRows } = table.getRowModel(); + + const virtualizer = useVirtualizer({ + count: tableRows.length, + getScrollElement: () => parentRef.current, + estimateSize: () => rowHeight, + overscan: 20, + }); + + return ( +
+ + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + ))} + + ))} + + + {virtualizer.getVirtualItems().length > 0 && ( + + + )} + {virtualizer.getVirtualItems().map((virtualRow) => { + const row = tableRows[virtualRow.index]; + return ( + onRowClick?.(virtualRow.index)} + onMouseEnter={() => onRowHover?.(virtualRow.index)} + onMouseLeave={() => onRowHover?.(null)} + > + {row.getVisibleCells().map((cell) => ( + + ))} + + ); + })} + {virtualizer.getVirtualItems().length > 0 && ( + + + )} + +
+
+ {header.isPlaceholder + ? null + : flexRender(header.column.columnDef.header, header.getContext())} + {header.column.getIsSorted() === 'asc' && ( + ▲ + )} + {header.column.getIsSorted() === 'desc' && ( + ▼ + )} +
+
+
+ {flexRender(cell.column.columnDef.cell, cell.getContext())} +
+
+
+ ); +} diff --git a/vortex-web/src/components/ThemePicker.tsx b/vortex-web/src/components/ThemePicker.tsx new file mode 100644 index 00000000000..00353bcbcbd --- /dev/null +++ b/vortex-web/src/components/ThemePicker.tsx @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useTheme, type ThemeChoice } from '../contexts/ThemeContext'; + +const cycle: ThemeChoice[] = ['dark', 'light', 'system']; + +const labels: Record = { + light: 'Light mode', + dark: 'Dark mode', + system: 'System theme', +}; + +function SunIcon() { + return ( + + + + + ); +} + +function MoonIcon() { + return ( + + + + ); +} + +function MonitorIcon() { + return ( + + + + + + ); +} + +const icons: Record JSX.Element> = { + light: SunIcon, + dark: MoonIcon, + system: MonitorIcon, +}; + +export function ThemePicker() { + const { theme, setTheme } = useTheme(); + + const next = () => { + const idx = cycle.indexOf(theme); + setTheme(cycle[(idx + 1) % cycle.length]); + }; + + const Icon = icons[theme]; + + return ( + + ); +} diff --git a/vortex-web/src/components/detail/ArraySummaryPane.tsx b/vortex-web/src/components/detail/ArraySummaryPane.tsx new file mode 100644 index 00000000000..06a2311118a --- /dev/null +++ b/vortex-web/src/components/detail/ArraySummaryPane.tsx @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import type { LayoutTreeNode } from '../swimlane/types'; +import { formatBytes, getNodeDisplayName, shortEncoding } from '../swimlane/utils'; + +interface ArraySummaryPaneProps { + node: LayoutTreeNode; +} + +export function ArraySummaryPane({ node }: ArraySummaryPaneProps) { + const name = getNodeDisplayName(node); + const totalBufferBytes = (node.bufferLengths ?? []).reduce((s, b) => s + b, 0); + + return ( +
+

{name}

+
+ Encoding + + {shortEncoding(node.encoding)} + + Metadata + + {formatBytes(node.metadataBytes)} + + Buffers + + {(node.bufferLengths ?? []).length} + + Buffer data + + {formatBytes(totalBufferBytes)} + + Children + {node.children.length} +
+
+ ); +} diff --git a/vortex-web/src/components/detail/BuffersPane.tsx b/vortex-web/src/components/detail/BuffersPane.tsx new file mode 100644 index 00000000000..d1d7f527139 --- /dev/null +++ b/vortex-web/src/components/detail/BuffersPane.tsx @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useCallback, useEffect, useMemo, useState } from 'react'; +import type { LayoutTreeNode } from '../swimlane/types'; +import { formatBytes, parseArrayNodeId } from '../swimlane/utils'; +import { useVortexFile } from '../../contexts/VortexFileContext'; + +interface BuffersPaneProps { + node: LayoutTreeNode; +} + +export function BuffersPane({ node }: BuffersPaneProps) { + const { fetchArrayBuffer } = useVortexFile(); + const bufferLengths = node.bufferLengths ?? []; + const [selectedBuffer, setSelectedBuffer] = useState(null); + const [bufferData, setBufferData] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const bufferNames: string[] = bufferLengths.map((_, i) => node.bufferNames?.[i] ?? `buffer ${i}`); + + const loadBuffer = useCallback( + async (index: number) => { + setSelectedBuffer(index); + setBufferData(null); + setError(null); + setLoading(true); + try { + const { layoutNodeId, arrayPath } = parseArrayNodeId(node.id); + const data = await fetchArrayBuffer(layoutNodeId, arrayPath, index); + setBufferData(data); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } finally { + setLoading(false); + } + }, + [node.id, fetchArrayBuffer], + ); + + // Auto-select first buffer when node changes. + useEffect(() => { + setBufferData(null); + setError(null); + if (bufferLengths.length > 0) { + loadBuffer(0); + } else { + setSelectedBuffer(null); + } + }, [node.id, bufferLengths.length, loadBuffer]); + + if (bufferLengths.length === 0) { + return
No buffers for this node.
; + } + + return ( +
+ {/* Buffer list — left side */} +
+ {bufferLengths.map((len, i) => ( + + ))} +
+ + {/* Hex viewer — right side */} +
+ {selectedBuffer === null && ( +
Select a buffer to view its contents.
+ )} + {loading &&
Loading…
} + {error &&
Error: {error}
} + {bufferData && } +
+
+ ); +} + +const BYTES_PER_ROW = 16; +const MAX_ROWS = 1024; + +interface HexRow { + offset: number; + bytes: { value: number; absIndex: number }[]; +} + +function HexView({ data }: { data: Uint8Array }) { + const [hovered, setHovered] = useState(null); + + const { rows, truncated } = useMemo(() => { + const result: HexRow[] = []; + for (let i = 0; i < data.length; i += BYTES_PER_ROW) { + const bytes: HexRow['bytes'] = []; + const end = Math.min(i + BYTES_PER_ROW, data.length); + for (let j = i; j < end; j++) { + bytes.push({ value: data[j], absIndex: j }); + } + result.push({ offset: i, bytes }); + } + const isTruncated = result.length > MAX_ROWS; + return { rows: isTruncated ? result.slice(0, MAX_ROWS) : result, truncated: isTruncated }; + }, [data]); + + const hlClass = 'bg-vortex-light-blue/20 rounded-sm'; + + return ( +
+ {rows.map((row) => ( +
+ {/* Offset */} + + {row.offset.toString(16).padStart(8, '0')} + + + {/* Hex bytes */} + + {Array.from({ length: BYTES_PER_ROW }, (_, j) => { + const b = row.bytes[j]; + if (!b) + return ( + + {' '} + + ); + const isHl = hovered === b.absIndex; + const gap = j === 8 ? ' ' : ''; + return ( + + {gap} + setHovered(b.absIndex)} + onMouseLeave={() => setHovered(null)} + > + {b.value.toString(16).padStart(2, '0')} + + + ); + })} + + + {/* ASCII */} + + {Array.from({ length: BYTES_PER_ROW }, (_, j) => { + const b = row.bytes[j]; + if (!b) return ; + const isHl = hovered === b.absIndex; + const ch = b.value >= 0x20 && b.value < 0x7f ? String.fromCharCode(b.value) : '.'; + const color = ch === '.' ? 'text-vortex-grey-dark' : 'text-vortex-light-blue'; + return ( + setHovered(b.absIndex)} + onMouseLeave={() => setHovered(null)} + > + {ch} + + ); + })} + +
+ ))} + {truncated && ( +
+ … {Math.ceil(data.length / BYTES_PER_ROW) - MAX_ROWS} more rows ( + {formatBytes(data.length)} total) +
+ )} +
+ ); +} diff --git a/vortex-web/src/components/detail/DetailPanel.stories.tsx b/vortex-web/src/components/detail/DetailPanel.stories.tsx new file mode 100644 index 00000000000..ce8a05c9c0c --- /dev/null +++ b/vortex-web/src/components/detail/DetailPanel.stories.tsx @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import type { Meta, StoryObj } from '@storybook/react-vite'; +import { DetailPanel } from './DetailPanel'; +import { withMockFileContext, withMockSelection } from '../../storybook/decorators'; +import { ordersMock } from '../../mocks/layouts'; +import { generateSegments } from '../../mocks/segments'; +import { generateFileStructure } from '../../mocks/fileStructure'; +import type { VortexFileState } from '../../contexts/VortexFileContext'; + +const layout = ordersMock(); +const segments = generateSegments(layout, 12_400_000); +const fileStructure = generateFileStructure(segments, 12_400_000); + +const mockFileState: VortexFileState = { + fileName: 'orders.vortex', + fileSize: 12_400_000, + rowCount: 100_000, + version: 1, + dtype: '{order_id=i64, ...}', + layoutTree: layout, + segments, + fileStructure, +}; + +const meta: Meta = { + component: DetailPanel, + decorators: [withMockFileContext(mockFileState), withMockSelection(layout)], + parameters: { + layout: 'padded', + }, +}; +export default meta; + +type Story = StoryObj; + +export const NoSelection: Story = {}; diff --git a/vortex-web/src/components/detail/DetailPanel.tsx b/vortex-web/src/components/detail/DetailPanel.tsx new file mode 100644 index 00000000000..a37ef08ac8d --- /dev/null +++ b/vortex-web/src/components/detail/DetailPanel.tsx @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useMemo, useState } from 'react'; +import { useVortexFile } from '../../contexts/VortexFileContext'; +import { useSelection } from '../../contexts/SelectionContext'; +import { + getNodeDisplayName, + findPathToNode, + getDtypeCategory, + shortEncoding, + DTYPE_COLORS, +} from '../swimlane/utils'; +import { SummaryPane } from './SummaryPane'; +import { ArraySummaryPane } from './ArraySummaryPane'; +import { EncodingPane } from './EncodingPane'; +import { SegmentsPane } from './SegmentsPane'; +import { TreemapPane } from './TreemapPane'; +import { BuffersPane } from './BuffersPane'; + +type TabId = 'encoding' | 'segments' | 'treemap' | 'buffers'; + +interface TabDef { + id: TabId; + label: string; +} + +export function DetailPanel() { + const file = useVortexFile(); + const { state: selection, selectNode, hoverNode } = useSelection(); + const [activeTab, setActiveTab] = useState('segments'); + + const isArrayNode = selection.selectedNode?.isArrayNode ?? false; + + const tabs = useMemo(() => { + const result: TabDef[] = []; + if (selection.selectedNode) { + if (isArrayNode) { + result.push({ id: 'treemap', label: 'Treemap' }); + if ((selection.selectedNode.bufferLengths ?? []).length > 0) { + result.push({ id: 'buffers', label: 'Buffers' }); + } + } else { + result.push({ id: 'treemap', label: 'Treemap' }); + result.push({ id: 'segments', label: 'Segments' }); + if (selection.selectedNode.children.length === 0) { + result.push({ id: 'encoding', label: 'Encoding' }); + } + } + } + return result; + }, [selection.selectedNode, isArrayNode]); + + const selectedPath = useMemo(() => { + if (!selection.selectedNodeId) return []; + return findPathToNode(file.layoutTree, selection.selectedNodeId); + }, [file.layoutTree, selection.selectedNodeId]); + + const hoveredPath = useMemo(() => { + if (!selection.hoveredNodeId) return []; + return findPathToNode(file.layoutTree, selection.hoveredNodeId); + }, [file.layoutTree, selection.hoveredNodeId]); + + const breadcrumb = hoveredPath.length > 0 ? hoveredPath : selectedPath; + + const selectedIdSet = useMemo(() => new Set(selectedPath.map((n) => n.id)), [selectedPath]); + const isHoverBreadcrumb = hoveredPath.length > 0; + + const currentTab = tabs.find((t) => t.id === activeTab) ? activeTab : tabs[0]?.id; + + return ( +
+ {/* Breadcrumb + tab bar */} +
+ {/* Tabs */} +
+ {tabs.map((tab) => ( + + ))} +
+ + {/* Breadcrumb — right of tabs */} + {breadcrumb.length > 1 && ( +
+ {breadcrumb.map((node, i) => { + const isLast = i === breadcrumb.length - 1; + const isShared = isHoverBreadcrumb && selectedIdSet.has(node.id); + const dimClass = isHoverBreadcrumb && !isShared ? 'opacity-50' : ''; + const prevNode = i > 0 ? breadcrumb[i - 1] : null; + const isArrayBoundary = node.isArrayNode && prevNode && !prevNode.isArrayNode; + return ( + + {isArrayBoundary && ( + + › + + )} + {i > 0 && !isArrayBoundary && /} + {isLast && !isHoverBreadcrumb ? ( + + {getNodeDisplayName(node)} + + ) : ( + + )} + + ); + })} + {(() => { + const tip = breadcrumb[breadcrumb.length - 1]; + if (!tip) return null; + const cat = getDtypeCategory(tip.dtype); + return ( + + {cat} + + ); + })()} +
+ )} +
+ + {/* Main content: tab content (left) + summary sidebar (right) */} +
+ {/* Tab content */} +
+ {currentTab === 'encoding' && selection.selectedNode && ( +
+ +
+ )} + {currentTab === 'segments' && selection.selectedNode && ( + + )} + {currentTab === 'treemap' && selection.selectedNode && ( + + )} + {currentTab === 'buffers' && selection.selectedNode && ( + + )} + {!currentTab && !selection.selectedNode && ( +
+ Select a node to view details. +
+ )} +
+ + {/* Summary sidebar — always visible */} +
+ {isArrayNode && selection.selectedNode ? ( + + ) : ( + + )} +
+
+
+ ); +} diff --git a/vortex-web/src/components/detail/EncodingPane.tsx b/vortex-web/src/components/detail/EncodingPane.tsx new file mode 100644 index 00000000000..d3b17886abd --- /dev/null +++ b/vortex-web/src/components/detail/EncodingPane.tsx @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useEffect, useState } from 'react'; +import { useVortexFile } from '../../contexts/VortexFileContext'; +import type { LayoutTreeNode, ArrayEncodingNode } from '../swimlane/types'; +import { shortEncoding, formatBytes } from '../swimlane/utils'; + +interface EncodingPaneProps { + node: LayoutTreeNode; +} + +/** + * Displays the array encoding tree inside a flat layout. + * If the tree is not inlined in the layout metadata, fetches it + * asynchronously from the segment data via the worker. + */ +export function EncodingPane({ node }: EncodingPaneProps) { + const { fetchEncodingTree } = useVortexFile(); + const [fetchedTree, setFetchedTree] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const tree = node.arrayEncodingTree ?? fetchedTree; + + useEffect(() => { + // If already inlined, nothing to fetch. + if (node.arrayEncodingTree) return; + + // Need at least one segment to fetch from. + if (node.segmentIds.length === 0) return; + + let cancelled = false; + setLoading(true); + setError(null); + setFetchedTree(null); + + fetchEncodingTree(node.id) + .then((result) => { + if (!cancelled) setFetchedTree(result); + }) + .catch((err) => { + if (!cancelled) setError(err instanceof Error ? err.message : String(err)); + }) + .finally(() => { + if (!cancelled) setLoading(false); + }); + + return () => { + cancelled = true; + }; + }, [node.id, node.arrayEncodingTree, node.segmentIds, fetchEncodingTree]); + + if (loading) { + return
Loading encoding tree…
; + } + + if (error) { + return
Error: {error}
; + } + + if (!tree) { + return ( +
+ No array encoding tree available for this node. +
+ ); + } + + return ( +
+ 
+        {renderArrayTree(tree, 0)}
+
+
+ ); +} + +function renderArrayTree(node: ArrayEncodingNode, indent: number): string { + const prefix = ' '.repeat(indent); + const enc = shortEncoding(node.encoding); + const parts = [enc]; + if (node.bufferLengths.length > 0) { + const bufs = node.bufferLengths.map((b) => formatBytes(b)).join(', '); + parts.push(`buffers: [${bufs}]`); + } + if (node.metadataBytes > 0) { + parts.push(`meta: ${formatBytes(node.metadataBytes)}`); + } + const line = `${prefix}${parts.join(' ')}`; + + if (node.children.length === 0) return line; + + const childLines = node.children.map((child) => renderArrayTree(child, indent + 1)); + return [line, ...childLines].join('\n'); +} diff --git a/vortex-web/src/components/detail/SegmentsPane.tsx b/vortex-web/src/components/detail/SegmentsPane.tsx new file mode 100644 index 00000000000..816bb07e1c9 --- /dev/null +++ b/vortex-web/src/components/detail/SegmentsPane.tsx @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useCallback, useMemo } from 'react'; +import type { LayoutTreeNode, SegmentMapEntry } from '../swimlane/types'; +import { collectSubtreeSegments, findPathToNode, getNodeDisplayName } from '../swimlane/utils'; +import { useVortexFile } from '../../contexts/VortexFileContext'; +import { useSelection } from '../../contexts/SelectionContext'; +import { DataTable, type CellRenderer } from '../DataTable'; + +interface SegmentsPaneProps { + node: LayoutTreeNode; + segments: SegmentMapEntry[]; +} + +export function SegmentsPane({ node, segments }: SegmentsPaneProps) { + const file = useVortexFile(); + const { selectNode, selectSegment, hoverSegment } = useSelection(); + const subtreeSegmentIds = useMemo(() => new Set(collectSubtreeSegments(node)), [node]); + + const { columns, rows } = useMemo(() => { + const filtered = segments.filter((s) => subtreeSegmentIds.has(s.index)); + const cols = ['index', 'byte_offset', 'byte_length', 'alignment', 'column', 'layout_path']; + const rowData = filtered.map((s) => ({ + index: s.index, + byte_offset: s.byteOffset, + byte_length: s.byteLength, + alignment: s.alignment, + column: s.column ?? '', + layout_path: s.layoutPath, + })); + return { columns: cols, rows: rowData }; + }, [segments, subtreeSegmentIds]); + + const pathRenderer: CellRenderer = useCallback( + (_value: unknown, row: Record) => { + const layoutPath = row.layout_path as string; + const pathNodes = findPathToNode(file.layoutTree, layoutPath); + return ( + + {pathNodes.map((pathNode, i) => ( + + {i > 0 && /} + + + ))} + + ); + }, + [file.layoutTree, selectNode], + ); + + const handleRowClick = useCallback( + (rowIndex: number) => { + const row = rows[rowIndex]; + if (row) selectSegment(row.index as number); + }, + [rows, selectSegment], + ); + + const handleRowHover = useCallback( + (rowIndex: number | null) => { + if (rowIndex == null) { + hoverSegment(null); + } else { + const row = rows[rowIndex]; + if (row) hoverSegment(row.index as number); + } + }, + [rows, hoverSegment], + ); + + if (rows.length === 0) { + return
No segments for this node.
; + } + + return ( +
+ +
+ ); +} diff --git a/vortex-web/src/components/detail/SummaryPane.tsx b/vortex-web/src/components/detail/SummaryPane.tsx new file mode 100644 index 00000000000..1a0f0eef336 --- /dev/null +++ b/vortex-web/src/components/detail/SummaryPane.tsx @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import type { LayoutTreeNode } from '../swimlane/types'; +import type { VortexFileState } from '../../contexts/VortexFileContext'; +import { + formatBytes, + formatRowCount, + getNodeDisplayName, + collectSubtreeSegments, + shortEncoding, +} from '../swimlane/utils'; + +function formatNum(n: number): string { + return n.toString().replace(/\B(?=(\d{3})+(?!\d))/g, '_'); +} + +interface SummaryPaneProps { + node: LayoutTreeNode | null; + file: VortexFileState; +} + +export function SummaryPane({ node, file }: SummaryPaneProps) { + if (!node) { + // File-level summary + return ( +
+

File Summary

+
+ File + {file.fileName} + Size + + {formatBytes(file.fileSize)} + + Rows + + {formatRowCount(file.rowCount)} + + Segments + {file.segments.length} + Version + v{file.version} +
+
+ ); + } + + // Node-level summary + const name = getNodeDisplayName(node); + const subtreeSegmentIds = collectSubtreeSegments(node); + const reachableSegments = file.segments.filter((s) => subtreeSegmentIds.includes(s.index)); + const totalBytes = reachableSegments.reduce((sum, s) => sum + s.byteLength, 0); + + return ( +
+

{name}

+
+ Encoding + + {shortEncoding(node.encoding)} + + Rows + {formatNum(node.rowCount)} + Row offset + + {formatNum(node.rowOffset)} + + Metadata + + {formatBytes(node.metadataBytes)} + + Data size + {formatBytes(totalBytes)} + Segments + {reachableSegments.length} + Children + {node.children.length} +
+
+ ); +} diff --git a/vortex-web/src/components/detail/TreemapPane.stories.tsx b/vortex-web/src/components/detail/TreemapPane.stories.tsx new file mode 100644 index 00000000000..9a97e9c2f52 --- /dev/null +++ b/vortex-web/src/components/detail/TreemapPane.stories.tsx @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import type { Meta, StoryObj } from '@storybook/react-vite'; +import { TreemapPane } from './TreemapPane'; +import { withMockFileContext, withMockSelection } from '../../storybook/decorators'; +import { ordersMock } from '../../mocks/layouts'; +import { generateSegments } from '../../mocks/segments'; +import { generateFileStructure } from '../../mocks/fileStructure'; +import type { VortexFileState } from '../../contexts/VortexFileContext'; + +const layout = ordersMock(); +const segments = generateSegments(layout, 12_400_000); +const fileStructure = generateFileStructure(segments, 12_400_000); + +const mockFileState: VortexFileState = { + fileName: 'orders.vortex', + fileSize: 12_400_000, + rowCount: 100_000, + version: 1, + dtype: '{order_id=i64, ...}', + layoutTree: layout, + segments, + fileStructure, +}; + +const meta: Meta = { + component: TreemapPane, + decorators: [withMockFileContext(mockFileState), withMockSelection(layout)], + parameters: { + layout: 'fullscreen', + }, +}; +export default meta; + +type Story = StoryObj; + +export const RootNode: Story = { + args: { + node: layout, + segments, + onSelectNode: (id: string) => console.log('select', id), + onHoverNode: (id: string | null) => console.log('hover', id), + }, + decorators: [ + (Story) => ( +
+ +
+ ), + ], +}; diff --git a/vortex-web/src/components/detail/TreemapPane.tsx b/vortex-web/src/components/detail/TreemapPane.tsx new file mode 100644 index 00000000000..9b91723eae3 --- /dev/null +++ b/vortex-web/src/components/detail/TreemapPane.tsx @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useMemo, useRef, useEffect, useCallback, useState } from 'react'; +import { hierarchy, treemap, treemapSquarify } from 'd3-hierarchy'; +import type { HierarchyRectangularNode } from 'd3-hierarchy'; +import type { LayoutTreeNode, SegmentMapEntry } from '../swimlane/types'; +import { + getNodeDisplayName, + getDtypeCategory, + collectSubtreeSegments, + DTYPE_COLORS, + formatBytes, +} from '../swimlane/utils'; +import { useTheme } from '../../contexts/ThemeContext'; + +interface TreemapPaneProps { + node: LayoutTreeNode; + segments: SegmentMapEntry[]; + onSelectNode: (nodeId: string) => void; + onHoverNode: (nodeId: string | null) => void; +} + +interface TreeNode { + name: string; + nodeId: string; + color: string; + bytes: number; + children?: TreeNode[]; +} + +type RectNode = HierarchyRectangularNode; + +/** Total buffer bytes for an array node subtree. */ +function arraySubtreeBytes(node: LayoutTreeNode): number { + const own = (node.bufferLengths ?? []).reduce((s, b) => s + b, 0); + const childBytes = node.children + .filter((c) => c.isArrayNode) + .reduce((s, c) => s + arraySubtreeBytes(c), 0); + return own + childBytes; +} + +function buildTree(node: LayoutTreeNode, segmentMap: Map): TreeNode { + const color = DTYPE_COLORS[getDtypeCategory(node.dtype)]; + const name = getNodeDisplayName(node); + + // For array nodes, size by buffer bytes; for layout nodes, by segment bytes. + // Layout-level treemaps skip array children to avoid eager expansion. + const isArray = node.isArrayNode ?? false; + const bytes = isArray + ? arraySubtreeBytes(node) + : collectSubtreeSegments(node).reduce( + (sum, id) => sum + (segmentMap.get(id)?.byteLength ?? 0), + 0, + ); + + // For layout nodes, skip array children of NON-flat layouts to avoid eager expansion. + // Flat layouts and array nodes show all their children (array tree is already fetched). + const isFlatOrArray = isArray || node.encoding === 'vortex.flat'; + const childrenToShow = isFlatOrArray + ? node.children + : node.children.filter((c) => !c.isArrayNode); + + if (childrenToShow.length === 0) { + return { name, nodeId: node.id, color, bytes: Math.max(bytes, 1) }; + } + + return { + name, + nodeId: node.id, + color, + bytes: Math.max(bytes, 1), + children: childrenToShow.map((c) => buildTree(c, segmentMap)), + }; +} + +function resolveThemeColors(choice: 'light' | 'dark' | 'system') { + let isDark: boolean; + if (choice === 'system') { + isDark = window.matchMedia('(prefers-color-scheme: dark)').matches; + } else { + isDark = choice === 'dark'; + } + return isDark + ? { fg: '#e4e4e8', dim: '#71717a', border: 'rgba(255,255,255,0.12)', highlight: '#2CB9D1' } + : { fg: '#18181b', dim: '#71717a', border: 'rgba(0,0,0,0.1)', highlight: '#2CB9D1' }; +} + +/** Find the deepest node containing point (px, py), preferring depth >= 1. */ +function hitTest(nodes: RectNode[], px: number, py: number): RectNode | null { + let best: RectNode | null = null; + for (const n of nodes) { + if (n.depth >= 1 && px >= n.x0 && px < n.x1 && py >= n.y0 && py < n.y1) { + if (!best || n.depth > best.depth) best = n; + } + } + return best; +} + +/** Walk up from a node to find its depth-1 ancestor. */ +function depth1Ancestor(n: RectNode): RectNode { + let cur = n; + while (cur.depth > 1 && cur.parent) cur = cur.parent; + return cur; +} + +/** Collect all nodeIds in a RectNode subtree. */ +function collectRectIds(n: RectNode): Set { + const ids = new Set(); + function walk(node: RectNode) { + ids.add(node.data.nodeId); + if (node.children) { + for (const c of node.children) walk(c); + } + } + walk(n); + return ids; +} + +export function TreemapPane({ node, segments, onSelectNode, onHoverNode }: TreemapPaneProps) { + const containerRef = useRef(null); + const svgRef = useRef(null); + const [size, setSize] = useState<{ w: number; h: number } | null>(null); + const [hoveredNodeId, setHoveredNodeId] = useState(null); + const [selectedNodeId, setSelectedNodeId] = useState(null); + + const segmentMap = useMemo(() => new Map(segments.map((s) => [s.index, s])), [segments]); + + const tree = useMemo(() => buildTree(node, segmentMap), [node, segmentMap]); + const { theme: themeChoice } = useTheme(); + const theme = useMemo(() => resolveThemeColors(themeChoice), [themeChoice]); + + useEffect(() => { + const el = containerRef.current; + if (!el) return; + const ro = new ResizeObserver(([entry]) => { + const { width, height } = entry.contentRect; + if (width > 0 && height > 0) setSize({ w: width, h: height }); + }); + ro.observe(el); + return () => ro.disconnect(); + }, []); + + // Reset local selection when the treemap root node changes. + useEffect(() => { + setSelectedNodeId(null); + }, [node.id]); + + const nodes = useMemo(() => { + if (!size) return []; + const root = hierarchy(tree) + .sum((d) => (d.children ? 0 : d.bytes)) + .sort((a, b) => (b.value ?? 0) - (a.value ?? 0)); + treemap() + .size([size.w, size.h]) + .tile(treemapSquarify) + .paddingTop(18) + .paddingInner(2) + .paddingOuter(1) + .round(true)(root); + return root.descendants(); + }, [tree, size]); + + // Set of nodeIds in the selected depth-1 subtree (for solid highlight). + const selectedSubtreeIds = useMemo>(() => { + if (!selectedNodeId) return new Set(); + const selected = nodes.find((n) => n.data.nodeId === selectedNodeId); + return selected ? collectRectIds(selected) : new Set(); + }, [selectedNodeId, nodes]); + + const handleMouseMove = useCallback( + (e: React.MouseEvent) => { + const svg = svgRef.current; + if (!svg) return; + const rect = svg.getBoundingClientRect(); + const px = e.clientX - rect.left; + const py = e.clientY - rect.top; + const hit = hitTest(nodes, px, py); + const nodeId = hit ? hit.data.nodeId : null; + setHoveredNodeId(nodeId); + onHoverNode(nodeId); + }, + [nodes, onHoverNode], + ); + + const handleMouseLeave = useCallback(() => { + setHoveredNodeId(null); + onHoverNode(null); + }, [onHoverNode]); + + const handleClick = useCallback( + (e: React.MouseEvent) => { + const svg = svgRef.current; + if (!svg) return; + const rect = svg.getBoundingClientRect(); + const px = e.clientX - rect.left; + const py = e.clientY - rect.top; + const hit = hitTest(nodes, px, py); + if (hit) { + e.stopPropagation(); + setSelectedNodeId(hit.data.nodeId); + } else { + setSelectedNodeId(null); + } + }, + [nodes], + ); + + const handleDoubleClick = useCallback( + (e: React.MouseEvent) => { + const svg = svgRef.current; + if (!svg) return; + const rect = svg.getBoundingClientRect(); + const px = e.clientX - rect.left; + const py = e.clientY - rect.top; + const hit = hitTest(nodes, px, py); + if (hit) { + e.stopPropagation(); + onSelectNode(hit.data.nodeId); + } + }, + [nodes, onSelectNode], + ); + + return ( +
+ {size && ( + + {nodes.map((n) => { + const w = n.x1 - n.x0; + const h = n.y1 - n.y0; + if (w < 1 || h < 1) return null; + + const isLeaf = !n.children || n.children.length === 0; + const d = n.data; + const isHovered = d.nodeId === hoveredNodeId; + const isSelected = selectedSubtreeIds.has(d.nodeId); + const maxChars = Math.floor(w / 6); + const label = + maxChars < 2 + ? '' + : d.name.length > maxChars + ? d.name.slice(0, maxChars - 1) + '\u2026' + : d.name; + + return ( + + + {n.depth === 1 && label && h > 14 && ( + + {label} + + )} + {n.depth === 1 && w > 50 && h > 28 && ( + + {formatBytes(d.bytes)} + + )} + + ); + })} + + )} +
+ ); +} diff --git a/vortex-web/src/components/explorer/DataPreview.tsx b/vortex-web/src/components/explorer/DataPreview.tsx new file mode 100644 index 00000000000..41e8efc9d81 --- /dev/null +++ b/vortex-web/src/components/explorer/DataPreview.tsx @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useEffect, useMemo, useState } from 'react'; +import { tableFromIPC } from 'apache-arrow'; +import { useSelection } from '../../contexts/SelectionContext'; +import { useVortexFile } from '../../contexts/VortexFileContext'; +import { parseArrayNodeId } from '../swimlane/utils'; +import { DataTable } from '../DataTable'; + +const ROW_LIMIT = 5000; + +function decodeArrow(ipcBytes: Uint8Array): { + columns: string[]; + rows: Record[]; +} { + const table = tableFromIPC(ipcBytes); + const columns = table.schema.fields.map((f) => f.name); + const rows: Record[] = []; + for (let i = 0; i < table.numRows; i++) { + const row: Record = {}; + for (const col of columns) { + row[col] = table.getChild(col)?.get(i) ?? null; + } + rows.push(row); + } + return { columns, rows }; +} + +export function DataPreview() { + const { state: selection } = useSelection(); + const { previewData, previewArrayData } = useVortexFile(); + const [ipcBytes, setIpcBytes] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const selectedNodeId = selection.selectedNodeId; + const isArrayNode = selection.selectedNode?.isArrayNode ?? false; + + useEffect(() => { + if (!selectedNodeId) { + setIpcBytes(null); + setError(null); + return; + } + + let cancelled = false; + setLoading(true); + setError(null); + + const fetchPromise = isArrayNode + ? (() => { + const { layoutNodeId, arrayPath } = parseArrayNodeId(selectedNodeId); + return previewArrayData(layoutNodeId, arrayPath, ROW_LIMIT); + })() + : previewData(selectedNodeId, ROW_LIMIT); + + fetchPromise + .then((bytes) => { + if (!cancelled) setIpcBytes(bytes); + }) + .catch((err) => { + if (!cancelled) setError(err instanceof Error ? err.message : String(err)); + }) + .finally(() => { + if (!cancelled) setLoading(false); + }); + + return () => { + cancelled = true; + }; + }, [selectedNodeId, isArrayNode, previewData, previewArrayData]); + + const decoded = useMemo(() => { + if (!ipcBytes) return null; + try { + return decodeArrow(ipcBytes); + } catch (err) { + console.error('Arrow decode error:', err); + return null; + } + }, [ipcBytes]); + + if (!selectedNodeId) { + return ( +
+ Select a node to preview data +
+ ); + } + + if (loading) { + return ( +
+ Loading preview… +
+ ); + } + + if (error) { + return ( +
+ {error} +
+ ); + } + + if (!decoded || decoded.rows.length === 0) { + return ( +
+ No data +
+ ); + } + + return ( + = ROW_LIMIT} + /> + ); +} diff --git a/vortex-web/src/components/explorer/ExplorerShell.stories.tsx b/vortex-web/src/components/explorer/ExplorerShell.stories.tsx new file mode 100644 index 00000000000..5e7cbbe339a --- /dev/null +++ b/vortex-web/src/components/explorer/ExplorerShell.stories.tsx @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import type { Meta, StoryObj } from '@storybook/react-vite'; +import { FileHeader } from './FileHeader'; +import { MainArea } from './MainArea'; +import { StatusBar } from './StatusBar'; +import { withMockFileContext, withMockSelection } from '../../storybook/decorators'; +import { ordersMock } from '../../mocks/layouts'; +import { generateSegments } from '../../mocks/segments'; +import { generateFileStructure } from '../../mocks/fileStructure'; +import type { VortexFileState } from '../../contexts/VortexFileContext'; + +const layout = ordersMock(); +const segments = generateSegments(layout, 12_400_000); +const fileStructure = generateFileStructure(segments, 12_400_000); + +const mockFileState: VortexFileState = { + fileName: 'orders.vortex', + fileSize: 12_400_000, + rowCount: 100_000, + version: 1, + dtype: + '{order_id=i64, is_active=bool, customer={id=i64, name=utf8}, items=list, amount=f64, metadata=struct, status=utf8}', + layoutTree: layout, + segments, + fileStructure, +}; + +/** Full page layout — mirrors App.tsx when a file is loaded */ +function ExplorerPage() { + return ( +
+ + + +
+ ); +} + +const meta: Meta = { + component: ExplorerPage, + parameters: { layout: 'fullscreen' }, + decorators: [withMockFileContext(mockFileState), withMockSelection(layout)], +}; +export default meta; + +type Story = StoryObj; + +export const Default: Story = {}; diff --git a/vortex-web/src/components/explorer/FileDropScreen.tsx b/vortex-web/src/components/explorer/FileDropScreen.tsx new file mode 100644 index 00000000000..14db31e0c03 --- /dev/null +++ b/vortex-web/src/components/explorer/FileDropScreen.tsx @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useState, useCallback, type DragEvent, type FormEvent } from 'react'; +import { ThemePicker } from '../ThemePicker'; + +interface FileDropScreenProps { + onFileLoaded: (file: File) => void; + loading: boolean; + error: string | null; +} + +export function FileDropScreen({ onFileLoaded, loading, error }: FileDropScreenProps) { + const [isDragging, setIsDragging] = useState(false); + const [url, setUrl] = useState(''); + const [fetchingUrl, setFetchingUrl] = useState(false); + const [urlError, setUrlError] = useState(null); + + const handleDragOver = useCallback((e: DragEvent) => { + e.preventDefault(); + setIsDragging(true); + }, []); + + const handleDragLeave = useCallback((e: DragEvent) => { + e.preventDefault(); + setIsDragging(false); + }, []); + + const handleDrop = useCallback( + (e: DragEvent) => { + e.preventDefault(); + setIsDragging(false); + const file = e.dataTransfer.files[0]; + if (file) onFileLoaded(file); + }, + [onFileLoaded], + ); + + const handleClick = useCallback(() => { + const input = document.createElement('input'); + input.type = 'file'; + input.accept = '.vortex,.vtx'; + input.onchange = () => { + const file = input.files?.[0]; + if (file) onFileLoaded(file); + }; + input.click(); + }, [onFileLoaded]); + + const handleUrlSubmit = useCallback( + async (e: FormEvent) => { + e.preventDefault(); + const trimmed = url.trim(); + if (!trimmed) return; + + setFetchingUrl(true); + setUrlError(null); + try { + const resp = await fetch(trimmed); + if (!resp.ok) throw new Error(`HTTP ${resp.status}: ${resp.statusText}`); + const blob = await resp.blob(); + const name = trimmed.split('/').pop() ?? 'remote.vortex'; + const file = new File([blob], name, { type: blob.type }); + onFileLoaded(file); + } catch (err) { + setUrlError(err instanceof Error ? err.message : String(err)); + } finally { + setFetchingUrl(false); + } + }, + [url, onFileLoaded], + ); + + const busy = loading || fetchingUrl; + + return ( +
+
+ +
+

+ Vortex Explorer +

+ +
+ {busy ? ( +

+ {fetchingUrl ? 'Fetching…' : 'Loading…'} +

+ ) : ( +
+

+ Drop a{' '} + + .vortex + {' '} + file here +

+

or click to browse

+
+ )} +
+ + {/* URL input */} +
e.stopPropagation()} + > + setUrl(e.target.value)} + placeholder="https://example.com/file.vortex" + disabled={busy} + className="flex-1 rounded border border-vortex-grey-light/40 dark:border-white/[0.08] bg-transparent px-3 py-1.5 font-mono text-sm text-vortex-fg-light dark:text-vortex-fg placeholder:text-vortex-grey-dark/40 focus:border-vortex-light-blue focus:outline-none disabled:opacity-50" + /> + +
+ + {(error || urlError) && ( +

{urlError || error}

+ )} +
+ ); +} diff --git a/vortex-web/src/components/explorer/FileHeader.stories.tsx b/vortex-web/src/components/explorer/FileHeader.stories.tsx new file mode 100644 index 00000000000..f01be01db41 --- /dev/null +++ b/vortex-web/src/components/explorer/FileHeader.stories.tsx @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import type { Meta, StoryObj } from '@storybook/react-vite'; +import { FileHeader } from './FileHeader'; +import { withMockFileContext } from '../../storybook/decorators'; +import { ordersMock } from '../../mocks/layouts'; +import { generateSegments } from '../../mocks/segments'; +import { generateFileStructure } from '../../mocks/fileStructure'; +import type { VortexFileState } from '../../contexts/VortexFileContext'; + +const layout = ordersMock(); +const segments = generateSegments(layout, 12_400_000); +const fileStructure = generateFileStructure(segments, 12_400_000); + +const mockFileState: VortexFileState = { + fileName: 'orders.vortex', + fileSize: 12_400_000, + rowCount: 100_000, + version: 1, + dtype: '{order_id=i64, ...}', + layoutTree: layout, + segments, + fileStructure, +}; + +const meta: Meta = { + component: FileHeader, + decorators: [withMockFileContext(mockFileState)], +}; +export default meta; + +type Story = StoryObj; + +export const Default: Story = {}; diff --git a/vortex-web/src/components/explorer/FileHeader.tsx b/vortex-web/src/components/explorer/FileHeader.tsx new file mode 100644 index 00000000000..9dd2f1335db --- /dev/null +++ b/vortex-web/src/components/explorer/FileHeader.tsx @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useVortexFile } from '../../contexts/VortexFileContext'; +import { ThemePicker } from '../ThemePicker'; + +interface FileHeaderProps { + onClose: () => void; +} + +export function FileHeader({ onClose }: FileHeaderProps) { + const file = useVortexFile(); + + return ( +
+ + {file.fileName} + + + v{file.version} + +
+ + +
+
+ ); +} diff --git a/vortex-web/src/components/explorer/FileMap.stories.tsx b/vortex-web/src/components/explorer/FileMap.stories.tsx new file mode 100644 index 00000000000..5f4bebf4798 --- /dev/null +++ b/vortex-web/src/components/explorer/FileMap.stories.tsx @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import type { Meta, StoryObj } from '@storybook/react-vite'; +import { FileMap } from './FileMap'; +import { withMockFileContext, withMockSelection } from '../../storybook/decorators'; +import { ordersMock } from '../../mocks/layouts'; +import { generateSegments } from '../../mocks/segments'; +import { generateFileStructure } from '../../mocks/fileStructure'; +import type { VortexFileState } from '../../contexts/VortexFileContext'; + +const layout = ordersMock(); +const segments = generateSegments(layout, 12_400_000); +const fileStructure = generateFileStructure(segments, 12_400_000); + +const mockFileState: VortexFileState = { + fileName: 'orders.vortex', + fileSize: 12_400_000, + rowCount: 100_000, + version: 1, + dtype: '{order_id=i64, ...}', + layoutTree: layout, + segments, + fileStructure, +}; + +const meta: Meta = { + component: FileMap, + decorators: [withMockFileContext(mockFileState), withMockSelection(layout)], +}; +export default meta; + +type Story = StoryObj; + +export const Default: Story = { + args: { height: 40 }, +}; + +export const Tall: Story = { + args: { height: 80 }, +}; diff --git a/vortex-web/src/components/explorer/FileMap.tsx b/vortex-web/src/components/explorer/FileMap.tsx new file mode 100644 index 00000000000..4f314c00f3f --- /dev/null +++ b/vortex-web/src/components/explorer/FileMap.tsx @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useRef, useEffect, useMemo, useState, useCallback } from 'react'; +import { collectSubtreeSegments } from '../swimlane/utils'; +import { useVortexFile } from '../../contexts/VortexFileContext'; +import { useSelection } from '../../contexts/SelectionContext'; +import type { SegmentMapEntry } from '../swimlane/types'; + +/** + * Pixel-level byte map of the file. Height matches one line of text (1lh). + * + * Three rendering tiers: + * 1. Selected segment (if any): bright blue, full opacity. + * 2. Other subtree segments: dim blue. + * 3. Everything else: neutral base. + */ +export function FileMap() { + const file = useVortexFile(); + const { state: selection } = useSelection(); + const canvasRef = useRef(null); + const containerRef = useRef(null); + const [crosshair, setCrosshair] = useState(null); + + // Segments belonging to the selected subtree, sorted by byte offset + const subtreeSegments = useMemo((): SegmentMapEntry[] => { + if (!selection.selectedNode) return []; + const segIds = new Set(collectSubtreeSegments(selection.selectedNode)); + return file.segments + .filter((s) => segIds.has(s.index)) + .sort((a, b) => a.byteOffset - b.byteOffset); + }, [selection.selectedNode, file.segments]); + + // The single selected segment (if any), sorted for scan + const focusedSegment = useMemo((): SegmentMapEntry | null => { + if (selection.selectedSegmentIndex == null) return null; + return file.segments.find((s) => s.index === selection.selectedSegmentIndex) ?? null; + }, [selection.selectedSegmentIndex, file.segments]); + + // Segments for hover preview (from hovered node or hovered segment) + const hoverSegments = useMemo((): SegmentMapEntry[] => { + if (selection.hoveredNode) { + const segIds = new Set(collectSubtreeSegments(selection.hoveredNode)); + return file.segments + .filter((s) => segIds.has(s.index)) + .sort((a, b) => a.byteOffset - b.byteOffset); + } + if (selection.hoveredSegmentIndex != null) { + const seg = file.segments.find((s) => s.index === selection.hoveredSegmentIndex); + return seg ? [seg] : []; + } + return []; + }, [selection.hoveredNode, selection.hoveredSegmentIndex, file.segments]); + + // Paint + useEffect(() => { + const canvas = canvasRef.current; + const container = containerRef.current; + if (!canvas || !container) return; + + const width = container.clientWidth; + const height = container.clientHeight; + if (width === 0 || height === 0) return; + + const dpr = window.devicePixelRatio || 1; + canvas.width = width * dpr; + canvas.height = height * dpr; + canvas.style.width = `${width}px`; + canvas.style.height = `${height}px`; + + const ctx = canvas.getContext('2d'); + if (!ctx) return; + + const fileSize = file.fileStructure.fileSize; + if (fileSize === 0) return; + + const imgData = ctx.createImageData(width * dpr, height * dpr); + const data = imgData.data; + const imgW = imgData.width; + const imgH = imgData.height; + + const dark = + document.documentElement.classList.contains('dark') || + window.matchMedia('(prefers-color-scheme: dark)').matches; + + // Base: neutral + const baseR = dark ? 50 : 210; + const baseG = dark ? 50 : 210; + const baseB = dark ? 50 : 210; + const baseA = dark ? 80 : 100; + + // Highlight color (vortex light blue #2CB9D1) + const hlR = 44, + hlG = 185, + hlB = 209; + + const hasSubtree = subtreeSegments.length > 0; + const hasHover = hoverSegments.length > 0; + const bytesPerPixel = fileSize / width; + + let segIdx = 0; + let hoverIdx = 0; + + for (let px = 0; px < width; px++) { + const byteStart = px * bytesPerPixel; + const byteEnd = (px + 1) * bytesPerPixel; + + // Advance subtree scan + while ( + segIdx < subtreeSegments.length && + subtreeSegments[segIdx].byteOffset + subtreeSegments[segIdx].byteLength <= byteStart + ) { + segIdx++; + } + + // Advance hover scan + while ( + hoverIdx < hoverSegments.length && + hoverSegments[hoverIdx].byteOffset + hoverSegments[hoverIdx].byteLength <= byteStart + ) { + hoverIdx++; + } + + // Check subtree hit + let isSubtree = false; + if (hasSubtree) { + for (let s = segIdx; s < subtreeSegments.length; s++) { + if (subtreeSegments[s].byteOffset >= byteEnd) break; + isSubtree = true; + break; + } + } + + // Check hover hit + let isHovered = false; + if (hasHover) { + for (let h = hoverIdx; h < hoverSegments.length; h++) { + if (hoverSegments[h].byteOffset >= byteEnd) break; + isHovered = true; + break; + } + } + + // Check focused segment hit + let isFocused = false; + if (focusedSegment) { + const fStart = focusedSegment.byteOffset; + const fEnd = fStart + focusedSegment.byteLength; + if (fStart < byteEnd && fEnd > byteStart) { + isFocused = true; + } + } + + let r: number, g: number, b: number, a: number; + if (isHovered) { + // Hover always wins — bright highlight + r = hlR; + g = hlG; + b = hlB; + a = 255; + } else if (isFocused) { + r = hlR; + g = hlG; + b = hlB; + a = 255; + } else if (isSubtree && !focusedSegment) { + // No segment selected: all subtree segments are bright + r = hlR; + g = hlG; + b = hlB; + a = dark ? 140 : 160; + } else if (isSubtree) { + // A segment is selected: other subtree segments are dimmed + r = hlR; + g = hlG; + b = hlB; + a = dark ? 90 : 110; + } else { + r = baseR; + g = baseG; + b = baseB; + a = baseA; + } + + const dprPxStart = Math.round(px * dpr); + const dprPxEnd = Math.round((px + 1) * dpr); + + for (let dpx = dprPxStart; dpx < dprPxEnd && dpx < imgW; dpx++) { + for (let dy = 0; dy < imgH; dy++) { + const idx = (dy * imgW + dpx) * 4; + data[idx] = r; + data[idx + 1] = g; + data[idx + 2] = b; + data[idx + 3] = a; + } + } + } + + ctx.putImageData(imgData, 0, 0); + }, [file, subtreeSegments, focusedSegment, hoverSegments]); + + // Resize observer + useEffect(() => { + const container = containerRef.current; + if (!container) return; + const observer = new ResizeObserver(() => { + const canvas = canvasRef.current; + if (canvas) canvas.dispatchEvent(new Event('resize')); + }); + observer.observe(container); + return () => observer.disconnect(); + }, []); + + const handleMouseMove = useCallback((e: React.MouseEvent) => { + const container = containerRef.current; + if (!container) return; + const rect = container.getBoundingClientRect(); + setCrosshair(e.clientX - rect.left); + }, []); + + const handleMouseLeave = useCallback(() => setCrosshair(null), []); + + return ( +
+ + {crosshair !== null && ( +
+ )} +
+ ); +} diff --git a/vortex-web/src/components/explorer/MainArea.tsx b/vortex-web/src/components/explorer/MainArea.tsx new file mode 100644 index 00000000000..a8029e1c0e8 --- /dev/null +++ b/vortex-web/src/components/explorer/MainArea.tsx @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useCallback, useRef, useState } from 'react'; +import { TreePanel } from './TreePanel'; +import { FileMap } from './FileMap'; +import { DataPreview } from './DataPreview'; +import { DetailPanel } from '../detail/DetailPanel'; + +const MIN_PANEL_HEIGHT = 120; +const DEFAULT_PREVIEW_HEIGHT = 200; + +/** + * Main explorer area: tree panel (left) | detail + filemap + preview (right). + * + * The preview panel at the bottom is vertically resizable via a drag handle. + */ +export function MainArea() { + const [previewHeight, setPreviewHeight] = useState(DEFAULT_PREVIEW_HEIGHT); + const dragging = useRef(false); + const startY = useRef(0); + const startHeight = useRef(0); + const containerRef = useRef(null); + + const onPointerDown = useCallback( + (e: React.PointerEvent) => { + e.preventDefault(); + dragging.current = true; + startY.current = e.clientY; + startHeight.current = previewHeight; + (e.target as HTMLElement).setPointerCapture(e.pointerId); + }, + [previewHeight], + ); + + const onPointerMove = useCallback((e: React.PointerEvent) => { + if (!dragging.current) return; + const containerHeight = containerRef.current?.clientHeight ?? 600; + const maxPreview = containerHeight - MIN_PANEL_HEIGHT; + const delta = startY.current - e.clientY; + const next = Math.min(maxPreview, Math.max(MIN_PANEL_HEIGHT, startHeight.current + delta)); + setPreviewHeight(next); + }, []); + + const onPointerUp = useCallback(() => { + dragging.current = false; + }, []); + + return ( +
+ {/* Left: tree panel — full height, fixed width */} +
+ +
+ + {/* Right: detail pane, file map, data preview — stacked vertically */} +
+ {/* Detail pane — fills available vertical space, scrolls internally */} + + + {/* File map strip */} +
+ +
+ + {/* Resize handle */} +
+ + {/* Data preview — resizable bottom section */} +
+ +
+
+
+ ); +} diff --git a/vortex-web/src/components/explorer/StatusBar.tsx b/vortex-web/src/components/explorer/StatusBar.tsx new file mode 100644 index 00000000000..95adf129795 --- /dev/null +++ b/vortex-web/src/components/explorer/StatusBar.tsx @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useMemo } from 'react'; +import { useVortexFile } from '../../contexts/VortexFileContext'; +import { useSelection } from '../../contexts/SelectionContext'; +import { formatBytes, formatRowCount, collectSubtreeSegments } from '../swimlane/utils'; + +export function StatusBar() { + const file = useVortexFile(); + const { state: selection } = useSelection(); + + // Show hovered node stats if hovering, otherwise selected node stats. + const activeNode = selection.hoveredNode ?? selection.selectedNode; + + const selectionStats = useMemo(() => { + if (!activeNode) return null; + + const segIds = new Set(collectSubtreeSegments(activeNode)); + const reachable = file.segments.filter((s) => segIds.has(s.index)); + const totalBytes = reachable.reduce((sum, s) => sum + s.byteLength, 0); + const pct = + file.fileStructure.fileSize > 0 ? (totalBytes / file.fileStructure.fileSize) * 100 : 0; + + return { + rows: activeNode.rowCount, + segments: reachable.length, + bytes: totalBytes, + pct, + }; + }, [activeNode, file.segments, file.fileStructure.fileSize]); + + return ( +
+ {/* File-level stats — left */} +
+ + + +
+ +
+ + {/* Selection stats — right */} + {selectionStats && ( +
+ + + +
+ )} +
+ ); +} + +function Stat({ label, value }: { label: string; value: string }) { + return ( + + {label}: + {value} + + ); +} diff --git a/vortex-web/src/components/explorer/TimelineTab.tsx b/vortex-web/src/components/explorer/TimelineTab.tsx new file mode 100644 index 00000000000..2df69fc6515 --- /dev/null +++ b/vortex-web/src/components/explorer/TimelineTab.tsx @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useState } from 'react'; +import { useVortexFile } from '../../contexts/VortexFileContext'; +import { useSelection } from '../../contexts/SelectionContext'; +import { LayoutSwimlane } from '../swimlane/LayoutSwimlane'; + +export function TimelineTab() { + const file = useVortexFile(); + const { state: selection, selectNode } = useSelection(); + const [mode, setMode] = useState<'schema' | 'layout'>('schema'); + + return ( +
+ {/* Mode toggle */} +
+ + +
+ + +
+ ); +} diff --git a/vortex-web/src/components/explorer/TreePanel.tsx b/vortex-web/src/components/explorer/TreePanel.tsx new file mode 100644 index 00000000000..1452b0ad253 --- /dev/null +++ b/vortex-web/src/components/explorer/TreePanel.tsx @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useState, useCallback, useMemo, useEffect, useRef } from 'react'; +import { useVortexFile } from '../../contexts/VortexFileContext'; +import { useSelection } from '../../contexts/SelectionContext'; +import { + flattenTree, + filterTreeBySearch, + findPathToNode, + isFlatLayout, + findNodeById, +} from '../swimlane/utils'; +import { TreeRow } from '../swimlane/TreeRow'; +import { TreeSearch } from '../swimlane/TreeSearch'; + +type TreeMode = 'schema' | 'layout'; + +export function TreePanel() { + const file = useVortexFile(); + const { state: selection, selectNode, hoverNode } = useSelection(); + const [mode, setMode] = useState('schema'); + const [expanded, setExpanded] = useState>(() => new Set(['root'])); + const [searchQuery, setSearchQuery] = useState(''); + + // Auto-expand ancestors so the selected node is visible, including synthetic group nodes. + useEffect(() => { + if (!selection.selectedNodeId) return; + const path = findPathToNode(file.layoutTree, selection.selectedNodeId); + if (path.length === 0) return; + + setExpanded((prev) => { + let next = new Set(prev); + for (const node of path) next.add(node.id); + + // Iteratively expand group nodes that contain the target until it's visible. + for (let attempt = 0; attempt < 5; attempt++) { + const rows = flattenTree(file.layoutTree, next, null, mode); + if (rows.some((r) => r.node.id === selection.selectedNodeId)) break; + + // Find group rows whose grouped children include the target. + let changed = false; + for (const row of rows) { + if (row.displayKind === 'group' && row.groupedChildren) { + if (row.groupedChildren.some((c) => c.id === selection.selectedNodeId)) { + next = new Set(next); + next.add(row.node.id); + changed = true; + } + } + } + if (!changed) break; + } + + return next; + }); + }, [selection.selectedNodeId, file.layoutTree, mode]); + + // Scroll the selected node into view only when the selection changes. + const scrollContainerRef = useRef(null); + const lastScrolledTo = useRef(null); + useEffect(() => { + if (!selection.selectedNodeId || selection.selectedNodeId === lastScrolledTo.current) return; + lastScrolledTo.current = selection.selectedNodeId; + // Defer to let the DOM update after expansion. + requestAnimationFrame(() => { + const container = scrollContainerRef.current; + if (!container) return; + const el = container.querySelector( + `[data-node-id="${CSS.escape(selection.selectedNodeId!)}"]`, + ); + el?.scrollIntoView({ block: 'center', behavior: 'smooth' }); + }); + }, [selection.selectedNodeId, expanded]); + + const allRows = useMemo( + () => flattenTree(file.layoutTree, expanded, null, mode), + [file.layoutTree, expanded, mode], + ); + + const visibleRows = useMemo( + () => filterTreeBySearch(allRows, searchQuery, file.layoutTree), + [allRows, searchQuery, file.layoutTree], + ); + + // When a flat layout node is expanded, lazily attach array encoding children. + // Track which nodes we've already requested to avoid re-triggering on tree updates. + const expandedArrayRequests = useRef(new Set()); + useEffect(() => { + for (const id of expanded) { + if (expandedArrayRequests.current.has(id)) continue; + const node = findNodeById(file.layoutTree, id); + if (node && isFlatLayout(node) && !node.children.some((c) => c.isArrayNode)) { + expandedArrayRequests.current.add(id); + file.expandArrayTree(id); + } + } + }, [expanded, file]); + + const toggleExpanded = useCallback((id: string) => { + setExpanded((prev) => { + const next = new Set(prev); + if (next.has(id)) next.delete(id); + else next.add(id); + return next; + }); + }, []); + + const handleNodeClick = useCallback( + (nodeId: string) => { + selectNode(selection.selectedNodeId === nodeId ? null : nodeId); + }, + [selection.selectedNodeId, selectNode], + ); + + return ( +
+ {/* Header: mode toggle + search */} +
+ +
+ +
+
+ + {/* Tree rows */} +
+ {visibleRows.map((row) => ( + toggleExpanded(row.node.id)} + onSelect={() => handleNodeClick(row.node.id)} + onHover={hoverNode} + /> + ))} +
+
+ ); +} + +/** Subtle segmented toggle for Schema / Layout mode */ +function ModeToggle({ mode, onChange }: { mode: TreeMode; onChange: (m: TreeMode) => void }) { + return ( +
+ {(['schema', 'layout'] as const).map((m) => ( + + ))} +
+ ); +} diff --git a/vortex-web/src/components/swimlane/AxisBar.tsx b/vortex-web/src/components/swimlane/AxisBar.tsx new file mode 100644 index 00000000000..27ad96081ff --- /dev/null +++ b/vortex-web/src/components/swimlane/AxisBar.tsx @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useMemo } from 'react'; +import { formatRowCount } from './utils'; + +interface AxisBarProps { + totalRows: number; + swimlaneMinWidth: number; + rulerPosition: { x: number; row: number } | null; + scrollLeft: number; + containerWidth: number; + axisRef: React.RefObject; +} + +export function AxisBar({ + totalRows, + swimlaneMinWidth, + rulerPosition, + scrollLeft, + containerWidth, + axisRef, +}: AxisBarProps) { + const axisTicks = useMemo(() => { + const ticks = []; + const step = totalRows / 5; + for (let i = 0; i <= 5; i++) { + ticks.push(Math.round(i * step)); + } + return ticks; + }, [totalRows]); + + return ( +
+
+ {axisTicks.map((tick) => ( +
+ {formatRowCount(tick)} +
+ ))} +
+ + {rulerPosition && ( +
+ {formatRowCount(rulerPosition.row)} +
+ )} +
+ ); +} diff --git a/vortex-web/src/components/swimlane/DtypeLegend.tsx b/vortex-web/src/components/swimlane/DtypeLegend.tsx new file mode 100644 index 00000000000..11ca38292b9 --- /dev/null +++ b/vortex-web/src/components/swimlane/DtypeLegend.tsx @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { DTYPE_CATEGORIES, DTYPE_COLORS } from './styles'; + +export function DtypeLegend() { + return ( +
+ {DTYPE_CATEGORIES.map((cat) => ( +
+
+ {cat} +
+ ))} +
+ ); +} diff --git a/vortex-web/src/components/swimlane/LayoutSwimlane.stories.tsx b/vortex-web/src/components/swimlane/LayoutSwimlane.stories.tsx new file mode 100644 index 00000000000..deacacab73d --- /dev/null +++ b/vortex-web/src/components/swimlane/LayoutSwimlane.stories.tsx @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useState } from 'react'; +import type { Meta, StoryObj } from '@storybook/react-vite'; +import { LayoutSwimlane } from './LayoutSwimlane'; +import { ordersMock, simpleMock, wideMock, deepMock, heavyChunksMock } from '../../mocks'; + +const meta: Meta = { + component: LayoutSwimlane, + parameters: { + layout: 'fullscreen', + }, + decorators: [ + (Story) => ( +
+ +
+ ), + ], +}; +export default meta; + +type Story = StoryObj; + +const ordersLayout = ordersMock(); + +export const Orders: Story = { + args: { + layout: ordersLayout, + totalRows: 100000, + defaultExpanded: ['root', 'root.customer', 'root.customer.id', 'root.status'], + height: 400, + }, +}; + +export const SchemaMode: Story = { + args: { + layout: ordersLayout, + totalRows: 100000, + mode: 'schema', + defaultExpanded: ['root', 'root.customer'], + height: 400, + }, +}; + +export const LayoutMode: Story = { + args: { + layout: ordersLayout, + totalRows: 100000, + mode: 'layout', + defaultExpanded: ['root', 'root.customer', 'root.customer.id', 'root.status'], + height: 400, + }, +}; + +export const SingleFlat: Story = { + args: { + layout: simpleMock(), + totalRows: 10000, + height: 100, + }, +}; + +export const WideSchema: Story = { + args: { + layout: wideMock(), + totalRows: 50000, + mode: 'schema', + defaultExpanded: ['root'], + height: 400, + }, +}; + +export const DeepNesting: Story = { + args: { + layout: deepMock(), + totalRows: 25000, + mode: 'schema', + defaultExpanded: ['root', 'root.user', 'root.user.profile', 'root.user.profile.address'], + height: 400, + }, +}; + +export const HeavyChunks: Story = { + args: { + layout: heavyChunksMock(), + totalRows: 1000000, + mode: 'layout', + defaultExpanded: ['root', 'root.values'], + height: 400, + }, +}; + +/** Interactive story demonstrating controlled selection */ +export const WithSelection: StoryObj = { + render: () => { + const [selectedId, setSelectedId] = useState(null); + return ( +
+
Selected: {selectedId ?? 'none'}
+ +
+ ); + }, +}; diff --git a/vortex-web/src/components/swimlane/LayoutSwimlane.tsx b/vortex-web/src/components/swimlane/LayoutSwimlane.tsx new file mode 100644 index 00000000000..65d095e9a53 --- /dev/null +++ b/vortex-web/src/components/swimlane/LayoutSwimlane.tsx @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useState, useRef, useCallback, useMemo, useEffect } from 'react'; +import type { LayoutTreeNode, FlattenedRow } from './types'; +import { flattenTree, filterTreeBySearch } from './utils'; +import { ROW_HEIGHT, TREE_WIDTH, DEFAULT_SWIMLANE_MIN_WIDTH } from './styles'; +import { TreeRow } from './TreeRow'; +import { TreeSearch } from './TreeSearch'; +import { SwimlaneBar } from './SwimlaneBar'; +import { AxisBar } from './AxisBar'; +import { Tooltip } from './Tooltip'; + +export interface LayoutSwimlaneProps { + /** The root layout tree node to visualize */ + layout: LayoutTreeNode; + /** Total number of rows in the dataset */ + totalRows: number; + /** Initially expanded node IDs */ + defaultExpanded?: string[]; + /** Currently selected node ID (controlled) */ + selectedNodeId?: string | null; + /** Callback when a tree/bar node is clicked */ + onNodeSelect?: (nodeId: string | null) => void; + /** Display mode: 'schema' shows logical columns, 'layout' shows full layout tree */ + mode?: 'schema' | 'layout'; + /** Minimum width of the swimlane panel */ + swimlaneMinWidth?: number; + /** Height of the scrollable area */ + height?: number; +} + +export function LayoutSwimlane({ + layout, + totalRows, + defaultExpanded = [], + selectedNodeId = null, + onNodeSelect, + mode = 'schema', + swimlaneMinWidth = DEFAULT_SWIMLANE_MIN_WIDTH, + height, +}: LayoutSwimlaneProps) { + const [expanded, setExpanded] = useState>(() => new Set(defaultExpanded)); + const [searchQuery, setSearchQuery] = useState(''); + const [tooltip, setTooltip] = useState<{ + node: LayoutTreeNode; + position: { x: number; y: number }; + } | null>(null); + const [rulerPosition, setRulerPosition] = useState<{ x: number; row: number } | null>(null); + + const treeScrollRef = useRef(null); + const swimlaneScrollRef = useRef(null); + const swimlanePanelRef = useRef(null); + const axisRef = useRef(null); + + // Flatten the tree + const allRows = useMemo( + () => flattenTree(layout, expanded, null, mode), + [layout, expanded, mode], + ); + + const visibleRows = useMemo( + () => filterTreeBySearch(allRows, searchQuery, layout), + [allRows, searchQuery, layout], + ); + + const toggleExpanded = useCallback((id: string) => { + setExpanded((prev) => { + const next = new Set(prev); + if (next.has(id)) next.delete(id); + else next.add(id); + return next; + }); + }, []); + + const handleNodeClick = useCallback( + (nodeId: string) => { + onNodeSelect?.(selectedNodeId === nodeId ? null : nodeId); + }, + [selectedNodeId, onNodeSelect], + ); + + const handleTooltip = useCallback( + (node: LayoutTreeNode | null, position: { x: number; y: number }) => { + setTooltip(node ? { node, position } : null); + }, + [], + ); + + // Sync vertical scroll between tree and swimlane + useEffect(() => { + const tree = treeScrollRef.current; + const swimlane = swimlaneScrollRef.current; + if (!tree || !swimlane) return; + + let syncing = false; + const syncScroll = (source: HTMLDivElement, target: HTMLDivElement) => () => { + if (syncing) return; + syncing = true; + target.scrollTop = source.scrollTop; + syncing = false; + }; + + const treeHandler = syncScroll(tree, swimlane); + const swimlaneHandler = syncScroll(swimlane, tree); + tree.addEventListener('scroll', treeHandler); + swimlane.addEventListener('scroll', swimlaneHandler); + return () => { + tree.removeEventListener('scroll', treeHandler); + swimlane.removeEventListener('scroll', swimlaneHandler); + }; + }, []); + + // Sync horizontal scroll between swimlane and axis + useEffect(() => { + const swimlane = swimlaneScrollRef.current; + const axis = axisRef.current; + if (!swimlane || !axis) return; + + const handleScroll = () => { + axis.style.transform = `translateX(-${swimlane.scrollLeft}px)`; + }; + swimlane.addEventListener('scroll', handleScroll); + return () => swimlane.removeEventListener('scroll', handleScroll); + }, []); + + // Ruler mouse tracking + const handleSwimlaneMouseMove = useCallback( + (e: React.MouseEvent) => { + const panel = swimlanePanelRef.current; + if (!panel) return; + const rect = panel.getBoundingClientRect(); + const x = e.clientX - rect.left; + const panelWidth = panel.offsetWidth; + if (x >= 0 && x <= panelWidth) { + const rowNum = (x / panelWidth) * totalRows; + setRulerPosition({ x, row: Math.max(0, Math.min(totalRows, rowNum)) }); + } + }, + [totalRows], + ); + + const handleSwimlaneMouseLeave = useCallback(() => setRulerPosition(null), []); + + const contentHeight = visibleRows.length * ROW_HEIGHT; + + return ( +
+ {/* Tree + Swimlane */} +
+ {/* Tree panel */} +
+ +
+
+ {visibleRows.map((row) => ( + toggleExpanded(row.node.id)} + onSelect={() => handleNodeClick(row.node.id)} + /> + ))} +
+
+
+ + {/* Swimlane panel */} +
+
+ {visibleRows.map((row) => ( + + ))} + + {rulerPosition && ( +
+ )} +
+
+
+ + {/* Axis */} +
+
+ +
+ + {tooltip && } +
+ ); +} + +/** A single swimlane row — just height + bar positioning, no decoration */ +function SwimlaneRow({ + row, + totalRows, + onHover, +}: { + row: FlattenedRow; + totalRows: number; + onHover: (node: LayoutTreeNode | null, position: { x: number; y: number }) => void; +}) { + return ( +
+ {row.displayKind !== 'hiddenIndicator' && ( + + )} +
+ ); +} + +export default LayoutSwimlane; diff --git a/vortex-web/src/components/swimlane/SplitRegion.tsx b/vortex-web/src/components/swimlane/SplitRegion.tsx new file mode 100644 index 00000000000..b882efb2506 --- /dev/null +++ b/vortex-web/src/components/swimlane/SplitRegion.tsx @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import React, { useState } from 'react'; +import type { Split } from './types'; +import { MIN_LABEL_WIDTH } from './styles'; + +interface SplitRegionProps { + split: Split; + totalRows: number; + swimlaneWidth: number; + isSelected: boolean; + onClick: (e: React.MouseEvent) => void; +} + +export function SplitRegion({ + split, + totalRows, + swimlaneWidth, + isSelected, + onClick, +}: SplitRegionProps) { + const [isHovered, setIsHovered] = useState(false); + + const left = (split.rowRange[0] / totalRows) * 100; + const width = ((split.rowRange[1] - split.rowRange[0]) / totalRows) * 100; + const widthPx = ((split.rowRange[1] - split.rowRange[0]) / totalRows) * swimlaneWidth; + const showLabel = widthPx >= MIN_LABEL_WIDTH || isSelected || isHovered; + + return ( +
setIsHovered(true)} + onMouseLeave={() => setIsHovered(false)} + > +
+ {split.id} +
+
+ ); +} diff --git a/vortex-web/src/components/swimlane/SwimlaneBar.tsx b/vortex-web/src/components/swimlane/SwimlaneBar.tsx new file mode 100644 index 00000000000..6c0beef5eb2 --- /dev/null +++ b/vortex-web/src/components/swimlane/SwimlaneBar.tsx @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import React from 'react'; +import type { LayoutTreeNode, FlattenedRow } from './types'; +import { getDtypeCategory, getNodeRowRange } from './utils'; +import { DTYPE_COLORS, getEncodingStyle } from './styles'; + +interface SwimlaneBarProps { + row: FlattenedRow; + totalRows: number; + onHover: (node: LayoutTreeNode | null, position: { x: number; y: number }) => void; +} + +export function SwimlaneBar({ row, totalRows, onHover }: SwimlaneBarProps) { + const { node, displayKind, groupedChildren } = row; + const isLeaf = node.children.length === 0; + const isGroup = displayKind === 'group'; + const style = getEncodingStyle(node.encoding); + + // Group nodes: render rolled-up bars from each grouped child + if (isGroup && groupedChildren) { + return ( + <> + {groupedChildren.map((child) => { + const range = getNodeRowRange(child); + const left = (range[0] / totalRows) * 100; + const width = ((range[1] - range[0]) / totalRows) * 100; + const dtypeCat = getDtypeCategory(child.dtype); + const dtypeColor = DTYPE_COLORS[dtypeCat]; + return ( +
+ ); + })} + + ); + } + + const rowRange = getNodeRowRange(node); + const left = (rowRange[0] / totalRows) * 100; + const width = ((rowRange[1] - rowRange[0]) / totalRows) * 100; + + let barClasses = 'absolute top-[3px] bottom-[3px] rounded transition-[filter] duration-100'; + const barStyle: React.CSSProperties = { + left: `calc(${left}% + 1px)`, + width: `calc(${width}% - 3px)`, + }; + + if (isLeaf) { + const dtypeCat = getDtypeCategory(node.dtype); + const dtypeColor = DTYPE_COLORS[dtypeCat]; + barStyle.backgroundColor = `${dtypeColor}40`; + barStyle.border = `1.5px solid ${dtypeColor}`; + barClasses += ' cursor-pointer'; + } else { + barStyle.border = `1.5px solid ${style.color}40`; + } + + const handleMouseEnter = (e: React.MouseEvent) => { + if (isLeaf) { + (e.currentTarget as HTMLDivElement).style.filter = 'brightness(1.15)'; + onHover(node, { x: e.clientX, y: e.clientY }); + } + }; + + const handleMouseMove = (e: React.MouseEvent) => { + if (isLeaf) { + onHover(node, { x: e.clientX, y: e.clientY }); + } + }; + + const handleMouseLeave = (e: React.MouseEvent) => { + if (isLeaf) { + (e.currentTarget as HTMLDivElement).style.filter = ''; + onHover(null, { x: 0, y: 0 }); + } + }; + + return ( +
+ ); +} diff --git a/vortex-web/src/components/swimlane/Tooltip.tsx b/vortex-web/src/components/swimlane/Tooltip.tsx new file mode 100644 index 00000000000..45aa1c78600 --- /dev/null +++ b/vortex-web/src/components/swimlane/Tooltip.tsx @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import type { LayoutTreeNode } from './types'; +import { + getDtypeCategory, + formatBytes, + getNodeDisplayName, + getNodeRowRange, + shortEncoding, +} from './utils'; +import { DTYPE_COLORS } from './styles'; + +interface TooltipProps { + node: LayoutTreeNode; + position: { x: number; y: number }; +} + +export function Tooltip({ node, position }: TooltipProps) { + const rowRange = getNodeRowRange(node); + const rows = rowRange[1] - rowRange[0]; + const dtypeCat = getDtypeCategory(node.dtype); + const dtypeColor = DTYPE_COLORS[dtypeCat]; + const name = getNodeDisplayName(node); + + return ( +
+
+ {name} + + {dtypeCat} + +
+
+ rows + {rows.toLocaleString()} + {node.dtype && ( + <> + dtype + {node.dtype} + + )} + {node.encoding && ( + <> + encoding + + {shortEncoding(node.encoding)} + + + )} + {node.metadataBytes > 0 && ( + <> + metadata + + {formatBytes(node.metadataBytes)} + + + )} +
+
+ ); +} diff --git a/vortex-web/src/components/swimlane/TreeRow.tsx b/vortex-web/src/components/swimlane/TreeRow.tsx new file mode 100644 index 00000000000..8c76344f4a4 --- /dev/null +++ b/vortex-web/src/components/swimlane/TreeRow.tsx @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import type { FlattenedRow } from './types'; +import { + ROW_HEIGHT, + DTYPE_COLORS, + getEncodingStyle, + getNodeDisplayName, + hasExpandableChildren, + formatRowRange, + getDtypeCategory, + shortEncoding, +} from './utils'; + +interface TreeRowProps { + row: FlattenedRow; + isExpanded: boolean; + isSelected: boolean; + mode: 'schema' | 'layout'; + onToggle: () => void; + onSelect: () => void; + onHover?: (nodeId: string | null) => void; +} + +export function TreeRow({ + row, + isExpanded, + isSelected, + mode, + onToggle, + onSelect, + onHover, +}: TreeRowProps) { + const { node, depth, displayKind } = row; + + if (displayKind === 'hiddenIndicator') { + const name = node.childType.kind === 'transparent' ? node.childType.name : 'hidden'; + return ( +
+ {name} +
+ ); + } + + const isGroup = displayKind === 'group'; + const expandable = hasExpandableChildren(node); + const isLeaf = !expandable; + const style = getEncodingStyle(node.encoding); + const name = getNodeDisplayName(node); + + let badgeText: string; + let badgeColor: string; + if (isGroup) { + badgeText = '···'; + badgeColor = style.color; + } else if (mode === 'schema') { + const dtypeCat = getDtypeCategory(node.dtype); + const dtypeStr = shortEncoding(node.dtype); + badgeText = + dtypeCat === 'struct' + ? 'struct' + : dtypeStr.length > 20 + ? dtypeStr.slice(0, 18) + '…' + : dtypeStr; + badgeColor = DTYPE_COLORS[dtypeCat]; + } else { + const ct = node.childType; + if (ct.kind === 'chunk') { + badgeText = formatRowRange(row.rowRange); + } else { + badgeText = style.label; + if (node.children.length > 0 && node.childType.kind !== 'field') { + badgeText += ` (${node.children.length})`; + } + } + badgeColor = style.color; + } + + const opacity = isGroup ? 'opacity-50' : isLeaf ? 'opacity-70' : ''; + const fontStyle = isGroup ? 'italic' : ''; + const selectedBg = isSelected ? 'bg-vortex-light-blue/10' : ''; + + return ( +
onHover?.(node.id)} + onMouseLeave={() => onHover?.(null)} + > + { + e.stopPropagation(); + if (expandable) onToggle(); + }} + > + {isExpanded ? '▼' : '▶'} + + + {name} + + + {badgeText} + +
+ ); +} diff --git a/vortex-web/src/components/swimlane/TreeSearch.tsx b/vortex-web/src/components/swimlane/TreeSearch.tsx new file mode 100644 index 00000000000..b02613db054 --- /dev/null +++ b/vortex-web/src/components/swimlane/TreeSearch.tsx @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { useState, useCallback } from 'react'; + +interface TreeSearchProps { + onSearch: (query: string) => void; +} + +export function TreeSearch({ onSearch }: TreeSearchProps) { + const [value, setValue] = useState(''); + + const handleChange = useCallback( + (e: React.ChangeEvent) => { + const q = e.target.value; + setValue(q); + onSearch(q); + }, + [onSearch], + ); + + const handleClear = useCallback(() => { + setValue(''); + onSearch(''); + }, [onSearch]); + + return ( +
+ ⌕ + + {value && ( + + )} +
+ ); +} diff --git a/vortex-web/src/components/swimlane/index.ts b/vortex-web/src/components/swimlane/index.ts new file mode 100644 index 00000000000..893b6ce2f66 --- /dev/null +++ b/vortex-web/src/components/swimlane/index.ts @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +// Main component +export { LayoutSwimlane, default } from './LayoutSwimlane'; +export type { LayoutSwimlaneProps } from './LayoutSwimlane'; + +// Types +export type { + LayoutTreeNode, + LayoutChildKind, + SegmentMapEntry, + FileStructureInfo, + FlattenedRow, + DisplayKind, + Split, + DtypeCategory, +} from './types'; + +// Sub-components +export { TreeRow } from './TreeRow'; +export { TreeSearch } from './TreeSearch'; +export { SwimlaneBar } from './SwimlaneBar'; +export { AxisBar } from './AxisBar'; +export { Tooltip } from './Tooltip'; +export { DtypeLegend } from './DtypeLegend'; +export { SplitRegion } from './SplitRegion'; + +// Utilities +export { + getDtypeCategory, + rangesOverlap, + createSplits, + formatBytes, + formatRowRange, + formatRowCount, + getNodeDisplayName, + getNodeRowRange, + getEncodingStyle, + hasExpandableChildren, + flattenTree, + filterTreeBySearch, + findNodeById, + findPathToNode, + collectSubtreeIds, + collectSubtreeSegments, + ENCODING_STYLES, + DTYPE_COLORS, + DTYPE_CATEGORIES, +} from './utils'; diff --git a/vortex-web/src/components/swimlane/styles.ts b/vortex-web/src/components/swimlane/styles.ts new file mode 100644 index 00000000000..0c82f2cbbf1 --- /dev/null +++ b/vortex-web/src/components/swimlane/styles.ts @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +// Re-export constants from utils for backward compat, plus any rendering-only constants. +export { + ENCODING_STYLES, + getEncodingStyle, + DTYPE_COLORS, + DTYPE_CATEGORIES, + ROW_HEIGHT, + MIN_LABEL_WIDTH, + GROUP_SIZE, +} from './utils'; + +export const TREE_WIDTH = 260; +export const DEFAULT_SWIMLANE_MIN_WIDTH = 800; +export const DEFAULT_HEIGHT = 360; diff --git a/vortex-web/src/components/swimlane/types.ts b/vortex-web/src/components/swimlane/types.ts new file mode 100644 index 00000000000..50919ad6aea --- /dev/null +++ b/vortex-web/src/components/swimlane/types.ts @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +// Unified layout tree type mirroring the Rust Layout trait. +// Each node represents a layout in the Vortex file format. + +export interface LayoutTreeNode { + /** Path-based ID, e.g. "root.customer.id.[0]" */ + id: string; + /** Encoding name, e.g. "vortex.flat", "vortex.chunked" */ + encoding: string; + /** DType string, e.g. "i64", "utf8", "{name=utf8, age=i32}" */ + dtype: string; + /** Number of rows in this layout */ + rowCount: number; + /** Absolute row offset in the file */ + rowOffset: number; + /** Size of metadata for this layout in bytes */ + metadataBytes: number; + /** Segment IDs referenced by this layout */ + segmentIds: number[]; + /** Relationship of this node to its parent */ + childType: LayoutChildKind; + /** Child layout nodes */ + children: LayoutTreeNode[]; + /** For flat layouts: the array encoding tree inside this layout */ + arrayEncodingTree?: ArrayEncodingNode; + /** True if this node represents an array encoding node (not a layout node) */ + isArrayNode?: boolean; + /** Buffer byte lengths for array nodes */ + bufferLengths?: number[]; + /** Buffer names for array nodes */ + bufferNames?: string[]; +} + +export interface ArrayEncodingNode { + encoding: string; + dtype: string; + metadataBytes: number; + numBuffers: number; + bufferLengths: number[]; + bufferNames: string[]; + children: ArrayEncodingNode[]; + childNames: string[]; +} + +export type LayoutChildKind = + | { kind: 'root' } + | { kind: 'field'; fieldName: string } + | { kind: 'chunk'; chunkIndex: number; rowOffset: number } + | { kind: 'transparent'; name: string } + | { kind: 'auxiliary'; name: string }; + +export interface SegmentMapEntry { + index: number; + byteOffset: number; + byteLength: number; + alignment: number; + column: string | null; + /** Node ID path in the layout tree */ + layoutPath: string; +} + +export interface FileStructureInfo { + fileSize: number; + version: number; + postscriptSize: number; + totalDataBytes: number; + totalMetadataBytes: number; +} + +// Rendering types (internal to swimlane) + +export type DisplayKind = 'normal' | 'group' | 'hiddenIndicator'; + +export interface FlattenedRow { + node: LayoutTreeNode; + depth: number; + displayKind: DisplayKind; + groupedChildren?: LayoutTreeNode[]; + rowRange: [number, number]; +} + +// Retained from original types +export type DtypeCategory = + | 'bool' + | 'int' + | 'float' + | 'utf8' + | 'datetime' + | 'struct' + | 'list' + | 'other'; + +export interface Split { + id: string; + rowRange: [number, number]; +} diff --git a/vortex-web/src/components/swimlane/utils.ts b/vortex-web/src/components/swimlane/utils.ts new file mode 100644 index 00000000000..605ebbfd30d --- /dev/null +++ b/vortex-web/src/components/swimlane/utils.ts @@ -0,0 +1,584 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import type { + LayoutTreeNode, + ArrayEncodingNode, + Split, + DtypeCategory, + FlattenedRow, + DisplayKind, +} from './types'; + +// Encoding styles — keyed by encoding string, with fallback for unknown encodings +export const ENCODING_STYLES: Record = { + 'vortex.struct': { color: '#5971FD', label: 'struct' }, + 'vortex.chunked': { color: '#CEE562', label: 'chunked' }, + 'vortex.flat': { color: '#2CB9D1', label: 'flat' }, + 'vortex.dict': { color: '#EEB3E1', label: 'dict' }, + 'vortex.zonemap': { color: '#FB863D', label: 'zonemap' }, + 'vortex.fsst': { color: '#EEB3E1', label: 'fsst' }, + 'vortex.roaring_bool': { color: '#EEB3E1', label: 'roaring' }, + 'vortex.roaring_int': { color: '#2CB9D1', label: 'roaring' }, + 'vortex.alp': { color: '#FB863D', label: 'alp' }, + 'vortex.alp_rd': { color: '#FB863D', label: 'alp-rd' }, + 'vortex.for': { color: '#CEE562', label: 'for' }, + 'vortex.bitpacked': { color: '#CEE562', label: 'bitpacked' }, + 'vortex.runend': { color: '#CEE562', label: 'run-end' }, + 'vortex.runend_bool': { color: '#CEE562', label: 'run-end' }, + 'vortex.zigzag': { color: '#2CB9D1', label: 'zigzag' }, + 'vortex.constant': { color: '#8F8F8F', label: 'const' }, + 'vortex.sparse': { color: '#8F8F8F', label: 'sparse' }, +}; + +const DEFAULT_ENCODING_STYLE = { color: '#8F8F8F', label: 'unknown' }; + +export function getEncodingStyle(encoding: string): { color: string; label: string } { + return ( + ENCODING_STYLES[encoding] ?? { + ...DEFAULT_ENCODING_STYLE, + label: encoding.split('.').pop() ?? encoding, + } + ); +} + +// Dtype colors (for flat chunk bars in swimlane) +export const DTYPE_COLORS: Record = { + bool: '#D97BC6', + int: '#2CB9D1', + float: '#FB863D', + utf8: '#5971FD', + datetime: '#8BB536', + list: '#A78BFA', + struct: '#999999', + other: '#777777', +}; + +export const DTYPE_CATEGORIES: DtypeCategory[] = [ + 'bool', + 'int', + 'float', + 'utf8', + 'datetime', + 'list', + 'struct', + 'other', +]; + +export const ROW_HEIGHT = 26; +export const MIN_LABEL_WIDTH = 36; +export const GROUP_SIZE = 10; + +/** + * Determine dtype category from dtype string + */ +export function getDtypeCategory(dtype?: string): DtypeCategory { + if (!dtype) return 'other'; + const d = dtype.toLowerCase(); + if (d === 'bool' || d === 'boolean') return 'bool'; + // Struct/list checks first — composite dtypes contain field names that would false-match others. + if (d.startsWith('{') || d === 'struct') return 'struct'; + if (d.includes('list') || d.includes('array')) return 'list'; + if (d.includes('utf8') || d.includes('string') || d.includes('binary')) return 'utf8'; + if (d.includes('timestamp') || d.includes('date') || d.includes('time')) return 'datetime'; + if (d.includes('int') || d.includes('uint') || d.startsWith('i') || d.startsWith('u')) + return 'int'; + if (d.includes('float') || d.includes('double') || d.includes('decimal') || d.startsWith('f')) + return 'float'; + return 'other'; +} + +/** + * Check if two ranges overlap + */ +export function rangesOverlap(a: [number, number], b: [number, number]): boolean { + return a[0] < b[1] && b[0] < a[1]; +} + +/** + * Collect all row boundaries from a LayoutTreeNode tree + */ +export function collectBoundaries(node: LayoutTreeNode, set: Set = new Set()): Set { + set.add(node.rowOffset); + set.add(node.rowOffset + node.rowCount); + + for (const child of node.children) { + collectBoundaries(child, set); + } + + return set; +} + +/** + * Create splits from layout boundaries + */ +export function createSplits(layout: LayoutTreeNode): Split[] { + const boundaries = Array.from(collectBoundaries(layout)).sort((a, b) => a - b); + return boundaries.slice(0, -1).map((start, i) => ({ + id: `s${i}`, + rowRange: [start, boundaries[i + 1]] as [number, number], + })); +} + +/** + * Get the combined row range of selected splits + */ +export function getSelectedRowRange( + splits: Split[], + selectedSplits: Set, +): [number, number] | null { + if (selectedSplits.size === 0) return null; + const selected = splits.filter((s) => selectedSplits.has(s.id)); + const min = Math.min(...selected.map((s) => s.rowRange[0])); + const max = Math.max(...selected.map((s) => s.rowRange[1])); + return [min, max]; +} + +/** + * Get the display name for a layout tree node based on its child type + */ +export function getNodeDisplayName(node: LayoutTreeNode): string { + const ct = node.childType; + switch (ct.kind) { + case 'root': + return 'root'; + case 'field': + return ct.fieldName; + case 'chunk': + return `[${ct.chunkIndex}]`; + case 'transparent': + return ct.name; + case 'auxiliary': + return ct.name; + } +} + +/** + * Get a badge label for schema mode + */ +export function getSchemaLabel(node: LayoutTreeNode): string { + return node.dtype; +} + +/** + * Get a badge label for layout mode + */ +export function getLayoutLabel(node: LayoutTreeNode): string { + const ct = node.childType; + switch (ct.kind) { + case 'root': + return getEncodingStyle(node.encoding).label; + case 'field': + return `[field] ${getEncodingStyle(node.encoding).label}`; + case 'chunk': + return `[chunk ${ct.chunkIndex}]`; + case 'transparent': + return `[transparent: ${ct.name}]`; + case 'auxiliary': + return `[aux: ${ct.name}]`; + } +} + +/** + * Check if a node has expandable children + */ +export function hasExpandableChildren(node: LayoutTreeNode): boolean { + return node.children.length > 0; +} + +/** + * Get the row range tuple for a node + */ +export function getNodeRowRange(node: LayoutTreeNode): [number, number] { + return [node.rowOffset, node.rowOffset + node.rowCount]; +} + +/** + * Check if a node is a "field" child — used in schema mode to identify logical columns + */ +function isFieldChild(node: LayoutTreeNode): boolean { + return node.childType.kind === 'field'; +} + +/** + * In schema mode, find the field-level children, skipping intermediate layout nodes. + * Returns the node itself if it has field children, or walks through transparent/chunk/aux + * nodes to find them. + */ +function collectSchemaChildren(node: LayoutTreeNode): LayoutTreeNode[] { + const fieldChildren = node.children.filter(isFieldChild); + if (fieldChildren.length > 0) return fieldChildren; + // No field children — this is a leaf or layout-only node + return []; +} + +/** + * Group children into groups of GROUP_SIZE when there are too many. + * Returns null if grouping is not needed. + */ +export function groupChildren( + children: LayoutTreeNode[], + parentId: string, +): { groups: LayoutTreeNode[]; isGrouped: true } | null { + if (children.length <= GROUP_SIZE) return null; + + const groups: LayoutTreeNode[] = []; + for (let i = 0; i < children.length; i += GROUP_SIZE) { + const groupNodes = children.slice(i, Math.min(i + GROUP_SIZE, children.length)); + const startIdx = i; + const endIdx = Math.min(i + GROUP_SIZE - 1, children.length - 1); + const firstNode = groupNodes[0]; + const lastNode = groupNodes[groupNodes.length - 1]; + + groups.push({ + id: `${parentId}_group_${startIdx}_${endIdx}`, + encoding: 'group', + dtype: '', + rowCount: lastNode.rowOffset + lastNode.rowCount - firstNode.rowOffset, + rowOffset: firstNode.rowOffset, + metadataBytes: 0, + segmentIds: [], + childType: { kind: 'transparent', name: `chunks ${startIdx}–${endIdx}` }, + children: groupNodes, + }); + } + return { groups, isGrouped: true }; +} + +/** + * Flatten a layout tree into rows for rendering. + * + * @param root - The root layout tree node + * @param expanded - Set of expanded node IDs + * @param selectedRange - Optional selected row range for filtering + * @param mode - 'schema' shows logical column hierarchy, 'layout' shows full layout tree + */ +export function flattenTree( + root: LayoutTreeNode, + expanded: Set, + selectedRange: [number, number] | null, + mode: 'schema' | 'layout', +): FlattenedRow[] { + const result: FlattenedRow[] = []; + + function walk(node: LayoutTreeNode, depth: number) { + const rowRange = getNodeRowRange(node); + result.push({ node, depth, displayKind: 'normal', rowRange }); + + if (!expanded.has(node.id)) return; + + let childrenToShow: LayoutTreeNode[]; + + if (mode === 'schema') { + // In schema mode, show field children at the top level. + // If a field child has no field sub-children, show its layout children when expanded. + const schemaChildren = collectSchemaChildren(node); + childrenToShow = schemaChildren.length > 0 ? schemaChildren : node.children; + } else { + childrenToShow = node.children; + } + + // Apply chunk grouping if there are many children of the same type + const chunkChildren = childrenToShow.filter((c) => c.childType.kind === 'chunk'); + const nonChunkChildren = childrenToShow.filter((c) => c.childType.kind !== 'chunk'); + + // Show non-chunk children first + for (const child of nonChunkChildren) { + walk(child, depth + 1); + } + + // Group chunk children if needed + if (chunkChildren.length > 0) { + const groupResult = groupChildren(chunkChildren, node.id); + + if (groupResult) { + const visibleGroups = selectedRange + ? groupResult.groups.filter((g) => rangesOverlap(getNodeRowRange(g), selectedRange)) + : groupResult.groups; + + for (const group of visibleGroups) { + const groupRowRange = getNodeRowRange(group); + result.push({ + node: group, + depth: depth + 1, + displayKind: 'group', + groupedChildren: group.children, + rowRange: groupRowRange, + }); + + if (expanded.has(group.id)) { + const visibleInGroup = selectedRange + ? group.children.filter((c) => rangesOverlap(getNodeRowRange(c), selectedRange)) + : group.children; + + for (const child of visibleInGroup) { + walk(child, depth + 2); + } + + if (selectedRange && visibleInGroup.length < group.children.length) { + addHiddenIndicator( + group, + group.children.length - visibleInGroup.length, + depth + 2, + 'in group', + ); + } + } + } + + if (selectedRange && visibleGroups.length < groupResult.groups.length) { + addHiddenIndicator( + node, + groupResult.groups.length - visibleGroups.length, + depth + 1, + 'groups', + ); + } + } else { + const visible = selectedRange + ? chunkChildren.filter((c) => rangesOverlap(getNodeRowRange(c), selectedRange)) + : chunkChildren; + + for (const child of visible) { + walk(child, depth + 1); + } + + if (selectedRange && visible.length < chunkChildren.length) { + addHiddenIndicator(node, chunkChildren.length - visible.length, depth + 1, 'chunks'); + } + } + } + } + + function addHiddenIndicator( + parent: LayoutTreeNode, + hiddenCount: number, + depth: number, + label: string, + ) { + const indicator: LayoutTreeNode = { + id: `${parent.id}_hidden_${label}`, + encoding: '', + dtype: '', + rowCount: parent.rowCount, + rowOffset: parent.rowOffset, + metadataBytes: 0, + segmentIds: [], + childType: { kind: 'transparent', name: `${hiddenCount} more ${label}` }, + children: [], + }; + result.push({ + node: indicator, + depth, + displayKind: 'hiddenIndicator' as DisplayKind, + rowRange: getNodeRowRange(parent), + }); + } + + walk(root, 0); + return result; +} + +/** + * Find a node by ID in a layout tree + */ +export function findNodeById(root: LayoutTreeNode, id: string): LayoutTreeNode | null { + if (root.id === id) return root; + for (const child of root.children) { + const found = findNodeById(child, id); + if (found) return found; + } + return null; +} + +/** + * Find the path from root to a node (inclusive of both endpoints). + * Returns an empty array if the node is not found. + */ +export function findPathToNode(root: LayoutTreeNode, id: string): LayoutTreeNode[] { + if (root.id === id) return [root]; + for (const child of root.children) { + const path = findPathToNode(child, id); + if (path.length > 0) return [root, ...path]; + } + return []; +} + +/** + * Collect all node IDs in a subtree + */ +export function collectSubtreeIds(node: LayoutTreeNode): Set { + const ids = new Set(); + function walk(n: LayoutTreeNode) { + ids.add(n.id); + for (const child of n.children) walk(child); + } + walk(node); + return ids; +} + +/** + * Collect all segment IDs reachable from a subtree + */ +export function collectSubtreeSegments(node: LayoutTreeNode): number[] { + const segments: number[] = []; + function walk(n: LayoutTreeNode) { + segments.push(...n.segmentIds); + for (const child of n.children) walk(child); + } + walk(node); + return segments; +} + +/** + * Filter nodes matching a search query (and their ancestors) + */ +export function filterTreeBySearch( + rows: FlattenedRow[], + query: string, + root: LayoutTreeNode, +): FlattenedRow[] { + if (!query.trim()) return rows; + + const lowerQuery = query.toLowerCase(); + const matchingIds = new Set(); + + // Find all matching nodes + function findMatches(node: LayoutTreeNode) { + const name = getNodeDisplayName(node).toLowerCase(); + const dtype = node.dtype.toLowerCase(); + const encoding = node.encoding.toLowerCase(); + if (name.includes(lowerQuery) || dtype.includes(lowerQuery) || encoding.includes(lowerQuery)) { + matchingIds.add(node.id); + } + for (const child of node.children) findMatches(child); + } + findMatches(root); + + // Collect ancestors of matching nodes + const ancestorIds = new Set(); + function collectAncestors(node: LayoutTreeNode, path: string[]) { + if (matchingIds.has(node.id)) { + for (const id of path) ancestorIds.add(id); + } + for (const child of node.children) { + collectAncestors(child, [...path, node.id]); + } + } + collectAncestors(root, []); + + // Collect descendants of matching nodes so expanded children are visible. + const descendantIds = new Set(); + function collectDescendants(node: LayoutTreeNode) { + descendantIds.add(node.id); + for (const child of node.children) collectDescendants(child); + } + function findDescendantsOfMatches(node: LayoutTreeNode) { + if (matchingIds.has(node.id)) { + collectDescendants(node); + } else { + for (const child of node.children) findDescendantsOfMatches(child); + } + } + findDescendantsOfMatches(root); + + const visibleIds = new Set([...matchingIds, ...ancestorIds, ...descendantIds]); + return rows.filter((row) => visibleIds.has(row.node.id)); +} + +/** + * Strip the `vortex.` prefix from an encoding name for display. + */ +export function shortEncoding(encoding: string): string { + return encoding.startsWith('vortex.') ? encoding.slice(7) : encoding; +} + +/** + * Format bytes to human readable string + */ +export function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`; +} + +/** + * Format row range to compact string (e.g., "0k–25k") + */ +export function formatRowRange(range: [number, number]): string { + const fmt = (n: number) => (n >= 1000 ? `${(n / 1000).toFixed(1)}k` : String(n)); + return `${fmt(range[0])}–${fmt(range[1])}`; +} + +/** + * Format row count to compact string (e.g., "27.1k") + */ +export function formatRowCount(n: number): string { + if (n < 1000) return String(n); + if (n < 1000000) return `${(n / 1000).toFixed(1)}k`; + return `${(n / 1000000).toFixed(1)}M`; +} + +/** + * Convert an ArrayEncodingNode tree into LayoutTreeNode children + * so they can appear in the layout tree under a flat layout node. + */ +export function arrayTreeToLayoutChildren( + arrayTree: ArrayEncodingNode, + parentNode: LayoutTreeNode, +): LayoutTreeNode[] { + function convert(node: ArrayEncodingNode, parentId: string, name: string): LayoutTreeNode { + const id = `${parentId}.$${name}`; + + const children = node.children.map((child, i) => { + const childName = node.childNames[i] ?? `child ${i}`; + return convert(child, id, childName); + }); + + return { + id, + encoding: node.encoding, + dtype: node.dtype || parentNode.dtype, + rowCount: parentNode.rowCount, + rowOffset: parentNode.rowOffset, + metadataBytes: node.metadataBytes, + segmentIds: [], + childType: { kind: 'field', fieldName: name }, + children, + isArrayNode: true, + bufferLengths: node.bufferLengths, + bufferNames: node.bufferNames, + }; + } + + // Wrap the entire array tree as a single "array" child of the flat layout. + return [convert(arrayTree, parentNode.id, 'array')]; +} + +/** + * Check if a layout node is a flat layout that can have array children. + */ +export function isFlatLayout(node: LayoutTreeNode): boolean { + return node.encoding === 'vortex.flat' && !node.isArrayNode; +} + +/** + * Parse an array node ID into its layout node ID and array child path. + * Array node IDs look like "root.col.$array.$values.$encoded". + * The first `$array` segment represents the root of the decoded array tree, + * so the WASM-side path skips it: → layoutNodeId: "root.col", arrayPath: ["values", "encoded"] + */ +export function parseArrayNodeId(nodeId: string): { layoutNodeId: string; arrayPath: string[] } { + const parts = nodeId.split('.'); + const firstArrayIdx = parts.findIndex((p) => p.startsWith('$')); + if (firstArrayIdx === -1) { + return { layoutNodeId: nodeId, arrayPath: [] }; + } + // Skip the first $array segment — it represents the decoded root, not a child to navigate to. + const arraySegments = parts.slice(firstArrayIdx).map((p) => p.slice(1)); + return { + layoutNodeId: parts.slice(0, firstArrayIdx).join('.'), + arrayPath: arraySegments.slice(1), + }; +} diff --git a/vortex-web/src/contexts/SelectionContext.tsx b/vortex-web/src/contexts/SelectionContext.tsx new file mode 100644 index 00000000000..e44ee6ef008 --- /dev/null +++ b/vortex-web/src/contexts/SelectionContext.tsx @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { createContext, useContext, useReducer, useCallback, type ReactNode } from 'react'; +import type { LayoutTreeNode } from '../components/swimlane/types'; +import { findNodeById } from '../components/swimlane/utils'; + +export interface SelectionState { + selectedNodeId: string | null; + selectedNode: LayoutTreeNode | null; + hoveredNodeId: string | null; + hoveredNode: LayoutTreeNode | null; + hoveredSegmentIndex: number | null; + selectedSegmentIndex: number | null; +} + +type SelectionAction = + | { type: 'selectNode'; nodeId: string | null; tree: LayoutTreeNode } + | { type: 'hoverNode'; nodeId: string | null; tree: LayoutTreeNode } + | { type: 'hoverSegment'; index: number | null } + | { type: 'selectSegment'; index: number | null } + | { type: 'clearSelection' }; + +function selectionReducer(state: SelectionState, action: SelectionAction): SelectionState { + switch (action.type) { + case 'selectNode': { + const nodeId = action.nodeId; + const node = nodeId ? findNodeById(action.tree, nodeId) : null; + return { ...state, selectedNodeId: nodeId, selectedNode: node, selectedSegmentIndex: null }; + } + case 'hoverNode': { + const nodeId = action.nodeId; + const node = nodeId ? findNodeById(action.tree, nodeId) : null; + return { ...state, hoveredNodeId: nodeId, hoveredNode: node, hoveredSegmentIndex: null }; + } + case 'hoverSegment': + return { ...state, hoveredSegmentIndex: action.index }; + case 'selectSegment': + return { ...state, selectedSegmentIndex: action.index }; + case 'clearSelection': + return { + selectedNodeId: null, + selectedNode: null, + hoveredNodeId: null, + hoveredNode: null, + hoveredSegmentIndex: null, + selectedSegmentIndex: null, + }; + } +} + +const initialState: SelectionState = { + selectedNodeId: null, + selectedNode: null, + hoveredNodeId: null, + hoveredNode: null, + hoveredSegmentIndex: null, + selectedSegmentIndex: null, +}; + +interface SelectionContextValue { + state: SelectionState; + selectNode: (nodeId: string | null) => void; + hoverNode: (nodeId: string | null) => void; + hoverSegment: (index: number | null) => void; + selectSegment: (index: number | null) => void; + clearSelection: () => void; +} + +const SelectionContext = createContext(null); + +export function SelectionProvider({ + tree, + children, +}: { + tree: LayoutTreeNode; + children: ReactNode; +}) { + const [state, dispatch] = useReducer(selectionReducer, initialState); + + const selectNode = useCallback( + (nodeId: string | null) => dispatch({ type: 'selectNode', nodeId, tree }), + [tree], + ); + + const hoverNode = useCallback( + (nodeId: string | null) => dispatch({ type: 'hoverNode', nodeId, tree }), + [tree], + ); + + const hoverSegment = useCallback( + (index: number | null) => dispatch({ type: 'hoverSegment', index }), + [], + ); + + const selectSegment = useCallback( + (index: number | null) => dispatch({ type: 'selectSegment', index }), + [], + ); + + const clearSelection = useCallback(() => dispatch({ type: 'clearSelection' }), []); + + return ( + + {children} + + ); +} + +export function useSelection(): SelectionContextValue { + const ctx = useContext(SelectionContext); + if (!ctx) throw new Error('useSelection must be used within SelectionProvider'); + return ctx; +} + +export { SelectionContext }; diff --git a/vortex-web/src/contexts/ThemeContext.tsx b/vortex-web/src/contexts/ThemeContext.tsx new file mode 100644 index 00000000000..4230ebef84d --- /dev/null +++ b/vortex-web/src/contexts/ThemeContext.tsx @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { createContext, useCallback, useContext, useEffect, useState, type ReactNode } from 'react'; + +export type ThemeChoice = 'light' | 'dark' | 'system'; + +interface ThemeContextValue { + theme: ThemeChoice; + setTheme: (theme: ThemeChoice) => void; +} + +const ThemeContext = createContext(null); + +const STORAGE_KEY = 'vortex-theme'; + +function applyTheme(choice: ThemeChoice) { + const root = document.documentElement; + root.classList.remove('light', 'dark'); + + if (choice === 'system') { + const prefersDark = window.matchMedia('(prefers-color-scheme: dark)').matches; + root.classList.add(prefersDark ? 'dark' : 'light'); + } else { + root.classList.add(choice); + } +} + +function getStoredTheme(): ThemeChoice { + const stored = localStorage.getItem(STORAGE_KEY); + if (stored === 'light' || stored === 'dark' || stored === 'system') return stored; + return 'dark'; +} + +export function ThemeProvider({ children }: { children: ReactNode }) { + const [theme, setThemeState] = useState(getStoredTheme); + + const setTheme = useCallback((choice: ThemeChoice) => { + localStorage.setItem(STORAGE_KEY, choice); + setThemeState(choice); + applyTheme(choice); + }, []); + + // Apply on mount and listen for system preference changes. + useEffect(() => { + applyTheme(theme); + + if (theme !== 'system') return; + + const mq = window.matchMedia('(prefers-color-scheme: dark)'); + const handler = () => applyTheme('system'); + mq.addEventListener('change', handler); + return () => mq.removeEventListener('change', handler); + }, [theme]); + + return {children}; +} + +export function useTheme(): ThemeContextValue { + const ctx = useContext(ThemeContext); + if (!ctx) throw new Error('useTheme must be used within a ThemeProvider'); + return ctx; +} diff --git a/vortex-web/src/contexts/VortexFileContext.tsx b/vortex-web/src/contexts/VortexFileContext.tsx new file mode 100644 index 00000000000..f3705aa5fd4 --- /dev/null +++ b/vortex-web/src/contexts/VortexFileContext.tsx @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { createContext, useContext } from 'react'; +import type { + LayoutTreeNode, + SegmentMapEntry, + FileStructureInfo, + ArrayEncodingNode, +} from '../components/swimlane/types'; + +export interface VortexFileState { + fileName: string; + fileSize: number; + rowCount: number; + version: number; + dtype: string; + layoutTree: LayoutTreeNode; + segments: SegmentMapEntry[]; + fileStructure: FileStructureInfo; +} + +export interface VortexFileContextValue extends VortexFileState { + fetchEncodingTree: (nodeId: string) => Promise; + previewData: (nodeId: string, rowLimit: number) => Promise; + /** Fetch and attach array encoding tree children to a flat layout node. */ + expandArrayTree: (nodeId: string) => Promise; + /** Fetch a buffer from a decoded array node. */ + fetchArrayBuffer: ( + layoutNodeId: string, + arrayPath: string[], + bufferIndex: number, + ) => Promise; + /** Preview data from a specific array node, returning Arrow IPC bytes. */ + previewArrayData: ( + layoutNodeId: string, + arrayPath: string[], + rowLimit: number, + ) => Promise; +} + +const VortexFileContext = createContext(null); + +export function VortexFileProvider({ + value, + children, +}: { + value: VortexFileContextValue; + children: React.ReactNode; +}) { + return {children}; +} + +export function useVortexFile(): VortexFileContextValue { + const ctx = useContext(VortexFileContext); + if (!ctx) throw new Error('useVortexFile must be used within VortexFileProvider'); + return ctx; +} + +export { VortexFileContext }; diff --git a/vortex-web/src/index.css b/vortex-web/src/index.css index 8e92ec8c9e5..432e388dca6 100644 --- a/vortex-web/src/index.css +++ b/vortex-web/src/index.css @@ -1,62 +1,91 @@ /* SPDX-License-Identifier: Apache-2.0 */ /* SPDX-FileCopyrightText: Copyright the Vortex contributors */ -@import "tailwindcss"; +@import 'tailwindcss'; -:root { - --background: #101010; -} +@custom-variant dark (&:where(.dark, .dark *)); @theme inline { - --color-background: var(--background); - - --color-grey: #D9D9D9; - --color-vortex-black: rgba(16, 16, 16, 1); - --color-vortex-white: rgba(255, 255, 255, 1); - --color-vortex-grey: rgba(236, 236, 236, 1); - --color-vortex-grey-dark: rgba(143, 143, 143, 1); - --color-vortex-grey-light: rgba(207, 207, 207, 1); - --color-vortex-grey-lightest: rgba(241, 241, 241, 1); - --color-vortex-green: rgba(206, 229, 98, 1); - --color-vortex-washed-green: rgba(241, 248, 208, 1); - --color-vortex-light-blue: rgba(44, 185, 209, 1); - --color-vortex-washed-light-blue: rgba(213, 241, 246, 1); - --color-vortex-washed-lighter-blue: rgba(234, 248, 250, 1); - --color-vortex-blue: rgba(89, 113, 253, 1); - --color-vortex-pink: rgba(238, 179, 225, 1); - --color-vortex-orange: rgba(251, 134, 61, 1); - --color-vortex-red: rgba(204, 51, 51, 1); - - --font-sans: "Geist", sans-serif; - --font-mono: "Geist Mono", monospace; - --font-funnel: "Funnel Display", sans-serif; + --color-grey: #d4d4d8; + + /* Surface palette */ + --color-vortex-black: #1a1a1e; + --color-vortex-white: #fafafa; + --color-vortex-grey: #e4e4e7; + --color-vortex-grey-dark: #71717a; + --color-vortex-grey-light: #d4d4d8; + --color-vortex-grey-lightest: #f4f4f5; + + /* Foreground — separate from surfaces so text contrast is tunable */ + --color-vortex-fg: #e4e4e8; + --color-vortex-fg-light: #18181b; + + /* Accent colors */ + --color-vortex-green: #cee562; + --color-vortex-washed-green: #f1f8d0; + --color-vortex-light-blue: #2cb9d1; + --color-vortex-washed-light-blue: #d5f1f6; + --color-vortex-washed-lighter-blue: #eaf8fa; + --color-vortex-blue: #5971fd; + --color-vortex-pink: #eeb3e1; + --color-vortex-orange: #fb863d; + --color-vortex-red: #dc2626; + + --font-sans: 'Geist', sans-serif; + --font-mono: 'Geist Mono', monospace; + --font-funnel: 'Funnel Display', sans-serif; } html { scrollbar-width: thin; - scrollbar-color: var(--color-vortex-black) var(--color-vortex-grey); + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +.light, +.light * { + scrollbar-color: var(--color-vortex-grey-light) var(--color-vortex-grey-lightest); +} + +.dark, +.dark * { + scrollbar-color: #333338 var(--color-vortex-black); } body { - background: var(--background); - font-family: theme("fontFamily.sans"); + font-family: theme('fontFamily.sans'); text-rendering: optimizeLegibility; } +/* Default app is dark themed */ +:root:not(.light) { + --array-node-bg: rgba(255, 255, 255, 0.03); +} +:root:not(.light) body { + background: var(--color-vortex-black); +} + +.light { + --array-node-bg: rgba(0, 0, 0, 0.025); +} +.light body { + background: var(--color-vortex-white); +} + @layer utilities { .dashed-bottom { - @apply relative after:absolute after:left-0 after:right-0 after:bottom-0 after:h-px after:bg-[length:16px_1px] after:bg-repeat-x after:bg-[linear-gradient(to_right,white_12px,transparent_4px)]; + @apply relative after:absolute after:left-0 after:right-0 after:bottom-0 after:h-px after:bg-[length:16px_1px] after:bg-repeat-x after:bg-[linear-gradient(to_right,currentColor_12px,transparent_4px)] after:opacity-20; } .dashed-top { - @apply relative before:absolute before:left-0 before:right-0 before:top-0 before:h-px before:bg-[length:16px_1px] before:bg-repeat-x before:bg-[linear-gradient(to_right,white_12px,transparent_4px)]; + @apply relative before:absolute before:left-0 before:right-0 before:top-0 before:h-px before:bg-[length:16px_1px] before:bg-repeat-x before:bg-[linear-gradient(to_right,currentColor_12px,transparent_4px)] before:opacity-20; } .dashed-right { - @apply relative after:absolute after:top-0 after:bottom-0 after:right-0 after:w-px after:bg-[length:1px_16px] after:bg-repeat-y after:bg-[linear-gradient(to_bottom,white_12px,transparent_4px)]; + @apply relative after:absolute after:top-0 after:bottom-0 after:right-0 after:w-px after:bg-[length:1px_16px] after:bg-repeat-y after:bg-[linear-gradient(to_bottom,currentColor_12px,transparent_4px)] after:opacity-20; } .dashed-left { - @apply relative before:absolute before:top-0 before:bottom-0 before:left-0 before:w-px before:bg-[length:1px_16px] before:bg-repeat-y before:bg-[linear-gradient(to_bottom,white_12px,transparent_4px)]; + @apply relative before:absolute before:top-0 before:bottom-0 before:left-0 before:w-px before:bg-[length:1px_16px] before:bg-repeat-y before:bg-[linear-gradient(to_bottom,currentColor_12px,transparent_4px)] before:opacity-20; } } diff --git a/vortex-web/src/main.tsx b/vortex-web/src/main.tsx index 1e31b504cd2..6fbdadbfa60 100644 --- a/vortex-web/src/main.tsx +++ b/vortex-web/src/main.tsx @@ -1,13 +1,16 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -import { StrictMode } from "react"; -import { createRoot } from "react-dom/client"; -import "./index.css"; -import App from "./App.tsx"; +import { StrictMode } from 'react'; +import { createRoot } from 'react-dom/client'; +import './index.css'; +import App from './App.tsx'; +import { ThemeProvider } from './contexts/ThemeContext.tsx'; -createRoot(document.getElementById("root")!).render( +createRoot(document.getElementById('root')!).render( - + + + , ); diff --git a/vortex-web/src/mocks/fileStructure.ts b/vortex-web/src/mocks/fileStructure.ts new file mode 100644 index 00000000000..02b14e2ece4 --- /dev/null +++ b/vortex-web/src/mocks/fileStructure.ts @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import type { FileStructureInfo, SegmentMapEntry } from '../components/swimlane/types'; + +/** + * Generate a FileStructureInfo from segment entries and a total file size. + */ +export function generateFileStructure( + segments: SegmentMapEntry[], + fileSize: number, +): FileStructureInfo { + const totalDataBytes = segments.reduce((sum, s) => sum + s.byteLength, 0); + + const eofSize = 8; + const postscriptSize = 64; + const footerSize = Math.max(256, segments.length * 16); + const layoutSize = Math.max(128, Math.floor(fileSize * 0.02)); + const dtypeSize = Math.max(64, Math.floor(fileSize * 0.01)); + const metadataTotal = eofSize + postscriptSize + footerSize + layoutSize + dtypeSize; + + return { + fileSize, + version: 1, + postscriptSize, + totalDataBytes, + totalMetadataBytes: metadataTotal, + }; +} diff --git a/vortex-web/src/mocks/generators.ts b/vortex-web/src/mocks/generators.ts new file mode 100644 index 00000000000..ac8b6fb64a7 --- /dev/null +++ b/vortex-web/src/mocks/generators.ts @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import type { LayoutTreeNode, LayoutChildKind } from '../components/swimlane/types'; + +let nextSegmentId = 0; + +export function resetSegmentIds() { + nextSegmentId = 0; +} + +function allocSegments(count: number): number[] { + const ids = []; + for (let i = 0; i < count; i++) { + ids.push(nextSegmentId++); + } + return ids; +} + +export function makeFlat(opts: { + id: string; + childType: LayoutChildKind; + dtype: string; + rowCount: number; + rowOffset: number; + metadataBytes?: number; +}): LayoutTreeNode { + return { + id: opts.id, + encoding: 'vortex.flat', + dtype: opts.dtype, + rowCount: opts.rowCount, + rowOffset: opts.rowOffset, + metadataBytes: opts.metadataBytes ?? 64, + segmentIds: allocSegments(1), + childType: opts.childType, + children: [], + }; +} + +export function makeChunked(opts: { + id: string; + childType: LayoutChildKind; + dtype: string; + rowCount: number; + rowOffset: number; + chunkCount: number; + childEncoding?: string; + metadataBytes?: number; +}): LayoutTreeNode { + const rowsPerChunk = Math.ceil(opts.rowCount / opts.chunkCount); + const children: LayoutTreeNode[] = []; + + for (let i = 0; i < opts.chunkCount; i++) { + const chunkOffset = opts.rowOffset + i * rowsPerChunk; + const chunkRows = Math.min(rowsPerChunk, opts.rowOffset + opts.rowCount - chunkOffset); + if (chunkRows <= 0) break; + + children.push( + makeFlat({ + id: `${opts.id}.[${i}]`, + childType: { kind: 'chunk', chunkIndex: i, rowOffset: chunkOffset }, + dtype: opts.dtype, + rowCount: chunkRows, + rowOffset: chunkOffset, + metadataBytes: 32, + }), + ); + } + + return { + id: opts.id, + encoding: 'vortex.chunked', + dtype: opts.dtype, + rowCount: opts.rowCount, + rowOffset: opts.rowOffset, + metadataBytes: opts.metadataBytes ?? 128, + segmentIds: allocSegments(1), + childType: opts.childType, + children, + }; +} + +export function makeStruct(opts: { + id: string; + childType: LayoutChildKind; + dtype: string; + rowCount: number; + rowOffset: number; + fields: LayoutTreeNode[]; + metadataBytes?: number; +}): LayoutTreeNode { + return { + id: opts.id, + encoding: 'vortex.struct', + dtype: opts.dtype, + rowCount: opts.rowCount, + rowOffset: opts.rowOffset, + metadataBytes: opts.metadataBytes ?? 96, + segmentIds: allocSegments(1), + childType: opts.childType, + children: opts.fields, + }; +} + +export function makeDict(opts: { + id: string; + childType: LayoutChildKind; + dtype: string; + rowCount: number; + rowOffset: number; + codesDtype?: string; + metadataBytes?: number; +}): LayoutTreeNode { + return { + id: opts.id, + encoding: 'vortex.dict', + dtype: opts.dtype, + rowCount: opts.rowCount, + rowOffset: opts.rowOffset, + metadataBytes: opts.metadataBytes ?? 64, + segmentIds: allocSegments(1), + childType: opts.childType, + children: [ + makeFlat({ + id: `${opts.id}.codes`, + childType: { kind: 'transparent', name: 'codes' }, + dtype: opts.codesDtype ?? 'u16', + rowCount: opts.rowCount, + rowOffset: opts.rowOffset, + }), + makeFlat({ + id: `${opts.id}.values`, + childType: { kind: 'transparent', name: 'values' }, + dtype: opts.dtype, + rowCount: opts.rowCount, + rowOffset: opts.rowOffset, + }), + ], + }; +} + +export function makeZoned(opts: { + id: string; + childType: LayoutChildKind; + dtype: string; + rowCount: number; + rowOffset: number; + zoneCount: number; + metadataBytes?: number; +}): LayoutTreeNode { + const rowsPerZone = Math.ceil(opts.rowCount / opts.zoneCount); + const children: LayoutTreeNode[] = []; + + // Zones auxiliary child + const zoneChildren: LayoutTreeNode[] = []; + for (let i = 0; i < opts.zoneCount; i++) { + const zoneOffset = opts.rowOffset + i * rowsPerZone; + const zoneRows = Math.min(rowsPerZone, opts.rowOffset + opts.rowCount - zoneOffset); + if (zoneRows <= 0) break; + + zoneChildren.push( + makeFlat({ + id: `${opts.id}.zones.[${i}]`, + childType: { kind: 'chunk', chunkIndex: i, rowOffset: zoneOffset }, + dtype: opts.dtype, + rowCount: zoneRows, + rowOffset: zoneOffset, + }), + ); + } + + children.push({ + id: `${opts.id}.data`, + encoding: 'vortex.chunked', + dtype: opts.dtype, + rowCount: opts.rowCount, + rowOffset: opts.rowOffset, + metadataBytes: 64, + segmentIds: allocSegments(1), + childType: { kind: 'transparent', name: 'data' }, + children: zoneChildren, + }); + + // Zone map stats (auxiliary) + children.push( + makeFlat({ + id: `${opts.id}.zone_map`, + childType: { kind: 'auxiliary', name: 'zone_map' }, + dtype: `{min=${opts.dtype}, max=${opts.dtype}}`, + rowCount: opts.zoneCount, + rowOffset: 0, + }), + ); + + return { + id: opts.id, + encoding: 'vortex.zonemap', + dtype: opts.dtype, + rowCount: opts.rowCount, + rowOffset: opts.rowOffset, + metadataBytes: opts.metadataBytes ?? 256, + segmentIds: allocSegments(1), + childType: opts.childType, + children, + }; +} diff --git a/vortex-web/src/mocks/index.ts b/vortex-web/src/mocks/index.ts new file mode 100644 index 00000000000..681e38aace8 --- /dev/null +++ b/vortex-web/src/mocks/index.ts @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +export { simpleMock, ordersMock, wideMock, deepMock, heavyChunksMock } from './layouts'; +export { generateSegments } from './segments'; +export { generateFileStructure } from './fileStructure'; +export { + makeFlat, + makeChunked, + makeStruct, + makeDict, + makeZoned, + resetSegmentIds, +} from './generators'; diff --git a/vortex-web/src/mocks/layouts.ts b/vortex-web/src/mocks/layouts.ts new file mode 100644 index 00000000000..e18c469c7ab --- /dev/null +++ b/vortex-web/src/mocks/layouts.ts @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import type { LayoutTreeNode } from '../components/swimlane/types'; +import { + resetSegmentIds, + makeFlat, + makeChunked, + makeStruct, + makeDict, + makeZoned, +} from './generators'; + +/** + * Simple: single flat column + */ +export function simpleMock(): LayoutTreeNode { + resetSegmentIds(); + return makeFlat({ + id: 'root', + childType: { kind: 'root' }, + dtype: 'i64', + rowCount: 10000, + rowOffset: 0, + metadataBytes: 64, + }); +} + +/** + * Orders: 7 columns, mixed encodings — the canonical demo dataset + */ +export function ordersMock(): LayoutTreeNode { + resetSegmentIds(); + const totalRows = 100_000; + + return makeStruct({ + id: 'root', + childType: { kind: 'root' }, + dtype: + '{order_id=i64, is_active=bool, customer={id=i64, name=utf8}, items=list, amount=f64, metadata=struct, status=utf8}', + rowCount: totalRows, + rowOffset: 0, + fields: [ + makeFlat({ + id: 'root.order_id', + childType: { kind: 'field', fieldName: 'order_id' }, + dtype: 'i64', + rowCount: totalRows, + rowOffset: 0, + metadataBytes: 64, + }), + makeFlat({ + id: 'root.is_active', + childType: { kind: 'field', fieldName: 'is_active' }, + dtype: 'bool', + rowCount: totalRows, + rowOffset: 0, + metadataBytes: 32, + }), + makeStruct({ + id: 'root.customer', + childType: { kind: 'field', fieldName: 'customer' }, + dtype: '{id=i64, name=utf8}', + rowCount: totalRows, + rowOffset: 0, + fields: [ + makeChunked({ + id: 'root.customer.id', + childType: { kind: 'field', fieldName: 'id' }, + dtype: 'i64', + rowCount: totalRows, + rowOffset: 0, + chunkCount: 50, + }), + makeChunked({ + id: 'root.customer.name', + childType: { kind: 'field', fieldName: 'name' }, + dtype: 'utf8', + rowCount: totalRows, + rowOffset: 0, + chunkCount: 8, + }), + ], + }), + makeFlat({ + id: 'root.items', + childType: { kind: 'field', fieldName: 'items' }, + dtype: 'list', + rowCount: totalRows, + rowOffset: 0, + metadataBytes: 128, + }), + makeZoned({ + id: 'root.amount', + childType: { kind: 'field', fieldName: 'amount' }, + dtype: 'f64', + rowCount: totalRows, + rowOffset: 0, + zoneCount: 5, + }), + makeFlat({ + id: 'root.metadata', + childType: { kind: 'field', fieldName: 'metadata' }, + dtype: 'struct', + rowCount: totalRows, + rowOffset: 0, + metadataBytes: 256, + }), + makeDict({ + id: 'root.status', + childType: { kind: 'field', fieldName: 'status' }, + dtype: 'utf8', + rowCount: totalRows, + rowOffset: 0, + }), + ], + }); +} + +/** + * Wide: 200 columns (to test search and scrolling) + */ +export function wideMock(): LayoutTreeNode { + resetSegmentIds(); + const totalRows = 50_000; + const dtypes = ['i32', 'i64', 'f32', 'f64', 'utf8', 'bool', 'u16', 'u32']; + + const fields: LayoutTreeNode[] = Array.from({ length: 200 }, (_, i) => { + const dtype = dtypes[i % dtypes.length]; + return makeFlat({ + id: `root.col_${i}`, + childType: { kind: 'field', fieldName: `col_${i}` }, + dtype, + rowCount: totalRows, + rowOffset: 0, + }); + }); + + return makeStruct({ + id: 'root', + childType: { kind: 'root' }, + dtype: '{...200 columns}', + rowCount: totalRows, + rowOffset: 0, + fields, + }); +} + +/** + * Deep: nested structs (3 levels deep) + */ +export function deepMock(): LayoutTreeNode { + resetSegmentIds(); + const totalRows = 25_000; + + return makeStruct({ + id: 'root', + childType: { kind: 'root' }, + dtype: + '{user={profile={address={street=utf8, city=utf8, zip=u32}, name=utf8, age=u8}, orders=list}}', + rowCount: totalRows, + rowOffset: 0, + fields: [ + makeStruct({ + id: 'root.user', + childType: { kind: 'field', fieldName: 'user' }, + dtype: '{profile={...}, orders=list}', + rowCount: totalRows, + rowOffset: 0, + fields: [ + makeStruct({ + id: 'root.user.profile', + childType: { kind: 'field', fieldName: 'profile' }, + dtype: '{address={...}, name=utf8, age=u8}', + rowCount: totalRows, + rowOffset: 0, + fields: [ + makeStruct({ + id: 'root.user.profile.address', + childType: { kind: 'field', fieldName: 'address' }, + dtype: '{street=utf8, city=utf8, zip=u32}', + rowCount: totalRows, + rowOffset: 0, + fields: [ + makeFlat({ + id: 'root.user.profile.address.street', + childType: { kind: 'field', fieldName: 'street' }, + dtype: 'utf8', + rowCount: totalRows, + rowOffset: 0, + }), + makeFlat({ + id: 'root.user.profile.address.city', + childType: { kind: 'field', fieldName: 'city' }, + dtype: 'utf8', + rowCount: totalRows, + rowOffset: 0, + }), + makeFlat({ + id: 'root.user.profile.address.zip', + childType: { kind: 'field', fieldName: 'zip' }, + dtype: 'u32', + rowCount: totalRows, + rowOffset: 0, + }), + ], + }), + makeFlat({ + id: 'root.user.profile.name', + childType: { kind: 'field', fieldName: 'name' }, + dtype: 'utf8', + rowCount: totalRows, + rowOffset: 0, + }), + makeFlat({ + id: 'root.user.profile.age', + childType: { kind: 'field', fieldName: 'age' }, + dtype: 'u8', + rowCount: totalRows, + rowOffset: 0, + }), + ], + }), + makeChunked({ + id: 'root.user.orders', + childType: { kind: 'field', fieldName: 'orders' }, + dtype: 'list', + rowCount: totalRows, + rowOffset: 0, + chunkCount: 5, + }), + ], + }), + ], + }); +} + +/** + * Heavy chunks: a single column with 500 chunks (to test grouping) + */ +export function heavyChunksMock(): LayoutTreeNode { + resetSegmentIds(); + const totalRows = 1_000_000; + + return makeStruct({ + id: 'root', + childType: { kind: 'root' }, + dtype: '{values=i64}', + rowCount: totalRows, + rowOffset: 0, + fields: [ + makeChunked({ + id: 'root.values', + childType: { kind: 'field', fieldName: 'values' }, + dtype: 'i64', + rowCount: totalRows, + rowOffset: 0, + chunkCount: 500, + }), + ], + }); +} diff --git a/vortex-web/src/mocks/segments.ts b/vortex-web/src/mocks/segments.ts new file mode 100644 index 00000000000..a1c6430d5f5 --- /dev/null +++ b/vortex-web/src/mocks/segments.ts @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import type { LayoutTreeNode, SegmentMapEntry } from '../components/swimlane/types'; +import { getNodeDisplayName } from '../components/swimlane/utils'; + +/** + * Generate SegmentMapEntry[] by walking a layout tree and assigning byte offsets. + */ +export function generateSegments(root: LayoutTreeNode, fileSize: number): SegmentMapEntry[] { + const entries: SegmentMapEntry[] = []; + + // Collect all segment IDs with their layout context + function walk(node: LayoutTreeNode, columnPath: string) { + const name = getNodeDisplayName(node); + const currentPath = columnPath ? `${columnPath}.${name}` : name; + + for (const segId of node.segmentIds) { + entries.push({ + index: segId, + byteOffset: 0, // filled in below + byteLength: 0, + alignment: 64, + column: node.childType.kind === 'field' ? node.childType.fieldName : null, + layoutPath: node.id, + }); + } + + for (const child of node.children) { + walk(child, currentPath); + } + } + + walk(root, ''); + + // Sort by index and assign byte offsets proportionally + entries.sort((a, b) => a.index - b.index); + const totalSegments = entries.length; + if (totalSegments === 0) return entries; + + // Reserve ~10% for metadata at the end + const dataRegionSize = Math.floor(fileSize * 0.9); + const avgSegmentSize = Math.floor(dataRegionSize / totalSegments); + + let offset = 0; + for (const entry of entries) { + // Add some variance: segments between 0.5x and 1.5x average size + const variance = 0.5 + hashIndex(entry.index) / 0xffff; + const segmentSize = Math.max(64, Math.floor(avgSegmentSize * variance)); + const alignment = 64; + // Align offset + offset = Math.ceil(offset / alignment) * alignment; + + entry.byteOffset = offset; + entry.byteLength = segmentSize; + offset += segmentSize; + } + + return entries; +} + +function hashIndex(n: number): number { + // Simple deterministic hash for variance + let h = n * 2654435761; + h = ((h >>> 16) ^ h) * 0x45d9f3b; + return (h >>> 0) & 0xffff; +} diff --git a/vortex-web/src/storybook/decorators.tsx b/vortex-web/src/storybook/decorators.tsx new file mode 100644 index 00000000000..28483564d18 --- /dev/null +++ b/vortex-web/src/storybook/decorators.tsx @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import type { Decorator } from '@storybook/react-vite'; +import type { VortexFileState } from '../contexts/VortexFileContext'; +import type { LayoutTreeNode } from '../components/swimlane/types'; +import { VortexFileProvider } from '../contexts/VortexFileContext'; +import { SelectionProvider } from '../contexts/SelectionContext'; + +const EMPTY_TREE: LayoutTreeNode = { + id: 'root', + encoding: 'vortex.struct', + dtype: 'struct', + rowCount: 0, + rowOffset: 0, + metadataBytes: 0, + segmentIds: [], + childType: { kind: 'root' }, + children: [], +}; + +/** + * Storybook decorator that wraps a story in VortexFileContext.Provider + */ +export function withMockFileContext(state: VortexFileState): Decorator { + const value = { + ...state, + fetchEncodingTree: () => Promise.reject(new Error('Not implemented in storybook')), + previewData: () => Promise.reject(new Error('Not implemented in storybook')), + expandArrayTree: () => Promise.resolve(), + fetchArrayBuffer: () => Promise.reject(new Error('Not implemented in storybook')), + previewArrayData: () => Promise.reject(new Error('Not implemented in storybook')), + }; + return (Story) => ( + + + + ); +} + +/** + * Storybook decorator that wraps a story in both VortexFileContext and SelectionContext + */ +export function withMockSelection(tree?: LayoutTreeNode): Decorator { + return (Story) => ( + + + + ); +} diff --git a/vortex-web/src/workers/VortexWorker.ts b/vortex-web/src/workers/VortexWorker.ts new file mode 100644 index 00000000000..29c8a8757a3 --- /dev/null +++ b/vortex-web/src/workers/VortexWorker.ts @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import type { + LayoutTreeNode, + SegmentMapEntry, + FileStructureInfo, + ArrayEncodingNode, +} from '../components/swimlane/types'; + +export interface OpenFileResult { + rowCount: number; + dtype: string; + layoutTree: LayoutTreeNode; + segments: SegmentMapEntry[]; + fileStructure: FileStructureInfo; +} + +interface PendingRequest { + resolve: (value: T) => void; + reject: (reason: Error) => void; +} + +/** Typed async wrapper around the Vortex WASM Web Worker. */ +export class VortexWorker { + private worker: Worker; + private pending = new Map>(); + private nextId = 0; + + constructor() { + this.worker = new Worker(new URL('./vortex.worker.ts', import.meta.url), { type: 'module' }); + this.worker.onmessage = (e: MessageEvent) => { + const { type, id, data, error } = e.data; + const req = this.pending.get(id); + if (!req) return; + this.pending.delete(id); + if (type === 'result') { + req.resolve(data); + } else { + req.reject(new Error(error ?? 'Unknown worker error')); + } + }; + } + + /** Open a Vortex file in the worker. The File is structured-cloneable. */ + openFile(file: File): Promise { + const id = this.nextId++; + return new Promise((resolve, reject) => { + this.pending.set(id, { resolve, reject }); + this.worker.postMessage({ type: 'open', id, file }); + }); + } + + /** Fetch the array encoding tree for a flat layout node by its ID. */ + fetchEncodingTree(nodeId: string): Promise { + const id = this.nextId++; + return new Promise((resolve, reject) => { + this.pending.set(id, { resolve, reject }); + this.worker.postMessage({ type: 'fetchEncodingTree', id, nodeId }); + }); + } + + /** Fetch a buffer from a decoded array node. */ + fetchArrayBuffer( + layoutNodeId: string, + arrayPath: string[], + bufferIndex: number, + ): Promise { + const id = this.nextId++; + return new Promise((resolve, reject) => { + this.pending.set(id, { resolve, reject }); + this.worker.postMessage({ + type: 'fetchArrayBuffer', + id, + layoutNodeId, + arrayPath, + bufferIndex, + }); + }); + } + + /** Preview data from a specific array node within a flat layout. */ + previewArrayData( + layoutNodeId: string, + arrayPath: string[], + rowLimit: number, + ): Promise { + const id = this.nextId++; + return new Promise((resolve, reject) => { + this.pending.set(id, { resolve, reject }); + this.worker.postMessage({ type: 'previewArrayData', id, layoutNodeId, arrayPath, rowLimit }); + }); + } + + /** Preview data from a specific layout node, returning Arrow IPC bytes. */ + previewData(nodeId: string, rowLimit: number): Promise { + const id = this.nextId++; + return new Promise((resolve, reject) => { + this.pending.set(id, { resolve, reject }); + this.worker.postMessage({ type: 'previewData', id, nodeId, rowLimit }); + }); + } + + terminate(): void { + this.worker.terminate(); + } +} diff --git a/vortex-web/src/workers/vortex.worker.ts b/vortex-web/src/workers/vortex.worker.ts new file mode 100644 index 00000000000..e296b5732f9 --- /dev/null +++ b/vortex-web/src/workers/vortex.worker.ts @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +/// + +import init, { open_vortex_file, type VortexFileHandle } from '../wasm/pkg/vortex_web_wasm.js'; + +let initialized = false; +let fileHandle: VortexFileHandle | null = null; + +self.onmessage = async (e: MessageEvent) => { + const { type, id } = e.data; + try { + if (!initialized) { + await init(); + initialized = true; + } + + if (type === 'open') { + // Free previous handle if any. + if (fileHandle) { + fileHandle.free(); + fileHandle = null; + } + fileHandle = await open_vortex_file(e.data.file); + const result = { + rowCount: Number(fileHandle.row_count), + dtype: fileHandle.dtype, + layoutTree: JSON.parse(fileHandle.layout_tree()), + segments: JSON.parse(fileHandle.segment_map()), + fileStructure: JSON.parse(fileHandle.file_structure()), + }; + self.postMessage({ type: 'result', id, data: result }); + } else if (type === 'fetchEncodingTree') { + if (!fileHandle) throw new Error('No file open'); + const json = await fileHandle.fetch_encoding_tree(e.data.nodeId); + self.postMessage({ type: 'result', id, data: JSON.parse(json) }); + } else if (type === 'fetchArrayBuffer') { + if (!fileHandle) throw new Error('No file open'); + const buf: Uint8Array = await fileHandle.fetch_array_buffer( + e.data.layoutNodeId, + e.data.arrayPath, + e.data.bufferIndex, + ); + self.postMessage({ type: 'result', id, data: buf }, [buf.buffer]); + } else if (type === 'previewArrayData') { + if (!fileHandle) throw new Error('No file open'); + const ipcBytes: Uint8Array = await fileHandle.preview_array_data( + e.data.layoutNodeId, + e.data.arrayPath, + e.data.rowLimit, + ); + self.postMessage({ type: 'result', id, data: ipcBytes }, [ipcBytes.buffer]); + } else if (type === 'previewData') { + if (!fileHandle) throw new Error('No file open'); + const ipcBytes: Uint8Array = await fileHandle.preview_data(e.data.nodeId, e.data.rowLimit); + self.postMessage({ type: 'result', id, data: ipcBytes }, [ipcBytes.buffer]); + } + } catch (err) { + self.postMessage({ type: 'error', id, error: String(err) }); + } +};