Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion encodings/alp/src/alp/compute/compare.rs
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ mod tests {
let expected = BoolArray::from_iter([true; 10]);
assert_arrays_eq!(r_lte, expected);

//0.0605_f32 < 0.06051_f32;
// 0.0605_f32 < 0.06051_f32;
let r_lt = alp_scalar_compare(encoded.as_view(), 0.06051_f32, CompareOperator::Lt)
.unwrap()
.unwrap();
Expand Down
1 change: 0 additions & 1 deletion encodings/fastlanes/src/bitpacking/array/unpack_iter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,6 @@ impl<T: PhysicalPType<Physical: BitPacking>> UnpackStrategy<T> for BitPackingStr
/// // handle partial trailing chunk
/// }
/// ```
///
pub struct UnpackedChunks<T: PhysicalPType, S: UnpackStrategy<T>> {
strategy: S,
bit_width: usize,
Expand Down
1 change: 0 additions & 1 deletion encodings/fsst/src/kernel.rs
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,6 @@ mod tests {

#[test]
fn issues_6034_test_fsst_filter_with_nulls_and_special_chars() -> VortexResult<()> {
//
// Test case with special characters and nulls
// Values: ["", "", "", "", "", "", "", "", "", "", "", ",", "A<<<<<<<", "", "", "", "", null, null, null, null, null, null]
// Mask: only the last element is selected (true at index 22)
Expand Down
1 change: 0 additions & 1 deletion encodings/zstd/src/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -935,7 +935,6 @@ impl ZstdData {
} else if dtype.is_nullable() && matches!(slice_validity, Validity::NonNullable) {
slice_validity = Validity::AllValid;
}
//
// END OF IMPORTANT BLOCK
//

Expand Down
7 changes: 4 additions & 3 deletions rustfmt.toml
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
condense_wildcard_suffixes = true
format_macro_matchers = true
format_macro_bodies = true
format_macro_matchers = true
group_imports = "StdExternalCrate"
unstable_features = true
use_field_init_shorthand = true
imports_granularity = "Item"
normalize_comments = true
style_edition = "2024"
unstable_features = true
use_field_init_shorthand = true
1 change: 0 additions & 1 deletion vortex-array/src/aggregate_fn/fns/is_constant/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ use crate::scalar_fn::fns::operators::Operator;
/// Check if two arrays of the same length have equal values at every position (null-safe).
///
/// Two positions are considered equal if they are both null, or both non-null with the same value.
///
// TODO(ngates): move this function out when we have any/all aggregate functions.
fn arrays_value_equal(a: &ArrayRef, b: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<bool> {
debug_assert_eq!(a.len(), b.len());
Expand Down
1 change: 0 additions & 1 deletion vortex-array/src/aggregate_fn/fns/sum/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,6 @@ pub struct SumPartial {
}

/// The accumulated sum value.
///
// TODO(ngates): instead of an enum, we should use a Box<dyn State> to avoid dispatcher over the
// input type every time? Perhaps?
pub enum SumState {
Expand Down
1 change: 0 additions & 1 deletion vortex-array/src/aliases/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
// SPDX-FileCopyrightText: Copyright the Vortex contributors

//! Re-exports of third-party crates we use in macros exported from vortex-array.
//!

pub mod paste {
//! Re-export of [`paste`](https://docs.rs/paste/latest/paste/).
Expand Down
6 changes: 3 additions & 3 deletions vortex-array/src/arrays/chunked/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -384,9 +384,9 @@ mod test {
// Create chunks where some are empty but all non-empty chunks have all invalid values
let chunks = vec![
PrimitiveArray::new(buffer![1u64, 2], Validity::AllInvalid).into_array(),
PrimitiveArray::new(buffer![0u64; 0], Validity::AllInvalid).into_array(), // empty chunk
PrimitiveArray::new(buffer![0u64; 0], Validity::AllInvalid).into_array(), /* empty chunk */
PrimitiveArray::new(buffer![3u64, 4, 5], Validity::AllInvalid).into_array(),
PrimitiveArray::new(buffer![0u64; 0], Validity::AllInvalid).into_array(), // empty chunk
PrimitiveArray::new(buffer![0u64; 0], Validity::AllInvalid).into_array(), /* empty chunk */
];

let chunked =
Expand All @@ -410,7 +410,7 @@ mod test {
PrimitiveArray::new(buffer![1u64, 2], Validity::AllValid).into_array(),
PrimitiveArray::new(buffer![0u64; 0], Validity::AllValid).into_array(), // empty chunk

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how did this survive

PrimitiveArray::new(buffer![3u64, 4], Validity::AllInvalid).into_array(),
PrimitiveArray::new(buffer![0u64; 0], Validity::AllInvalid).into_array(), // empty chunk
PrimitiveArray::new(buffer![0u64; 0], Validity::AllInvalid).into_array(), /* empty chunk */

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this one is horrible

];

let chunked =
Expand Down
2 changes: 1 addition & 1 deletion vortex-array/src/arrays/fixed_size_list/tests/filter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ fn test_filter_with_nulls() {
let expected = FixedSizeListArray::new(
expected_elements.into_array(),
2,
Validity::from_iter([true, true]), // Both selected lists are valid, but type is still nullable.
Validity::from_iter([true, true]), /* Both selected lists are valid, but type is still nullable. */
2,
);

Expand Down
2 changes: 1 addition & 1 deletion vortex-array/src/arrays/patched/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
//! `indices` and `values` are aligned and accessed together.
//!
//! ```text
//!
//!

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

?

//! chunk 0 chunk 0 chunk 0 chunk 0 chunk 0 chunk 0
//! lane 0 lane 1 lane 2 lane 3 lane 4 lane 5
//! ┌────────────┬────────────┬────────────┬────────────┬────────────┬────────────┐
Expand Down
2 changes: 1 addition & 1 deletion vortex-array/src/arrays/primitive/compute/slice.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ impl SliceReduce for Primitive {
let validity = array.validity()?.slice(range)?;

// SAFETY:
//slicing an existing PrimitiveArray on element boundaries preserves the buffer
// slicing an existing PrimitiveArray on element boundaries preserves the buffer
// alignment, ptype, length, and validity invariants.
let array = unsafe {
PrimitiveArray::new_unchecked_from_handle(values, array.ptype(), validity).into_array()
Expand Down
2 changes: 1 addition & 1 deletion vortex-array/src/arrays/varbin/compute/compare.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ impl CompareKernel for VarBin {

if rhs_is_empty {
let buffer = match operator {
CompareOperator::Gte => BitBuffer::new_set(len), // Every possible value is >= ""
CompareOperator::Gte => BitBuffer::new_set(len), /* Every possible value is >= "" */
CompareOperator::Lt => BitBuffer::new_unset(len), // No value is < ""
CompareOperator::Eq | CompareOperator::Lte => {
let lhs_offsets = lhs.offsets().clone().execute::<PrimitiveArray>(ctx)?;
Expand Down
1 change: 0 additions & 1 deletion vortex-array/src/dtype/field.rs
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,6 @@ impl Display for Field {
/// let dtype_i32 = DType::Primitive(PType::I32, Nullability::NonNullable);
/// assert_eq!(dtype_i32, FieldPath::root().resolve(dtype_i32.clone()).unwrap());
/// ```
///
// TODO(ngates): we should probably reverse the path. Or better yet, store a Arc<[Field]> along
// with a positional index to allow cheap step_into.
#[derive(Clone, Default, Debug, PartialEq, Eq, Hash)]
Expand Down
1 change: 0 additions & 1 deletion vortex-array/src/expr/analysis/labeling.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ use crate::expr::traversal::TraversalOrder;
/// - `merge_child`: Mutable function that folds child labels into an accumulator.
/// Takes `(self_label, child_label)` and returns the updated accumulator.
/// Called once per child, with the initial accumulator being the node's self-label.
///
pub fn label_tree<L: Clone>(
expr: &Expression,
self_label: impl Fn(&Expression) -> L,
Expand Down
1 change: 0 additions & 1 deletion vortex-cxx/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ use write::*;

/// By default, the C++ API uses a current-thread runtime, providing control of the threading
/// model to the C++ side.
///
// TODO(ngates): in the future, we could expose an API for C++ to spawn threads that can drive
// this runtime.
pub(crate) static RUNTIME: LazyLock<CurrentThreadRuntime> =
Expand Down
1 change: 0 additions & 1 deletion vortex-datafusion/src/v2/table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,6 @@ impl TableProvider for VortexTable {
///
/// We should not (and actually, cannot) perform I/O here, so the best we can do is return
/// cardinality and byte size estimates.
///
// NOTE(ngates): it's not obvious these are actually used? I think DataFusion does join
// planning over stats from the physical plan?
fn statistics(&self) -> Option<Statistics> {
Expand Down
26 changes: 12 additions & 14 deletions vortex-duckdb/src/convert/expr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -117,20 +117,18 @@ pub(super) fn try_from_bound_expression_with_col_sub(
try_from_expression_inner(value, Some(col_sub))
}

/*
* Called before pushdown_complex_filter or a table filter expression call.
* As we support complex filter pushdown, Duckdb pushes expressions to Vortex.
* However, it doesn't know what type of expressions we can handle. Here we list
* all expressions that are quaranteed to be converted to Vortex expressions.
*
* If we return true here, and expression is in the list for
* pushdown_complex_filter, we must handle it, or query engine will break.
*
* Example: we don't support substr() expression so we tell Duckdb we can't
* push it.
* Example: optional filters may fail to parse on our side (we return
* Ok(None)), so we don't allow pushing these.
*/
// Called before pushdown_complex_filter or a table filter expression call.
// As we support complex filter pushdown, Duckdb pushes expressions to Vortex.
// However, it doesn't know what type of expressions we can handle. Here we list
// all expressions that are quaranteed to be converted to Vortex expressions.
//
// If we return true here, and expression is in the list for
// pushdown_complex_filter, we must handle it, or query engine will break.
//
// Example: we don't support substr() expression so we tell Duckdb we can't
// push it.
// Example: optional filters may fail to parse on our side (we return
// Ok(None)), so we don't allow pushing these.
pub fn can_push_expression(value: &duckdb::ExpressionRef) -> bool {
let Some(value) = value.as_class() else {
return false;
Expand Down
2 changes: 1 addition & 1 deletion vortex-duckdb/src/exporter/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -480,7 +480,7 @@ mod tests {
assert_eq!(copy_from_slice(&mut target, &source, 10, 64), 28,);
assert_eq!(
target[0],
0xff_99_59_0d_0c_cc_80_c0_u64, // Python: hex(0xff_fe_65_64_34_33_32_03_02 >> 2), then remove the high two hexits
0xff_99_59_0d_0c_cc_80_c0_u64, /* Python: hex(0xff_fe_65_64_34_33_32_03_02 >> 2), then remove the high two hexits */
"{:#08x} == {:#08x}",
target[0],
0xff_99_59_0d_0c_cc_80_c0_u64
Expand Down
20 changes: 9 additions & 11 deletions vortex-duckdb/src/table_function.rs
Original file line number Diff line number Diff line change
Expand Up @@ -451,17 +451,15 @@ pub fn statistics(bind_data: &TableFunctionBind, column_index: usize) -> Option<
Some(ColumnStatistics::from(&stats_aggregate, dtype))
}

/**
* Duckdb requires post-filter cardinality estimates, otherwise join planner
* may flip join sides which is a huge regression for some queries i.e. 1000x
* for tpcds 85.
*
* See duckdb/src/optimizer/join_order/relation_statistics_helper.cpp
*
* As we don't report distinct values (same as Parquet), the only heuristic
* duckdb uses is a 0.2 filter if there is any non-optional filter. We mimic it
* here.
*/
/// Duckdb requires post-filter cardinality estimates, otherwise join planner
/// may flip join sides which is a huge regression for some queries i.e. 1000x
/// for tpcds 85.
///
/// See duckdb/src/optimizer/join_order/relation_statistics_helper.cpp
///
/// As we don't report distinct values (same as Parquet), the only heuristic
/// duckdb uses is a 0.2 filter if there is any non-optional filter. We mimic it
/// here.
const DEFAULT_SELECTIVITY: f64 = 0.2;
pub fn cardinality(bind_data: &TableFunctionBind) -> Cardinality {
match bind_data.data_source.row_count() {
Expand Down
9 changes: 0 additions & 9 deletions vortex-ffi/cinclude/vortex.h
Original file line number Diff line number Diff line change
Expand Up @@ -654,7 +654,6 @@ bool vx_array_is_nullable(const vx_array *array);
* const vx_array* array = vx_array_new_null(1);
* assert(vx_array_has_dtype(array, DTYPE_NULL));
* vx_array_free(array);
*
*/
bool vx_array_has_dtype(const vx_array *array, vx_dtype_variant variant);

Expand All @@ -664,7 +663,6 @@ bool vx_array_has_dtype(const vx_array *array, vx_dtype_variant variant);
* const vx_array* array = vx_array_new_null(1);
* assert(!vx_array_is_primitive(array, PTYPE_U32));
* vx_array_free(array);
*
*/
bool vx_array_is_primitive(const vx_array *array, vx_ptype ptype);

Expand Down Expand Up @@ -722,7 +720,6 @@ const vx_array *vx_array_new_null(size_t len);
* const vx_array* array = vx_array_new_primitive(PTYPE_U32, buffer, 3,
* &validity, &error);
* vx_array_free(array);
*
*/
const vx_array *vx_array_new_primitive(vx_ptype ptype,
const void *ptr,
Expand Down Expand Up @@ -753,7 +750,6 @@ const vx_array *vx_array_new_primitive(vx_ptype ptype,
* const vx_array* vx = vx_array_from_arrow(&array, &schema, false, &error);
* // ... push it to a sink or write it ...
* vx_array_free(vx);
*
*/
const vx_array *
vx_array_from_arrow(FFI_ArrowArray *array, FFI_ArrowSchema *schema, bool nullable, vx_error **error_out);
Expand Down Expand Up @@ -1088,7 +1084,6 @@ void vx_expression_free(vx_expression *ptr);
* vx_array_free(applied_array);
* vx_expression_free(root);
* vx_array_free(array);
*
*/
vx_expression *vx_expression_root(void);

Expand Down Expand Up @@ -1122,7 +1117,6 @@ vx_expression *vx_expression_root(void);
* vx_expression_free(threshold);
* vx_expression_free(age);
* vx_expression_free(root);
*
*/
vx_expression *vx_expression_literal(const vx_scalar *scalar, vx_error **err);

Expand All @@ -1140,7 +1134,6 @@ vx_expression *vx_expression_literal(const vx_scalar *scalar, vx_error **err);
* vx_expression* select = vx_expression_select(names, 2, root);
* vx_expression_free(select);
* vx_expression_free(root);
*
*/
vx_expression *vx_expression_select(const char *const *names, size_t len, const vx_expression *child);

Expand Down Expand Up @@ -1177,7 +1170,6 @@ vx_expression *vx_expression_or(const vx_expression *const *expressions, size_t
* ) {
* return vx_expression_binary(VX_OPERATOR_EQ, lhs, rhs);
* }
*
*/
vx_expression *
vx_expression_binary(vx_binary_operator operator_, const vx_expression *lhs, const vx_expression *rhs);
Expand Down Expand Up @@ -1679,7 +1671,6 @@ void vx_struct_column_builder_add_field(vx_struct_column_builder *builder,
*
* vx_array_free(struct_array);
* vx_array_free(field_array);
*
*/
const vx_array *vx_struct_column_builder_finalize(vx_struct_column_builder *builder, vx_error **error);

Expand Down
2 changes: 1 addition & 1 deletion vortex-ffi/examples/hello_vortex.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
//! You can invoke this example from a checkout by running
//!
//! ```ignore
//!cargo run -p vortex-ffi --example hello_vortex
//! cargo run -p vortex-ffi --example hello_vortex
//! ```

use std::clone::Clone;
Expand Down
4 changes: 0 additions & 4 deletions vortex-ffi/src/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,6 @@ pub unsafe extern "C" fn vx_array_is_nullable(array: *const vx_array) -> bool {
/// const vx_array* array = vx_array_new_null(1);
/// assert(vx_array_has_dtype(array, DTYPE_NULL));
/// vx_array_free(array);
///
#[unsafe(no_mangle)]
pub unsafe extern "C-unwind" fn vx_array_has_dtype(
array: *const vx_array,
Expand All @@ -109,7 +108,6 @@ pub unsafe extern "C-unwind" fn vx_array_has_dtype(
/// const vx_array* array = vx_array_new_null(1);
/// assert(!vx_array_is_primitive(array, PTYPE_U32));
/// vx_array_free(array);
///
#[unsafe(no_mangle)]
pub unsafe extern "C-unwind" fn vx_array_is_primitive(
array: *const vx_array,
Expand Down Expand Up @@ -318,7 +316,6 @@ unsafe fn primitive_from_raw<T: vortex::dtype::NativePType>(
/// const vx_array* array = vx_array_new_primitive(PTYPE_U32, buffer, 3,
/// &validity, &error);
/// vx_array_free(array);
///
#[unsafe(no_mangle)]
pub extern "C-unwind" fn vx_array_new_primitive(
ptype: vx_ptype,
Expand Down Expand Up @@ -370,7 +367,6 @@ pub extern "C-unwind" fn vx_array_new_primitive(
/// const vx_array* vx = vx_array_from_arrow(&array, &schema, false, &error);
/// // ... push it to a sink or write it ...
/// vx_array_free(vx);
///
#[unsafe(no_mangle)]
pub unsafe extern "C-unwind" fn vx_array_from_arrow(
array: *mut FFI_ArrowArray,
Expand Down
4 changes: 0 additions & 4 deletions vortex-ffi/src/expression.rs
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@ crate::box_wrapper!(
/// vx_array_free(applied_array);
/// vx_expression_free(root);
/// vx_array_free(array);
///
#[unsafe(no_mangle)]
pub unsafe extern "C" fn vx_expression_root() -> *mut vx_expression {
vx_expression::new(root())
Expand Down Expand Up @@ -96,7 +95,6 @@ pub unsafe extern "C" fn vx_expression_root() -> *mut vx_expression {
/// vx_expression_free(threshold);
/// vx_expression_free(age);
/// vx_expression_free(root);
///
#[unsafe(no_mangle)]
pub unsafe extern "C-unwind" fn vx_expression_literal(
scalar: *const vx_scalar,
Expand All @@ -121,7 +119,6 @@ pub unsafe extern "C-unwind" fn vx_expression_literal(
/// vx_expression* select = vx_expression_select(names, 2, root);
/// vx_expression_free(select);
/// vx_expression_free(root);
///
#[unsafe(no_mangle)]
pub unsafe extern "C" fn vx_expression_select(
names: *const *const c_char,
Expand Down Expand Up @@ -245,7 +242,6 @@ impl From<vx_binary_operator> for Operator {
/// ) {
/// return vx_expression_binary(VX_OPERATOR_EQ, lhs, rhs);
/// }
///
#[unsafe(no_mangle)]
pub unsafe extern "C" fn vx_expression_binary(
operator: vx_binary_operator,
Expand Down
1 change: 0 additions & 1 deletion vortex-ffi/src/macros.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@
//! These macros don't require any Send + Sync bounds. We could try and define this behaviour here
//! to make it clearer, but for now, it's important to just be careful when documenting the thread
//! safety of the functions that use these types.
//!

/// Define a native FFI type that wraps an [`std::sync::Arc<T>`] type with unsized T.
///
Expand Down
1 change: 0 additions & 1 deletion vortex-ffi/src/struct_array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,6 @@ pub unsafe extern "C" fn vx_struct_column_builder_add_field(
///
/// vx_array_free(struct_array);
/// vx_array_free(field_array);
///
#[unsafe(no_mangle)]
pub extern "C-unwind" fn vx_struct_column_builder_finalize(
builder: *mut vx_struct_column_builder,
Expand Down
1 change: 0 additions & 1 deletion vortex-file/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,6 @@
//! goals of locality or parallelism. For example, one may write a column in a Struct Layout with
//! or without chunking, or completely elide statistics to save space or if they are not needed, for
//! example if the metadata is being stored in an external index.
//!

mod counting;
mod file;
Expand Down
Loading
Loading