diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 2bb0d20c8f4833..29df2db66a414f 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -165,6 +165,27 @@ if (DEFINED ENV{PAIMON_HOME} AND NOT PAIMON_HOME) set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "" FORCE) endif() +option(BUILD_RUST_READERS "Build Rust-based format readers (Lance, etc.)" OFF) +if (DEFINED ENV{BUILD_RUST_READERS}) + set(BUILD_RUST_READERS "$ENV{BUILD_RUST_READERS}" CACHE BOOL "" FORCE) +endif() +# Auto-enable if pre-built Rust library exists (from zigbuild or manylinux2014) +if (NOT BUILD_RUST_READERS) + if (EXISTS "${SRC_DIR}/rust/doris-native/target/x86_64-unknown-linux-gnu/release/libdoris_ffi.a" + OR EXISTS "${SRC_DIR}/rust/doris-native/target/release/libdoris_ffi.a") + set(BUILD_RUST_READERS ON) + message(STATUS "Auto-enabling BUILD_RUST_READERS: pre-built library found") + endif() +endif() +if (BUILD_RUST_READERS) + # rust.cmake detects pre-built .a or builds via Corrosion. + # If neither exists, it sets BUILD_RUST_READERS=OFF and returns. + include(cmake/rust.cmake) +endif() +if (BUILD_RUST_READERS) + add_definitions(-DBUILD_RUST_READERS) +endif() + set(CMAKE_SKIP_RPATH TRUE) set(Boost_USE_STATIC_LIBS ON) set(Boost_USE_STATIC_RUNTIME ON) @@ -686,6 +707,10 @@ if (ENABLE_PAIMON_CPP) set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} tbb_paimon) endif() +if (BUILD_RUST_READERS) + set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} doris_ffi_lib) +endif() + set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_END_GROUP}) # Add all external dependencies. They should come after the palo libs. diff --git a/be/cmake/rust.cmake b/be/cmake/rust.cmake new file mode 100644 index 00000000000000..1bdc0120590a57 --- /dev/null +++ b/be/cmake/rust.cmake @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Rust integration for doris-ffi static library. +# +# Two modes: +# 1. Pre-built: If libdoris_ffi.a already exists (e.g., built by cargo-zigbuild +# on the CI host), use it directly. No Rust toolchain needed inside the +# build container. +# 2. Corrosion: If no pre-built .a found and cargo is available, build via +# Corrosion (FetchContent). Used on developer machines. + +# Check for pre-built .a (from zigbuild, manylinux2014, or manual build) +set(PREBUILT_RUST_PATHS + "${SRC_DIR}/rust/doris-native/target/x86_64-unknown-linux-gnu/release/libdoris_ffi.a" + "${SRC_DIR}/rust/doris-native/target/release/libdoris_ffi.a" + "${CMAKE_BINARY_DIR}/libdoris_ffi.a" +) + +set(RUST_LIB_PATH "") +foreach(p ${PREBUILT_RUST_PATHS}) + if (EXISTS "${p}") + set(RUST_LIB_PATH "${p}") + message(STATUS "Rust readers: using pre-built library at ${p}") + break() + endif() +endforeach() + +if (RUST_LIB_PATH) + # Mode 1: Pre-built .a found — no Corrosion needed + add_library(doris_ffi_lib STATIC IMPORTED GLOBAL) + set_target_properties(doris_ffi_lib PROPERTIES + IMPORTED_LOCATION "${RUST_LIB_PATH}" + IMPORTED_LINK_INTERFACE_LIBRARIES "m;dl;pthread" + ) + message(STATUS "Rust readers enabled (pre-built)") +else() + # Mode 2: Build via Corrosion (developer machines with cargo) + find_program(CARGO_EXECUTABLE cargo) + if (NOT CARGO_EXECUTABLE) + message(WARNING "BUILD_RUST_READERS=ON but no pre-built libdoris_ffi.a and no cargo in PATH. Disabling.") + set(BUILD_RUST_READERS OFF PARENT_SCOPE) + return() + endif() + + include(FetchContent) + FetchContent_Declare( + Corrosion + GIT_REPOSITORY https://github.com/corrosion-rs/corrosion.git + GIT_TAG v0.5.1 + ) + FetchContent_MakeAvailable(Corrosion) + + set(RUST_MANIFEST_PATH "${SRC_DIR}/rust/doris-native/Cargo.toml") + corrosion_import_crate( + MANIFEST_PATH ${RUST_MANIFEST_PATH} + CRATES doris-ffi + ) + + add_library(doris_ffi_lib STATIC IMPORTED GLOBAL) + set_target_properties(doris_ffi_lib PROPERTIES + IMPORTED_LOCATION "${CMAKE_BINARY_DIR}/libdoris_ffi.a" + IMPORTED_LINK_INTERFACE_LIBRARIES "m;dl;pthread" + ) + add_dependencies(doris_ffi_lib cargo-build_doris_ffi) + message(STATUS "Rust readers enabled (Corrosion)") +endif() diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index 19c594536b1246..cff29c68e8b09a 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -79,6 +79,9 @@ #include "format/table/transactional_hive_reader.h" #include "format/table/trino_connector_jni_reader.h" #include "format/text/text_reader.h" +#ifdef BUILD_RUST_READERS +#include "format/lance/lance_rust_reader.h" +#endif #include "io/cache/block_file_cache_profile.h" #include "load/group_commit/wal/wal_reader.h" #include "runtime/descriptors.h" @@ -1140,6 +1143,16 @@ Status FileScanner::_get_next_reader() { } break; } +#ifdef BUILD_RUST_READERS + case TFileFormatType::FORMAT_LANCE: { + auto lance_reader = LanceRustReader::create_unique(_file_slot_descs, _state, _profile, + range, _params); + init_status = lance_reader->init_reader(); + _cur_reader = std::move(lance_reader); + need_to_get_parsed_schema = true; + break; + } +#endif default: return Status::NotSupported("Not supported create reader for file format: {}.", to_string(_params->format_type)); diff --git a/be/src/format/CMakeLists.txt b/be/src/format/CMakeLists.txt index 64e7b4f14fe05e..ef9dab92c00f97 100644 --- a/be/src/format/CMakeLists.txt +++ b/be/src/format/CMakeLists.txt @@ -22,6 +22,13 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/format") set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/format") file(GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS *.cpp) + +# Lance reader requires Rust static library (BUILD_RUST_READERS=ON) +if (NOT BUILD_RUST_READERS) + file(GLOB_RECURSE LANCE_FILES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/lance/*.cpp) + list(REMOVE_ITEM SRC_FILES ${LANCE_FILES}) +endif() + add_library(Format STATIC ${SRC_FILES}) pch_reuse(Format) diff --git a/be/src/format/lance/lance_ffi.h b/be/src/format/lance/lance_ffi.h new file mode 100644 index 00000000000000..0f846f8da06a1d --- /dev/null +++ b/be/src/format/lance/lance_ffi.h @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#ifdef BUILD_RUST_READERS + +#include + +#include +#include + +namespace doris::lance_ffi { + +// FFI status codes (must match Rust error.rs) +constexpr int32_t LANCE_FFI_OK = 0; +constexpr int32_t LANCE_FFI_EOF = 1; +constexpr int32_t LANCE_FFI_ERR_LANCE = -1; +constexpr int32_t LANCE_FFI_ERR_ARROW = -2; +constexpr int32_t LANCE_FFI_ERR_IO = -3; +constexpr int32_t LANCE_FFI_ERR_PANIC = -4; +constexpr int32_t LANCE_FFI_ERR_INVALID_ARG = -5; + +} // namespace doris::lance_ffi + +// Opaque handle to a Rust LanceReader. +using LanceReaderHandle = void*; + +extern "C" { + +/// Open a Lance dataset and create a reader. +int32_t lance_reader_open(const uint8_t* uri_ptr, size_t uri_len, + const uint8_t* const* column_names_ptr, + const size_t* column_names_len_ptr, size_t num_columns, size_t batch_size, + LanceReaderHandle* handle_out); + +/// Read the next batch via Arrow C Data Interface. +/// Returns LANCE_FFI_OK with data, LANCE_FFI_EOF on end, negative on error. +int32_t lance_reader_next_batch(LanceReaderHandle handle, ArrowSchema* schema_out, + ArrowArray* array_out, bool* eof_out, int64_t* bytes_out); + +/// Get the schema of the scan output. +int32_t lance_reader_get_schema(LanceReaderHandle handle, ArrowSchema* schema_out); + +/// Close the reader and free resources. Safe to call with null handle. +void lance_reader_close(LanceReaderHandle handle); + +/// Retrieve the last error message. Returns bytes written (excluding null terminator). +size_t lance_reader_last_error(uint8_t* buf, size_t buf_len); + +/// Open a Lance dataset from a JSON config string. +/// Config JSON: {"uri":"...", "columns":[], "batch_size":N, "version":N, "storage_options":{}} +/// storage_options carries S3 credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, etc.) +int32_t lance_reader_open_json(const uint8_t* config_json_ptr, size_t config_json_len, + LanceReaderHandle* handle_out); + +/// Phase 0 echo function for build verification. +int32_t rust_echo(int32_t x); + +/// Create a test Lance dataset at the given path. For GTests only. +/// Dataset has 5 rows: id(INT32), name(UTF8), score(FLOAT64). +int32_t lance_test_create_dataset(const uint8_t* path_ptr, size_t path_len); + +/// Create a multi-fragment test dataset. 3 fragments, 5 rows each = 15 total. +/// Columns: id(INT32), name(UTF8), value(FLOAT64). +int32_t lance_test_create_multi_fragment_dataset(const uint8_t* path_ptr, size_t path_len); + +} // extern "C" + +#endif // BUILD_RUST_READERS diff --git a/be/src/format/lance/lance_rust_reader.cpp b/be/src/format/lance/lance_rust_reader.cpp new file mode 100644 index 00000000000000..166bbd52dcc519 --- /dev/null +++ b/be/src/format/lance/lance_rust_reader.cpp @@ -0,0 +1,403 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifdef BUILD_RUST_READERS + +#include "format/lance/lance_rust_reader.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "core/block/block.h" +#include "core/block/column_with_type_and_name.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/primitive_type.h" +#include "format/lance/lance_ffi.h" +#include "runtime/descriptors.h" +#include "runtime/runtime_state.h" +#include "util/timezone_utils.h" + +namespace doris { +#include "common/compile_check_avoid_begin.h" + +const std::vector LanceRustReader::_empty_slot_descs; + +LanceRustReader::LanceRustReader(const std::vector& file_slot_descs, + RuntimeState* state, RuntimeProfile* /*profile*/, + const TFileRangeDesc& range, + const TFileScanRangeParams* range_params) + : _file_slot_descs(file_slot_descs), _state(state), _range(range), _params(range_params) { + TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, _ctzz); +} + +LanceRustReader::LanceRustReader(const TFileScanRangeParams& params, const TFileRangeDesc& range, + io::IOContext* /*io_ctx*/) + : _file_slot_descs(_empty_slot_descs), + _state(nullptr), + _range(range), + _params(¶ms), + _schema_only(true) { + TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, _ctzz); +} + +LanceRustReader::~LanceRustReader() { + static_cast(close()); +} + +Status LanceRustReader::init_reader() { + return _open_with_json(false); +} + +Status LanceRustReader::init_schema_reader() { + return _open_with_json(true); +} + +Status LanceRustReader::_open_with_json(bool schema_only) { + std::string uri = _range.path; + if (uri.empty()) { + return Status::InvalidArgument("Lance reader: dataset URI is empty"); + } + + // Lance datasets are directories (e.g., data.lance/). + // The TVF path may point to a file inside (e.g., data.lance/data/xxx.lance). + // Strip back to the .lance dataset root and extract the fragment file name + // so we only read the specific fragment assigned to this scan range. + std::string fragment_file; + auto lance_pos = uri.find(".lance"); + if (lance_pos != std::string::npos) { + auto end = lance_pos + 6; // ".lance" is 6 chars + if (end < uri.size() && uri[end] == '/') { + // Extract the relative path after the dataset root (e.g., "data/xxx.lance") + fragment_file = uri.substr(end + 1); + uri = uri.substr(0, end); + } + } + + // Build JSON config for the Rust FFI + rapidjson::Document doc(rapidjson::kObjectType); + auto& alloc = doc.GetAllocator(); + + doc.AddMember("uri", rapidjson::Value(uri.c_str(), alloc), alloc); + + // Pass fragment file name so Rust reads only this fragment, not all + if (!fragment_file.empty() && !schema_only) { + doc.AddMember("fragment_file", rapidjson::Value(fragment_file.c_str(), alloc), alloc); + } + + // Columns (only for data reads, not schema-only) + rapidjson::Value cols(rapidjson::kArrayType); + if (!schema_only) { + for (const auto* slot : _file_slot_descs) { + cols.PushBack(rapidjson::Value(slot->col_name().c_str(), alloc), alloc); + } + } + doc.AddMember("columns", cols, alloc); + + // Batch size + size_t batch_size = schema_only ? 1 : 4096; + if (!schema_only && _state) { + size_t bs = static_cast(_state->query_options().batch_size); + if (bs > 0) batch_size = bs; + } + doc.AddMember("batch_size", static_cast(batch_size), alloc); + + // Version (time travel) from TLanceFileDesc + uint64_t version = 0; + if (_range.__isset.table_format_params && _range.table_format_params.__isset.lance_params && + _range.table_format_params.lance_params.__isset.version) { + version = static_cast(_range.table_format_params.lance_params.version); + } + doc.AddMember("version", version, alloc); + + // Storage options (S3 credentials from scan range params properties) + rapidjson::Value storage_opts(rapidjson::kObjectType); + if (_range.__isset.table_format_params && _range.table_format_params.__isset.lance_params && + _range.table_format_params.lance_params.__isset.dataset_uri) { + // If a specific dataset_uri is set in lance_params, use it instead + const auto& lance_uri = _range.table_format_params.lance_params.dataset_uri; + if (!lance_uri.empty()) { + doc.RemoveMember("uri"); + doc.AddMember("uri", rapidjson::Value(lance_uri.c_str(), alloc), alloc); + } + } + // Map Doris S3 property keys to lance/object_store standard keys + static const std::vector> s3_key_mapping = { + {"AWS_ACCESS_KEY", "aws_access_key_id"}, + {"AWS_SECRET_KEY", "aws_secret_access_key"}, + {"AWS_TOKEN", "aws_session_token"}, + {"AWS_ENDPOINT", "aws_endpoint"}, + {"AWS_REGION", "aws_region"}, + }; + // Read S3 properties from TFileScanRangeParams.properties + if (_params && _params->__isset.properties) { + for (const auto& [k, v] : _params->properties) { + // Map Doris key names to object_store key names + bool mapped = false; + for (const auto& [doris_key, lance_key] : s3_key_mapping) { + if (k == doris_key) { + storage_opts.AddMember(rapidjson::Value(lance_key.c_str(), alloc), + rapidjson::Value(v.c_str(), alloc), alloc); + mapped = true; + break; + } + } + // Pass through any unrecognized keys as-is + if (!mapped && !v.empty()) { + storage_opts.AddMember(rapidjson::Value(k.c_str(), alloc), + rapidjson::Value(v.c_str(), alloc), alloc); + } + } + } + doc.AddMember("storage_options", storage_opts, alloc); + + // Serialize to JSON string + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc.Accept(writer); + std::string config_json = buffer.GetString(); + + LanceReaderHandle handle = nullptr; + int32_t rc = lance_reader_open_json(reinterpret_cast(config_json.data()), + config_json.size(), &handle); + if (rc != lance_ffi::LANCE_FFI_OK) { + return _get_ffi_error(rc); + } + + _reader_handle = handle; + return Status::OK(); +} + +Status LanceRustReader::_do_get_next_block(Block* block, size_t* read_rows, bool* eof) { + if (!_reader_handle) { + return Status::InternalError("Lance reader is not initialized"); + } + + if (_col_name_to_block_idx.empty()) { + _col_name_to_block_idx = block->get_name_to_pos_map(); + } + + ArrowSchema c_schema {}; + ArrowArray c_array {}; + bool is_eof = false; + int64_t batch_bytes = 0; + + int32_t rc = + lance_reader_next_batch(_reader_handle, &c_schema, &c_array, &is_eof, &batch_bytes); + + if (rc == lance_ffi::LANCE_FFI_EOF || is_eof) { + *read_rows = 0; + *eof = true; + return Status::OK(); + } + if (rc != lance_ffi::LANCE_FFI_OK) { + return _get_ffi_error(rc); + } + + // Import Arrow C Data Interface into C++ RecordBatch + arrow::Result> import_result = + arrow::ImportRecordBatch(&c_array, &c_schema); + if (!import_result.ok()) { + return Status::InternalError("Failed to import Lance arrow batch: {}", + import_result.status().message()); + } + + auto record_batch = std::move(import_result).ValueUnsafe(); + const auto num_rows = static_cast(record_batch->num_rows()); + const auto num_columns = record_batch->num_columns(); + + // Convert Arrow columns to Doris Block columns (same pattern as PaimonCppReader) + for (int c = 0; c < num_columns; ++c) { + const auto& field = record_batch->schema()->field(c); + + auto it = _col_name_to_block_idx.find(field->name()); + if (it == _col_name_to_block_idx.end()) { + continue; + } + + const ColumnWithTypeAndName& column_with_name = block->get_by_position(it->second); + try { + RETURN_IF_ERROR(column_with_name.type->get_serde()->read_column_from_arrow( + column_with_name.column->assume_mutable_ref(), record_batch->column(c).get(), 0, + num_rows, _ctzz)); + } catch (Exception& e) { + return Status::InternalError("Failed to convert Lance arrow to block: {}", e.what()); + } + } + + *read_rows = num_rows; + *eof = false; + return Status::OK(); +} + +Status LanceRustReader::_get_columns_impl( + std::unordered_map* name_to_type) { + if (_schema_only && _reader_handle) { + // In schema-only mode, get columns from the Lance schema + ArrowSchema c_schema {}; + int32_t rc = lance_reader_get_schema(_reader_handle, &c_schema); + if (rc != lance_ffi::LANCE_FFI_OK) { + return _get_ffi_error(rc); + } + auto import_result = arrow::ImportSchema(&c_schema); + if (!import_result.ok()) { + return Status::InternalError("Failed to import Lance schema: {}", + import_result.status().message()); + } + auto schema = std::move(import_result).ValueUnsafe(); + for (const auto& field : schema->fields()) { + auto doris_type = _arrow_type_to_doris_type(field->type()); + if (doris_type) { + name_to_type->emplace(field->name(), make_nullable(doris_type)); + } + } + return Status::OK(); + } + + for (const auto* slot : _file_slot_descs) { + name_to_type->emplace(slot->col_name(), slot->type()); + } + return Status::OK(); +} + +Status LanceRustReader::get_parsed_schema(std::vector* col_names, + std::vector* col_types) { + if (!_reader_handle) { + return Status::InternalError("Lance reader is not initialized"); + } + + ArrowSchema c_schema {}; + int32_t rc = lance_reader_get_schema(_reader_handle, &c_schema); + if (rc != lance_ffi::LANCE_FFI_OK) { + return _get_ffi_error(rc); + } + + auto import_result = arrow::ImportSchema(&c_schema); + if (!import_result.ok()) { + return Status::InternalError("Failed to import Lance schema: {}", + import_result.status().message()); + } + + auto schema = std::move(import_result).ValueUnsafe(); + for (const auto& field : schema->fields()) { + auto doris_type = _arrow_type_to_doris_type(field->type()); + if (doris_type) { + col_names->push_back(field->name()); + col_types->push_back(make_nullable(doris_type)); + } + } + return Status::OK(); +} + +Status LanceRustReader::close() { + if (_reader_handle) { + lance_reader_close(_reader_handle); + _reader_handle = nullptr; + } + return Status::OK(); +} + +Status LanceRustReader::_get_ffi_error(int32_t status_code) const { + constexpr size_t kBufSize = 1024; + uint8_t buf[kBufSize]; + size_t len = lance_reader_last_error(buf, kBufSize); + + std::string msg; + if (len > 0) { + msg.assign(reinterpret_cast(buf), len); + } else { + msg = fmt::format("Lance FFI error code: {}", status_code); + } + return Status::InternalError("Rust Lance reader: {}", msg); +} + +DataTypePtr LanceRustReader::_arrow_type_to_doris_type( + const std::shared_ptr& arrow_type) { + // Arrow type IDs: STRING and UTF8 are the same value in arrow-cpp. + // LARGE_STRING and LARGE_UTF8 are the same. Doris has no unsigned int types + // except UInt8 (used for BOOLEAN), so unsigned ints are widened to signed. + switch (arrow_type->id()) { + case arrow::Type::BOOL: + return std::make_shared(); + case arrow::Type::INT8: + case arrow::Type::UINT8: + return std::make_shared(); + case arrow::Type::INT16: + case arrow::Type::UINT16: + return std::make_shared(); + case arrow::Type::INT32: + case arrow::Type::UINT32: + return std::make_shared(); + case arrow::Type::INT64: + case arrow::Type::UINT64: + return std::make_shared(); + case arrow::Type::HALF_FLOAT: + case arrow::Type::FLOAT: + return std::make_shared(); + case arrow::Type::DOUBLE: + return std::make_shared(); + case arrow::Type::STRING: + case arrow::Type::LARGE_STRING: + return std::make_shared(); + case arrow::Type::BINARY: + case arrow::Type::LARGE_BINARY: + return std::make_shared(); + case arrow::Type::DATE32: + case arrow::Type::DATE64: + return std::make_shared(); + case arrow::Type::TIMESTAMP: + return std::make_shared(); + case arrow::Type::DECIMAL128: { + auto decimal_type = std::static_pointer_cast(arrow_type); + return create_decimal(decimal_type->precision(), decimal_type->scale(), false); + } + case arrow::Type::LIST: + case arrow::Type::LARGE_LIST: { + auto list_type = std::static_pointer_cast(arrow_type); + auto inner = _arrow_type_to_doris_type(list_type->value_type()); + if (inner) { + return std::make_shared(make_nullable(inner)); + } + return nullptr; + } + case arrow::Type::FIXED_SIZE_LIST: { + auto fsl_type = std::static_pointer_cast(arrow_type); + auto inner = _arrow_type_to_doris_type(fsl_type->value_type()); + if (inner) { + return std::make_shared(make_nullable(inner)); + } + return nullptr; + } + default: + // Unsupported types fall back to string + return std::make_shared(); + } +} + +#include "common/compile_check_avoid_end.h" +} // namespace doris + +#endif // BUILD_RUST_READERS diff --git a/be/src/format/lance/lance_rust_reader.h b/be/src/format/lance/lance_rust_reader.h new file mode 100644 index 00000000000000..29e82809773859 --- /dev/null +++ b/be/src/format/lance/lance_rust_reader.h @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#ifdef BUILD_RUST_READERS + +#include + +#include +#include +#include +#include + +#include "common/factory_creator.h" +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "format/generic_reader.h" + +namespace arrow { +class DataType; +} + +namespace doris { +namespace io { +struct IOContext; +} + +class RuntimeProfile; +class RuntimeState; +class SlotDescriptor; +class Block; +class TFileRangeDesc; +class TFileScanRangeParams; + +/// Reads Lance format datasets via Rust FFI (lance-rs). +/// +/// Data exchange uses the Arrow C Data Interface: the Rust side exports +/// ArrowSchema + ArrowArray, which the C++ side imports as an +/// arrow::RecordBatch, then converts column-by-column to Doris Block. +/// +/// Each reader instance owns an opaque Rust handle that holds a +/// single-threaded tokio runtime and a lance::Scanner stream. +class LanceRustReader : public GenericReader { + ENABLE_FACTORY_CREATOR(LanceRustReader); + +public: + LanceRustReader(const std::vector& file_slot_descs, RuntimeState* state, + RuntimeProfile* profile, const TFileRangeDesc& range, + const TFileScanRangeParams* range_params); + + ~LanceRustReader() override; + + /// Constructor for schema-only mode (used by fetch_table_schema RPC). + /// Only needs params and range, no slot descs or runtime state. + LanceRustReader(const TFileScanRangeParams& params, const TFileRangeDesc& range, + io::IOContext* io_ctx); + + Status init_reader(); + + /// Initialize reader in schema-only mode (open dataset, read schema, no scan). + Status init_schema_reader() override; + + Status get_parsed_schema(std::vector* col_names, + std::vector* col_types) override; + + Status close() override; + +protected: + Status _do_get_next_block(Block* block, size_t* read_rows, bool* eof) override; + + Status _get_columns_impl(std::unordered_map* name_to_type) override; + +private: + /// Open via JSON config — shared by init_reader() and init_schema_reader(). + Status _open_with_json(bool schema_only); + + /// Build a Doris Status from the Rust FFI error code + thread-local error message. + Status _get_ffi_error(int32_t status_code) const; + + /// Convert an Arrow DataType to a Doris DataTypePtr. + static DataTypePtr _arrow_type_to_doris_type( + const std::shared_ptr& arrow_type); + + const std::vector& _file_slot_descs; + RuntimeState* _state; + const TFileRangeDesc& _range; + const TFileScanRangeParams* _params; + + void* _reader_handle = nullptr; + std::unordered_map _col_name_to_block_idx; + cctz::time_zone _ctzz; + bool _schema_only = false; + static const std::vector _empty_slot_descs; +}; + +} // namespace doris + +#include "common/compile_check_avoid_begin.h" +#include "common/compile_check_avoid_end.h" + +#endif // BUILD_RUST_READERS diff --git a/be/src/rust/doris-native/Cargo.lock b/be/src/rust/doris-native/Cargo.lock new file mode 100644 index 00000000000000..0f0ec472af7e33 --- /dev/null +++ b/be/src/rust/doris-native/Cargo.lock @@ -0,0 +1,5409 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrow" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3a3ec4fe573f9d1f59d99c085197ef669b00b088ba1d7bb75224732d9357a74" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dcf19f07792d8c7f91086c67b574a79301e367029b17fcf63fb854332246a10" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "num", +] + +[[package]] +name = "arrow-array" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7845c32b41f7053e37a075b3c2f29c6f5ea1b3ca6e5df7a2d325ee6e1b4a63cf" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.15.5", + "num", +] + +[[package]] +name = "arrow-buffer" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b5c681a99606f3316f2a99d9c8b6fa3aad0b1d34d8f6d7a1b471893940219d8" +dependencies = [ + "bytes", + "half", + "num", +] + +[[package]] +name = "arrow-cast" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6365f8527d4f87b133eeb862f9b8093c009d41a210b8f101f91aa2392f61daac" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30dac4d23ac769300349197b845e0fd18c7f9f15d260d4659ae6b5a9ca06f586" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "lazy_static", + "lexical-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd962fc3bf7f60705b25bcaa8eb3318b2545aa1d528656525ebdd6a17a6cd6fb" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num", +] + +[[package]] +name = "arrow-ipc" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3527365b24372f9c948f16e53738eb098720eea2093ae73c7af04ac5e30a39b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "flatbuffers", + "lz4_flex", + "zstd", +] + +[[package]] +name = "arrow-json" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdec0024749fc0d95e025c0b0266d78613727b3b3a5d4cf8ea47eb6d38afdd1" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "indexmap", + "lexical-core", + "num", + "serde", + "serde_json", +] + +[[package]] +name = "arrow-ord" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79af2db0e62a508d34ddf4f76bfd6109b6ecc845257c9cba6f939653668f89ac" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "half", + "num", +] + +[[package]] +name = "arrow-row" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da30e9d10e9c52f09ea0cf15086d6d785c11ae8dcc3ea5f16d402221b6ac7735" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35b0f9c0c3582dd55db0f136d3b44bfa0189df07adcf7dc7f2f2e74db0f52eb8" +dependencies = [ + "bitflags 2.11.0", +] + +[[package]] +name = "arrow-select" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92fc337f01635218493c23da81a364daf38c694b05fc20569c3193c11c561984" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num", +] + +[[package]] +name = "arrow-string" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d596a9fc25dae556672d5069b090331aca8acb93cae426d8b7dcdf1c558fa0ce" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num", + "regex", + "regex-syntax", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener 5.4.1", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-priority-channel" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acde96f444d31031f760c5c43dc786b97d3e1cb2ee49dd06898383fe9a999758" +dependencies = [ + "event-listener 4.0.3", +] + +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "async_cell" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "447ab28afbb345f5408b120702a44e5529ebf90b1796ec76e9528df8e288e6c2" +dependencies = [ + "loom", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "aws-config" +version = "1.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11493b0bad143270fb8ad284a096dd529ba91924c5409adeac856cc1bf047dbc" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 1.4.0", + "sha1", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-lc-rs" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "aws-runtime" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc0651c57e384202e47153c1260b84a9936e19803d747615edf199dc3b98d17" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "bytes-utils", + "fastrand", + "http 1.4.0", + "http-body 1.0.1", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.97.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aadc669e184501caaa6beafb28c6267fc1baef0810fb58f9b205485ca3f2567" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.99.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1342a7db8f358d3de0aed2007a0b54e875458e39848d54cc1d46700b2bfcb0a8" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.101.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab41ad64e4051ecabeea802d6a17845a91e83287e1dd249e6963ea1ba78c428a" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.0", + "percent-encoding", + "sha2", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-http" +version = "0.63.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a2f165a7feee6f263028b899d0a181987f4fa7179a6411a32a439fba7c5f769" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2", + "http 1.4.0", + "hyper", + "hyper-rustls", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.62.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9648b0bb82a2eedd844052c6ad2a1a822d1f8e3adee5fbf668366717e428856a" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "028999056d2d2fd58a697232f9eec4a643cf73a71cf327690a7edad1d2af2110" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876ab3c9c29791ba4ba02b780a3049e21ec63dabda09268b175272c3733a79e6" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.4.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d73dbfbaa8e4bc57b9045137680b958d274823509a360abfd8e1d514d40c95c" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47c8323699dd9b3c8d5b3c13051ae9cdef58fd179957c882f8374dd8725962d9" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "bitpacking" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019" +dependencies = [ + "crunchy", +] + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytemuck" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + +[[package]] +name = "cc" +version = "1.2.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "census" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "serde", + "windows-targets 0.52.6", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "datafusion" +version = "42.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae5f2abc725737d6e87b6d348a5aa2d0a77e4cf873045f004546da946e6e619" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-ipc", + "arrow-schema", + "async-trait", + "bytes", + "chrono", + "dashmap 6.1.0", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-functions-window", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-sql", + "futures", + "glob", + "half", + "hashbrown 0.14.5", + "indexmap", + "itertools 0.13.0", + "log", + "num_cpus", + "object_store 0.11.2", + "parking_lot", + "paste", + "pin-project-lite", + "rand 0.8.5", + "sqlparser", + "tempfile", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "datafusion-catalog" +version = "42.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "998761705551f11ffa4ee692cc285b44eb1def6e0d28c4eaf5041b9e2810dc1e" +dependencies = [ + "arrow-schema", + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-common" +version = "42.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11986f191e88d950f10a5cc512a598afba27d92e04a0201215ad60785005115a" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.14.5", + "instant", + "libc", + "num_cpus", + "object_store 0.11.2", + "paste", + "sqlparser", + "tokio", +] + +[[package]] +name = "datafusion-common-runtime" +version = "42.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "694c9d7ea1b82f95768215c4cb5c2d5c613690624e832a7ee64be563139d582f" +dependencies = [ + "log", + "tokio", +] + +[[package]] +name = "datafusion-execution" +version = "42.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b4cedcd98151e0a297f34021b6b232ff0ebc0f2f18ea5e7446b5ebda99b1a1" +dependencies = [ + "arrow", + "chrono", + "dashmap 6.1.0", + "datafusion-common", + "datafusion-expr", + "futures", + "hashbrown 0.14.5", + "log", + "object_store 0.11.2", + "parking_lot", + "rand 0.8.5", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "42.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8dd114dc0296cacaee98ad3165724529fcca9a65b2875abcd447b9cc02b2b74" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "chrono", + "datafusion-common", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", + "paste", + "serde_json", + "sqlparser", + "strum", + "strum_macros", +] + +[[package]] +name = "datafusion-expr-common" +version = "42.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1ba2bb018218d9260bbd7de6a46a20f61b93d4911dba8aa07735625004c4fb" +dependencies = [ + "arrow", + "datafusion-common", + "paste", +] + +[[package]] +name = "datafusion-functions" +version = "42.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "547cb780a4ac51fd8e52c0fb9188bc16cea4e35aebf6c454bda0b82a7a417304" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "chrono", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "hashbrown 0.14.5", + "hex", + "itertools 0.13.0", + "log", + "rand 0.8.5", + "regex", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "42.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e68cf5aa7ebcac08bd04bb709a9a6d4963eafd227da62b628133bc509c40f5a0" +dependencies = [ + "ahash", + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "half", + "log", + "paste", + "sqlparser", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "42.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2285d080dfecdfb8605b0ab2f1a41e2473208dc8e9bd6f5d1dbcfe97f517e6f" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", + "rand 0.8.5", +] + +[[package]] +name = "datafusion-functions-nested" +version = "42.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b6ffbbb7cf7bf0c0e05eb6207023fef341cac83a593a5365a6fc83803c572a9" +dependencies = [ + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-physical-expr-common", + "itertools 0.13.0", + "log", + "paste", + "rand 0.8.5", +] + +[[package]] +name = "datafusion-functions-window" +version = "42.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e78d30ebd6e9f74d4aeddec32744f5a18b5f9584591bc586fb5259c4848bac5" +dependencies = [ + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr-common", + "log", +] + +[[package]] +name = "datafusion-optimizer" +version = "42.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be172c44bf344df707e0c041fa3f41e6dc5fb0976f539c68bc442bca150ee58c" +dependencies = [ + "arrow", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "hashbrown 0.14.5", + "indexmap", + "itertools 0.13.0", + "log", + "paste", + "regex-syntax", +] + +[[package]] +name = "datafusion-physical-expr" +version = "42.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b86b7fa0b8161c49b0f005b0df193fc6d9b65ceec675f155422cda5d1583ca" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "arrow-string", + "base64", + "chrono", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.14.5", + "hex", + "indexmap", + "itertools 0.13.0", + "log", + "paste", + "petgraph 0.6.5", + "regex", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "42.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "242ba8a26351d9ca16295814c46743b0d1b00ec372174bdfbba991d0953dd596" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "hashbrown 0.14.5", + "rand 0.8.5", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "42.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ca088eb904bf1cfc9c5e5653110c70a6eaba43164085a9d180b35b77ce3b8b" +dependencies = [ + "arrow-schema", + "datafusion-common", + "datafusion-execution", + "datafusion-physical-expr", + "datafusion-physical-plan", + "itertools 0.13.0", +] + +[[package]] +name = "datafusion-physical-plan" +version = "42.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4989a53b824abc759685eb643f4d604c2fc2fea4e2c309ac3473bea263ecbbeb" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "futures", + "half", + "hashbrown 0.14.5", + "indexmap", + "itertools 0.13.0", + "log", + "once_cell", + "parking_lot", + "pin-project-lite", + "rand 0.8.5", + "tokio", +] + +[[package]] +name = "datafusion-sql" +version = "42.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66b9b75b9da10ed656073ac0553708f17eb8fa5a7b065ef9848914c93150ab9e" +dependencies = [ + "arrow", + "arrow-array", + "arrow-schema", + "datafusion-common", + "datafusion-expr", + "log", + "regex", + "sqlparser", + "strum", +] + +[[package]] +name = "deepsize" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cdb987ec36f6bf7bfbea3f928b75590b736fc42af8e54d97592481351b2b96c" +dependencies = [ + "deepsize_derive", +] + +[[package]] +name = "deepsize_derive" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990101d41f3bc8c1a45641024377ee284ecc338e5ecf3ea0f0e236d897c72796" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", + "serde_core", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "doc-comment" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780955b8b195a21ab8e4ac6b60dd1dbdcec1dc6c51c0617964b08c81785e12c9" + +[[package]] +name = "doris-ffi" +version = "0.1.0" +dependencies = [ + "arrow", + "arrow-array", + "arrow-schema", + "cc", + "futures", + "lance", + "lance-index", + "lance-io", + "lance-linalg", + "serde", + "serde_json", + "tempfile", + "tokio", +] + +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener 5.4.1", + "pin-project-lite", +] + +[[package]] +name = "fastdivide" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" + +[[package]] +name = "fastrand" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a043dc74da1e37d6afe657061213aa6f425f855399a11d3463c6ecccc4dfda1f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "24.12.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs4" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" +dependencies = [ + "rustix 0.38.44", + "windows-sys 0.52.0", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "fsst" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed8ebb43289bd8fe56619ffeebf3d8c8f146df3c70e6f879a3deedd02195dae" +dependencies = [ + "rand 0.8.5", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generator" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f04ae4152da20c76fe800fa48659201d5cf627c5149ca0b707b69d7eef6cf9" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.4.0", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.4.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http 1.4.0", + "http-body 1.0.1", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "hyper" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http 1.4.0", + "http-body 1.0.1", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http 1.4.0", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "hyperloglogplus" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "621debdf94dcac33e50475fdd76d34d5ea9c0362a834b9db08c3024696c1fbe3" +dependencies = [ + "serde", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iri-string" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lance" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cceab54ac337e206bab80de6438f036fa26c84b4721a45efdbd08be0bd2f9e30" +dependencies = [ + "arrow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "async-recursion", + "async-trait", + "async_cell", + "aws-credential-types", + "byteorder", + "bytes", + "chrono", + "dashmap 5.5.3", + "datafusion", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "deepsize", + "futures", + "half", + "itertools 0.13.0", + "lance-arrow", + "lance-core", + "lance-datafusion", + "lance-encoding", + "lance-file", + "lance-index", + "lance-io", + "lance-linalg", + "lance-table", + "lazy_static", + "log", + "moka", + "object_store 0.10.2", + "permutation", + "pin-project", + "prost", + "prost-build", + "prost-types", + "rand 0.8.5", + "roaring", + "serde", + "serde_json", + "snafu 0.7.5", + "tantivy", + "tempfile", + "tokio", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "lance-arrow" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd57ed4892da295c9e97c9065e606c35b45f2515a6dab67b80b9647fbd8887b5" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", + "bytes", + "getrandom 0.2.17", + "half", + "num-traits", + "rand 0.8.5", +] + +[[package]] +name = "lance-core" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae9e04a71eef248add2f9450b32c726fa28835d7a77a0cebed0833564b59344" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "async-trait", + "byteorder", + "bytes", + "chrono", + "datafusion-common", + "datafusion-sql", + "deepsize", + "futures", + "lance-arrow", + "lazy_static", + "libc", + "log", + "mock_instant", + "moka", + "num_cpus", + "object_store 0.10.2", + "pin-project", + "prost", + "rand 0.8.5", + "roaring", + "serde_json", + "snafu 0.7.5", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", + "url", +] + +[[package]] +name = "lance-datafusion" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67e7b231b130f5e755c86423ce3532b7b433f990c80aeb77ac305bdab3227c1" +dependencies = [ + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "arrow-select", + "async-trait", + "datafusion", + "datafusion-common", + "datafusion-functions", + "datafusion-physical-expr", + "futures", + "lance-arrow", + "lance-core", + "lazy_static", + "log", + "prost", + "snafu 0.7.5", + "tokio", +] + +[[package]] +name = "lance-encoding" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8057c9ca7f3d84640514531c5dfe81fb288a24e414f086284c8bac8b3539e650" +dependencies = [ + "arrayref", + "arrow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", + "bytemuck", + "byteorder", + "bytes", + "fsst", + "futures", + "hex", + "hyperloglogplus", + "itertools 0.13.0", + "lance-arrow", + "lance-core", + "lazy_static", + "log", + "num-traits", + "paste", + "prost", + "prost-build", + "prost-types", + "rand 0.8.5", + "seq-macro", + "snafu 0.7.5", + "tokio", + "tracing", + "zstd", +] + +[[package]] +name = "lance-file" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2f5a12060b6a1626a52fc7821d53a9f1e9c7d0b87cfaa4a9b08e5a7762ed98a" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "async-recursion", + "async-trait", + "byteorder", + "bytes", + "datafusion-common", + "deepsize", + "futures", + "lance-arrow", + "lance-core", + "lance-encoding", + "lance-io", + "log", + "num-traits", + "object_store 0.10.2", + "prost", + "prost-build", + "prost-types", + "roaring", + "snafu 0.7.5", + "tempfile", + "tokio", + "tracing", +] + +[[package]] +name = "lance-index" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e078a57929129cb4f42788596c81da007fd467c7991cdd76a7abe5180f31043e" +dependencies = [ + "arrow", + "arrow-array", + "arrow-ord", + "arrow-schema", + "arrow-select", + "async-recursion", + "async-trait", + "bitvec", + "bytes", + "crossbeam-queue", + "datafusion", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-sql", + "deepsize", + "futures", + "half", + "itertools 0.13.0", + "lance-arrow", + "lance-core", + "lance-datafusion", + "lance-encoding", + "lance-file", + "lance-io", + "lance-linalg", + "lance-table", + "lazy_static", + "log", + "moka", + "num-traits", + "object_store 0.10.2", + "prost", + "prost-build", + "rand 0.8.5", + "rayon", + "roaring", + "serde", + "serde_json", + "snafu 0.7.5", + "tantivy", + "tempfile", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "lance-io" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17e6739ec725950dbe77814d34b03bd29a03c1807c1406994fcf57ff87f94ef" +dependencies = [ + "arrow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", + "async-priority-channel", + "async-recursion", + "async-trait", + "aws-config", + "aws-credential-types", + "byteorder", + "bytes", + "chrono", + "deepsize", + "futures", + "lance-arrow", + "lance-core", + "lazy_static", + "log", + "object_store 0.10.2", + "path_abs", + "pin-project", + "prost", + "prost-build", + "rand 0.8.5", + "shellexpand", + "snafu 0.7.5", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "lance-linalg" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6430b82c3c7d4bb822dc297b85ec1d5f764420e37a697590ce2a7a040ebb5131" +dependencies = [ + "arrow-array", + "arrow-ord", + "arrow-schema", + "bitvec", + "cc", + "deepsize", + "futures", + "half", + "lance-arrow", + "lance-core", + "lazy_static", + "log", + "num-traits", + "rand 0.8.5", + "rayon", + "tokio", + "tracing", +] + +[[package]] +name = "lance-table" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6da33f07d4c347f2ec5d8e1e324590c2bfd243ee8a656936d5a978247716d77" +dependencies = [ + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ipc", + "arrow-schema", + "async-trait", + "aws-credential-types", + "byteorder", + "bytes", + "chrono", + "deepsize", + "futures", + "lance-arrow", + "lance-core", + "lance-file", + "lance-io", + "log", + "object_store 0.10.2", + "prost", + "prost-build", + "prost-types", + "rand 0.8.5", + "rangemap", + "roaring", + "serde", + "serde_json", + "snafu 0.7.5", + "tokio", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "levenshtein_automata" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libc" +version = "0.2.184" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" +dependencies = [ + "libc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "measure_time" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbefd235b0aadd181626f281e1d684e116972988c14c264e42069d5e8a5775cc" +dependencies = [ + "instant", + "log", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "mio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "mock_instant" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9366861eb2a2c436c20b12c8dbec5f798cea6b47ad99216be0282942e2c81ea0" +dependencies = [ + "once_cell", +] + +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener 5.4.1", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "murmurhash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object_store" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6da452820c715ce78221e8202ccc599b4a52f3e1eb3eedb487b680c81a8e3f3" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "futures", + "humantime", + "hyper", + "itertools 0.13.0", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.8.5", + "reqwest", + "ring", + "rustls-pemfile", + "serde", + "serde_json", + "snafu 0.7.5", + "tokio", + "tracing", + "url", + "walkdir", +] + +[[package]] +name = "object_store" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures", + "humantime", + "itertools 0.13.0", + "parking_lot", + "percent-encoding", + "snafu 0.8.9", + "tokio", + "tracing", + "url", + "walkdir", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oneshot" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "ownedbytes" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "path_abs" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ef02f6342ac01d8a93b65f96db53fe68a92a15f41144f97fb00a9e669633c3" +dependencies = [ + "serde", + "serde_derive", + "std_prelude", + "stfu8", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "permutation" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df202b0b0f5b8e389955afd5f27b007b00fb948162953f1db9c70d2c7e3157d7" + +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset 0.4.2", + "indexmap", +] + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset 0.5.7", + "indexmap", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.117", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck 0.5.0", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph 0.7.1", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.117", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quick-xml" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash 2.1.2", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.2", + "ring", + "rustc-hash 2.1.2", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand 0.8.5", +] + +[[package]] +name = "rangemap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.11.0", +] + +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 2.0.18", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "roaring" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41589aba99537475bf697f2118357cad1c31590c5a1b9f6d9fc4ad6d07503661" +dependencies = [ + "bytemuck", + "byteorder", +] + +[[package]] +name = "rust-stemmers" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.11.0", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.11.0", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +dependencies = [ + "aws-lc-rs", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.11.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shellexpand" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" +dependencies = [ + "dirs", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + +[[package]] +name = "sketches-ddsketch" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" +dependencies = [ + "serde", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "snafu" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" +dependencies = [ + "doc-comment", + "snafu-derive 0.7.5", +] + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive 0.8.9", +] + +[[package]] +name = "snafu-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "sqlparser" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2e5b515a2bd5168426033e9efbfd05500114833916f1d5c268f938b4ee130ac" +dependencies = [ + "log", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "std_prelude" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8207e78455ffdf55661170876f88daf85356e4edd54e0a3dbc79586ca1e50cbe" + +[[package]] +name = "stfu8" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51f1e89f093f99e7432c491c382b88a6860a5adbe6bf02574bf0a08efff1978" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.117", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tantivy" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96599ea6fccd844fc833fed21d2eecac2e6a7c1afd9e044057391d78b1feb141" +dependencies = [ + "aho-corasick", + "arc-swap", + "base64", + "bitpacking", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "downcast-rs", + "fastdivide", + "fnv", + "fs4", + "htmlescape", + "itertools 0.12.1", + "levenshtein_automata", + "log", + "lru", + "lz4_flex", + "measure_time", + "memmap2", + "num_cpus", + "once_cell", + "oneshot", + "rayon", + "regex", + "rust-stemmers", + "rustc-hash 1.1.0", + "serde", + "serde_json", + "sketches-ddsketch", + "smallvec", + "tantivy-bitpacker", + "tantivy-columnar", + "tantivy-common", + "tantivy-fst", + "tantivy-query-grammar", + "tantivy-stacker", + "tantivy-tokenizer-api", + "tempfile", + "thiserror 1.0.69", + "time", + "uuid", + "winapi", +] + +[[package]] +name = "tantivy-bitpacker" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df" +dependencies = [ + "bitpacking", +] + +[[package]] +name = "tantivy-columnar" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e" +dependencies = [ + "downcast-rs", + "fastdivide", + "itertools 0.12.1", + "serde", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-sstable", + "tantivy-stacker", +] + +[[package]] +name = "tantivy-common" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4" +dependencies = [ + "async-trait", + "byteorder", + "ownedbytes", + "serde", + "time", +] + +[[package]] +name = "tantivy-fst" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" +dependencies = [ + "byteorder", + "regex-syntax", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82" +dependencies = [ + "nom", +] + +[[package]] +name = "tantivy-sstable" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e" +dependencies = [ + "tantivy-bitpacker", + "tantivy-common", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-stacker" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8" +dependencies = [ + "murmurhash32", + "rand_distr", + "tantivy-common", +] + +[[package]] +name = "tantivy-tokenizer-api" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04" +dependencies = [ + "serde", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bd1c4c0fc4a7ab90fc15ef6daaa3ec3b893f004f915f2392557ed23237820cd" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags 2.11.0", + "bytes", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.117", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.11.0", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck 0.5.0", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck 0.5.0", + "indexmap", + "prettyplease", + "syn 2.0.117", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.117", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.11.0", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zerofrom" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/be/src/rust/doris-native/Cargo.toml b/be/src/rust/doris-native/Cargo.toml new file mode 100644 index 00000000000000..239bb86b158bdf --- /dev/null +++ b/be/src/rust/doris-native/Cargo.toml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[workspace] +members = ["crates/doris-ffi"] +resolver = "2" + +[workspace.package] +edition = "2021" +license = "Apache-2.0" + +[workspace.dependencies] +lance = "0.20" +lance-io = "0.20" +lance-index = "0.20" +lance-linalg = "0.20" +arrow = { version = "53", features = ["ffi"] } +arrow-array = "53" +arrow-schema = { version = "53", features = ["ffi"] } +tokio = { version = "1", features = ["rt"] } +futures = "0.3" + +[profile.release] +panic = "unwind" +lto = "fat" +codegen-units = 1 +strip = "symbols" + +[profile.dev] +panic = "unwind" diff --git a/be/src/rust/doris-native/README.md b/be/src/rust/doris-native/README.md new file mode 100644 index 00000000000000..3fc251aed86e36 --- /dev/null +++ b/be/src/rust/doris-native/README.md @@ -0,0 +1,181 @@ +# doris-native: Rust Native Readers for Apache Doris + +This workspace contains the Rust-based format readers for Doris BE, starting with Lance support. + +## Architecture + +``` +C++ (Doris BE) Rust (doris-native) +┌─────────────────┐ ┌──────────────────┐ +│ LanceRustReader │──JSON config─>│ lance_reader_open │ +│ (GenericReader) │ │ lance_reader_next │──> lance-rs +│ │<─Arrow C ABI──│ lance_reader_close│ Dataset::scan() +└─────────────────┘ └──────────────────┘ +``` + +Data exchange uses the Arrow C Data Interface (zero-copy between Rust and C++). +Each reader owns a single-threaded tokio runtime (`block_on()` on the scanner thread). + +## Prerequisites + +- Rust stable toolchain (see `rust-toolchain.toml`) +- For BE integration: `BUILD_RUST_READERS=ON` in CMake + +## Quick Start + +### Run Rust tests + +```bash +cd be/src/rust/doris-native +cargo test +``` + +Expected output: 24 tests passing (error handling, lance reader, FFI bridge). + +### Build release library + +```bash +cargo build --release +# Output: target/release/libdoris_ffi.a (linked into doris_be) +``` + +### Build with Doris BE + +```bash +# From repo root: +export DORIS_HOME=$PWD +export DORIS_THIRDPARTY=/path/to/thirdparty +export BUILD_RUST_READERS=ON + +# Via build.sh: +./build.sh --be + +# Or via cmake directly: +cd be/build_Release +cmake -DBUILD_RUST_READERS=ON ... +make -j$(nproc) doris_be +``` + +## Crate Structure + +``` +doris-native/ +├── Cargo.toml # Workspace root +├── rust-toolchain.toml # Rust version pin +└── crates/ + └── doris-ffi/ # Static library linked into doris_be + ├── Cargo.toml + └── src/ + ├── lib.rs # Module root + rust_echo FFI + ├── error.rs # Thread-local error handling (FFI_OK, FFI_ERR_*) + ├── lance_reader.rs # LanceReader + LanceReaderConfig + └── ffi.rs # extern "C" functions (lance_reader_open, etc.) +``` + +## FFI Functions + +| Function | Purpose | +|----------|---------| +| `lance_reader_open(uri, columns, batch_size, handle_out)` | Open dataset (simple API) | +| `lance_reader_open_json(config_json, len, handle_out)` | Open with full config (S3 creds, version, vector search) | +| `lance_reader_next_batch(handle, schema, array, eof, bytes)` | Read next Arrow batch | +| `lance_reader_get_schema(handle, schema_out)` | Get dataset schema | +| `lance_reader_close(handle)` | Free resources | +| `lance_reader_last_error(buf, len)` | Get error message | +| `lance_test_create_dataset(path, len)` | Create 5-row test dataset | +| `lance_test_create_multi_fragment_dataset(path, len)` | Create 15-row, 3-fragment test dataset | + +## JSON Config + +The `lance_reader_open_json` accepts a JSON config string: + +```json +{ + "uri": "s3://bucket/data.lance", + "columns": ["id", "name"], + "batch_size": 4096, + "version": 0, + "storage_options": { + "AWS_ACCESS_KEY_ID": "...", + "AWS_SECRET_ACCESS_KEY": "..." + }, + "filter": "category = 'shoes'", + "vector_search": { + "column": "embedding", + "query": [0.1, 0.2, 0.3], + "k": 10, + "metric": "cosine", + "nprobes": 20, + "ef": 100 + }, + "full_text_search": "machine learning", + "limit": 100, + "offset": 0, + "fragment_ids": [0, 1, 2] +} +``` + +## Running E2E Tests + +### Standalone C++ test (no Doris cluster needed) + +```bash +# Build test binary: +RUST_LIB=be/src/rust/doris-native/target/release/libdoris_ffi.a +ARROW_LIB=/path/to/thirdparty/installed/lib64 +clang++ -std=c++20 -O2 \ + -I/path/to/thirdparty/installed/include \ + be/test/format/lance/standalone_lance_test.cpp \ + $RUST_LIB -Wl,--start-group $ARROW_LIB/libarrow.a ... -Wl,--end-group \ + -lpthread -ldl -lm -lrt -o lance_test + +./lance_test +# All 8 tests PASSED! +``` + +### Live Doris cluster test + +```bash +# 1. Create test datasets on BE: +./lance_create single /opt/apache-doris/be/lance_test_data/single.lance +./lance_create multi /opt/apache-doris/be/lance_test_data/multi.lance + +# 2. Query via MySQL client: +mysql -h 127.0.0.1 -P 9030 -u root -e " +SELECT * FROM local( + \"file_path\" = \"lance_test_data/single.lance/data/\", + \"backend_id\" = \"\", + \"format\" = \"lance\" +) ORDER BY id;" + +# Expected: +# id name score +# 1 alice 90.5 +# 2 bob 85.0 +# 3 carol 92.3 +# 4 dave 78.1 +# 5 eve 88.7 +``` + +### Regression test + +```bash +# Run the lance TVF regression test suite: +./run-regression-test.sh --run -s test_lance_tvf + +# Run by file: +./run-regression-test.sh --run \ + -f regression-test/suites/external_table_p0/tvf/lance/test_lance_tvf.groovy + +# Generate expected output (first time): +./run-regression-test.sh --run -s test_lance_tvf -genOut +``` + +## Test Summary + +| Layer | Tests | What's verified | +|-------|-------|----------------| +| Rust unit (24) | `cargo test` | Error handling, LanceReader open/read/close, FFI lifecycle, JSON config | +| C++ standalone (8) | `lance_test` binary | FFI bridge, Arrow import, schema inference, data verification, multi-fragment | +| Live cluster (8) | MySQL queries | Full TVF: SELECT *, projection, COUNT, WHERE, LIMIT, multi-fragment, aggregation | +| Regression (9) | `test_lance_tvf.groovy` | Automated CI-ready version of live cluster tests | diff --git a/be/src/rust/doris-native/crates/doris-ffi/Cargo.toml b/be/src/rust/doris-native/crates/doris-ffi/Cargo.toml new file mode 100644 index 00000000000000..8f4e215ba39bf2 --- /dev/null +++ b/be/src/rust/doris-native/crates/doris-ffi/Cargo.toml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "doris-ffi" +version = "0.1.0" +edition.workspace = true +license.workspace = true + +[lib] +# "lib" is needed so `cargo test` can link the test harness. +# "staticlib" produces the .a for C++ linkage. +crate-type = ["staticlib", "lib"] + +[dependencies] +lance = { workspace = true } +lance-io = { workspace = true } +lance-index = { workspace = true } +lance-linalg = { workspace = true } +arrow = { workspace = true } +arrow-array = { workspace = true } +arrow-schema = { workspace = true } +tokio = { workspace = true } +futures = { workspace = true } +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[build-dependencies] +cc = "1" + +[dev-dependencies] +tempfile = "3" +tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros"] } diff --git a/be/src/rust/doris-native/crates/doris-ffi/build.rs b/be/src/rust/doris-native/crates/doris-ffi/build.rs new file mode 100644 index 00000000000000..dfe5b0e6fc4e8c --- /dev/null +++ b/be/src/rust/doris-native/crates/doris-ffi/build.rs @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Build script: compiles glibc compatibility stubs for older Linux systems. +//! +//! On glibc < 2.38, aws-lc-rs references symbols like __isoc23_sscanf that +//! don't exist. This build script detects the glibc version and compiles +//! stub implementations when needed. +//! +//! On newer glibc (>= 2.38), the stubs are skipped to avoid duplicate symbols. + +fn main() { + println!("cargo:rerun-if-changed=glibc_compat.c"); + + // Only needed on Linux + if std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default() != "linux" { + return; + } + + // Detect glibc version by parsing the output of `ldd --version` + let needs_compat = match detect_glibc_version() { + Some((major, minor)) => { + println!( + "cargo:warning=Detected glibc {}.{} — {}", + major, + minor, + if major < 2 || (major == 2 && minor < 38) { + "compiling compat stubs" + } else { + "skipping compat stubs (not needed)" + } + ); + major < 2 || (major == 2 && minor < 38) + } + None => { + // Can't detect — compile stubs to be safe (use weak symbols to avoid conflicts) + println!("cargo:warning=Cannot detect glibc version — compiling compat stubs with weak symbols"); + true + } + }; + + if needs_compat { + cc::Build::new() + .file("glibc_compat.c") + .warnings(false) + .compile("glibc_compat"); + } +} + +fn detect_glibc_version() -> Option<(u32, u32)> { + // Try ldd --version (works on most Linux) + let output = std::process::Command::new("ldd") + .arg("--version") + .output() + .ok()?; + + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let text = if stdout.contains("GLIBC") || stdout.contains("glibc") || stdout.contains("ldd") { + stdout + } else { + stderr + }; + + // Parse "ldd (GNU libc) 2.17" or "ldd (Ubuntu GLIBC 2.35-0ubuntu3) 2.35" + for line in text.lines() { + // Find version number pattern: major.minor + let parts: Vec<&str> = line.split_whitespace().collect(); + for part in parts.iter().rev() { + if let Some((major_str, minor_str)) = part.split_once('.') { + if let (Ok(major), Ok(minor)) = (major_str.parse::(), minor_str.parse::()) + { + if major >= 2 && minor < 100 { + return Some((major, minor)); + } + } + } + } + } + None +} diff --git a/be/src/rust/doris-native/crates/doris-ffi/glibc_compat.c b/be/src/rust/doris-native/crates/doris-ffi/glibc_compat.c new file mode 100644 index 00000000000000..c003395061dd97 --- /dev/null +++ b/be/src/rust/doris-native/crates/doris-ffi/glibc_compat.c @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Compatibility stubs for glibc symbols that aws-lc-rs (Rust TLS crypto) +// references but are only available in newer glibc versions. +// +// All functions use __attribute__((weak)) so that if the real glibc +// provides them, the real versions take precedence. This prevents +// duplicate symbol errors on newer systems. + +#include +#include +#include +#include +#include + +#define WEAK __attribute__((weak)) + +// glibc 2.38+: ISO C23 scanning/parsing functions +WEAK int __isoc23_sscanf(const char* s, const char* fmt, ...) { + va_list ap; + va_start(ap, fmt); + int r = vsscanf(s, fmt, ap); + va_end(ap); + return r; +} + +WEAK long __isoc23_strtol(const char* s, char** endp, int base) { + return strtol(s, endp, base); +} + +WEAK unsigned long __isoc23_strtoul(const char* s, char** endp, int base) { + return strtoul(s, endp, base); +} + +WEAK unsigned long long __isoc23_strtoull(const char* s, char** endp, int base) { + return strtoull(s, endp, base); +} + +// glibc 2.32+: thread safety indicator +WEAK char __libc_single_threaded = 0; + +// glibc 2.30+: clock-aware pthread functions +WEAK int pthread_cond_clockwait(pthread_cond_t* cond, pthread_mutex_t* mutex, clockid_t clock_id, + const struct timespec* abstime) { + (void)clock_id; + return pthread_cond_timedwait(cond, mutex, abstime); +} + +WEAK int pthread_mutex_clocklock(pthread_mutex_t* mutex, clockid_t clock_id, + const struct timespec* abstime) { + (void)clock_id; + return pthread_mutex_timedlock(mutex, abstime); +} diff --git a/be/src/rust/doris-native/crates/doris-ffi/src/error.rs b/be/src/rust/doris-native/crates/doris-ffi/src/error.rs new file mode 100644 index 00000000000000..4ee102195708f5 --- /dev/null +++ b/be/src/rust/doris-native/crates/doris-ffi/src/error.rs @@ -0,0 +1,189 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::cell::RefCell; +use std::fmt; + +/// FFI status codes returned to C++. +pub const FFI_OK: i32 = 0; +pub const FFI_EOF: i32 = 1; +pub const FFI_ERR_LANCE: i32 = -1; +pub const FFI_ERR_ARROW: i32 = -2; +pub const FFI_ERR_IO: i32 = -3; +pub const FFI_ERR_PANIC: i32 = -4; +pub const FFI_ERR_INVALID_ARG: i32 = -5; + +/// Errors that can occur at the FFI boundary. +#[derive(Debug)] +pub enum FfiError { + Lance(String), + Arrow(String), + Io(String), + InvalidArg(String), +} + +impl fmt::Display for FfiError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + FfiError::Lance(msg) => write!(f, "Lance error: {}", msg), + FfiError::Arrow(msg) => write!(f, "Arrow error: {}", msg), + FfiError::Io(msg) => write!(f, "IO error: {}", msg), + FfiError::InvalidArg(msg) => write!(f, "Invalid argument: {}", msg), + } + } +} + +impl From for FfiError { + fn from(e: lance::Error) -> Self { + FfiError::Lance(e.to_string()) + } +} + +impl From for FfiError { + fn from(e: arrow::error::ArrowError) -> Self { + FfiError::Arrow(e.to_string()) + } +} + +impl From for FfiError { + fn from(e: std::io::Error) -> Self { + FfiError::Io(e.to_string()) + } +} + +impl FfiError { + pub fn status_code(&self) -> i32 { + match self { + FfiError::Lance(_) => FFI_ERR_LANCE, + FfiError::Arrow(_) => FFI_ERR_ARROW, + FfiError::Io(_) => FFI_ERR_IO, + FfiError::InvalidArg(_) => FFI_ERR_INVALID_ARG, + } + } +} + +pub type FfiResult = Result; + +// Thread-local storage for the last error message. +// The C++ side retrieves this via lance_reader_last_error() only on error paths. +thread_local! { + static LAST_ERROR: RefCell> = const { RefCell::new(None) }; +} + +/// Store an error message in thread-local storage. +pub fn set_last_error(msg: String) { + LAST_ERROR.with(|e| { + *e.borrow_mut() = Some(msg); + }); +} + +/// Clear the last error. +pub fn clear_last_error() { + LAST_ERROR.with(|e| { + *e.borrow_mut() = None; + }); +} + +/// Copy the last error message into the provided buffer. +/// Returns the number of bytes written (excluding null terminator), +/// or 0 if no error is stored. +pub fn get_last_error(buf: &mut [u8]) -> usize { + LAST_ERROR.with(|e| { + let borrowed = e.borrow(); + match borrowed.as_ref() { + Some(msg) => { + let bytes = msg.as_bytes(); + let copy_len = bytes.len().min(buf.len().saturating_sub(1)); + if copy_len > 0 { + buf[..copy_len].copy_from_slice(&bytes[..copy_len]); + } + if buf.len() > copy_len { + buf[copy_len] = 0; // null terminator + } + copy_len + } + None => 0, + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_set_and_get_last_error() { + clear_last_error(); + set_last_error("test error message".to_string()); + + let mut buf = [0u8; 256]; + let len = get_last_error(&mut buf); + assert_eq!(len, 18); + assert_eq!(&buf[..len], b"test error message"); + assert_eq!(buf[len], 0); // null terminator + } + + #[test] + fn test_clear_last_error() { + set_last_error("some error".to_string()); + clear_last_error(); + + let mut buf = [0u8; 64]; + let len = get_last_error(&mut buf); + assert_eq!(len, 0); + } + + #[test] + fn test_get_last_error_truncation() { + set_last_error("a]long error message that exceeds buffer".to_string()); + + let mut buf = [0u8; 10]; + let len = get_last_error(&mut buf); + // Should copy at most buf_len - 1 = 9 bytes + assert_eq!(len, 9); + assert_eq!(&buf[..9], b"a]long er"); + assert_eq!(buf[9], 0); // null terminator + } + + #[test] + fn test_get_last_error_empty_buffer() { + set_last_error("error".to_string()); + let mut buf = [0u8; 0]; + let len = get_last_error(&mut buf); + assert_eq!(len, 0); + } + + #[test] + fn test_error_status_codes() { + assert_eq!(FfiError::Lance("x".into()).status_code(), FFI_ERR_LANCE); + assert_eq!(FfiError::Arrow("x".into()).status_code(), FFI_ERR_ARROW); + assert_eq!(FfiError::Io("x".into()).status_code(), FFI_ERR_IO); + assert_eq!( + FfiError::InvalidArg("x".into()).status_code(), + FFI_ERR_INVALID_ARG + ); + } + + #[test] + fn test_error_display() { + let e = FfiError::Lance("dataset not found".into()); + assert_eq!(e.to_string(), "Lance error: dataset not found"); + + let e = FfiError::Arrow("schema mismatch".into()); + assert_eq!(e.to_string(), "Arrow error: schema mismatch"); + } +} diff --git a/be/src/rust/doris-native/crates/doris-ffi/src/ffi.rs b/be/src/rust/doris-native/crates/doris-ffi/src/ffi.rs new file mode 100644 index 00000000000000..e7e03fbb02fdaa --- /dev/null +++ b/be/src/rust/doris-native/crates/doris-ffi/src/ffi.rs @@ -0,0 +1,864 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! C FFI functions for the Lance reader. +//! +//! All functions are wrapped in `catch_unwind` to prevent Rust panics from +//! unwinding across the FFI boundary (which is undefined behavior). +//! Errors are stored in thread-local storage and retrieved via +//! `lance_reader_last_error`. +//! +//! Data is exchanged via the Arrow C Data Interface (ArrowSchema + ArrowArray), +//! which is version-stable and allows zero-copy transfer between Rust and C++. + +use std::panic::AssertUnwindSafe; +use std::ptr; + +use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; + +use crate::error::{self, FFI_EOF, FFI_ERR_ARROW, FFI_ERR_INVALID_ARG, FFI_ERR_PANIC, FFI_OK}; +use crate::lance_reader::{LanceReader, LanceReaderConfig}; + +/// Opaque handle to a LanceReader. Allocated on the heap, freed by `lance_reader_close`. +type LanceReaderHandle = *mut LanceReader; + +/// Open a Lance dataset and create a reader handle. +/// +/// # Arguments +/// * `uri_ptr` - Pointer to UTF-8 encoded dataset URI +/// * `uri_len` - Length of the URI in bytes +/// * `column_names_ptr` - Array of pointers to UTF-8 column name strings +/// * `column_names_len_ptr` - Array of lengths for each column name +/// * `num_columns` - Number of columns (0 = read all) +/// * `batch_size` - Maximum rows per batch +/// * `handle_out` - Output: pointer to the created reader handle +/// +/// # Returns +/// FFI_OK on success, negative error code on failure. +#[no_mangle] +pub extern "C" fn lance_reader_open( + uri_ptr: *const u8, + uri_len: usize, + column_names_ptr: *const *const u8, + column_names_len_ptr: *const usize, + num_columns: usize, + batch_size: usize, + handle_out: *mut LanceReaderHandle, +) -> i32 { + error::clear_last_error(); + + std::panic::catch_unwind(AssertUnwindSafe(|| { + // Validate arguments + if uri_ptr.is_null() || handle_out.is_null() { + error::set_last_error("uri_ptr and handle_out must not be null".to_string()); + return FFI_ERR_INVALID_ARG; + } + + let uri = unsafe { + let slice = std::slice::from_raw_parts(uri_ptr, uri_len); + match std::str::from_utf8(slice) { + Ok(s) => s, + Err(e) => { + error::set_last_error(format!("Invalid UTF-8 in URI: {}", e)); + return FFI_ERR_INVALID_ARG; + } + } + }; + + let columns: Vec = + if num_columns > 0 && !column_names_ptr.is_null() && !column_names_len_ptr.is_null() { + (0..num_columns) + .map(|i| unsafe { + let name_ptr = *column_names_ptr.add(i); + let name_len = *column_names_len_ptr.add(i); + let slice = std::slice::from_raw_parts(name_ptr, name_len); + String::from_utf8_lossy(slice).into_owned() + }) + .collect() + } else { + Vec::new() + }; + + let batch_size = if batch_size == 0 { 4096 } else { batch_size }; + + match LanceReader::open(uri, &columns, batch_size) { + Ok(reader) => { + let boxed = Box::new(reader); + unsafe { + ptr::write(handle_out, Box::into_raw(boxed)); + } + FFI_OK + } + Err(e) => { + let code = e.status_code(); + error::set_last_error(e.to_string()); + code + } + } + })) + .unwrap_or_else(|panic| { + let msg = format_panic(&panic); + error::set_last_error(msg); + FFI_ERR_PANIC + }) +} + +/// Open a Lance dataset from a JSON config string. +/// +/// The config JSON contains: uri, columns, batch_size, version, storage_options. +/// storage_options carries S3 credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, etc.) +/// +/// Example config: +/// ```json +/// { +/// "uri": "s3://bucket/data.lance", +/// "columns": ["id", "name"], +/// "batch_size": 4096, +/// "version": 0, +/// "storage_options": { +/// "AWS_ACCESS_KEY_ID": "...", +/// "AWS_SECRET_ACCESS_KEY": "...", +/// "AWS_ENDPOINT": "...", +/// "AWS_REGION": "us-east-1" +/// } +/// } +/// ``` +#[no_mangle] +pub extern "C" fn lance_reader_open_json( + config_json_ptr: *const u8, + config_json_len: usize, + handle_out: *mut LanceReaderHandle, +) -> i32 { + error::clear_last_error(); + + std::panic::catch_unwind(AssertUnwindSafe(|| { + if config_json_ptr.is_null() || handle_out.is_null() { + error::set_last_error("config_json_ptr and handle_out must not be null".to_string()); + return FFI_ERR_INVALID_ARG; + } + + let json_str = unsafe { + let slice = std::slice::from_raw_parts(config_json_ptr, config_json_len); + match std::str::from_utf8(slice) { + Ok(s) => s, + Err(e) => { + error::set_last_error(format!("Invalid UTF-8 in config JSON: {}", e)); + return FFI_ERR_INVALID_ARG; + } + } + }; + + let config: LanceReaderConfig = match serde_json::from_str(json_str) { + Ok(c) => c, + Err(e) => { + error::set_last_error(format!("Invalid config JSON: {}", e)); + return FFI_ERR_INVALID_ARG; + } + }; + + match LanceReader::open_with_config(&config) { + Ok(reader) => { + let boxed = Box::new(reader); + unsafe { + ptr::write(handle_out, Box::into_raw(boxed)); + } + FFI_OK + } + Err(e) => { + let code = e.status_code(); + error::set_last_error(e.to_string()); + code + } + } + })) + .unwrap_or_else(|panic| { + let msg = format_panic(&panic); + error::set_last_error(msg); + FFI_ERR_PANIC + }) +} + +/// Read the next batch from the Lance reader. +/// +/// # Arguments +/// * `handle` - Reader handle from `lance_reader_open` +/// * `schema_out` - Output: Arrow C schema (caller allocates, Rust fills) +/// * `array_out` - Output: Arrow C array (caller allocates, Rust fills) +/// * `eof_out` - Output: set to true when no more data +/// * `bytes_out` - Output: approximate byte size of the batch (for memory tracking) +/// +/// # Returns +/// FFI_OK on success with data, FFI_EOF on end of stream, negative on error. +#[no_mangle] +pub extern "C" fn lance_reader_next_batch( + handle: LanceReaderHandle, + schema_out: *mut FFI_ArrowSchema, + array_out: *mut FFI_ArrowArray, + eof_out: *mut bool, + bytes_out: *mut i64, +) -> i32 { + error::clear_last_error(); + + std::panic::catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() || schema_out.is_null() || array_out.is_null() || eof_out.is_null() { + error::set_last_error("All output pointers must not be null".to_string()); + return FFI_ERR_INVALID_ARG; + } + + let reader = unsafe { &mut *handle }; + + match reader.next_batch() { + Ok(Some(batch)) => { + // Calculate approximate byte size for memory tracking + let batch_bytes: i64 = batch + .columns() + .iter() + .map(|col| col.get_array_memory_size() as i64) + .sum(); + + // Export via Arrow C Data Interface + let struct_array: arrow::array::StructArray = batch.into(); + let data = arrow::ffi::to_ffi(&struct_array.into()); + match data { + Ok((ffi_array, ffi_schema)) => { + unsafe { + ptr::write(array_out, ffi_array); + ptr::write(schema_out, ffi_schema); + ptr::write(eof_out, false); + if !bytes_out.is_null() { + ptr::write(bytes_out, batch_bytes); + } + } + FFI_OK + } + Err(e) => { + error::set_last_error(format!("Arrow FFI export failed: {}", e)); + FFI_ERR_ARROW + } + } + } + Ok(None) => { + unsafe { + ptr::write(eof_out, true); + if !bytes_out.is_null() { + ptr::write(bytes_out, 0); + } + } + FFI_EOF + } + Err(e) => { + let code = e.status_code(); + error::set_last_error(e.to_string()); + code + } + } + })) + .unwrap_or_else(|panic| { + let msg = format_panic(&panic); + error::set_last_error(msg); + FFI_ERR_PANIC + }) +} + +/// Get the schema of the scan output. +/// +/// # Arguments +/// * `handle` - Reader handle +/// * `schema_out` - Output: Arrow C schema +/// +/// # Returns +/// FFI_OK on success, negative on error. +#[no_mangle] +pub extern "C" fn lance_reader_get_schema( + handle: LanceReaderHandle, + schema_out: *mut FFI_ArrowSchema, +) -> i32 { + error::clear_last_error(); + + std::panic::catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() || schema_out.is_null() { + error::set_last_error("handle and schema_out must not be null".to_string()); + return FFI_ERR_INVALID_ARG; + } + + let reader = unsafe { &*handle }; + let schema = reader.schema(); + + match arrow::ffi::FFI_ArrowSchema::try_from(schema.as_ref()) { + Ok(ffi_schema) => { + unsafe { + ptr::write(schema_out, ffi_schema); + } + FFI_OK + } + Err(e) => { + error::set_last_error(format!("Schema export failed: {}", e)); + FFI_ERR_ARROW + } + } + })) + .unwrap_or_else(|panic| { + let msg = format_panic(&panic); + error::set_last_error(msg); + FFI_ERR_PANIC + }) +} + +/// Close the reader and free all resources. +/// +/// Safe to call with a null handle (no-op). +/// After this call, the handle must not be used again. +#[no_mangle] +pub extern "C" fn lance_reader_close(handle: LanceReaderHandle) { + if handle.is_null() { + return; + } + // catch_unwind to prevent panic on drop from crossing FFI + let _ = std::panic::catch_unwind(AssertUnwindSafe(|| unsafe { + drop(Box::from_raw(handle)); + })); +} + +/// Retrieve the last error message. +/// +/// Copies the error string into the provided buffer. Returns the number of +/// bytes written (excluding null terminator), or 0 if no error is stored. +/// +/// # Arguments +/// * `buf` - Buffer to write the error message into +/// * `buf_len` - Size of the buffer +#[no_mangle] +pub extern "C" fn lance_reader_last_error(buf: *mut u8, buf_len: usize) -> usize { + if buf.is_null() || buf_len == 0 { + return 0; + } + let slice = unsafe { std::slice::from_raw_parts_mut(buf, buf_len) }; + error::get_last_error(slice) +} + +/// Create a small test Lance dataset at the given path. +/// The dataset has 3 columns: id (INT32), name (UTF8), score (FLOAT64) +/// with 5 rows. Used by C++ GTests for end-to-end testing. +/// +/// Returns FFI_OK on success, negative on error. +#[no_mangle] +pub extern "C" fn lance_test_create_dataset(path_ptr: *const u8, path_len: usize) -> i32 { + error::clear_last_error(); + + std::panic::catch_unwind(AssertUnwindSafe(|| { + if path_ptr.is_null() { + error::set_last_error("path must not be null".to_string()); + return FFI_ERR_INVALID_ARG; + } + + let path = unsafe { + let slice = std::slice::from_raw_parts(path_ptr, path_len); + match std::str::from_utf8(slice) { + Ok(s) => s, + Err(e) => { + error::set_last_error(format!("Invalid UTF-8 in path: {}", e)); + return FFI_ERR_INVALID_ARG; + } + } + }; + + let rt = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(rt) => rt, + Err(e) => { + error::set_last_error(format!("Failed to create runtime: {}", e)); + return error::FFI_ERR_IO; + } + }; + + let result = rt.block_on(async { + use arrow::array::{Float64Array, Int32Array, StringArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use std::sync::Arc; + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("score", DataType::Float64, false), + ])); + + let batch = arrow::array::RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])), + Arc::new(StringArray::from(vec![ + "alice", "bob", "carol", "dave", "eve", + ])), + Arc::new(Float64Array::from(vec![90.5, 85.0, 92.3, 78.1, 88.7])), + ], + ) + .map_err(|e| crate::error::FfiError::Arrow(e.to_string()))?; + + let batches = arrow_array::RecordBatchIterator::new(vec![Ok(batch)], schema); + let write_result: Result = + lance::Dataset::write(batches, path, None::).await; + write_result.map_err(|e| crate::error::FfiError::Lance(e.to_string()))?; + + Ok::<_, crate::error::FfiError>(()) + }); + + match result { + Ok(()) => FFI_OK, + Err(e) => { + let code = e.status_code(); + error::set_last_error(e.to_string()); + code + } + } + })) + .unwrap_or_else(|panic| { + let msg = format_panic(&panic); + error::set_last_error(msg); + FFI_ERR_PANIC + }) +} + +fn format_panic(panic: &Box) -> String { + if let Some(s) = panic.downcast_ref::<&str>() { + format!("Rust panic: {}", s) + } else if let Some(s) = panic.downcast_ref::() { + format!("Rust panic: {}", s) + } else { + "Rust panic: unknown".to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{Array, Float64Array, Int32Array, StringArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use std::sync::Arc; + + /// Helper: create a small Lance dataset and return the URI. + fn create_test_dataset(dir: &std::path::Path) -> String { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("value", DataType::Float64, false), + ])); + + let batch = arrow::array::RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![10, 20, 30])), + Arc::new(StringArray::from(vec!["x", "y", "z"])), + Arc::new(Float64Array::from(vec![1.1, 2.2, 3.3])), + ], + ) + .unwrap(); + + let uri = dir.join("ffi_test.lance").to_string_lossy().to_string(); + + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + rt.block_on(async { + let batches = arrow_array::RecordBatchIterator::new(vec![Ok(batch)], schema); + lance::Dataset::write(batches, &uri, None::) + .await + .unwrap(); + }); + + uri + } + + #[test] + fn test_ffi_open_null_uri() { + let mut handle: LanceReaderHandle = ptr::null_mut(); + let rc = lance_reader_open( + ptr::null(), + 0, + ptr::null(), + ptr::null(), + 0, + 1024, + &mut handle, + ); + assert_eq!(rc, FFI_ERR_INVALID_ARG); + + // Verify error message is set + let mut buf = [0u8; 256]; + let len = lance_reader_last_error(buf.as_mut_ptr(), buf.len()); + assert!(len > 0); + let msg = std::str::from_utf8(&buf[..len]).unwrap(); + assert!(msg.contains("null")); + } + + #[test] + fn test_ffi_open_null_handle_out() { + let uri = b"/some/path"; + let rc = lance_reader_open( + uri.as_ptr(), + uri.len(), + ptr::null(), + ptr::null(), + 0, + 1024, + ptr::null_mut(), + ); + assert_eq!(rc, FFI_ERR_INVALID_ARG); + } + + #[test] + fn test_ffi_open_nonexistent_path() { + let uri = b"/nonexistent/dataset.lance"; + let mut handle: LanceReaderHandle = ptr::null_mut(); + let rc = lance_reader_open( + uri.as_ptr(), + uri.len(), + ptr::null(), + ptr::null(), + 0, + 1024, + &mut handle, + ); + assert!(rc < 0, "Expected error, got {}", rc); + assert!(handle.is_null()); + } + + #[test] + fn test_ffi_close_null_handle() { + // Should not crash + lance_reader_close(ptr::null_mut()); + } + + #[test] + fn test_ffi_last_error_null_buf() { + let len = lance_reader_last_error(ptr::null_mut(), 0); + assert_eq!(len, 0); + } + + #[test] + fn test_ffi_next_batch_null_handle() { + let mut schema = FFI_ArrowSchema::empty(); + let mut array = FFI_ArrowArray::empty(); + let mut eof = false; + let mut bytes: i64 = 0; + + let rc = lance_reader_next_batch( + ptr::null_mut(), + &mut schema, + &mut array, + &mut eof, + &mut bytes, + ); + assert_eq!(rc, FFI_ERR_INVALID_ARG); + } + + #[test] + fn test_ffi_get_schema_null_handle() { + let mut schema = FFI_ArrowSchema::empty(); + let rc = lance_reader_get_schema(ptr::null_mut(), &mut schema); + assert_eq!(rc, FFI_ERR_INVALID_ARG); + } + + #[test] + fn test_ffi_full_lifecycle() { + let tmp = tempfile::tempdir().unwrap(); + let uri = create_test_dataset(tmp.path()); + + // Open + let mut handle: LanceReaderHandle = ptr::null_mut(); + let rc = lance_reader_open( + uri.as_ptr(), + uri.len(), + ptr::null(), + ptr::null(), + 0, + 1024, + &mut handle, + ); + assert_eq!(rc, FFI_OK, "open failed"); + assert!(!handle.is_null()); + + // Get schema + let mut schema = FFI_ArrowSchema::empty(); + let rc = lance_reader_get_schema(handle, &mut schema); + assert_eq!(rc, FFI_OK, "get_schema failed"); + + // Import and verify schema + let imported_schema = arrow::datatypes::Schema::try_from(&schema).unwrap(); + assert_eq!(imported_schema.fields().len(), 3); + assert_eq!(imported_schema.field(0).name(), "id"); + assert_eq!(imported_schema.field(1).name(), "name"); + assert_eq!(imported_schema.field(2).name(), "value"); + + // Read first batch + let mut out_schema = FFI_ArrowSchema::empty(); + let mut out_array = FFI_ArrowArray::empty(); + let mut eof = false; + let mut bytes: i64 = 0; + + let rc = lance_reader_next_batch( + handle, + &mut out_schema, + &mut out_array, + &mut eof, + &mut bytes, + ); + assert_eq!(rc, FFI_OK, "next_batch failed"); + assert!(!eof); + assert!(bytes > 0, "bytes should be positive"); + + // Import and verify batch data via Arrow C Data Interface + let imported_array = unsafe { arrow::ffi::from_ffi(out_array, &out_schema) }.unwrap(); + let struct_array = arrow::array::StructArray::from(imported_array); + let record_batch = arrow::array::RecordBatch::from(struct_array); + assert_eq!(record_batch.num_rows(), 3); + assert_eq!(record_batch.num_columns(), 3); + + let ids = record_batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(ids.values(), &[10, 20, 30]); + + // Read next batch — should be EOF + let mut out_schema2 = FFI_ArrowSchema::empty(); + let mut out_array2 = FFI_ArrowArray::empty(); + let mut eof2 = false; + let mut bytes2: i64 = 0; + let rc = lance_reader_next_batch( + handle, + &mut out_schema2, + &mut out_array2, + &mut eof2, + &mut bytes2, + ); + assert_eq!(rc, FFI_EOF); + assert!(eof2); + + // Close + lance_reader_close(handle); + } + + #[test] + fn test_ffi_open_with_column_projection() { + let tmp = tempfile::tempdir().unwrap(); + let uri = create_test_dataset(tmp.path()); + + let col_name = b"name"; + let col_ptrs = [col_name.as_ptr()]; + let col_lens = [col_name.len()]; + + let mut handle: LanceReaderHandle = ptr::null_mut(); + let rc = lance_reader_open( + uri.as_ptr(), + uri.len(), + col_ptrs.as_ptr(), + col_lens.as_ptr(), + 1, + 1024, + &mut handle, + ); + assert_eq!(rc, FFI_OK); + assert!(!handle.is_null()); + + // Verify schema has only 1 column + let mut schema = FFI_ArrowSchema::empty(); + let rc = lance_reader_get_schema(handle, &mut schema); + assert_eq!(rc, FFI_OK); + + let imported_schema = arrow::datatypes::Schema::try_from(&schema).unwrap(); + assert_eq!(imported_schema.fields().len(), 1); + assert_eq!(imported_schema.field(0).name(), "name"); + + lance_reader_close(handle); + } + + #[test] + fn test_ffi_echo() { + assert_eq!(crate::rust_echo(42), 42); + assert_eq!(crate::rust_echo(0), 0); + assert_eq!(crate::rust_echo(-999), -999); + } + + #[test] + fn test_ffi_open_json_config() { + let tmp = tempfile::tempdir().unwrap(); + let uri = create_test_dataset(tmp.path()); + + let config = serde_json::json!({ + "uri": uri, + "columns": ["name", "value"], + "batch_size": 1024, + "version": 0, + "storage_options": {} + }); + let config_str = config.to_string(); + + let mut handle: LanceReaderHandle = ptr::null_mut(); + let rc = lance_reader_open_json(config_str.as_ptr(), config_str.len(), &mut handle); + assert_eq!(rc, FFI_OK, "open_json failed: {}", { + let mut buf = [0u8; 256]; + let len = lance_reader_last_error(buf.as_mut_ptr(), buf.len()); + std::str::from_utf8(&buf[..len]).unwrap_or("?").to_string() + }); + assert!(!handle.is_null()); + + // Verify schema has 2 projected columns + let mut schema = FFI_ArrowSchema::empty(); + let rc = lance_reader_get_schema(handle, &mut schema); + assert_eq!(rc, FFI_OK); + let imported = arrow::datatypes::Schema::try_from(&schema).unwrap(); + assert_eq!(imported.fields().len(), 2); + assert_eq!(imported.field(0).name(), "name"); + assert_eq!(imported.field(1).name(), "value"); + + lance_reader_close(handle); + } + + #[test] + fn test_ffi_open_json_invalid() { + let mut handle: LanceReaderHandle = ptr::null_mut(); + let bad_json = b"not valid json"; + let rc = lance_reader_open_json(bad_json.as_ptr(), bad_json.len(), &mut handle); + assert_eq!(rc, FFI_ERR_INVALID_ARG); + } +} + +/// Create a multi-fragment test Lance dataset at the given path. +/// Writes 3 separate batches as 3 fragments, 5 rows each = 15 total rows. +/// Used by regression tests to verify fragment-level parallelism. +#[no_mangle] +pub extern "C" fn lance_test_create_multi_fragment_dataset( + path_ptr: *const u8, + path_len: usize, +) -> i32 { + error::clear_last_error(); + + std::panic::catch_unwind(AssertUnwindSafe(|| { + if path_ptr.is_null() { + error::set_last_error("path must not be null".to_string()); + return FFI_ERR_INVALID_ARG; + } + + let path = unsafe { + let slice = std::slice::from_raw_parts(path_ptr, path_len); + match std::str::from_utf8(slice) { + Ok(s) => s, + Err(e) => { + error::set_last_error(format!("Invalid UTF-8: {}", e)); + return FFI_ERR_INVALID_ARG; + } + } + }; + + let rt = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(rt) => rt, + Err(e) => { + error::set_last_error(format!("Runtime: {}", e)); + return error::FFI_ERR_IO; + } + }; + + let result = rt.block_on(async { + use arrow::array::{Float64Array, Int32Array, StringArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use std::sync::Arc; + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("value", DataType::Float64, false), + ])); + + // Fragment 1: ids 1-5 + let batch1 = arrow::array::RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])), + Arc::new(StringArray::from(vec!["a1", "a2", "a3", "a4", "a5"])), + Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0, 4.0, 5.0])), + ], + ) + .map_err(|e| crate::error::FfiError::Arrow(e.to_string()))?; + + // Write first fragment (create dataset) + let batches = arrow_array::RecordBatchIterator::new(vec![Ok(batch1)], schema.clone()); + let write_result: Result = + lance::Dataset::write(batches, path, None::).await; + write_result.map_err(|e| crate::error::FfiError::Lance(e.to_string()))?; + + // Fragment 2: ids 6-10 + let batch2 = arrow::array::RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![6, 7, 8, 9, 10])), + Arc::new(StringArray::from(vec!["b1", "b2", "b3", "b4", "b5"])), + Arc::new(Float64Array::from(vec![6.0, 7.0, 8.0, 9.0, 10.0])), + ], + ) + .map_err(|e| crate::error::FfiError::Arrow(e.to_string()))?; + + let batches = arrow_array::RecordBatchIterator::new(vec![Ok(batch2)], schema.clone()); + let mut params = lance::dataset::WriteParams::default(); + params.mode = lance::dataset::WriteMode::Append; + let write_result: Result = + lance::Dataset::write(batches, path, Some(params)).await; + write_result.map_err(|e| crate::error::FfiError::Lance(e.to_string()))?; + + // Fragment 3: ids 11-15 + let batch3 = arrow::array::RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![11, 12, 13, 14, 15])), + Arc::new(StringArray::from(vec!["c1", "c2", "c3", "c4", "c5"])), + Arc::new(Float64Array::from(vec![11.0, 12.0, 13.0, 14.0, 15.0])), + ], + ) + .map_err(|e| crate::error::FfiError::Arrow(e.to_string()))?; + + let batches = arrow_array::RecordBatchIterator::new(vec![Ok(batch3)], schema); + let mut params = lance::dataset::WriteParams::default(); + params.mode = lance::dataset::WriteMode::Append; + let write_result: Result = + lance::Dataset::write(batches, path, Some(params)).await; + write_result.map_err(|e| crate::error::FfiError::Lance(e.to_string()))?; + + Ok::<_, crate::error::FfiError>(()) + }); + + match result { + Ok(()) => FFI_OK, + Err(e) => { + let code = e.status_code(); + error::set_last_error(e.to_string()); + code + } + } + })) + .unwrap_or_else(|panic| { + let msg = format_panic(&panic); + error::set_last_error(msg); + FFI_ERR_PANIC + }) +} diff --git a/be/src/rust/doris-native/crates/doris-ffi/src/lance_reader.rs b/be/src/rust/doris-native/crates/doris-ffi/src/lance_reader.rs new file mode 100644 index 00000000000000..be9e89ad409982 --- /dev/null +++ b/be/src/rust/doris-native/crates/doris-ffi/src/lance_reader.rs @@ -0,0 +1,444 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use arrow::array::RecordBatch; +use arrow::datatypes::SchemaRef; +use futures::stream::BoxStream; +use futures::StreamExt; +use serde::Deserialize; +use std::collections::HashMap; + +use crate::error::{FfiError, FfiResult}; + +/// Configuration for opening a Lance dataset, deserialized from JSON. +/// Passed from C++ as a JSON string via the FFI. +#[derive(Debug, Default, Deserialize)] +pub struct LanceReaderConfig { + /// Dataset URI (s3://bucket/path.lance or file:///path.lance) + pub uri: String, + /// Column names to project (empty = all columns) + #[serde(default)] + pub columns: Vec, + /// Maximum rows per batch + #[serde(default = "default_batch_size")] + pub batch_size: usize, + /// Dataset version for time travel (0 = latest) + #[serde(default)] + pub version: u64, + /// Storage options (S3 credentials, etc.) + #[serde(default)] + pub storage_options: HashMap, + + // --- Index-accelerated search --- + /// SQL-like filter expression pushed down to Lance (uses scalar indexes if available). + /// Example: "category = 'shoes' AND price < 100" + #[serde(default)] + pub filter: Option, + + /// Vector nearest-neighbor search config. + #[serde(default)] + pub vector_search: Option, + + /// Full-text search query string (uses FTS index if available). + #[serde(default)] + pub full_text_search: Option, + + /// LIMIT pushdown (0 = no limit) + #[serde(default)] + pub limit: Option, + + /// OFFSET pushdown + #[serde(default)] + pub offset: Option, + + /// Specific fragment IDs to scan (empty = all fragments) + #[serde(default)] + pub fragment_ids: Vec, + + /// Fragment data file path relative to dataset root (e.g., "data/xxx.lance"). + /// When set, only the fragment whose data file matches this path is scanned. + /// This prevents duplicate reads when TVF creates multiple scan ranges. + #[serde(default)] + pub fragment_file: Option, +} + +/// Vector ANN search configuration. +#[derive(Debug, Deserialize)] +pub struct VectorSearchConfig { + /// Column name containing the vector + pub column: String, + /// Query vector as flat f32 array + pub query: Vec, + /// Number of nearest neighbors to return + pub k: usize, + /// Distance metric: "l2", "cosine", "dot" + #[serde(default = "default_metric")] + pub metric: String, + /// Number of IVF probes (higher = more accurate, slower) + #[serde(default)] + pub nprobes: Option, + /// HNSW ef search parameter + #[serde(default)] + pub ef: Option, + /// Refine factor for re-ranking + #[serde(default)] + pub refine_factor: Option, +} + +fn default_metric() -> String { + "l2".to_string() +} + +fn default_batch_size() -> usize { + 4096 +} + +/// A synchronous wrapper around lance-rs async APIs. +/// +/// Each reader owns a single-threaded tokio runtime (`new_current_thread`). +/// `block_on()` is called from the Doris scanner thread, which is allowed to block. +/// This creates zero additional OS threads — the runtime is purely a +/// future-polling state machine running inline on the calling thread. +pub struct LanceReader { + rt: tokio::runtime::Runtime, + stream: Option>>, + schema: SchemaRef, +} + +impl LanceReader { + /// Open a Lance dataset and prepare a scan stream. + /// + /// - `uri`: Dataset path (local file:/// or s3://) + /// - `columns`: Column names to project (empty = all columns) + /// - `batch_size`: Maximum rows per RecordBatch + pub fn open(uri: &str, columns: &[String], batch_size: usize) -> FfiResult { + Self::open_with_config(&LanceReaderConfig { + uri: uri.to_string(), + columns: columns.to_vec(), + batch_size, + ..Default::default() + }) + } + + /// Open a Lance dataset from a full config (supports S3 creds, version, etc.) + pub fn open_with_config(config: &LanceReaderConfig) -> FfiResult { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| FfiError::Io(format!("Failed to create tokio runtime: {}", e)))?; + + let (stream, schema) = rt.block_on(async { + // Build dataset open params with storage options (S3 creds, etc.) + let mut builder = lance::dataset::builder::DatasetBuilder::from_uri(&config.uri); + + // Set storage options (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, etc.) + for (k, v) in &config.storage_options { + builder = builder.with_storage_option(k, v); + } + + // Time travel: open a specific version + if config.version > 0 { + builder = builder.with_version(config.version); + } + + let dataset = builder.load().await.map_err(FfiError::from)?; + + let mut scanner = dataset.scan(); + if !config.columns.is_empty() { + let col_refs: Vec<&str> = config.columns.iter().map(|s| s.as_str()).collect(); + scanner + .project(&col_refs) + .map_err(|e| FfiError::Lance(e.to_string()))?; + } + scanner.batch_size(config.batch_size); + + // Filter pushdown (uses scalar indexes: BTree, Bitmap) + if let Some(ref filter_expr) = config.filter { + scanner + .filter(filter_expr) + .map_err(|e| FfiError::Lance(e.to_string()))?; + } + + // Vector ANN search (uses IVF-PQ, IVF-HNSW indexes) + if let Some(ref vs) = config.vector_search { + let query_array = arrow::array::Float32Array::from(vs.query.clone()); + scanner + .nearest(&vs.column, &query_array, vs.k) + .map_err(|e| FfiError::Lance(e.to_string()))?; + // Distance metric + let metric = match vs.metric.to_lowercase().as_str() { + "cosine" => lance_linalg::distance::MetricType::Cosine, + "dot" => lance_linalg::distance::MetricType::Dot, + _ => lance_linalg::distance::MetricType::L2, + }; + scanner.distance_metric(metric); + if let Some(n) = vs.nprobes { + scanner.nprobs(n); + } + if let Some(ef) = vs.ef { + scanner.ef(ef); + } + if let Some(rf) = vs.refine_factor { + scanner.refine(rf); + } + } + + // Full-text search (uses tantivy FTS index) + if let Some(ref fts_query) = config.full_text_search { + scanner + .full_text_search(lance_index::scalar::FullTextSearchQuery::new( + fts_query.clone(), + )) + .map_err(|e| FfiError::Lance(e.to_string()))?; + } + + // LIMIT/OFFSET pushdown + if config.limit.is_some() || config.offset.is_some() { + scanner + .limit(config.limit, config.offset) + .map_err(|e| FfiError::Lance(e.to_string()))?; + } + + // Fragment-level parallelism: scan only specific fragments by ID + if !config.fragment_ids.is_empty() { + let all_frags = dataset.get_fragments(); + let selected: Vec<_> = all_frags + .into_iter() + .filter(|f| config.fragment_ids.contains(&(f.id() as u64))) + .map(|f| f.metadata().clone()) + .collect(); + if !selected.is_empty() { + scanner.with_fragments(selected); + } + } + + // Filter to a single fragment by data file path. + // When TVF creates multiple scan ranges (one per .lance file), each range + // passes its fragment file path so we only read that specific fragment. + if let Some(ref frag_file) = config.fragment_file { + let all_frags = dataset.get_fragments(); + let selected: Vec<_> = all_frags + .into_iter() + .filter(|f| { + f.metadata() + .files + .iter() + .any(|df| frag_file.ends_with(&df.path)) + }) + .map(|f| f.metadata().clone()) + .collect(); + if !selected.is_empty() { + scanner.with_fragments(selected); + } + } + + let schema_ref = scanner + .schema() + .await + .map_err(|e| FfiError::Lance(e.to_string()))?; + + let stream = scanner + .try_into_stream() + .await + .map_err(|e| FfiError::Lance(e.to_string()))?; + + Ok::<_, FfiError>((stream.boxed(), schema_ref)) + })?; + + Ok(Self { + rt, + stream: Some(stream), + schema, + }) + } + + /// Read the next batch. Returns `None` on EOF. + pub fn next_batch(&mut self) -> FfiResult> { + let stream = self + .stream + .as_mut() + .ok_or_else(|| FfiError::Lance("Reader stream is closed".to_string()))?; + + let result = self.rt.block_on(stream.next()); + match result { + Some(Ok(batch)) => Ok(Some(batch)), + Some(Err(e)) => Err(FfiError::from(e)), + None => Ok(None), + } + } + + /// Get the schema of the scan output. + pub fn schema(&self) -> &SchemaRef { + &self.schema + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{Float64Array, Int32Array, StringArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use std::sync::Arc; + + /// Helper: create a small Lance dataset in a temp directory and return its path. + fn create_test_dataset(dir: &std::path::Path) -> String { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("score", DataType::Float64, false), + ])); + + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])), + Arc::new(StringArray::from(vec![ + "alice", "bob", "carol", "dave", "eve", + ])), + Arc::new(Float64Array::from(vec![90.5, 85.0, 92.3, 78.1, 88.7])), + ], + ) + .unwrap(); + + let uri = dir.join("test.lance").to_string_lossy().to_string(); + + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + rt.block_on(async { + let batches = arrow_array::RecordBatchIterator::new(vec![Ok(batch)], schema); + lance::Dataset::write(batches, &uri, None::) + .await + .unwrap(); + }); + + uri + } + + #[test] + fn test_open_and_read_all_columns() { + let tmp = tempfile::tempdir().unwrap(); + let uri = create_test_dataset(tmp.path()); + + let mut reader = LanceReader::open(&uri, &[], 1024).unwrap(); + + // Schema should have 3 fields + assert_eq!(reader.schema().fields().len(), 3); + assert_eq!(reader.schema().field(0).name(), "id"); + assert_eq!(reader.schema().field(1).name(), "name"); + assert_eq!(reader.schema().field(2).name(), "score"); + + // Read first batch — should contain all 5 rows + let batch = reader.next_batch().unwrap(); + assert!(batch.is_some()); + let batch = batch.unwrap(); + assert_eq!(batch.num_rows(), 5); + assert_eq!(batch.num_columns(), 3); + + // Verify values + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(ids.values(), &[1, 2, 3, 4, 5]); + + let names = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(names.value(0), "alice"); + assert_eq!(names.value(4), "eve"); + + // Next batch should be EOF + let batch = reader.next_batch().unwrap(); + assert!(batch.is_none()); + } + + #[test] + fn test_open_with_column_projection() { + let tmp = tempfile::tempdir().unwrap(); + let uri = create_test_dataset(tmp.path()); + + let columns = vec!["name".to_string(), "score".to_string()]; + let mut reader = LanceReader::open(&uri, &columns, 1024).unwrap(); + + assert_eq!(reader.schema().fields().len(), 2); + assert_eq!(reader.schema().field(0).name(), "name"); + assert_eq!(reader.schema().field(1).name(), "score"); + + let batch = reader.next_batch().unwrap().unwrap(); + assert_eq!(batch.num_columns(), 2); + assert_eq!(batch.num_rows(), 5); + + let scores = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert!((scores.value(0) - 90.5).abs() < f64::EPSILON); + } + + #[test] + fn test_open_with_small_batch_size() { + let tmp = tempfile::tempdir().unwrap(); + let uri = create_test_dataset(tmp.path()); + + let mut reader = LanceReader::open(&uri, &[], 2).unwrap(); + + let mut total_rows = 0; + let mut batch_count = 0; + loop { + match reader.next_batch().unwrap() { + Some(batch) => { + total_rows += batch.num_rows(); + batch_count += 1; + } + None => break, + } + } + assert_eq!(total_rows, 5); + assert!( + batch_count >= 2, + "Expected multiple batches, got {}", + batch_count + ); + } + + #[test] + fn test_open_nonexistent_path() { + let result = LanceReader::open("/nonexistent/path/to/dataset.lance", &[], 1024); + assert!(result.is_err()); + match result { + Err(e) => assert_eq!(e.status_code(), crate::error::FFI_ERR_LANCE), + Ok(_) => panic!("Expected error"), + } + } + + #[test] + fn test_open_invalid_column_name() { + let tmp = tempfile::tempdir().unwrap(); + let uri = create_test_dataset(tmp.path()); + + let columns = vec!["nonexistent_column".to_string()]; + let result = LanceReader::open(&uri, &columns, 1024); + assert!(result.is_err()); + } +} diff --git a/be/src/rust/doris-native/crates/doris-ffi/src/lib.rs b/be/src/rust/doris-native/crates/doris-ffi/src/lib.rs new file mode 100644 index 00000000000000..1bb7b455ce3f39 --- /dev/null +++ b/be/src/rust/doris-native/crates/doris-ffi/src/lib.rs @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// FFI functions necessarily work with raw pointers +#![allow(clippy::not_unsafe_ptr_arg_deref)] + +pub mod error; +pub mod ffi; +pub mod lance_reader; + +use std::panic::AssertUnwindSafe; + +/// Trivial FFI round-trip function for build verification (Phase 0). +/// Returns the input value unchanged. +#[no_mangle] +pub extern "C" fn rust_echo(x: i32) -> i32 { + std::panic::catch_unwind(AssertUnwindSafe(|| x)).unwrap_or(-1) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_echo() { + assert_eq!(rust_echo(42), 42); + assert_eq!(rust_echo(0), 0); + assert_eq!(rust_echo(-1), -1); + } +} diff --git a/be/src/rust/doris-native/rust-toolchain.toml b/be/src/rust/doris-native/rust-toolchain.toml new file mode 100644 index 00000000000000..70b51cc7b99861 --- /dev/null +++ b/be/src/rust/doris-native/rust-toolchain.toml @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[toolchain] +channel = "stable" diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp index e4ac5b79bca7db..9f9faf90a2bd93 100644 --- a/be/src/service/internal_service.cpp +++ b/be/src/service/internal_service.cpp @@ -78,6 +78,9 @@ #include "format/orc/vorc_reader.h" #include "format/parquet/vparquet_reader.h" #include "format/text/text_reader.h" +#ifdef BUILD_RUST_READERS +#include "format/lance/lance_rust_reader.h" +#endif #include "io/fs/local_file_system.h" #include "io/fs/stream_load_pipe.h" #include "io/io_common.h" @@ -866,6 +869,12 @@ void PInternalService::fetch_table_schema(google::protobuf::RpcController* contr io_ctx.get(), io_ctx); break; } +#ifdef BUILD_RUST_READERS + case TFileFormatType::FORMAT_LANCE: { + reader = LanceRustReader::create_unique(params, range, io_ctx.get()); + break; + } +#endif default: st = Status::InternalError("Not supported file format in fetch table schema: {}", params.format_type); diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt index b44e5de64a60ae..d52c72b91994a6 100644 --- a/be/test/CMakeLists.txt +++ b/be/test/CMakeLists.txt @@ -30,6 +30,12 @@ file(GLOB_RECURSE UT_FILES CONFIGURE_DEPENDS *.cpp) file(GLOB_RECURSE VECTOR_FILES CONFIGURE_DEPENDS storage/index/ann/*.cpp) list(REMOVE_ITEM UT_FILES ${VECTOR_FILES}) +# Lance reader tests require Rust static library (BUILD_RUST_READERS=ON) +if (NOT BUILD_RUST_READERS) + file(GLOB_RECURSE LANCE_TEST_FILES CONFIGURE_DEPENDS format/lance/*.cpp) + list(REMOVE_ITEM UT_FILES ${LANCE_TEST_FILES}) +endif() + if(NOT DEFINED DORIS_WITH_LZO) list(REMOVE_ITEM UT_FILES ${CMAKE_CURRENT_SOURCE_DIR}/exec/plain_text_line_reader_lzop_test.cpp) endif() diff --git a/be/test/format/lance/lance_rust_reader_test.cpp b/be/test/format/lance/lance_rust_reader_test.cpp new file mode 100644 index 00000000000000..e319204626c0d6 --- /dev/null +++ b/be/test/format/lance/lance_rust_reader_test.cpp @@ -0,0 +1,218 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifdef BUILD_RUST_READERS + +#include + +#include +#include +#include + +#include "format/lance/lance_ffi.h" + +namespace doris { + +class LanceFfiTest : public testing::Test { +protected: + void SetUp() override { + // Create a unique temp directory for each test + _test_dir = std::filesystem::temp_directory_path() / "doris_lance_test_XXXXXX"; + _test_dir = std::filesystem::path(mkdtemp(const_cast(_test_dir.string().c_str()))); + + // Create a test Lance dataset via Rust FFI + std::string dataset_path = (_test_dir / "test.lance").string(); + int32_t rc = lance_test_create_dataset( + reinterpret_cast(dataset_path.data()), dataset_path.size()); + ASSERT_EQ(rc, lance_ffi::LANCE_FFI_OK) + << "Failed to create test dataset: " << _get_last_error(); + + _dataset_path = dataset_path; + } + + void TearDown() override { + std::error_code ec; + std::filesystem::remove_all(_test_dir, ec); + } + + std::string _get_last_error() { + uint8_t buf[1024]; + size_t len = lance_reader_last_error(buf, sizeof(buf)); + if (len > 0) { + return std::string(reinterpret_cast(buf), len); + } + return "(no error)"; + } + + std::filesystem::path _test_dir; + std::string _dataset_path; +}; + +// ==================== Raw FFI tests ==================== + +TEST_F(LanceFfiTest, EchoRoundTrip) { + EXPECT_EQ(rust_echo(42), 42); + EXPECT_EQ(rust_echo(0), 0); + EXPECT_EQ(rust_echo(-1), -1); +} + +TEST_F(LanceFfiTest, OpenAndClose) { + LanceReaderHandle handle = nullptr; + int32_t rc = lance_reader_open(reinterpret_cast(_dataset_path.data()), + _dataset_path.size(), nullptr, nullptr, 0, 1024, &handle); + ASSERT_EQ(rc, lance_ffi::LANCE_FFI_OK) << _get_last_error(); + ASSERT_NE(handle, nullptr); + + lance_reader_close(handle); +} + +TEST_F(LanceFfiTest, OpenNonexistentPath) { + std::string bad_path = "/nonexistent/path/dataset.lance"; + LanceReaderHandle handle = nullptr; + int32_t rc = lance_reader_open(reinterpret_cast(bad_path.data()), + bad_path.size(), nullptr, nullptr, 0, 1024, &handle); + EXPECT_LT(rc, 0); + EXPECT_EQ(handle, nullptr); + + // Error message should be available + std::string err = _get_last_error(); + EXPECT_FALSE(err.empty()); +} + +TEST_F(LanceFfiTest, CloseNullHandle) { + // Should not crash + lance_reader_close(nullptr); +} + +TEST_F(LanceFfiTest, GetSchema) { + LanceReaderHandle handle = nullptr; + int32_t rc = lance_reader_open(reinterpret_cast(_dataset_path.data()), + _dataset_path.size(), nullptr, nullptr, 0, 1024, &handle); + ASSERT_EQ(rc, lance_ffi::LANCE_FFI_OK); + + ArrowSchema c_schema {}; + rc = lance_reader_get_schema(handle, &c_schema); + ASSERT_EQ(rc, lance_ffi::LANCE_FFI_OK) << _get_last_error(); + + // The test dataset has 3 columns: id, name, score + // Arrow struct schema: n_children = number of columns + EXPECT_EQ(c_schema.n_children, 3); + EXPECT_STREQ(c_schema.children[0]->name, "id"); + EXPECT_STREQ(c_schema.children[1]->name, "name"); + EXPECT_STREQ(c_schema.children[2]->name, "score"); + + // Release schema + if (c_schema.release) c_schema.release(&c_schema); + lance_reader_close(handle); +} + +TEST_F(LanceFfiTest, ReadAllBatches) { + LanceReaderHandle handle = nullptr; + int32_t rc = lance_reader_open(reinterpret_cast(_dataset_path.data()), + _dataset_path.size(), nullptr, nullptr, 0, 1024, &handle); + ASSERT_EQ(rc, lance_ffi::LANCE_FFI_OK); + + int64_t total_rows = 0; + int batch_count = 0; + + while (true) { + ArrowSchema c_schema {}; + ArrowArray c_array {}; + bool eof = false; + int64_t bytes = 0; + + rc = lance_reader_next_batch(handle, &c_schema, &c_array, &eof, &bytes); + + if (rc == lance_ffi::LANCE_FFI_EOF || eof) { + break; + } + ASSERT_EQ(rc, lance_ffi::LANCE_FFI_OK) << _get_last_error(); + ASSERT_GT(bytes, 0); + + total_rows += c_array.length; + batch_count++; + + // Release Arrow C ABI ownership + if (c_array.release) c_array.release(&c_array); + if (c_schema.release) c_schema.release(&c_schema); + } + + EXPECT_EQ(total_rows, 5); + EXPECT_GE(batch_count, 1); + + lance_reader_close(handle); +} + +TEST_F(LanceFfiTest, ReadWithColumnProjection) { + // Project only "name" column + const uint8_t* col_name = reinterpret_cast("name"); + const uint8_t* col_ptrs[] = {col_name}; + size_t col_lens[] = {4}; + + LanceReaderHandle handle = nullptr; + int32_t rc = lance_reader_open(reinterpret_cast(_dataset_path.data()), + _dataset_path.size(), col_ptrs, col_lens, 1, 1024, &handle); + ASSERT_EQ(rc, lance_ffi::LANCE_FFI_OK) << _get_last_error(); + + // Verify schema has only 1 column + ArrowSchema c_schema {}; + rc = lance_reader_get_schema(handle, &c_schema); + ASSERT_EQ(rc, lance_ffi::LANCE_FFI_OK); + EXPECT_EQ(c_schema.n_children, 1); + EXPECT_STREQ(c_schema.children[0]->name, "name"); + if (c_schema.release) c_schema.release(&c_schema); + + // Read batch — should have 1 column, 5 rows + ArrowSchema batch_schema {}; + ArrowArray batch_array {}; + bool eof = false; + int64_t bytes = 0; + + rc = lance_reader_next_batch(handle, &batch_schema, &batch_array, &eof, &bytes); + ASSERT_EQ(rc, lance_ffi::LANCE_FFI_OK); + EXPECT_EQ(batch_array.length, 5); + EXPECT_EQ(batch_array.n_children, 1); + + if (batch_array.release) batch_array.release(&batch_array); + if (batch_schema.release) batch_schema.release(&batch_schema); + + lance_reader_close(handle); +} + +TEST_F(LanceFfiTest, ErrorMessageRetrieval) { + // Trigger an error + std::string bad_path = "/does/not/exist.lance"; + LanceReaderHandle handle = nullptr; + lance_reader_open(reinterpret_cast(bad_path.data()), bad_path.size(), nullptr, + nullptr, 0, 1024, &handle); + + // Retrieve error message + uint8_t buf[1024]; + size_t len = lance_reader_last_error(buf, sizeof(buf)); + EXPECT_GT(len, 0u); + + std::string msg(reinterpret_cast(buf), len); + // Should contain something about the path not existing + EXPECT_FALSE(msg.empty()); + + // Null buffer should return 0 + EXPECT_EQ(lance_reader_last_error(nullptr, 0), 0u); +} + +} // namespace doris + +#endif // BUILD_RUST_READERS diff --git a/be/test/format/lance/standalone_lance_test.cpp b/be/test/format/lance/standalone_lance_test.cpp new file mode 100644 index 00000000000000..4ddfa2542a3d1a --- /dev/null +++ b/be/test/format/lance/standalone_lance_test.cpp @@ -0,0 +1,376 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Standalone Lance FFI test — links only against libdoris_ffi.a and arrow. +// Does NOT depend on the full Doris BE build. +// Build: see Makefile target below or CMake standalone_lance_test target. + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +// FFI declarations (matching lance_ffi.h without Doris deps) +extern "C" { +int32_t rust_echo(int32_t x); +int32_t lance_test_create_dataset(const uint8_t* path_ptr, size_t path_len); +int32_t lance_reader_open(const uint8_t* uri_ptr, size_t uri_len, + const uint8_t* const* column_names_ptr, + const size_t* column_names_len_ptr, size_t num_columns, size_t batch_size, + void** handle_out); +int32_t lance_reader_next_batch(void* handle, ArrowSchema* schema_out, ArrowArray* array_out, + bool* eof_out, int64_t* bytes_out); +int32_t lance_reader_get_schema(void* handle, ArrowSchema* schema_out); +void lance_reader_close(void* handle); +size_t lance_reader_last_error(uint8_t* buf, size_t buf_len); +int32_t lance_reader_open_json(const uint8_t* config_json_ptr, size_t config_json_len, + void** handle_out); +} + +static std::string get_last_error() { + uint8_t buf[1024]; + size_t len = lance_reader_last_error(buf, sizeof(buf)); + return len > 0 ? std::string(reinterpret_cast(buf), len) : "(no error)"; +} + +#define ASSERT_EQ(a, b, msg) \ + do { \ + if ((a) != (b)) { \ + std::cerr << "FAIL: " << msg << ": " << (a) << " != " << (b) << std::endl; \ + return 1; \ + } \ + } while (0) +#define ASSERT_TRUE(a, msg) \ + do { \ + if (!(a)) { \ + std::cerr << "FAIL: " << msg << std::endl; \ + return 1; \ + } \ + } while (0) +#define ASSERT_OK(rc, msg) \ + do { \ + if ((rc) < 0) { \ + std::cerr << "FAIL: " << msg << ": " << get_last_error() << std::endl; \ + return 1; \ + } \ + } while (0) + +int test_echo() { + std::cout << " test_echo... "; + ASSERT_EQ(rust_echo(42), 42, "echo 42"); + ASSERT_EQ(rust_echo(0), 0, "echo 0"); + ASSERT_EQ(rust_echo(-1), -1, "echo -1"); + std::cout << "OK" << std::endl; + return 0; +} + +int test_open_close(const std::string& uri) { + std::cout << " test_open_close... "; + void* handle = nullptr; + int32_t rc = lance_reader_open(reinterpret_cast(uri.data()), uri.size(), + nullptr, nullptr, 0, 1024, &handle); + ASSERT_OK(rc, "open"); + ASSERT_TRUE(handle != nullptr, "handle not null"); + lance_reader_close(handle); + std::cout << "OK" << std::endl; + return 0; +} + +int test_get_schema(const std::string& uri) { + std::cout << " test_get_schema... "; + void* handle = nullptr; + int32_t rc = lance_reader_open(reinterpret_cast(uri.data()), uri.size(), + nullptr, nullptr, 0, 1024, &handle); + ASSERT_OK(rc, "open"); + + ArrowSchema c_schema {}; + rc = lance_reader_get_schema(handle, &c_schema); + ASSERT_OK(rc, "get_schema"); + ASSERT_EQ(c_schema.n_children, 3L, "3 columns"); + ASSERT_TRUE(strcmp(c_schema.children[0]->name, "id") == 0, "col0=id"); + ASSERT_TRUE(strcmp(c_schema.children[1]->name, "name") == 0, "col1=name"); + ASSERT_TRUE(strcmp(c_schema.children[2]->name, "score") == 0, "col2=score"); + + if (c_schema.release) c_schema.release(&c_schema); + lance_reader_close(handle); + std::cout << "OK" << std::endl; + return 0; +} + +int test_read_batches(const std::string& uri) { + std::cout << " test_read_batches... "; + void* handle = nullptr; + int32_t rc = lance_reader_open(reinterpret_cast(uri.data()), uri.size(), + nullptr, nullptr, 0, 1024, &handle); + ASSERT_OK(rc, "open"); + + int64_t total_rows = 0; + int batch_count = 0; + while (true) { + ArrowSchema c_schema {}; + ArrowArray c_array {}; + bool eof = false; + int64_t bytes = 0; + rc = lance_reader_next_batch(handle, &c_schema, &c_array, &eof, &bytes); + if (rc == 1 || eof) break; // FFI_EOF = 1 + ASSERT_OK(rc, "next_batch"); + ASSERT_TRUE(bytes > 0, "bytes > 0"); + + // Import via Arrow C Data Interface and verify + auto import_result = arrow::ImportRecordBatch(&c_array, &c_schema); + ASSERT_TRUE(import_result.ok(), "ImportRecordBatch"); + auto batch = import_result.ValueUnsafe(); + total_rows += batch->num_rows(); + batch_count++; + } + ASSERT_EQ(total_rows, 5L, "5 total rows"); + ASSERT_TRUE(batch_count >= 1, "at least 1 batch"); + + lance_reader_close(handle); + std::cout << "OK (" << batch_count << " batch, " << total_rows << " rows)" << std::endl; + return 0; +} + +int test_column_projection(const std::string& uri) { + std::cout << " test_column_projection... "; + const uint8_t* col = reinterpret_cast("name"); + const uint8_t* cols[] = {col}; + size_t lens[] = {4}; + + void* handle = nullptr; + int32_t rc = lance_reader_open(reinterpret_cast(uri.data()), uri.size(), cols, + lens, 1, 1024, &handle); + ASSERT_OK(rc, "open with projection"); + + ArrowSchema c_schema {}; + rc = lance_reader_get_schema(handle, &c_schema); + ASSERT_OK(rc, "get_schema"); + ASSERT_EQ(c_schema.n_children, 1L, "1 projected column"); + ASSERT_TRUE(strcmp(c_schema.children[0]->name, "name") == 0, "col=name"); + if (c_schema.release) c_schema.release(&c_schema); + + lance_reader_close(handle); + std::cout << "OK" << std::endl; + return 0; +} + +int test_error_path() { + std::cout << " test_error_path... "; + std::string bad = "/nonexistent/path.lance"; + void* handle = nullptr; + int32_t rc = lance_reader_open(reinterpret_cast(bad.data()), bad.size(), + nullptr, nullptr, 0, 1024, &handle); + ASSERT_TRUE(rc < 0, "error code negative"); + ASSERT_TRUE(handle == nullptr, "handle null on error"); + std::string err = get_last_error(); + ASSERT_TRUE(!err.empty(), "error message non-empty"); + + // null handle close should not crash + lance_reader_close(nullptr); + std::cout << "OK" << std::endl; + return 0; +} + +int test_json_config(const std::string& uri) { + std::cout << " test_json_config... "; + // Build JSON config with storage_options (empty for local) and version=0 + std::string config = + R"({"uri":")" + uri + + R"(","columns":["id","score"],"batch_size":4096,"version":0,"storage_options":{}})"; + + void* handle = nullptr; + int32_t rc = lance_reader_open_json(reinterpret_cast(config.data()), + config.size(), &handle); + ASSERT_OK(rc, "open_json"); + ASSERT_TRUE(handle != nullptr, "handle not null"); + + // Verify 2 projected columns + ArrowSchema c_schema {}; + rc = lance_reader_get_schema(handle, &c_schema); + ASSERT_OK(rc, "get_schema"); + ASSERT_EQ(c_schema.n_children, 2L, "2 projected columns"); + ASSERT_TRUE(strcmp(c_schema.children[0]->name, "id") == 0, "col0=id"); + ASSERT_TRUE(strcmp(c_schema.children[1]->name, "score") == 0, "col1=score"); + if (c_schema.release) c_schema.release(&c_schema); + + // Read data + ArrowSchema batch_schema {}; + ArrowArray batch_array {}; + bool eof = false; + int64_t bytes = 0; + rc = lance_reader_next_batch(handle, &batch_schema, &batch_array, &eof, &bytes); + ASSERT_OK(rc, "next_batch"); + ASSERT_EQ(batch_array.length, 5L, "5 rows"); + ASSERT_EQ(batch_array.n_children, 2L, "2 columns in batch"); + if (batch_array.release) batch_array.release(&batch_array); + if (batch_schema.release) batch_schema.release(&batch_schema); + + lance_reader_close(handle); + std::cout << "OK" << std::endl; + return 0; +} + +// Simulates the full TVF query path: +// 1. fetch_table_schema: open with no columns → get schema → return col names/types +// 2. FileScanner: open with projected columns → read batches → verify data values +int test_tvf_simulation(const std::string& uri) { + std::cout << " test_tvf_simulation... " << std::flush; + + // ========== Phase 1: Schema Inference (fetch_table_schema RPC) ========== + // FE sends PFetchTableSchemaRequest to BE. BE opens dataset, reads schema, returns it. + { + std::string config = R"({"uri":")" + uri + + R"(","columns":[],"batch_size":1,"version":0,"storage_options":{}})"; + void* handle = nullptr; + int32_t rc = lance_reader_open_json(reinterpret_cast(config.data()), + config.size(), &handle); + ASSERT_OK(rc, "schema: open"); + + ArrowSchema c_schema {}; + rc = lance_reader_get_schema(handle, &c_schema); + ASSERT_OK(rc, "schema: get_schema"); + + // Verify schema: 3 columns (id:int32, name:utf8, score:float64) + ASSERT_EQ(c_schema.n_children, 3L, "schema: 3 columns"); + + // id column - Arrow int32 format "i" + ASSERT_TRUE(strcmp(c_schema.children[0]->name, "id") == 0, "schema: col0=id"); + ASSERT_TRUE(strcmp(c_schema.children[0]->format, "i") == 0, "schema: id is int32"); + + // name column - Arrow utf8 format "u" + ASSERT_TRUE(strcmp(c_schema.children[1]->name, "name") == 0, "schema: col1=name"); + ASSERT_TRUE(strcmp(c_schema.children[1]->format, "u") == 0, "schema: name is utf8"); + + // score column - Arrow float64 format "g" + ASSERT_TRUE(strcmp(c_schema.children[2]->name, "score") == 0, "schema: col2=score"); + ASSERT_TRUE(strcmp(c_schema.children[2]->format, "g") == 0, "schema: score is float64"); + + if (c_schema.release) c_schema.release(&c_schema); + lance_reader_close(handle); + } + + // ========== Phase 2: Data Scan (FileScanner::get_next_block) ========== + // FE plans query with schema from Phase 1, sends scan range to BE. + // BE opens dataset with projected columns, reads batches, converts to Block. + { + std::string config = + R"({"uri":")" + uri + + R"(","columns":["id","name","score"],"batch_size":4096,"version":0,"storage_options":{}})"; + void* handle = nullptr; + int32_t rc = lance_reader_open_json(reinterpret_cast(config.data()), + config.size(), &handle); + ASSERT_OK(rc, "scan: open"); + + // Read first (and only) batch + ArrowSchema c_schema {}; + ArrowArray c_array {}; + bool eof = false; + int64_t bytes = 0; + rc = lance_reader_next_batch(handle, &c_schema, &c_array, &eof, &bytes); + ASSERT_OK(rc, "scan: next_batch"); + ASSERT_TRUE(!eof, "scan: not eof on first batch"); + ASSERT_TRUE(bytes > 0, "scan: bytes > 0"); + + // Import via Arrow C Data Interface (same as LanceRustReader::get_next_block) + auto import_result = arrow::ImportRecordBatch(&c_array, &c_schema); + ASSERT_TRUE(import_result.ok(), "scan: ImportRecordBatch"); + auto batch = import_result.ValueUnsafe(); + + ASSERT_EQ(batch->num_rows(), 5L, "scan: 5 rows"); + ASSERT_EQ(batch->num_columns(), 3L, "scan: 3 columns"); + + // Verify actual data values (this is what Doris Block would contain) + // Column 0: id (int32) = [1, 2, 3, 4, 5] + auto id_array = std::dynamic_pointer_cast(batch->column(0)); + ASSERT_TRUE(id_array != nullptr, "scan: id is Int32Array"); + ASSERT_EQ(id_array->Value(0), 1, "scan: id[0]=1"); + ASSERT_EQ(id_array->Value(1), 2, "scan: id[1]=2"); + ASSERT_EQ(id_array->Value(4), 5, "scan: id[4]=5"); + + // Column 1: name (utf8) = ["alice", "bob", "carol", "dave", "eve"] + auto name_array = std::dynamic_pointer_cast(batch->column(1)); + ASSERT_TRUE(name_array != nullptr, "scan: name is StringArray"); + ASSERT_TRUE(name_array->GetString(0) == "alice", "scan: name[0]=alice"); + ASSERT_TRUE(name_array->GetString(1) == "bob", "scan: name[1]=bob"); + ASSERT_TRUE(name_array->GetString(4) == "eve", "scan: name[4]=eve"); + + // Column 2: score (float64) = [90.5, 85.0, 92.3, 78.1, 88.7] + auto score_array = std::dynamic_pointer_cast(batch->column(2)); + ASSERT_TRUE(score_array != nullptr, "scan: score is DoubleArray"); + ASSERT_TRUE(std::abs(score_array->Value(0) - 90.5) < 0.01, "scan: score[0]=90.5"); + ASSERT_TRUE(std::abs(score_array->Value(1) - 85.0) < 0.01, "scan: score[1]=85.0"); + ASSERT_TRUE(std::abs(score_array->Value(4) - 88.7) < 0.01, "scan: score[4]=88.7"); + + // Verify EOF on next call + ArrowSchema eof_schema {}; + ArrowArray eof_array {}; + bool is_eof = false; + int64_t eof_bytes = 0; + rc = lance_reader_next_batch(handle, &eof_schema, &eof_array, &is_eof, &eof_bytes); + ASSERT_TRUE(rc == 1 || is_eof, "scan: EOF after last batch"); + + lance_reader_close(handle); + } + + std::cout << "OK (schema inference + full data scan verified)" << std::endl; + return 0; +} + +int main() { + // Create test dataset + auto tmpdir = std::filesystem::temp_directory_path() / "lance_e2e_test"; + std::filesystem::create_directories(tmpdir); + std::string dataset_path = (tmpdir / "test.lance").string(); + + std::cout << "Creating test dataset at " << dataset_path << std::endl; + int32_t rc = lance_test_create_dataset(reinterpret_cast(dataset_path.data()), + dataset_path.size()); + if (rc != 0) { + std::cerr << "Failed to create dataset: " << get_last_error() << std::endl; + return 1; + } + + std::cout << "Running Lance FFI E2E tests:" << std::endl; + int failures = 0; + failures += test_echo(); + failures += test_open_close(dataset_path); + failures += test_get_schema(dataset_path); + failures += test_read_batches(dataset_path); + failures += test_column_projection(dataset_path); + failures += test_error_path(); + failures += test_json_config(dataset_path); + failures += test_tvf_simulation(dataset_path); + + // Cleanup + std::filesystem::remove_all(tmpdir); + + if (failures == 0) { + std::cout << "\nAll 8 tests PASSED!" << std::endl; + } else { + std::cerr << "\n" << failures << " test(s) FAILED!" << std::endl; + } + return failures; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java index 774ee4e6e838f6..71f888618b8a12 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java @@ -31,6 +31,7 @@ public class FileFormatConstants { public static final String FORMAT_WAL = "wal"; public static final String FORMAT_ARROW = "arrow"; public static final String FORMAT_NATIVE = "native"; + public static final String FORMAT_LANCE = "lance"; public static final String PROP_FORMAT = "format"; public static final String PROP_COLUMN_SEPARATOR = "column_separator"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java index ac2512d88b51c3..67ca640dd56e29 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java @@ -40,6 +40,7 @@ public abstract class FileFormatProperties { public static final String FORMAT_WAL = "wal"; public static final String FORMAT_ARROW = "arrow"; public static final String FORMAT_NATIVE = "native"; + public static final String FORMAT_LANCE = "lance"; public static final String PROP_COMPRESS_TYPE = "compress_type"; protected String formatName; @@ -105,6 +106,8 @@ public static FileFormatProperties createFileFormatProperties(String formatStrin return new ArrowFileFormatProperties(); case FORMAT_NATIVE: return new NativeFileFormatProperties(); + case FORMAT_LANCE: + return new LanceFileFormatProperties(); default: throw new AnalysisException("format:" + formatString + " is not supported."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/LanceFileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/LanceFileFormatProperties.java new file mode 100644 index 00000000000000..a2fca1a3972e11 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/LanceFileFormatProperties.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.fileformat; + +import org.apache.doris.thrift.TFileAttributes; +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TFileTextScanRangeParams; +import org.apache.doris.thrift.TResultFileSinkOptions; + +import java.util.Map; + +public class LanceFileFormatProperties extends FileFormatProperties { + + public LanceFileFormatProperties() { + super(TFileFormatType.FORMAT_LANCE, FileFormatProperties.FORMAT_LANCE); + } + + @Override + public void analyzeFileFormatProperties(Map formatProperties, + boolean isRemoveOriginProperty) { + // Lance format has no special format properties to parse in Phase 1. + } + + @Override + public void fullTResultFileSinkOptions(TResultFileSinkOptions sinkOptions) { + // Lance write is not supported in Phase 1. + } + + @Override + public TFileAttributes toTFileAttributes() { + TFileAttributes fileAttributes = new TFileAttributes(); + TFileTextScanRangeParams fileTextScanRangeParams = new TFileTextScanRangeParams(); + fileAttributes.setTextParams(fileTextScanRangeParams); + return fileAttributes; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index f5db95b72959a8..f0124581df76dc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -763,6 +763,8 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_PAIMON_CPP_READER = "enable_paimon_cpp_reader"; + public static final String ENABLE_RUST_LANCE_READER = "enable_rust_lance_reader"; + public static final String ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE = "enable_count_push_down_for_external_table"; public static final String FETCH_ALL_FE_FOR_SYSTEM_TABLE = "fetch_all_fe_for_system_table"; @@ -2873,6 +2875,12 @@ public static boolean isEagerAggregationOnJoin() { description = {"Paimon 非原生文件读取使用 paimon-cpp", "Use paimon-cpp for non-native Paimon reads"}) private boolean enablePaimonCppReader = false; + @VarAttrDef.VarAttr(name = ENABLE_RUST_LANCE_READER, + fuzzy = true, + description = {"使用 Rust Lance 读取器读取 Lance 格式数据", + "Use Rust-based Lance reader for Lance format data"}) + private boolean enableRustLanceReader = false; + @VarAttrDef.VarAttr(name = ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE, fuzzy = true, description = {"对外表启用 count(*) 下推优化", "enable count(*) pushdown optimization for external table"}) @@ -5498,6 +5506,7 @@ public TQueryOptions toThrift() { tResult.setEnableOrcFilterByMinMax(enableOrcFilterByMinMax); tResult.setEnablePaimonCppReader(enablePaimonCppReader); tResult.setFilePresignedUrlTtlSeconds(filePresignedUrlTtlSeconds); + tResult.setEnableRustLanceReader(enableRustLanceReader); tResult.setEmbedMaxBatchSize(embedMaxBatchSize); tResult.setAiContextWindowSize(aiContextWindowSize); tResult.setCheckOrcInitSargsSuccess(checkOrcInitSargsSuccess); diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 783a0f0fe84cbb..4cfa19d7fca394 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -480,6 +480,8 @@ struct TQueryOptions { 214: optional i32 embed_max_batch_size = 5; 215: optional i64 ai_context_window_size = 131072; + // Use Rust-based Lance reader for FORMAT_LANCE scan ranges + 216: optional bool enable_rust_lance_reader = false; // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query. diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index eb58021b74414e..93a5e0c660d39a 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -112,7 +112,8 @@ enum TFileFormatType { FORMAT_WAL = 15, FORMAT_ARROW = 16, FORMAT_TEXT = 17, - FORMAT_NATIVE = 18 + FORMAT_NATIVE = 18, + FORMAT_LANCE = 19 } // In previous versions, the data compression format and file format were stored together, as TFileFormatType, @@ -426,6 +427,15 @@ struct TRemoteDorisFileDesc { 6: optional string password } +struct TLanceFileDesc { + // URI of the Lance dataset (s3://..., file:///..., etc.) + 1: optional string dataset_uri + // Specific fragment IDs to read (for split-level parallelism) + 2: optional list fragment_ids + // Dataset version for time travel + 3: optional i64 version +} + struct TTableFormatFileDesc { 1: optional string table_format_type 2: optional TIcebergFileDesc iceberg_params @@ -439,6 +449,7 @@ struct TTableFormatFileDesc { 10: optional TRemoteDorisFileDesc remote_doris_params // JDBC connection parameters (used when table_format_type == "jdbc") 11: optional map jdbc_params + 12: optional TLanceFileDesc lance_params } // Deprecated, hive text talbe is a special format, not a serde type diff --git a/regression-test/data/external_table_p0/tvf/lance/multi.lance/_transactions/0-055c1ba4-03ab-4a0e-8829-15bb1a6d2d08.txn b/regression-test/data/external_table_p0/tvf/lance/multi.lance/_transactions/0-055c1ba4-03ab-4a0e-8829-15bb1a6d2d08.txn new file mode 100644 index 00000000000000..19c6dcd615c02d Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/lance/multi.lance/_transactions/0-055c1ba4-03ab-4a0e-8829-15bb1a6d2d08.txn differ diff --git a/regression-test/data/external_table_p0/tvf/lance/multi.lance/_transactions/1-242b9a64-71b6-4ce9-bc7d-a09c7bd82089.txn b/regression-test/data/external_table_p0/tvf/lance/multi.lance/_transactions/1-242b9a64-71b6-4ce9-bc7d-a09c7bd82089.txn new file mode 100644 index 00000000000000..97b040e771b6de Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/lance/multi.lance/_transactions/1-242b9a64-71b6-4ce9-bc7d-a09c7bd82089.txn differ diff --git a/regression-test/data/external_table_p0/tvf/lance/multi.lance/_transactions/2-43117771-3274-4c35-bbf7-948403ce6510.txn b/regression-test/data/external_table_p0/tvf/lance/multi.lance/_transactions/2-43117771-3274-4c35-bbf7-948403ce6510.txn new file mode 100644 index 00000000000000..9c53f36897b099 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/lance/multi.lance/_transactions/2-43117771-3274-4c35-bbf7-948403ce6510.txn differ diff --git a/regression-test/data/external_table_p0/tvf/lance/multi.lance/_versions/1.manifest b/regression-test/data/external_table_p0/tvf/lance/multi.lance/_versions/1.manifest new file mode 100644 index 00000000000000..ff7eaef31babe0 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/lance/multi.lance/_versions/1.manifest differ diff --git a/regression-test/data/external_table_p0/tvf/lance/multi.lance/_versions/2.manifest b/regression-test/data/external_table_p0/tvf/lance/multi.lance/_versions/2.manifest new file mode 100644 index 00000000000000..e4af9e217e7e76 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/lance/multi.lance/_versions/2.manifest differ diff --git a/regression-test/data/external_table_p0/tvf/lance/multi.lance/_versions/3.manifest b/regression-test/data/external_table_p0/tvf/lance/multi.lance/_versions/3.manifest new file mode 100644 index 00000000000000..53790c3f9ac028 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/lance/multi.lance/_versions/3.manifest differ diff --git a/regression-test/data/external_table_p0/tvf/lance/multi.lance/data/5605bf04-7859-468c-8f08-7c95a5444617.lance b/regression-test/data/external_table_p0/tvf/lance/multi.lance/data/5605bf04-7859-468c-8f08-7c95a5444617.lance new file mode 100644 index 00000000000000..b4813da537d430 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/lance/multi.lance/data/5605bf04-7859-468c-8f08-7c95a5444617.lance differ diff --git a/regression-test/data/external_table_p0/tvf/lance/multi.lance/data/82a4ff91-1bdd-487d-a008-9c5682a1dca0.lance b/regression-test/data/external_table_p0/tvf/lance/multi.lance/data/82a4ff91-1bdd-487d-a008-9c5682a1dca0.lance new file mode 100644 index 00000000000000..2383820fa9e187 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/lance/multi.lance/data/82a4ff91-1bdd-487d-a008-9c5682a1dca0.lance differ diff --git a/regression-test/data/external_table_p0/tvf/lance/multi.lance/data/ca28aa58-00ae-48cc-ac5a-dc1746d1680b.lance b/regression-test/data/external_table_p0/tvf/lance/multi.lance/data/ca28aa58-00ae-48cc-ac5a-dc1746d1680b.lance new file mode 100644 index 00000000000000..0fb873d0e04424 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/lance/multi.lance/data/ca28aa58-00ae-48cc-ac5a-dc1746d1680b.lance differ diff --git a/regression-test/data/external_table_p0/tvf/lance/single.lance/_transactions/0-c6b23c0d-3ce1-45af-852f-25eeddb0fa8f.txn b/regression-test/data/external_table_p0/tvf/lance/single.lance/_transactions/0-c6b23c0d-3ce1-45af-852f-25eeddb0fa8f.txn new file mode 100644 index 00000000000000..618889ae1d13c7 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/lance/single.lance/_transactions/0-c6b23c0d-3ce1-45af-852f-25eeddb0fa8f.txn differ diff --git a/regression-test/data/external_table_p0/tvf/lance/single.lance/_versions/1.manifest b/regression-test/data/external_table_p0/tvf/lance/single.lance/_versions/1.manifest new file mode 100644 index 00000000000000..c4e31100689880 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/lance/single.lance/_versions/1.manifest differ diff --git a/regression-test/data/external_table_p0/tvf/lance/single.lance/data/f80e031c-321c-4729-849b-e4338790114e.lance b/regression-test/data/external_table_p0/tvf/lance/single.lance/data/f80e031c-321c-4729-849b-e4338790114e.lance new file mode 100644 index 00000000000000..6701e79413b514 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/lance/single.lance/data/f80e031c-321c-4729-849b-e4338790114e.lance differ diff --git a/regression-test/data/external_table_p0/tvf/lance/test_lance_tvf.out b/regression-test/data/external_table_p0/tvf/lance/test_lance_tvf.out new file mode 100644 index 00000000000000..743a732e0fb7f2 --- /dev/null +++ b/regression-test/data/external_table_p0/tvf/lance/test_lance_tvf.out @@ -0,0 +1,40 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_all -- +1 alice 90.5 +2 bob 85 +3 carol 92.3 +4 dave 78.09999999999999 +5 eve 88.7 + +-- !projection -- +alice 90.5 +bob 85 +carol 92.3 +dave 78.09999999999999 +eve 88.7 + +-- !count -- +5 + +-- !filter -- +1 alice 90.5 +3 carol 92.3 +5 eve 88.7 + +-- !limit -- +1 alice 90.5 +2 bob 85 + +-- !multi_count -- +15 1 15 + +-- !multi_filter -- +11 c1 11 +12 c2 12 +13 c3 13 +14 c4 14 +15 c5 15 + +-- !multi_agg -- +120 8 + diff --git a/regression-test/suites/external_table_p0/tvf/lance/test_lance_tvf.groovy b/regression-test/suites/external_table_p0/tvf/lance/test_lance_tvf.groovy new file mode 100644 index 00000000000000..36d73c23a12380 --- /dev/null +++ b/regression-test/suites/external_table_p0/tvf/lance/test_lance_tvf.groovy @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_lance_tvf", "p0,external,lance") { + + List> backends = sql """ show backends """ + assertTrue(backends.size() > 0) + def be_id = backends[0][0] + + // Copy lance test datasets to all BE nodes (fromDst=false means upload TO BE) + def dataFilePath = context.config.dataPath + "/external_table_p0/tvf/lance" + def outFilePath = "/" + for (List backend : backends) { + def be_host = backend[1] + scpFiles("root", be_host, dataFilePath + "/single.lance", outFilePath, false) + scpFiles("root", be_host, dataFilePath + "/multi.lance", outFilePath, false) + } + + def single_path = "single.lance/data/*.lance" + def multi_path = "multi.lance/data/*.lance" + + // --- Test 1: SELECT * (single fragment, 5 rows) --- + order_qt_select_all """ + select * from local( + "file_path" = "${single_path}", + "backend_id" = "${be_id}", + "format" = "lance" + ) order by id + """ + + // --- Test 2: Column projection --- + order_qt_projection """ + select name, score from local( + "file_path" = "${single_path}", + "backend_id" = "${be_id}", + "format" = "lance" + ) order by name + """ + + // --- Test 3: COUNT(*) --- + qt_count """ + select count(*) as cnt from local( + "file_path" = "${single_path}", + "backend_id" = "${be_id}", + "format" = "lance" + ) + """ + + // --- Test 4: WHERE filter --- + order_qt_filter """ + select * from local( + "file_path" = "${single_path}", + "backend_id" = "${be_id}", + "format" = "lance" + ) where score > 88 order by id + """ + + // --- Test 5: LIMIT --- + qt_limit """ + select * from local( + "file_path" = "${single_path}", + "backend_id" = "${be_id}", + "format" = "lance" + ) order by id limit 2 + """ + + // --- Test 6: Multi-fragment COUNT (3 fragments, 15 rows total, no duplicates) --- + qt_multi_count """ + select count(*) as cnt, min(id) as min_id, max(id) as max_id from local( + "file_path" = "${multi_path}", + "backend_id" = "${be_id}", + "format" = "lance" + ) + """ + + // --- Test 7: Multi-fragment with WHERE --- + order_qt_multi_filter """ + select * from local( + "file_path" = "${multi_path}", + "backend_id" = "${be_id}", + "format" = "lance" + ) where id > 10 order by id + """ + + // --- Test 8: Multi-fragment aggregation --- + qt_multi_agg """ + select sum(value) as total, avg(value) as avg_val from local( + "file_path" = "${multi_path}", + "backend_id" = "${be_id}", + "format" = "lance" + ) + """ + + // --- Test 9: Error case - nonexistent path --- + test { + sql """ + select * from local( + "file_path" = "nonexistent/path.lance", + "backend_id" = "${be_id}", + "format" = "lance" + ) + """ + exception "No matches found" + } +}