[feature](lance) Add Rust-based Lance format reader for AI-native dat…

be/CMakeLists.txt

@@ -165,6 +165,27 @@ if (DEFINED ENV{PAIMON_HOME} AND NOT PAIMON_HOME)
     set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "" FORCE)
 endif()
 
+option(BUILD_RUST_READERS "Build Rust-based format readers (Lance, etc.)" OFF)
+if (DEFINED ENV{BUILD_RUST_READERS})
+    set(BUILD_RUST_READERS "$ENV{BUILD_RUST_READERS}" CACHE BOOL "" FORCE)
+endif()
+# Auto-enable if pre-built Rust library exists (from zigbuild or manylinux2014)
+if (NOT BUILD_RUST_READERS)
+    if (EXISTS "${SRC_DIR}/rust/doris-native/target/x86_64-unknown-linux-gnu/release/libdoris_ffi.a"
+        OR EXISTS "${SRC_DIR}/rust/doris-native/target/release/libdoris_ffi.a")
+        set(BUILD_RUST_READERS ON)
+        message(STATUS "Auto-enabling BUILD_RUST_READERS: pre-built library found")
+    endif()
+endif()
+if (BUILD_RUST_READERS)
+    # rust.cmake detects pre-built .a or builds via Corrosion.
+    # If neither exists, it sets BUILD_RUST_READERS=OFF and returns.
+    include(cmake/rust.cmake)
+endif()
+if (BUILD_RUST_READERS)
+    add_definitions(-DBUILD_RUST_READERS)
+endif()
+
 set(CMAKE_SKIP_RPATH TRUE)
 set(Boost_USE_STATIC_LIBS ON)
 set(Boost_USE_STATIC_RUNTIME ON)
@@ -686,6 +707,10 @@ if (ENABLE_PAIMON_CPP)
     set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} tbb_paimon)
 endif()
 
+if (BUILD_RUST_READERS)
+    set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} doris_ffi_lib)
+endif()
+
 set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_END_GROUP})
 
 # Add all external dependencies. They should come after the palo libs.

be/cmake/rust.cmake

@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Rust integration for doris-ffi static library.
+#
+# Two modes:
+#   1. Pre-built: If libdoris_ffi.a already exists (e.g., built by cargo-zigbuild
+#      on the CI host), use it directly. No Rust toolchain needed inside the
+#      build container.
+#   2. Corrosion: If no pre-built .a found and cargo is available, build via
+#      Corrosion (FetchContent). Used on developer machines.
+
+# Check for pre-built .a (from zigbuild, manylinux2014, or manual build)
+set(PREBUILT_RUST_PATHS
+    "${SRC_DIR}/rust/doris-native/target/x86_64-unknown-linux-gnu/release/libdoris_ffi.a"
+    "${SRC_DIR}/rust/doris-native/target/release/libdoris_ffi.a"
+    "${CMAKE_BINARY_DIR}/libdoris_ffi.a"
+)
+
+set(RUST_LIB_PATH "")
+foreach(p ${PREBUILT_RUST_PATHS})
+    if (EXISTS "${p}")
+        set(RUST_LIB_PATH "${p}")
+        message(STATUS "Rust readers: using pre-built library at ${p}")
+        break()
+    endif()
+endforeach()
+
+if (RUST_LIB_PATH)
+    # Mode 1: Pre-built .a found — no Corrosion needed
+    add_library(doris_ffi_lib STATIC IMPORTED GLOBAL)
+    set_target_properties(doris_ffi_lib PROPERTIES
+        IMPORTED_LOCATION "${RUST_LIB_PATH}"
+        IMPORTED_LINK_INTERFACE_LIBRARIES "m;dl;pthread"
+    )
+    message(STATUS "Rust readers enabled (pre-built)")
+else()
+    # Mode 2: Build via Corrosion (developer machines with cargo)
+    find_program(CARGO_EXECUTABLE cargo)
+    if (NOT CARGO_EXECUTABLE)
+        message(WARNING "BUILD_RUST_READERS=ON but no pre-built libdoris_ffi.a and no cargo in PATH. Disabling.")
+        set(BUILD_RUST_READERS OFF PARENT_SCOPE)
+        return()
+    endif()
+
+    include(FetchContent)
+    FetchContent_Declare(
+        Corrosion
+        GIT_REPOSITORY https://github.com/corrosion-rs/corrosion.git
+        GIT_TAG v0.5.1
+    )
+    FetchContent_MakeAvailable(Corrosion)
+
+    set(RUST_MANIFEST_PATH "${SRC_DIR}/rust/doris-native/Cargo.toml")
+    corrosion_import_crate(
+        MANIFEST_PATH ${RUST_MANIFEST_PATH}
+        CRATES doris-ffi
+    )
+
+    add_library(doris_ffi_lib STATIC IMPORTED GLOBAL)
+    set_target_properties(doris_ffi_lib PROPERTIES
+        IMPORTED_LOCATION "${CMAKE_BINARY_DIR}/libdoris_ffi.a"
+        IMPORTED_LINK_INTERFACE_LIBRARIES "m;dl;pthread"
+    )
+    add_dependencies(doris_ffi_lib cargo-build_doris_ffi)
+    message(STATUS "Rust readers enabled (Corrosion)")
+endif()

be/src/exec/scan/file_scanner.cpp

@@ -79,6 +79,9 @@
 #include "format/table/transactional_hive_reader.h"
 #include "format/table/trino_connector_jni_reader.h"
 #include "format/text/text_reader.h"
+#ifdef BUILD_RUST_READERS
+#include "format/lance/lance_rust_reader.h"
+#endif
 #include "io/cache/block_file_cache_profile.h"
 #include "load/group_commit/wal/wal_reader.h"
 #include "runtime/descriptors.h"
@@ -1140,6 +1143,16 @@ Status FileScanner::_get_next_reader() {
             }
             break;
         }
+#ifdef BUILD_RUST_READERS
+        case TFileFormatType::FORMAT_LANCE: {
+            auto lance_reader = LanceRustReader::create_unique(_file_slot_descs, _state, _profile,
+                                                               range, _params);
+            init_status = lance_reader->init_reader();
+            _cur_reader = std::move(lance_reader);
+            need_to_get_parsed_schema = true;
+            break;
+        }
+#endif
         default:
             return Status::NotSupported("Not supported create reader for file format: {}.",
                                         to_string(_params->format_type));

be/src/format/CMakeLists.txt

@@ -22,6 +22,13 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/format")
 set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/format")
 
 file(GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS *.cpp)
+
+# Lance reader requires Rust static library (BUILD_RUST_READERS=ON)
+if (NOT BUILD_RUST_READERS)
+    file(GLOB_RECURSE LANCE_FILES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/lance/*.cpp)
+    list(REMOVE_ITEM SRC_FILES ${LANCE_FILES})
+endif()
+
 add_library(Format STATIC ${SRC_FILES})
 
 pch_reuse(Format)

be/src/format/lance/lance_ffi.h

@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#ifdef BUILD_RUST_READERS
+
+#include <arrow/c/abi.h>
+
+#include <cstddef>
+#include <cstdint>
+
+namespace doris::lance_ffi {
+
+// FFI status codes (must match Rust error.rs)
+constexpr int32_t LANCE_FFI_OK = 0;
+constexpr int32_t LANCE_FFI_EOF = 1;
+constexpr int32_t LANCE_FFI_ERR_LANCE = -1;
+constexpr int32_t LANCE_FFI_ERR_ARROW = -2;
+constexpr int32_t LANCE_FFI_ERR_IO = -3;
+constexpr int32_t LANCE_FFI_ERR_PANIC = -4;
+constexpr int32_t LANCE_FFI_ERR_INVALID_ARG = -5;
+
+} // namespace doris::lance_ffi
+
+// Opaque handle to a Rust LanceReader.
+using LanceReaderHandle = void*;
+
+extern "C" {
+
+/// Open a Lance dataset and create a reader.
+int32_t lance_reader_open(const uint8_t* uri_ptr, size_t uri_len,
+                          const uint8_t* const* column_names_ptr,
+                          const size_t* column_names_len_ptr, size_t num_columns, size_t batch_size,
+                          LanceReaderHandle* handle_out);
+
+/// Read the next batch via Arrow C Data Interface.
+/// Returns LANCE_FFI_OK with data, LANCE_FFI_EOF on end, negative on error.
+int32_t lance_reader_next_batch(LanceReaderHandle handle, ArrowSchema* schema_out,
+                                ArrowArray* array_out, bool* eof_out, int64_t* bytes_out);
+
+/// Get the schema of the scan output.
+int32_t lance_reader_get_schema(LanceReaderHandle handle, ArrowSchema* schema_out);
+
+/// Close the reader and free resources. Safe to call with null handle.
+void lance_reader_close(LanceReaderHandle handle);
+
+/// Retrieve the last error message. Returns bytes written (excluding null terminator).
+size_t lance_reader_last_error(uint8_t* buf, size_t buf_len);
+
+/// Open a Lance dataset from a JSON config string.
+/// Config JSON: {"uri":"...", "columns":[], "batch_size":N, "version":N, "storage_options":{}}
+/// storage_options carries S3 credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, etc.)
+int32_t lance_reader_open_json(const uint8_t* config_json_ptr, size_t config_json_len,
+                               LanceReaderHandle* handle_out);
+
+/// Phase 0 echo function for build verification.
+int32_t rust_echo(int32_t x);
+
+/// Create a test Lance dataset at the given path. For GTests only.
+/// Dataset has 5 rows: id(INT32), name(UTF8), score(FLOAT64).
+int32_t lance_test_create_dataset(const uint8_t* path_ptr, size_t path_len);
+
+/// Create a multi-fragment test dataset. 3 fragments, 5 rows each = 15 total.
+/// Columns: id(INT32), name(UTF8), value(FLOAT64).
+int32_t lance_test_create_multi_fragment_dataset(const uint8_t* path_ptr, size_t path_len);
+
+} // extern "C"
+
+#endif // BUILD_RUST_READERS