[test](lance) Add indexed coverage for cosine, dot and non-Float32 element types

.gitignore

@@ -142,6 +142,12 @@ docker/thirdparties/docker-compose/*/*.yaml
 docker/thirdparties/docker-compose/*/*.env
 docker/thirdparties/docker-compose/*/cache/
 docker/thirdparties/docker-compose/*/scripts/SUCCESS
+# Rendered from the .tpl next to them by run-thirdparties-docker.sh, which substitutes
+# CONTAINER_UID; the templates are tracked, the rendered copies are not.
+docker/thirdparties/docker-compose/*/entrypoint.sh
+docker/thirdparties/docker-compose/*/scripts/entrypoint.sh
+# Left behind if lance_build_preinstalled_catalog.py fails while promoting a rebuild.
+docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance.old/
 docker/runtime/be/resource/apache-doris/
 
 # other

docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl

@@ -192,6 +192,13 @@ services:
     volumes:
       - ./data:/mnt/data
       - ./scripts/preinstalled_data/:/mnt/preinstalled_data
+    # The lance fixture is mirrored, not copied: it is a Lance Directory (V2) catalog whose
+    # __manifest version is a commit count, so a rebuild can lower it (it went 35 -> 27 when
+    # the vector table matrix grew). Lance names version files u64::MAX - version so that a
+    # listing returns the newest first, which means a leftover higher-versioned manifest from
+    # an earlier fixture would win over the one just published and point readers at tables
+    # that no longer exist. This bucket is not recreated between runs, so a plain copy would
+    # merge the two catalogs. --remove makes the prefix match the committed fixture exactly.
     entrypoint: >
       /bin/sh -c "
       until (/usr/bin/mc config host add minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
@@ -203,7 +210,7 @@ services:
         /usr/bin/mc policy set public minio/warehouse;
         /usr/bin/mc cp -r /mnt/data/input/minio/warehouse/* minio/warehouse/;
       fi;
-      /usr/bin/mc cp -r /mnt/preinstalled_data/lance/ minio/warehouse/lance/;
+      /usr/bin/mc mirror --overwrite --remove /mnt/preinstalled_data/lance/ minio/warehouse/lance/;
       /usr/bin/mc cp -r /mnt/preinstalled_data/iceberg/ minio/warehouse/wh/multi_catalog/;
       "
 

docker/thirdparties/docker-compose/iceberg/scripts/lance_fixture_requirements.txt

@@ -19,21 +19,38 @@
 #
 # The fixture has several readers and they do not all run the same Lance version:
 #
-#   lance-c v0.1.6 (lance-rs 7.0.0-beta)  BE, both from source and from the prebuilt
-#                                         doris-thirdparty package - thirdparty/vars.sh
-#   lance-java 4.0.0                      Spark, via lance-spark-bundle 0.4.0; it registers
-#                                         runtime tables into the same __manifest
-#   lance-java (FE)                       Doris FE Directory Namespace client
-#
-# So pin the writer to the oldest Lance in that set rather than to whichever one the BE
-# happens to use: an older writer is readable by every reader above, while a newer one
-# would leave the Spark cell unverified. Verified for the committed fixture, which now
-# carries one index per ANN algorithm: pylance 7.0.0 - the same Lance generation the BE
-# reads through lance-c v0.1.6 - reproduces every line of the generator self-check that
-# pylance 4.0.1 produces, for all six tables. Same index type and coverage, same exact and
-# refined top-k, same nprobes=1 boundary rows, same IVF partition ranges, same ef=5 vs
-# ef=50 graph rows. The regression goldens therefore do not depend on which Lance version
-# reads the fixture, only on the frozen bytes this writer produced.
-pylance==4.0.1
-lance-namespace==0.6.1
+#   lance-c v0.1.6 (lance-rs 7.0.0-beta.7)  BE, both from source and from the prebuilt
+#                                           doris-thirdparty package - thirdparty/vars.sh
+#   lance-java 4.0.0                        Spark, via lance-spark-bundle 0.4.0; it
+#                                           registers runtime tables into the same __manifest
+#   lance-java 9.1.0-beta.3                 Doris FE Directory Namespace client - fe/pom.xml
+#
+# The writer is pinned to the Lance generation the BE actually reads. It used to be pinned
+# to the oldest reader instead, on the theory that an older writer is readable by every
+# reader above; that stopped being possible once the fixture had to carry non-Float32
+# indexes. Measured on pylance 4.0.1 (Lance 4): IVF_HNSW_FLAT + float64 and
+# IVF_HNSW_FLAT + uint8/hamming panic during index creation, and IVF_FLAT + float64 builds
+# but every search against it fails with "Task was aborted". All of those work on Lance 7.
+#
+# So Spark/lance-java 4.0.0 compatibility with this fixture is no longer something the pin
+# guarantees; it is something the regression run verifies. external_table_p0/lance covers
+# it: the Spark and REST catalog suites read and register into the same __manifest this
+# writer produces, and they must pass for a fixture rebuild to be acceptable.
+#
+# The pylance and lance-namespace pins are enforced rather than advisory:
+# check_pinned_writer() refuses to build or verify the fixture unless the installed versions
+# match them exactly. That check has to exist because the generator's own self-check cannot
+# catch a wrong writer - it reads the fixture back with the very Lance that produced it, so
+# it would pass by construction while emitting something the BE's lance-c or Spark's
+# lance-java cannot open. Those two decide what lands on disk: pylance writes the data and
+# index files, lance-namespace writes the __manifest and picks the hash-prefixed directory
+# names.
+#
+# pyarrow is pinned for reproducibility but deliberately NOT asserted. It only supplies the
+# in-memory Arrow buffers handed to Lance, and whether it changed a stored value is checked
+# directly - check_data_shapes pins a digest of every vector, and check_vector_dataset reads
+# rows back and compares them through the element type. Verifying the bytes beats pinning a
+# proxy for them.
+pylance==7.0.0
+lance-namespace==0.7.7
 pyarrow==25.0.0

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/05008fb7_doris$vs_ivf_hnsw_sq_f32/_versions/latest_version_hint.json

@@ -0,0 +1 @@
+{"version":3}

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/06352138_doris$vs_ivf_pq_f32_cosine/_versions/latest_version_hint.json

@@ -0,0 +1 @@
+{"version":3}

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/198a557b_doris$vs_index_matrix/_versions/latest_version_hint.json

@@ -0,0 +1 @@
+{"version":46}

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/3c75792e_doris$vs_ivf_flat_u8/_versions/latest_version_hint.json

@@ -0,0 +1 @@
+{"version":3}

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/4197a0e9_doris$vs_ivf_flat_f32_cosine/_versions/latest_version_hint.json

@@ -0,0 +1 @@
+{"version":3}

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/482a6218_doris$nested_index/_transactions/3-a6deb520-c206-4777-81d2-da8d8421d2b9.txn

@@ -0,0 +1 @@
+$a6deb520-c206-4777-81d2-da8d8421d2b9Ú

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/482a6218_doris$nested_index/_versions/latest_version_hint.json

@@ -0,0 +1 @@
+{"version":5}

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/57b2c9e6_doris$vs_ivf_flat_f32/_versions/latest_version_hint.json

@@ -0,0 +1 @@
+{"version":3}

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/75d47c69_doris$vs_ivf_hnsw_flat_f32/_versions/latest_version_hint.json

@@ -0,0 +1 @@
+{"version":3}

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/7d876d35_doris$vs_ivf_flat_f64/_versions/latest_version_hint.json

@@ -0,0 +1 @@
+{"version":3}

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/831a8785_doris$vs_ivf_pq_f32_dot/_versions/latest_version_hint.json

@@ -0,0 +1 @@
+{"version":3}

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/8be5010b_doris$vs_ivf_hnsw_pq_f32/_versions/latest_version_hint.json

@@ -0,0 +1 @@
+{"version":3}

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/latest_version_hint.json

@@ -1 +1 @@
-{"version":40}
+{"version":29}

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/b3c9ce40_doris$vs_ivf_pq_f32/_versions/latest_version_hint.json

@@ -0,0 +1 @@
+{"version":3}